2022-03-08 02:19:18 +00:00
|
|
|
#!/usr/bin/env python
|
|
|
|
|
|
|
|
import torch
|
|
|
|
import torch.nn as nn
|
2022-11-29 05:42:06 +00:00
|
|
|
|
2023-09-11 08:24:28 +00:00
|
|
|
from colossalai.legacy.nn import CheckpointModule
|
2022-11-29 05:42:06 +00:00
|
|
|
|
2022-03-08 02:19:18 +00:00
|
|
|
from .registry import non_distributed_component_funcs
|
2022-11-29 05:42:06 +00:00
|
|
|
from .utils.dummy_data_generator import DummyDataGenerator
|
2022-03-08 02:19:18 +00:00
|
|
|
|
|
|
|
|
|
|
|
class NetWithRepeatedlyComputedLayers(CheckpointModule):
|
|
|
|
"""
|
|
|
|
This model is to test with layers which go through forward pass multiple times.
|
|
|
|
In this model, the fc1 and fc2 call forward twice
|
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(self, checkpoint=False) -> None:
|
|
|
|
super().__init__(checkpoint=checkpoint)
|
|
|
|
self.fc1 = nn.Linear(5, 5)
|
|
|
|
self.fc2 = nn.Linear(5, 5)
|
|
|
|
self.fc3 = nn.Linear(5, 2)
|
|
|
|
self.layers = [self.fc1, self.fc2, self.fc1, self.fc2, self.fc3]
|
|
|
|
|
|
|
|
def forward(self, x):
|
|
|
|
for layer in self.layers:
|
|
|
|
x = layer(x)
|
|
|
|
return x
|
|
|
|
|
|
|
|
|
|
|
|
class DummyDataLoader(DummyDataGenerator):
|
|
|
|
|
|
|
|
def generate(self):
|
|
|
|
data = torch.rand(16, 5)
|
|
|
|
label = torch.randint(low=0, high=2, size=(16,))
|
|
|
|
return data, label
|
|
|
|
|
|
|
|
|
|
|
|
@non_distributed_component_funcs.register(name='repeated_computed_layers')
|
|
|
|
def get_training_components():
|
2022-03-08 06:45:01 +00:00
|
|
|
|
2022-11-29 05:42:06 +00:00
|
|
|
def model_builder(checkpoint=False):
|
2022-03-08 06:45:01 +00:00
|
|
|
return NetWithRepeatedlyComputedLayers(checkpoint)
|
|
|
|
|
2022-03-08 02:19:18 +00:00
|
|
|
trainloader = DummyDataLoader()
|
|
|
|
testloader = DummyDataLoader()
|
2022-03-08 06:45:01 +00:00
|
|
|
|
2022-03-08 02:19:18 +00:00
|
|
|
criterion = torch.nn.CrossEntropyLoss()
|
2022-03-14 12:48:41 +00:00
|
|
|
return model_builder, trainloader, testloader, torch.optim.Adam, criterion
|