cifar10 exmaple change model to Resnet, Evaluate accuracy is very low.
shou123 opened this issue · 0 comments
❓ Questions and Help
Until we move the questions to another medium, feel free to use this to submit your question:
Question
I run the FLsim cifar10 example, change the model from simple CNN to Resnet, and run the same dataset. The evaluate accuracy is very low. The report as:
Train finished Global Round: 2
(round = 2, epoch = 1, global round = 2), Loss/Training: 1.919056011840796
(round = 2, epoch = 1, global round = 2), Accuracy/Training: 29.19
(round = 2, epoch = 1, global round = 2), Loss/Aggregation: 2.3162186018220936
(round = 2, epoch = 1, global round = 2), Accuracy/Aggregation: 12.362
(round = 2, epoch = 1, global round = 2): Evaluates global model on all data of eval users
(round = 2, epoch = 1, global round = 2), Loss/Eval: 2.315063210050012
(round = 2, epoch = 1, global round = 2), Accuracy/Eval: 12.2
Current eval accuracy: {'Accuracy': 12.2}%, Best so far: {'Accuracy': 10.01}%
IMAGE_SIZE = 32
def build_data_provider(local_batch_size, examples_per_user, drop_last: bool = False):
#============================================iid===============================================================
transform = transforms.Compose(
[
transforms.Resize(IMAGE_SIZE),
transforms.CenterCrop(IMAGE_SIZE),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
]
)
train_dataset = CIFAR10(
root="/home/shiyue/FLsim/cifar10", train=True, download=True, transform=transform
)
test_dataset = CIFAR10(
root="/home/shiyue/FLsim/cifar10", train=False, download=True, transform=transform
)
sharder = SequentialSharder(examples_per_shard=examples_per_user)
fl_data_loader = DataLoader(train_dataset, test_dataset, test_dataset, sharder, local_batch_size, drop_last)
data_provider = DataProvider(fl_data_loader)
return data_provider
def main(
trainer_config,
data_config,
use_cuda_if_available: bool = True,
) -> None:
cuda_enabled = torch.cuda.is_available() and use_cuda_if_available
device = torch.device(f"cuda:{0}" if cuda_enabled else "cpu")
model = torch.hub.load('pytorch/vision:v0.10.0', 'resnet18', pretrained=False)
# pyre-fixme[6]: Expected `Optional[str]` for 2nd param but got `device`.
global_model = FLModel(model, device)
if cuda_enabled:
global_model.fl_cuda()
trainer = instantiate(trainer_config, model=global_model, cuda_enabled=cuda_enabled)
data_provider = build_data_provider(
local_batch_size=data_config.local_batch_size,
examples_per_user=data_config.examples_per_user,
# examples_per_user = trainer_config.users_per_round,
drop_last=False,
)
metrics_reporter = MetricsReporter([Channel.TENSORBOARD, Channel.STDOUT])
final_model, eval_score = trainer.train(
data_provider=data_provider,
metrics_reporter=metrics_reporter,
num_total_users=data_provider.num_train_users(),
distributed_world_size=1,
)
trainer.test(
data_provider=data_provider,
metrics_reporter=MetricsReporter([Channel.STDOUT]),
)
@hydra.main(config_path=None, config_name="cifar10_tutorial")
def run(cfg: DictConfig) -> None:
print(OmegaConf.to_yaml(cfg))
trainer_config = cfg.trainer
data_config = cfg.data
main(
trainer_config,
data_config,
)
if __name__ == "__main__":
cfg = maybe_parse_json_config()
run(cfg)