diff --git a/docs/Femnist_stats.md b/docs/Femnist_stats.md index 475055fc..5b84b93b 100644 --- a/docs/Femnist_stats.md +++ b/docs/Femnist_stats.md @@ -15,10 +15,10 @@ import matplotlib.pyplot as plt import numpy as np # Make sure you have downloaded and installed FedScale +import fedscale.core.config_parser as parser from fedscale.dataloaders.femnist import FEMNIST from fedscale.dataloaders.utils_data import get_data_transform from fedscale.dataloaders.divide_data import DataPartitioner -from fedscale.core.config_parser import args ``` @@ -33,10 +33,10 @@ test_dataset = FEMNIST('./benchmark/dataset/data/femnist', dataset='test', trans Partition the dataset by the `client_data_mapping` file, which gives the real-world client-level heterogeneity. ```{code-cell} -args.task = 'cv' -training_sets = DataPartitioner(data=train_dataset, args=args, numOfClass=62) +parser.args.task = 'cv' +training_sets = DataPartitioner(data=train_dataset, args=parser.args, numOfClass=62) training_sets.partition_data_helper(num_clients=None, data_map_file='./benchmark/dataset/data/femnist/client_data_mapping/train.csv') -#testing_sets = DataPartitioner(data=test_dataset, args=args, numOfClass=62, isTest=True) +#testing_sets = DataPartitioner(data=test_dataset, args=parser.args, numOfClass=62, isTest=True) #testing_sets.partition_data_helper(num_clients=None, data_map_file='./benchmark/dataset/data/femnist/client_data_mapping/train.csv') ``` @@ -76,7 +76,7 @@ rank=1 isTest = False dropLast = True partition = training_sets.use(rank - 1, isTest) -num_loaders = min(int(len(partition)/ args.batch_size/2), args.num_loaders) +num_loaders = min(int(len(partition)/ parser.args.batch_size/2), parser.args.num_loaders) dataloader = DataLoader(partition, batch_size=16, shuffle=True, pin_memory=True, timeout=60, num_workers=num_loaders, drop_last=dropLast) ``` diff --git a/examples/heterofl/customized_fllibs.py b/examples/heterofl/customized_fllibs.py index 920642a6..f538aa19 100644 --- a/examples/heterofl/customized_fllibs.py +++ b/examples/heterofl/customized_fllibs.py @@ -9,14 +9,14 @@ import torch.nn.functional as F from resnet_heterofl import resnet18 -from fedscale.core.config_parser import args +import fedscale.core.config_parser as parser def init_model(): global tokenizer logging.info("Initializing the model ...") - if args.model == 'resnet_heterofl': + if parser.args.model == 'resnet_heterofl': model = resnet18() return model diff --git a/examples/notebook/fedscale_demo_client.ipynb b/examples/notebook/fedscale_demo_client.ipynb index 7b330c60..08916b10 100644 --- a/examples/notebook/fedscale_demo_client.ipynb +++ b/examples/notebook/fedscale_demo_client.ipynb @@ -26,9 +26,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "\n", - "(03-05) 19:51:46 INFO [executor.py:50] (EXECUTOR:1) is setting up environ ...\n", - "(03-05) 19:51:46 INFO [fllibs.py:84] Initializing the model ...\n" + "(10-05) 10:56:27 INFO [executor.py:61] (EXECUTOR:1) is setting up environ ...\n" ] }, { @@ -43,743 +41,174 @@ "name": "stderr", "output_type": "stream", "text": [ - "(03-05) 19:51:48 INFO [executor.py:119] Data partitioner starts ...\n", - "(03-05) 19:51:48 INFO [divide_data.py:94] Randomly partitioning data, 50000 samples...\n", - "(03-05) 19:51:48 INFO [divide_data.py:94] Randomly partitioning data, 10000 samples...\n", - "(03-05) 19:51:48 INFO [executor.py:127] Data partitioner completes ...\n", - "(03-05) 19:51:48 INFO [executor.py:87] Connecting to Coordinator (127.0.0.1) for control plane communication ...\n", - "(03-05) 19:51:48 INFO [executor.py:100] Started GRPC server at [::]:10002 for control plane\n", - "(03-05) 19:51:48 INFO [executor.py:302] Start monitoring events ...\n", - "(03-05) 19:52:02 INFO [executor.py:196] Received GRPC ReportExecutorInfo request\n", - "(03-05) 19:52:03 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:52:03 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:03 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:52:06 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 6.166736531928269, 'trained_size': 600, 'success': True, 'utility': 14002.723716596103}\n", - "(03-05) 19:52:06 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:06 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:52:08 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 8.450328796211208, 'trained_size': 600, 'success': True, 'utility': 27784.575430198744}\n", - "(03-05) 19:52:08 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:08 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:52:12 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 5.541807488564527, 'trained_size': 600, 'success': True, 'utility': 2746.5351530115886}\n", - "(03-05) 19:52:12 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:12 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:52:16 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 5.133763529590654, 'trained_size': 600, 'success': True, 'utility': 2885.5709383391113}\n", - "(03-05) 19:52:16 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:52:16 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:17 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:52:20 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 4.097920617766082, 'trained_size': 600, 'success': True, 'utility': 1503.5819372413082}\n", - "(03-05) 19:52:20 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:20 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:52:24 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 5.534401371750412, 'trained_size': 600, 'success': True, 'utility': 11159.077982035868}\n", - "(03-05) 19:52:24 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:24 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:52:28 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 5.330126642058009, 'trained_size': 600, 'success': True, 'utility': 6170.8408601980755}\n", - "(03-05) 19:52:28 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:28 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:52:32 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 7.039049797966295, 'trained_size': 600, 'success': True, 'utility': 29348.86824140578}\n", - "(03-05) 19:52:33 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:52:33 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:33 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:52:37 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 5.081957636442971, 'trained_size': 600, 'success': True, 'utility': 1992.5550867611482}\n", - "(03-05) 19:52:37 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:37 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:52:41 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 6.0582187644090615, 'trained_size': 600, 'success': True, 'utility': 4488.897376691365}\n", - "(03-05) 19:52:41 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:41 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:52:45 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 7.132398456906873, 'trained_size': 600, 'success': True, 'utility': 3649.875449565315}\n", - "(03-05) 19:52:45 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:45 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:52:49 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 4.333043703100524, 'trained_size': 600, 'success': True, 'utility': 2171.3751901872533}\n", - "(03-05) 19:52:50 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:52:50 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:50 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:52:54 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 4.7663854069129075, 'trained_size': 600, 'success': True, 'utility': 2540.748670849144}\n", - "(03-05) 19:52:54 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:54 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:52:58 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 3.698958115841707, 'trained_size': 600, 'success': True, 'utility': 2274.132320557489}\n", - "(03-05) 19:52:58 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:52:58 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:53:02 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 5.585061749169056, 'trained_size': 600, 'success': True, 'utility': 5945.434712182724}\n", - "(03-05) 19:53:02 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:02 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:53:06 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 4.955509286161629, 'trained_size': 600, 'success': True, 'utility': 5305.577428286064}\n", - "(03-05) 19:53:06 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:53:07 INFO [executor.py:205] Received GRPC Test request\n", - "(03-05) 19:53:11 INFO [utils_model.py:365] Rank 1: Test set: Average loss: 2.7138, Top-1 Accuracy: 1000.0/10000 (0.1), Top-5 Accuracy: 0.5\n", - "(03-05) 19:53:11 INFO [executor.py:293] After aggregation epoch 5, CumulTime 84.8237, eval_time 4.5167, test_loss 2.7138, test_accuracy 10.00%, test_5_accuracy 50.00% \n", - "\n", - "(03-05) 19:53:11 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:11 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:53:15 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 3.2707472533574076, 'trained_size': 600, 'success': True, 'utility': 2227.6138230670535}\n", - "(03-05) 19:53:15 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:15 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:53:19 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 3.3100625099085925, 'trained_size': 600, 'success': True, 'utility': 2481.3448414820673}\n", - "(03-05) 19:53:19 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:19 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:53:23 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 3.401243356481161, 'trained_size': 600, 'success': True, 'utility': 2405.9800404336133}\n", - "(03-05) 19:53:23 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:23 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:53:27 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.862476760428658, 'trained_size': 600, 'success': True, 'utility': 1963.7636954736715}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 19:53:28 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:53:28 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:28 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:53:32 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 3.266853203517368, 'trained_size': 600, 'success': True, 'utility': 1732.60615963363}\n", - "(03-05) 19:53:32 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:32 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:53:36 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.855396778266464, 'trained_size': 600, 'success': True, 'utility': 1653.382626893338}\n", - "(03-05) 19:53:36 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:36 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:53:40 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.831089434529156, 'trained_size': 600, 'success': True, 'utility': 2011.3035998657697}\n", - "(03-05) 19:53:40 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:40 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:53:44 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 3.042358627351506, 'trained_size': 600, 'success': True, 'utility': 1806.6110785048413}\n", - "(03-05) 19:53:44 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:53:44 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:44 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:53:48 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.5548434176129193, 'trained_size': 600, 'success': True, 'utility': 1740.3220917255137}\n", - "(03-05) 19:53:48 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:48 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:53:52 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.537354787879475, 'trained_size': 600, 'success': True, 'utility': 1490.9903545763884}\n", - "(03-05) 19:53:52 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:52 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:53:56 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.7060836902554817, 'trained_size': 600, 'success': True, 'utility': 1910.714230959362}\n", - "(03-05) 19:53:56 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:53:56 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:54:00 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.5772996829733055, 'trained_size': 600, 'success': True, 'utility': 1621.9429855720678}\n", - "(03-05) 19:54:01 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:54:01 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:01 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:54:04 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.5003501887888038, 'trained_size': 600, 'success': True, 'utility': 1684.2653642459697}\n", - "(03-05) 19:54:04 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:05 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:54:09 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.335390408034984, 'trained_size': 600, 'success': True, 'utility': 1778.1469914553643}\n", - "(03-05) 19:54:09 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:09 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:54:13 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.497344985942037, 'trained_size': 600, 'success': True, 'utility': 1546.2605788910164}\n", - "(03-05) 19:54:13 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:13 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:54:16 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.5425368854360304, 'trained_size': 600, 'success': True, 'utility': 1660.8008981256883}\n", - "(03-05) 19:54:17 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:54:17 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:17 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:54:21 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.415195213033262, 'trained_size': 600, 'success': True, 'utility': 1786.98568992865}\n", - "(03-05) 19:54:21 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:21 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:54:25 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.4208433446094118, 'trained_size': 600, 'success': True, 'utility': 1353.127631099204}\n", - "(03-05) 19:54:25 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:25 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:54:29 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.418253155498097, 'trained_size': 600, 'success': True, 'utility': 1981.8749601143572}\n", - "(03-05) 19:54:29 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:29 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:54:33 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.340269607021022, 'trained_size': 600, 'success': True, 'utility': 1483.7491945987733}\n", - "(03-05) 19:54:33 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:54:33 INFO [executor.py:205] Received GRPC Test request\n", - "(03-05) 19:54:38 INFO [utils_model.py:365] Rank 1: Test set: Average loss: 2.2961, Top-1 Accuracy: 1501.0/10000 (0.1501), Top-5 Accuracy: 0.6402\n", - "(03-05) 19:54:38 INFO [executor.py:293] After aggregation epoch 10, CumulTime 171.6946, eval_time 4.431, test_loss 2.2961, test_accuracy 15.01%, test_5_accuracy 64.02% \n", - "\n", - "(03-05) 19:54:38 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:38 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:54:42 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.3549062860273127, 'trained_size': 600, 'success': True, 'utility': 1466.2666661366375}\n", - "(03-05) 19:54:42 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:42 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:54:46 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.1950617674504946, 'trained_size': 600, 'success': True, 'utility': 1506.5332875586885}\n", - "(03-05) 19:54:46 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:46 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:54:50 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.3166805121342007, 'trained_size': 600, 'success': True, 'utility': 1325.011609419012}\n", - "(03-05) 19:54:50 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:51 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:54:54 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.4723577792963387, 'trained_size': 600, 'success': True, 'utility': 1776.1354232948072}\n", - "(03-05) 19:54:55 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:54:55 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:55 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:54:59 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.304801041390954, 'trained_size': 600, 'success': True, 'utility': 1535.8823206695893}\n", - "(03-05) 19:54:59 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:54:59 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:55:03 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.3828599152910512, 'trained_size': 600, 'success': True, 'utility': 1514.6184584414445}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 19:55:03 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:03 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:55:07 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.375393155409973, 'trained_size': 600, 'success': True, 'utility': 1456.2818429807087}\n", - "(03-05) 19:55:07 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:07 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:55:11 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.3304627750416005, 'trained_size': 600, 'success': True, 'utility': 1535.5820931533742}\n", - "(03-05) 19:55:11 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:55:11 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:11 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:55:15 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.1908794326446266, 'trained_size': 600, 'success': True, 'utility': 1314.7367253243654}\n", - "(03-05) 19:55:15 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:16 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:55:19 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.2584207735292345, 'trained_size': 600, 'success': True, 'utility': 1508.9038467215157}\n", - "(03-05) 19:55:19 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:19 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:55:23 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.295303288840891, 'trained_size': 600, 'success': True, 'utility': 1482.1924533448782}\n", - "(03-05) 19:55:23 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:23 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:55:27 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.3188039399682707, 'trained_size': 600, 'success': True, 'utility': 1481.1851231294972}\n", - "(03-05) 19:55:28 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:55:28 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:28 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:55:32 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.2343263967437323, 'trained_size': 600, 'success': True, 'utility': 1578.1255704852008}\n", - "(03-05) 19:55:32 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:32 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:55:36 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.2647111955608263, 'trained_size': 600, 'success': True, 'utility': 1668.398073681557}\n", - "(03-05) 19:55:36 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:36 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:55:40 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.250899268712639, 'trained_size': 600, 'success': True, 'utility': 1434.2170489987004}\n", - "(03-05) 19:55:40 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:40 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:55:44 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.385804536986689, 'trained_size': 600, 'success': True, 'utility': 1563.52496627369}\n", - "(03-05) 19:55:44 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:55:44 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:44 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:55:48 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.261032069521521, 'trained_size': 600, 'success': True, 'utility': 1405.5182046006125}\n", - "(03-05) 19:55:48 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:48 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:55:52 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.2476757832233494, 'trained_size': 600, 'success': True, 'utility': 1504.5573392711372}\n", - "(03-05) 19:55:52 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:53 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:55:56 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.264389253164051, 'trained_size': 600, 'success': True, 'utility': 1564.0058241145477}\n", - "(03-05) 19:55:56 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:55:56 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:56:01 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.345817700846023, 'trained_size': 600, 'success': True, 'utility': 1382.9925349358778}\n", - "(03-05) 19:56:01 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:56:01 INFO [executor.py:205] Received GRPC Test request\n", - "(03-05) 19:56:06 INFO [utils_model.py:365] Rank 1: Test set: Average loss: 2.2084, Top-1 Accuracy: 1913.0/10000 (0.1913), Top-5 Accuracy: 0.6651\n", - "(03-05) 19:56:06 INFO [executor.py:293] After aggregation epoch 15, CumulTime 259.4408, eval_time 4.4238, test_loss 2.2084, test_accuracy 19.13%, test_5_accuracy 66.51% \n", - "\n", - "(03-05) 19:56:06 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:06 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:56:10 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.123716077123185, 'trained_size': 600, 'success': True, 'utility': 1430.0789941802589}\n", - "(03-05) 19:56:10 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:10 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:56:14 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.269949128839266, 'trained_size': 600, 'success': True, 'utility': 1572.8569959001973}\n", - "(03-05) 19:56:14 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:14 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:56:18 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.154669296875331, 'trained_size': 600, 'success': True, 'utility': 1420.5006313147305}\n", - "(03-05) 19:56:18 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:18 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:56:22 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.144359698100791, 'trained_size': 600, 'success': True, 'utility': 1365.29748396522}\n", - "(03-05) 19:56:22 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:56:22 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:22 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:56:26 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.102855806035112, 'trained_size': 600, 'success': True, 'utility': 1274.6341017680618}\n", - "(03-05) 19:56:26 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:26 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:56:30 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.394839001783787, 'trained_size': 600, 'success': True, 'utility': 1645.1710697777712}\n", - "(03-05) 19:56:30 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:30 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:56:34 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.21188927581919, 'trained_size': 600, 'success': True, 'utility': 1396.9577181697314}\n", - "(03-05) 19:56:34 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:34 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:56:38 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.107037647722496, 'trained_size': 600, 'success': True, 'utility': 1344.8555227178956}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 19:56:38 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:56:39 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:39 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:56:43 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.1720451101022604, 'trained_size': 600, 'success': True, 'utility': 1380.5815644659626}\n", - "(03-05) 19:56:43 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:43 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:56:47 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.1201380600425055, 'trained_size': 600, 'success': True, 'utility': 1275.2148493159186}\n", - "(03-05) 19:56:47 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:47 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:56:51 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.1821282381523934, 'trained_size': 600, 'success': True, 'utility': 1459.9747272258583}\n", - "(03-05) 19:56:51 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:51 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:56:55 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.1758023567242, 'trained_size': 600, 'success': True, 'utility': 1476.699989569273}\n", - "(03-05) 19:56:55 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:56:55 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:55 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:56:59 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.035437998136092, 'trained_size': 600, 'success': True, 'utility': 1364.2217616224796}\n", - "(03-05) 19:56:59 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:56:59 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:57:03 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.1701135820429296, 'trained_size': 600, 'success': True, 'utility': 1518.8415417186968}\n", - "(03-05) 19:57:03 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:03 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:57:07 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.085135807932731, 'trained_size': 600, 'success': True, 'utility': 1731.0151182853706}\n", - "(03-05) 19:57:07 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:07 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:57:11 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.1527981822556526, 'trained_size': 600, 'success': True, 'utility': 1342.2939543433033}\n", - "(03-05) 19:57:11 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:57:12 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:12 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:57:16 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.014719032433945, 'trained_size': 600, 'success': True, 'utility': 1311.8465221580182}\n", - "(03-05) 19:57:16 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:16 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:57:20 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.281043129862417, 'trained_size': 600, 'success': True, 'utility': 1451.1360896981923}\n", - "(03-05) 19:57:20 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:20 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:57:24 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.0633856769043533, 'trained_size': 600, 'success': True, 'utility': 1368.8653777734285}\n", - "(03-05) 19:57:24 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:24 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:57:28 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.090589558293021, 'trained_size': 600, 'success': True, 'utility': 1230.4730862374379}\n", - "(03-05) 19:57:28 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:57:28 INFO [executor.py:205] Received GRPC Test request\n", - "(03-05) 19:57:32 INFO [utils_model.py:365] Rank 1: Test set: Average loss: 1.9998, Top-1 Accuracy: 2700.0/10000 (0.27), Top-5 Accuracy: 0.794\n", - "(03-05) 19:57:32 INFO [executor.py:293] After aggregation epoch 20, CumulTime 346.0395, eval_time 4.3024, test_loss 1.9998, test_accuracy 27.00%, test_5_accuracy 79.40% \n", - "\n", - "(03-05) 19:57:32 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:32 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:57:36 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.0481677621921603, 'trained_size': 600, 'success': True, 'utility': 1478.4253933249317}\n", - "(03-05) 19:57:36 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:36 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:57:40 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.056033548831132, 'trained_size': 600, 'success': True, 'utility': 1297.3380999680578}\n", - "(03-05) 19:57:40 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:40 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:57:44 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.060891474855394, 'trained_size': 600, 'success': True, 'utility': 1475.9031094150932}\n", - "(03-05) 19:57:44 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:44 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:57:48 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.2158287006742388, 'trained_size': 600, 'success': True, 'utility': 1291.0846382172736}\n", - "(03-05) 19:57:49 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:57:49 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:49 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:57:53 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.09353642927621, 'trained_size': 600, 'success': True, 'utility': 1424.8737624746068}\n", - "(03-05) 19:57:53 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:53 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:57:57 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.1035526290990383, 'trained_size': 600, 'success': True, 'utility': 1378.6350780575794}\n", - "(03-05) 19:57:57 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:57:57 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:58:01 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.1579087784150435, 'trained_size': 600, 'success': True, 'utility': 1396.0633464126092}\n", - "(03-05) 19:58:01 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:01 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:58:05 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.037446342941732, 'trained_size': 600, 'success': True, 'utility': 1154.0421149385427}\n", - "(03-05) 19:58:05 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:58:05 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:05 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:58:09 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.066558621155848, 'trained_size': 600, 'success': True, 'utility': 1338.3511201292906}\n", - "(03-05) 19:58:09 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:09 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:58:13 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.207615832531525, 'trained_size': 600, 'success': True, 'utility': 1602.5983151855885}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 19:58:13 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:13 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:58:17 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.038092442291111, 'trained_size': 600, 'success': True, 'utility': 1289.460686787126}\n", - "(03-05) 19:58:17 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:17 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:58:21 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.0531947550882927, 'trained_size': 600, 'success': True, 'utility': 1361.3531408998242}\n", - "(03-05) 19:58:21 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:58:21 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:21 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:58:25 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.026284462005707, 'trained_size': 600, 'success': True, 'utility': 1246.3758564618656}\n", - "(03-05) 19:58:25 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:25 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:58:29 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.9268038063247972, 'trained_size': 600, 'success': True, 'utility': 1295.8970141129037}\n", - "(03-05) 19:58:29 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:29 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:58:33 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.0349676225029287, 'trained_size': 600, 'success': True, 'utility': 1259.66646952994}\n", - "(03-05) 19:58:33 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:34 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:58:37 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.0335668811397376, 'trained_size': 600, 'success': True, 'utility': 1497.292954195968}\n", - "(03-05) 19:58:37 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:58:38 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:38 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:58:42 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.0359278051240572, 'trained_size': 600, 'success': True, 'utility': 1470.2886276668564}\n", - "(03-05) 19:58:42 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:42 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:58:45 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.109886064693276, 'trained_size': 600, 'success': True, 'utility': 1191.1139724312536}\n", - "(03-05) 19:58:46 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:46 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:58:50 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.0413252151263945, 'trained_size': 600, 'success': True, 'utility': 1399.1919617279573}\n", - "(03-05) 19:58:50 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:50 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:58:54 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.063987534519153, 'trained_size': 600, 'success': True, 'utility': 1280.1896187955108}\n", - "(03-05) 19:58:54 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:58:54 INFO [executor.py:205] Received GRPC Test request\n", - "(03-05) 19:58:59 INFO [utils_model.py:365] Rank 1: Test set: Average loss: 2.0768, Top-1 Accuracy: 2386.0/10000 (0.2386), Top-5 Accuracy: 0.7521\n", - "(03-05) 19:58:59 INFO [executor.py:293] After aggregation epoch 25, CumulTime 432.5197, eval_time 4.4762, test_loss 2.0768, test_accuracy 23.86%, test_5_accuracy 75.21% \n", - "\n", - "(03-05) 19:58:59 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:58:59 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:59:03 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.116516560301071, 'trained_size': 600, 'success': True, 'utility': 1402.2633183446426}\n", - "(03-05) 19:59:03 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:03 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:59:07 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.0701645234837036, 'trained_size': 600, 'success': True, 'utility': 1376.9202554481833}\n", - "(03-05) 19:59:07 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:07 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:59:11 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.0667062290212903, 'trained_size': 600, 'success': True, 'utility': 1439.1957985134122}\n", - "(03-05) 19:59:11 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:11 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:59:14 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.0114117999769174, 'trained_size': 600, 'success': True, 'utility': 1305.351123900438}\n", - "(03-05) 19:59:15 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:59:15 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:15 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:59:19 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.9733778578327215, 'trained_size': 600, 'success': True, 'utility': 1257.6658382659584}\n", - "(03-05) 19:59:19 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:19 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:59:23 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.002598832367345, 'trained_size': 600, 'success': True, 'utility': 1138.6594520139593}\n", - "(03-05) 19:59:23 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:23 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:59:27 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.0041657660853898, 'trained_size': 600, 'success': True, 'utility': 1300.0235158909077}\n", - "(03-05) 19:59:27 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:27 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:59:31 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.2596350499005604, 'trained_size': 600, 'success': True, 'utility': 1528.513945378608}\n", - "(03-05) 19:59:31 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:59:31 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:31 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:59:35 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.9714559314104667, 'trained_size': 600, 'success': True, 'utility': 1264.4251114602284}\n", - "(03-05) 19:59:35 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:35 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:59:39 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.0917116661729107, 'trained_size': 600, 'success': True, 'utility': 1293.9057448495837}\n", - "(03-05) 19:59:39 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:39 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:59:43 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.0267999837076633, 'trained_size': 600, 'success': True, 'utility': 1304.679444949351}\n", - "(03-05) 19:59:43 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:43 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 19:59:47 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.104859545471251, 'trained_size': 600, 'success': True, 'utility': 1515.3129181396878}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 19:59:47 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 19:59:47 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:47 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 19:59:51 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.0207189386044533, 'trained_size': 600, 'success': True, 'utility': 1418.8653047783168}\n", - "(03-05) 19:59:51 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:51 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 19:59:55 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.9922192840576463, 'trained_size': 600, 'success': True, 'utility': 1165.6195607929883}\n", - "(03-05) 19:59:55 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:55 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 19:59:59 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.082599841291858, 'trained_size': 600, 'success': True, 'utility': 1508.5314490511985}\n", - "(03-05) 19:59:59 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 19:59:59 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:00:03 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.978436176537425, 'trained_size': 600, 'success': True, 'utility': 1418.9808203257705}\n", - "(03-05) 20:00:03 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:00:03 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:03 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:00:07 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.0355327106376686, 'trained_size': 600, 'success': True, 'utility': 1486.5715034046652}\n", - "(03-05) 20:00:07 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:08 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:00:12 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.0312636384942397, 'trained_size': 600, 'success': True, 'utility': 1246.4683177021514}\n", - "(03-05) 20:00:12 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:12 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:00:15 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.0613371425112135, 'trained_size': 600, 'success': True, 'utility': 1479.4599265851532}\n", - "(03-05) 20:00:15 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:16 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:00:19 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.0011325796015322, 'trained_size': 600, 'success': True, 'utility': 1289.576032642941}\n", - "(03-05) 20:00:20 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:00:20 INFO [executor.py:205] Received GRPC Test request\n", - "(03-05) 20:00:24 INFO [utils_model.py:365] Rank 1: Test set: Average loss: 1.8732, Top-1 Accuracy: 3208.0/10000 (0.3208), Top-5 Accuracy: 0.8372\n", - "(03-05) 20:00:24 INFO [executor.py:293] After aggregation epoch 30, CumulTime 517.9716, eval_time 4.4725, test_loss 1.8732, test_accuracy 32.08%, test_5_accuracy 83.72% \n", + "(10-05) 10:56:29 INFO [executor.py:117] Data partitioner starts ...\n", + "(10-05) 10:56:29 INFO [divide_data.py:106] Randomly partitioning data, 50000 samples...\n", + "(10-05) 10:56:29 INFO [divide_data.py:106] Randomly partitioning data, 10000 samples...\n", + "(10-05) 10:56:29 INFO [executor.py:128] Data partitioner completes ...\n", + "(10-05) 10:56:29 INFO [channel_context.py:21] %%%%%%%%%% Opening grpc connection to 127.0.0.1 %%%%%%%%%%\n", + "(10-05) 10:56:29 INFO [executor.py:414] Start monitoring events ...\n", + "(10-05) 10:56:31 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 10:56:38 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 6.385307352168408, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:56:39 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 10:56:46 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 5.383798982996671, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:56:46 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 10:56:53 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 5.385428038255249, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:56:53 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 10:57:00 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 5.255024715696292, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:57:02 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 10:57:09 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 4.171052567362976, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:57:09 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 10:57:16 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 5.34112758911072, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:57:17 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 10:57:23 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 6.283451225568975, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:57:24 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 10:57:31 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 5.414917396137178, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:57:33 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 10:57:40 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 4.728196259173867, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:57:40 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 10:57:47 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 4.245132194605622, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:57:47 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 10:57:55 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 4.600091122132928, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:57:55 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 10:58:02 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 4.824014905330292, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:58:04 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 10:58:11 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 4.123896865647862, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:58:11 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 10:58:18 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 4.305955504712801, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:58:18 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 10:58:25 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 5.471528879133904, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:58:25 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 10:58:32 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 3.8462213926046678, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:58:53 INFO [model_test_module.py:306] Rank 1: Test set: Average loss: 2.4887, Top-1 Accuracy: 1000.0/10000 (0.1), Top-5 Accuracy: 0.5142\n", + "(10-05) 10:58:53 INFO [executor.py:376] After aggregation round 5, CumulTime 146.3842, eval_time 18.7353, test_loss 2.4887, test_accuracy 10.00%, test_5_accuracy 51.42% \n", "\n", - "(03-05) 20:00:24 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:24 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:00:28 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.0305383274740967, 'trained_size': 600, 'success': True, 'utility': 1462.0001204328105}\n", - "(03-05) 20:00:28 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:28 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:00:32 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.9336619912446902, 'trained_size': 600, 'success': True, 'utility': 1294.62441163821}\n", - "(03-05) 20:00:32 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:32 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:00:36 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.0401554220008205, 'trained_size': 600, 'success': True, 'utility': 1583.712187130973}\n", - "(03-05) 20:00:36 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:36 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:00:40 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.9479474672323027, 'trained_size': 600, 'success': True, 'utility': 1158.5322834130304}\n", - "(03-05) 20:00:41 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:00:41 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:41 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:00:45 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.9797072131826452, 'trained_size': 600, 'success': True, 'utility': 1235.4327240333362}\n", - "(03-05) 20:00:45 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:45 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:00:49 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.9243440673473557, 'trained_size': 600, 'success': True, 'utility': 1157.828717293125}\n", - "(03-05) 20:00:49 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:49 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:00:53 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.8893330867185034, 'trained_size': 600, 'success': True, 'utility': 1385.401167493123}\n", - "(03-05) 20:00:53 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:53 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:00:57 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.9204015343510246, 'trained_size': 600, 'success': True, 'utility': 1378.105586198975}\n", - "(03-05) 20:00:57 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:00:57 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:00:57 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:01:01 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.0028544047650727, 'trained_size': 600, 'success': True, 'utility': 1269.792534376649}\n", - "(03-05) 20:01:01 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:01 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:01:05 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.9416689845929862, 'trained_size': 600, 'success': True, 'utility': 1410.38517768992}\n", - "(03-05) 20:01:05 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:05 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:01:09 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.8792578827039252, 'trained_size': 600, 'success': True, 'utility': 1254.1446072295475}\n", - "(03-05) 20:01:09 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:09 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:01:13 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.9281330442669924, 'trained_size': 600, 'success': True, 'utility': 1134.995492566882}\n", - "(03-05) 20:01:13 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:01:13 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:13 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:01:17 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.9528769921552596, 'trained_size': 600, 'success': True, 'utility': 1261.4099705353692}\n", - "(03-05) 20:01:17 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:17 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:01:21 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.908636734145627, 'trained_size': 600, 'success': True, 'utility': 1338.0405667005566}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 20:01:21 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:21 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:01:25 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.950787108638224, 'trained_size': 600, 'success': True, 'utility': 1408.0140742954177}\n", - "(03-05) 20:01:25 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:25 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:01:29 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.845970159112002, 'trained_size': 600, 'success': True, 'utility': 1220.194690076604}\n", - "(03-05) 20:01:29 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:01:30 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:30 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:01:33 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.9863053709823113, 'trained_size': 600, 'success': True, 'utility': 1231.6260659080713}\n", - "(03-05) 20:01:34 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:34 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:01:38 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.861819415630533, 'trained_size': 600, 'success': True, 'utility': 1194.2565125975225}\n", - "(03-05) 20:01:38 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:38 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:01:42 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.941379630269865, 'trained_size': 600, 'success': True, 'utility': 1399.2833312306332}\n", - "(03-05) 20:01:42 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:42 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:01:45 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.8634286008164356, 'trained_size': 600, 'success': True, 'utility': 1048.5507242801202}\n", - "(03-05) 20:01:46 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:01:46 INFO [executor.py:205] Received GRPC Test request\n", - "(03-05) 20:01:50 INFO [utils_model.py:365] Rank 1: Test set: Average loss: 1.8117, Top-1 Accuracy: 3444.0/10000 (0.3444), Top-5 Accuracy: 0.8485\n", - "(03-05) 20:01:50 INFO [executor.py:293] After aggregation epoch 35, CumulTime 603.8085, eval_time 4.2813, test_loss 1.8117, test_accuracy 34.44%, test_5_accuracy 84.85% \n", + "(10-05) 10:58:54 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 10:59:01 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 4.542471038460385, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:59:01 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 10:59:08 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 3.3273613208657293, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:59:09 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 10:59:15 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 4.458196635807365, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:59:16 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 10:59:22 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 3.2138776843070707, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:59:25 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 10:59:32 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 3.80448099041252, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:59:32 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 10:59:39 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 3.3631498863039644, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:59:39 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 10:59:46 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 3.59191329329352, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:59:46 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 10:59:53 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.848148092967869, 'trained_size': 600, 'success': True}\n", + "(10-05) 10:59:56 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:00:03 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.872066663595638, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:00:03 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:00:10 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 3.5066577600041056, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:00:10 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:00:17 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.966638272894974, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:00:17 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:00:24 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.885736187690472, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:00:26 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:00:33 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 3.212822436472684, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:00:33 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:00:40 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.6508591739685192, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:00:40 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:00:47 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.887190611917579, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:00:47 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:00:54 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.704264756492174, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:00:57 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:01:04 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 3.152766867797803, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:01:04 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:01:11 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.4352049035450603, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:01:11 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:01:17 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.9751960506664665, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:01:18 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:01:24 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.7658717371408987, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:01:45 INFO [model_test_module.py:306] Rank 1: Test set: Average loss: 2.3363, Top-1 Accuracy: 1766.0/10000 (0.1766), Top-5 Accuracy: 0.6799\n", + "(10-05) 11:01:45 INFO [executor.py:376] After aggregation round 10, CumulTime 318.6031, eval_time 18.3159, test_loss 2.3363, test_accuracy 17.66%, test_5_accuracy 67.99% \n", "\n", - "(03-05) 20:01:50 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:50 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:01:54 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.9646369689717271, 'trained_size': 600, 'success': True, 'utility': 1322.2798583511626}\n", - "(03-05) 20:01:54 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:54 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:01:58 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.8917567231858734, 'trained_size': 600, 'success': True, 'utility': 1103.9877050268901}\n", - "(03-05) 20:01:58 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:01:58 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:02:02 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.8350389869315298, 'trained_size': 600, 'success': True, 'utility': 1188.0746131097214}\n", - "(03-05) 20:02:02 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:02 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:02:06 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.9889167591087267, 'trained_size': 600, 'success': True, 'utility': 1410.8204330119336}\n", - "(03-05) 20:02:06 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:02:06 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:06 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:02:10 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.9780086589109884, 'trained_size': 600, 'success': True, 'utility': 1307.9577249874935}\n", - "(03-05) 20:02:10 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:10 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:02:14 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.8255280024615868, 'trained_size': 600, 'success': True, 'utility': 1255.3792583284048}\n", - "(03-05) 20:02:14 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:14 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:02:18 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.007929334189772, 'trained_size': 600, 'success': True, 'utility': 1270.7919856117635}\n", - "(03-05) 20:02:18 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:18 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:02:22 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.8512500879198506, 'trained_size': 600, 'success': True, 'utility': 1192.4447233426754}\n", - "(03-05) 20:02:22 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:02:22 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:22 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:02:26 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.8468676694631463, 'trained_size': 600, 'success': True, 'utility': 1228.652877445791}\n", - "(03-05) 20:02:26 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:26 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:02:30 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.962533348157884, 'trained_size': 600, 'success': True, 'utility': 1302.59502029458}\n", - "(03-05) 20:02:30 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:30 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:02:34 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.8322774289943549, 'trained_size': 600, 'success': True, 'utility': 1240.5338798649514}\n", - "(03-05) 20:02:34 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:34 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:02:38 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.9053949229812972, 'trained_size': 600, 'success': True, 'utility': 1393.4351700000734}\n", - "(03-05) 20:02:39 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:02:39 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:39 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:02:42 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.8713817782323285, 'trained_size': 600, 'success': True, 'utility': 1177.2978988424259}\n", - "(03-05) 20:02:42 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:43 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:02:46 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.006376325710853, 'trained_size': 600, 'success': True, 'utility': 1504.6742805738331}\n", - "(03-05) 20:02:46 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:46 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:02:50 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.847246472874439, 'trained_size': 600, 'success': True, 'utility': 1153.7598219364834}\n", - "(03-05) 20:02:50 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:51 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:02:55 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.897780200336217, 'trained_size': 600, 'success': True, 'utility': 1268.9127627217445}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 20:02:55 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:02:55 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:55 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:02:59 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.779492240222068, 'trained_size': 600, 'success': True, 'utility': 1088.8340586980605}\n", - "(03-05) 20:02:59 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:02:59 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:03:03 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.927591189541765, 'trained_size': 600, 'success': True, 'utility': 1385.9930124283794}\n", - "(03-05) 20:03:03 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:03 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:03:07 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.823808782631677, 'trained_size': 600, 'success': True, 'utility': 1235.7291308921199}\n", - "(03-05) 20:03:07 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:07 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:03:11 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.8759606059262108, 'trained_size': 600, 'success': True, 'utility': 1250.8558040972227}\n", - "(03-05) 20:03:12 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:03:12 INFO [executor.py:205] Received GRPC Test request\n", - "(03-05) 20:03:16 INFO [utils_model.py:365] Rank 1: Test set: Average loss: 1.7142, Top-1 Accuracy: 3687.0/10000 (0.3687), Top-5 Accuracy: 0.8758\n", - "(03-05) 20:03:16 INFO [executor.py:293] After aggregation epoch 40, CumulTime 689.744, eval_time 4.3356, test_loss 1.7142, test_accuracy 36.87%, test_5_accuracy 87.58% \n", - "\n", - "(03-05) 20:03:16 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:16 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:03:20 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.8480782673562577, 'trained_size': 600, 'success': True, 'utility': 1349.2873695828011}\n", - "(03-05) 20:03:20 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:20 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:03:24 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.7929372761111932, 'trained_size': 600, 'success': True, 'utility': 1192.4202139105294}\n", - "(03-05) 20:03:24 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:24 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:03:28 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.7362577744435619, 'trained_size': 600, 'success': True, 'utility': 1151.6871665446422}\n", - "(03-05) 20:03:28 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:28 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:03:32 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.9925194381759908, 'trained_size': 600, 'success': True, 'utility': 1223.6800289368216}\n", - "(03-05) 20:03:32 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:03:32 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:32 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:03:36 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.830512400462969, 'trained_size': 600, 'success': True, 'utility': 1497.5041506589423}\n", - "(03-05) 20:03:36 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:36 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:03:40 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.8349068294544677, 'trained_size': 600, 'success': True, 'utility': 1400.6255865146293}\n", - "(03-05) 20:03:40 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:40 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:03:44 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.9418907999570807, 'trained_size': 600, 'success': True, 'utility': 1323.0535252059765}\n", - "(03-05) 20:03:44 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:44 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:03:48 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.8719995287896458, 'trained_size': 600, 'success': True, 'utility': 1248.070292408145}\n", - "(03-05) 20:03:48 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:03:48 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:48 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:03:52 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.8989385305273616, 'trained_size': 600, 'success': True, 'utility': 1252.9275700778478}\n", - "(03-05) 20:03:52 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:52 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:03:56 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.775787281583479, 'trained_size': 600, 'success': True, 'utility': 1148.2196153196924}\n", - "(03-05) 20:03:56 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:03:56 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:04:00 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.8291749204061216, 'trained_size': 600, 'success': True, 'utility': 1573.2588188051598}\n", - "(03-05) 20:04:00 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:00 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:04:04 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.817014992881873, 'trained_size': 600, 'success': True, 'utility': 1343.8027938668965}\n", - "(03-05) 20:04:04 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:04:04 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:04 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:04:08 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.7653575749297026, 'trained_size': 600, 'success': True, 'utility': 1071.7437489588135}\n", - "(03-05) 20:04:08 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:08 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:04:12 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.7762739099131988, 'trained_size': 600, 'success': True, 'utility': 1125.4640787204673}\n", - "(03-05) 20:04:12 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:12 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:04:16 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.847365836964642, 'trained_size': 600, 'success': True, 'utility': 1311.266167072203}\n", - "(03-05) 20:04:16 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:16 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:04:20 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.835720740653963, 'trained_size': 600, 'success': True, 'utility': 1205.3348487562137}\n", - "(03-05) 20:04:20 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:04:20 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:20 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:04:24 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.7371126621593311, 'trained_size': 600, 'success': True, 'utility': 1077.966875395407}\n", - "(03-05) 20:04:24 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:24 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:04:28 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.8277168456818187, 'trained_size': 600, 'success': True, 'utility': 980.834935755168}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 20:04:28 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:28 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:04:32 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.7426662545213896, 'trained_size': 600, 'success': True, 'utility': 1190.0903242930522}\n", - "(03-05) 20:04:32 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:32 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:04:36 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.7995705533812911, 'trained_size': 600, 'success': True, 'utility': 1096.5121526956664}\n", - "(03-05) 20:04:36 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:04:36 INFO [executor.py:205] Received GRPC Test request\n", - "(03-05) 20:04:41 INFO [utils_model.py:365] Rank 1: Test set: Average loss: 1.7289, Top-1 Accuracy: 3732.0/10000 (0.3732), Top-5 Accuracy: 0.8726\n", - "(03-05) 20:04:41 INFO [executor.py:293] After aggregation epoch 45, CumulTime 774.456, eval_time 4.3539, test_loss 1.7289, test_accuracy 37.32%, test_5_accuracy 87.26% \n", + "(10-05) 11:01:46 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:01:53 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.471355666196691, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:01:53 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:02:00 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.4356494496458128, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:02:00 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:02:07 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.576381177196222, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:02:07 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:02:15 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.3470000365460093, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:02:17 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:02:24 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.509514877440946, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:02:24 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:02:31 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.2464047509950236, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:02:31 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:02:38 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.4089918757961515, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:02:38 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:02:45 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.35296483668845, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:02:48 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:02:54 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.430718451410085, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:02:55 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:03:01 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.4424235744065412, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:03:02 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:03:09 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.3308809210055044, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:03:09 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:03:16 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.302724640744112, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:03:18 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:03:25 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.5397319132390055, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:03:25 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:03:32 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.447979369095947, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:03:33 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:03:39 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.412356183045489, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:03:39 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:03:46 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.2552800768384884, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:03:49 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:03:56 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.3954310684108044, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:03:56 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:04:02 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.356739756125249, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:04:03 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:04:10 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.3305684278581604, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:04:10 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:04:17 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.2867733053916672, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:04:38 INFO [model_test_module.py:306] Rank 1: Test set: Average loss: 2.2923, Top-1 Accuracy: 2431.0/10000 (0.2431), Top-5 Accuracy: 0.7641\n", + "(10-05) 11:04:38 INFO [executor.py:376] After aggregation round 15, CumulTime 491.1768, eval_time 18.6568, test_loss 2.2923, test_accuracy 24.31%, test_5_accuracy 76.41% \n", "\n", - "(03-05) 20:04:41 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:41 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:04:45 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.8155192238236744, 'trained_size': 600, 'success': True, 'utility': 1223.4512171144706}\n", - "(03-05) 20:04:45 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:45 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:04:49 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.6942830882404383, 'trained_size': 600, 'success': True, 'utility': 1220.0262536952941}\n", - "(03-05) 20:04:49 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:49 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:04:53 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.758519491861119, 'trained_size': 600, 'success': True, 'utility': 1198.2531382610441}\n", - "(03-05) 20:04:53 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:53 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:04:57 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.7202173000410665, 'trained_size': 600, 'success': True, 'utility': 1110.2569567005607}\n", - "(03-05) 20:04:57 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:04:57 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:04:57 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:05:01 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.7759415210484475, 'trained_size': 600, 'success': True, 'utility': 1210.253651626164}\n", - "(03-05) 20:05:01 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:01 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:05:05 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.747010003322145, 'trained_size': 600, 'success': True, 'utility': 1103.5466093429604}\n", - "(03-05) 20:05:05 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:05 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:05:09 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.878420482771366, 'trained_size': 600, 'success': True, 'utility': 1268.764633241904}\n", - "(03-05) 20:05:09 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:09 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:05:13 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.7935136843749835, 'trained_size': 600, 'success': True, 'utility': 1320.5959884231283}\n", - "(03-05) 20:05:14 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:05:14 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:14 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:05:18 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.8927883399908343, 'trained_size': 600, 'success': True, 'utility': 1241.9022626200242}\n", - "(03-05) 20:05:18 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:18 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:05:22 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.7426845588770072, 'trained_size': 600, 'success': True, 'utility': 1203.1526005188289}\n", - "(03-05) 20:05:22 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:22 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:05:25 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.7940250114440794, 'trained_size': 600, 'success': True, 'utility': 1373.3160368345784}\n", - "(03-05) 20:05:25 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:25 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:05:29 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.692213951236378, 'trained_size': 600, 'success': True, 'utility': 1060.3130446814387}\n", - "(03-05) 20:05:30 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:05:30 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:30 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:05:33 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.6825047396255601, 'trained_size': 600, 'success': True, 'utility': 1056.03719964548}\n", - "(03-05) 20:05:33 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:34 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:05:37 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.8255833605761063, 'trained_size': 600, 'success': True, 'utility': 1203.3196667939758}\n", - "(03-05) 20:05:37 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:38 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:05:42 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.8659166042251347, 'trained_size': 600, 'success': True, 'utility': 1360.2930216675932}\n", - "(03-05) 20:05:42 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:42 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:05:45 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.7575865862914926, 'trained_size': 600, 'success': True, 'utility': 1134.9229748027444}\n", - "(03-05) 20:05:46 INFO [executor.py:149] Received GRPC UpdateModel request\n", - "(03-05) 20:05:46 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:46 INFO [client.py:17] Start to train (CLIENT: 1) ...\n", - "(03-05) 20:05:50 INFO [client.py:187] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 1.7481981175983927, 'trained_size': 600, 'success': True, 'utility': 1201.7558842457438}\n", - "(03-05) 20:05:50 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:50 INFO [client.py:17] Start to train (CLIENT: 2) ...\n", - "(03-05) 20:05:54 INFO [client.py:187] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 1.763054232942623, 'trained_size': 600, 'success': True, 'utility': 1303.9145071883454}\n", - "(03-05) 20:05:54 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:54 INFO [client.py:17] Start to train (CLIENT: 3) ...\n", - "(03-05) 20:05:58 INFO [client.py:187] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 1.9435487088980432, 'trained_size': 600, 'success': True, 'utility': 1465.3837427969681}\n", - "(03-05) 20:05:58 INFO [executor.py:167] Received GRPC Train request\n", - "(03-05) 20:05:59 INFO [client.py:17] Start to train (CLIENT: 4) ...\n", - "(03-05) 20:06:02 INFO [client.py:187] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 1.8471025003468946, 'trained_size': 600, 'success': True, 'utility': 1383.5083075734728}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 20:06:03 INFO [executor.py:186] Received GRPC Stop request\n", - "(03-05) 20:06:03 INFO [executor.py:306] Terminating (Executor 1) ...\n" + "(10-05) 11:04:39 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:04:46 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.3175615307159316, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:04:46 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:04:53 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.198226276340273, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:04:53 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:05:00 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.317745547848251, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:05:00 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:05:07 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.2811246385331883, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:05:10 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:05:16 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.200125896722598, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:05:17 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:05:23 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.2880915137374966, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:05:23 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:05:30 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.2059975263981326, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:05:30 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:05:37 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.34091048582892, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:05:40 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:05:46 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.149645458253341, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:05:47 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:05:54 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.113924674635463, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:05:54 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:06:01 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.2351507674495643, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:06:01 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:06:08 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.2538349455169575, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:06:10 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:06:17 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.1228796089115454, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:06:17 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:06:25 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.079776040856923, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:06:25 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:06:31 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.064791808439913, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:06:31 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:06:38 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.107674873224351, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:06:41 INFO [client.py:32] Start to train (CLIENT: 1) ...\n", + "(10-05) 11:06:48 INFO [client.py:67] Training of (CLIENT: 1) completes, {'clientId': 1, 'moving_loss': 2.1673746026503267, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:06:48 INFO [client.py:32] Start to train (CLIENT: 2) ...\n", + "(10-05) 11:06:55 INFO [client.py:67] Training of (CLIENT: 2) completes, {'clientId': 2, 'moving_loss': 2.0539934263054516, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:06:55 INFO [client.py:32] Start to train (CLIENT: 3) ...\n", + "(10-05) 11:07:02 INFO [client.py:67] Training of (CLIENT: 3) completes, {'clientId': 3, 'moving_loss': 2.0785532106098277, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:07:02 INFO [client.py:32] Start to train (CLIENT: 4) ...\n", + "(10-05) 11:07:09 INFO [client.py:67] Training of (CLIENT: 4) completes, {'clientId': 4, 'moving_loss': 2.1002533861294888, 'trained_size': 600, 'success': True}\n", + "(10-05) 11:07:10 INFO [channel_context.py:33] %%%%%%%%%% Closing grpc connection to the aggregator %%%%%%%%%%\n" ] } ], @@ -792,21 +221,21 @@ "\n", "import sys, os\n", "\n", + "import fedscale.core.config_parser as parser\n", "from fedscale.core.execution.client import Client\n", "from fedscale.core.execution.executor import Executor\n", - "from fedscale.core.logger.execution import args\n", "### On CPU\n", - "args.use_cuda = \"False\"\n", - "Demo_Executor = Executor(args)\n", + "parser.args.use_cuda = \"False\"\n", + "Demo_Executor = Executor(parser.args)\n", "Demo_Executor.run()" ] } ], "metadata": { "kernelspec": { - "display_name": "fedscale", + "display_name": "Python 3.7.13 ('fedscale': conda)", "language": "python", - "name": "fedscale" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -818,7 +247,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.7.13" + }, + "vscode": { + "interpreter": { + "hash": "c61f092c149070809c0e629644d57c0873f394886eddd38a7b619aa7d2ef8d75" + } } }, "nbformat": 4, diff --git a/examples/notebook/fedscale_demo_server.ipynb b/examples/notebook/fedscale_demo_server.ipynb index 3890719c..15b206d3 100644 --- a/examples/notebook/fedscale_demo_server.ipynb +++ b/examples/notebook/fedscale_demo_server.ipynb @@ -26,262 +26,299 @@ "name": "stderr", "output_type": "stream", "text": [ - "(03-05) 19:51:42 INFO [aggregator.py:21] Job args Namespace(adam_epsilon=1e-08, backbone='./resnet50.pth', backend='gloo', base_port=10001, batch_size=30, bidirectional=True, blacklist_max_len=0.3, blacklist_rounds=-1, block_size=64, cfg_file='./utils/rcnn/cfgs/res101.yml', clf_block_size=32, clip_bound=0.9, clock_factor=1.1624548736462095, conf_path='~/dataset/', cuda_device=None, cut_off_util=0.05, data_cache='', data_dir='~/cifar10/', data_map_file=None, data_set='cifar10', decay_epoch=10, decay_factor=0.98, device_avail_file=None, device_conf_file='/tmp/client.cfg', dump_epoch=10000000000.0, embedding_file='glove.840B.300d.txt', epochs=50, epsilon=0.9, eval_interval=5, executor_configs='127.0.0.1:[1]', exploration_alpha=0.3, exploration_decay=0.98, exploration_factor=0.9, exploration_min=0.3, filter_less=32, filter_more=1000000000000000.0, finetune=False, gamma=0.9, gradient_policy=None, hidden_layers=7, hidden_size=256, input_dim=0, job_name='demo_job', labels_path='labels.json', learning_rate=0.05, line_by_line=False, local_steps=20, log_path='./', loss_decay=0.2, malicious_factor=1000000000000000.0, manager_port=9005, memory_capacity=2000, min_learning_rate=5e-05, mlm=False, mlm_probability=0.15, model='shufflenet_v2_x2_0', model_size=65536, n_actions=2, n_states=4, noise_dir=None, noise_factor=0.1, noise_max=0.5, noise_min=0.0, noise_prob=0.4, num_class=10, num_classes=35, num_executors=1, num_loaders=2, output_dim=0, overcommitment=1.3, overwrite_cache=False, pacer_delta=5, pacer_step=20, proxy_mu=0.1, ps_ip='127.0.0.1', ps_port='29501', rnn_type='lstm', round_penalty=2.0, round_threshold=30, sample_mode='random', sample_rate=16000, sample_seed=233, sample_window=5.0, spec_augment=False, speed_volume_perturb=False, target_replace_iter=15, task='cv', test_bsz=128, test_manifest='data/test_manifest.csv', test_output_dir='./logs/server', test_ratio=1.0, test_size_file='', this_rank=1, time_stamp='logs', num_participants=4, train_manifest='data/train_manifest.csv', train_size_file='', train_uniform=False, upload_epoch=20, use_cuda=True, vocab_tag_size=500, vocab_token_size=10000, weight_decay=0, window='hamming', window_size=0.02, window_stride=0.01, yogi_beta=0.9, yogi_beta2=0.99, yogi_eta=0.003, yogi_tau=1e-08)\n", - "(03-05) 19:51:42 INFO [aggregator.py:109] Initiating control plane communication ...\n", - "(03-05) 19:51:42 INFO [fllibs.py:84] Initializing the model ...\n", - "(03-05) 19:51:42 INFO [aggregator.py:454] Start monitoring events ...\n", - "(03-05) 19:52:02 INFO [channelcontext.py:38] %%%%%%%%%% Opening grpc connection to 127.0.0.1:10002 %%%%%%%%%%\n", - "(03-05) 19:52:02 INFO [aggregator.py:157] Received executor 1 information, 1/1\n", - "(03-05) 19:52:02 INFO [aggregator.py:165] Loading 4 client traces ...\n", - "(03-05) 19:52:02 INFO [aggregator.py:176] Info of all feasible clients {'total_feasible_clients': 4, 'total_num_samples': 50000}\n", - "(03-05) 19:52:03 INFO [aggregator.py:307] Wall clock: 0 s, Epoch: 1, Planned participants: 0, Succeed participants: 0, Training loss: 0.0\n", - "(03-05) 19:52:03 INFO [client_manager.py:163] Wall clock time: 0, 4 clients online, 0 clients offline\n", - "(03-05) 19:52:03 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:52:03 INFO [aggregator.py:472] Have received all executor information\n", - "(03-05) 19:52:16 INFO [aggregator.py:307] Wall clock: 339811 s, Epoch: 2, Planned participants: 4, Succeed participants: 4, Training loss: 6.323159086573664\n", - "(03-05) 19:52:16 INFO [client_manager.py:163] Wall clock time: 339811, 4 clients online, 0 clients offline\n", - "(03-05) 19:52:16 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:52:33 INFO [aggregator.py:307] Wall clock: 679622 s, Epoch: 3, Planned participants: 4, Succeed participants: 4, Training loss: 5.9117668469794316\n", - "(03-05) 19:52:33 INFO [client_manager.py:163] Wall clock time: 679622, 4 clients online, 0 clients offline\n", - "(03-05) 19:52:33 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:52:50 INFO [aggregator.py:307] Wall clock: 1019433 s, Epoch: 4, Planned participants: 4, Succeed participants: 4, Training loss: 5.8249794447245735\n", - "(03-05) 19:52:50 INFO [client_manager.py:163] Wall clock time: 1019433, 4 clients online, 0 clients offline\n", - "(03-05) 19:52:50 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:53:06 INFO [aggregator.py:307] Wall clock: 1359244 s, Epoch: 5, Planned participants: 4, Succeed participants: 4, Training loss: 5.556604243423761\n", - "(03-05) 19:53:06 INFO [client_manager.py:163] Wall clock time: 1359244, 4 clients online, 0 clients offline\n", - "(03-05) 19:53:06 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:53:11 INFO [aggregator.py:391] FL Testing in epoch: 5, virtual_clock: 1359243.825, top_1: 10.0 %, top_5: 50.0 %, test loss: 2.7138, test len: 10000\n", - "(03-05) 19:53:27 INFO [aggregator.py:307] Wall clock: 1699055 s, Epoch: 6, Planned participants: 4, Succeed participants: 4, Training loss: 5.0875098887478005\n", - "(03-05) 19:53:27 INFO [client_manager.py:163] Wall clock time: 1699055, 4 clients online, 0 clients offline\n", - "(03-05) 19:53:27 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:53:44 INFO [aggregator.py:307] Wall clock: 2038866 s, Epoch: 7, Planned participants: 4, Succeed participants: 4, Training loss: 4.739412325775855\n", - "(03-05) 19:53:44 INFO [client_manager.py:163] Wall clock time: 2038866, 4 clients online, 0 clients offline\n", - "(03-05) 19:53:44 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:54:00 INFO [aggregator.py:307] Wall clock: 2378677 s, Epoch: 8, Planned participants: 4, Succeed participants: 4, Training loss: 4.432909907047917\n", - "(03-05) 19:54:00 INFO [client_manager.py:163] Wall clock time: 2378677, 4 clients online, 0 clients offline\n", - "(03-05) 19:54:00 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:54:17 INFO [aggregator.py:307] Wall clock: 2718488 s, Epoch: 9, Planned participants: 4, Succeed participants: 4, Training loss: 4.187409370798236\n", - "(03-05) 19:54:17 INFO [client_manager.py:163] Wall clock time: 2718488, 4 clients online, 0 clients offline\n", - "(03-05) 19:54:17 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n" + "(10-05) 10:56:16 INFO [aggregator.py:33] Job args Namespace(adam_epsilon=1e-08, arrival_interval=3, async_buffer=10, async_mode=False, backbone='./resnet50.pth', backend='gloo', batch_size=30, bidirectional=True, blacklist_max_len=0.3, blacklist_rounds=-1, block_size=64, cfg_file='./utils/rcnn/cfgs/res101.yml', checkin_period=50, clf_block_size=32, clip_bound=0.9, clip_threshold=3.0, clock_factor=1.1624548736462095, conf_path='~/dataset/', connection_timeout=60, cuda_device=None, cut_off_util=0.05, data_cache='', data_dir='~/cifar10/', data_map_file=None, data_set='cifar10', decay_factor=0.98, decay_round=10, device_avail_file=None, device_conf_file='/tmp/client.cfg', dump_epoch=10000000000.0, embedding_file='glove.840B.300d.txt', engine='pytorch', epsilon=0.9, eval_interval=5, executor_configs='127.0.0.1:[1]', experiment_mode='simulation', exploration_alpha=0.3, exploration_decay=0.98, exploration_factor=0.9, exploration_min=0.3, filter_less=32, filter_more=1000000000000000.0, finetune=False, gamma=0.9, gradient_policy=None, hidden_layers=7, hidden_size=256, input_dim=0, job_name='demo_job', labels_path='labels.json', learning_rate=0.05, line_by_line=False, local_steps=20, log_path='./', loss_decay=0.2, malicious_factor=1000000000000000.0, max_concurrency=100, max_staleness=5, memory_capacity=2000, min_learning_rate=5e-05, mlm=False, mlm_probability=0.15, model='shufflenet_v2_x2_0', model_size=65536, model_zoo='torchcv', n_actions=2, n_states=4, noise_dir=None, noise_factor=0.1, noise_max=0.5, noise_min=0.0, noise_prob=0.4, num_class=10, num_classes=35, num_executors=1, num_loaders=2, num_participants=4, output_dim=0, overcommitment=1.3, overwrite_cache=False, pacer_delta=5, pacer_step=20, proxy_mu=0.1, ps_ip='127.0.0.1', ps_port='29501', rnn_type='lstm', round_penalty=2.0, round_threshold=30, rounds=20, sample_mode='random', sample_rate=16000, sample_seed=233, sample_window=5.0, spec_augment=False, speed_volume_perturb=False, target_delta=0.0001, target_replace_iter=15, task='cv', test_bsz=128, test_manifest='data/test_manifest.csv', test_output_dir='./logs/server', test_ratio=1.0, test_size_file='', this_rank=1, time_stamp='logs', train_manifest='data/train_manifest.csv', train_size_file='', train_uniform=False, upload_step=20, use_cuda=True, vocab_tag_size=500, vocab_token_size=10000, weight_decay=0, window='hamming', window_size=0.02, window_stride=0.01, yogi_beta=0.9, yogi_beta2=0.99, yogi_eta=0.003, yogi_tau=1e-08)\n", + "2022-10-05 10:56:16.673184: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n", + "2022-10-05 10:56:16.673223: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n", + "2022-10-05 10:56:16.732819: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2022-10-05 10:56:17.784849: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n", + "2022-10-05 10:56:17.784957: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n", + "2022-10-05 10:56:17.784973: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n", + "(10-05) 10:56:19 INFO [aggregator.py:129] Initiating control plane communication ...\n", + "(10-05) 10:56:19 INFO [aggregator.py:153] %%%%%%%%%% Opening aggregator sever using port [::]:29501 %%%%%%%%%%\n", + "(10-05) 10:56:19 INFO [fllibs.py:120] Initializing the model ...\n", + "(10-05) 10:56:19 INFO [aggregator.py:871] Start monitoring events ...\n", + "(10-05) 10:56:29 INFO [aggregator.py:263] Received executor 1 information, 1/1\n", + "(10-05) 10:56:29 INFO [aggregator.py:234] Loading 4 client traces ...\n", + "(10-05) 10:56:29 INFO [aggregator.py:251] Info of all feasible clients {'total_feasible_clients': 4, 'total_num_samples': 50000}\n", + "(10-05) 10:56:29 INFO [aggregator.py:529] Wall clock: 0 s, round: 1, Planned participants: 0, Succeed participants: 0, Training loss: 0.0\n", + "(10-05) 10:56:29 INFO [client_manager.py:202] Wall clock time: 0, 4 clients online, 0 clients offline\n", + "(10-05) 10:56:29 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 10:56:30 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 10:56:30 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 10:56:31 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:56:31 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:56:38 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:56:38 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:56:46 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:56:46 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:56:53 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:56:53 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:57:00 INFO [aggregator.py:529] Wall clock: 339847 s, round: 2, Planned participants: 4, Succeed participants: 4, Training loss: 5.602389772279155\n", + "(10-05) 10:57:00 INFO [client_manager.py:202] Wall clock time: 339847, 4 clients online, 0 clients offline\n", + "(10-05) 10:57:00 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 10:57:01 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 10:57:01 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 10:57:02 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:57:02 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:57:09 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:57:09 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:57:16 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:57:16 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:57:23 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:57:23 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:57:31 INFO [aggregator.py:529] Wall clock: 679693 s, round: 3, Planned participants: 4, Succeed participants: 4, Training loss: 5.302637194544962\n", + "(10-05) 10:57:31 INFO [client_manager.py:202] Wall clock time: 679693, 4 clients online, 0 clients offline\n", + "(10-05) 10:57:31 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 10:57:32 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 10:57:32 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 10:57:33 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:57:33 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:57:40 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:57:40 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:57:47 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:57:47 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:57:55 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:57:55 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:58:02 INFO [aggregator.py:529] Wall clock: 1019540 s, round: 4, Planned participants: 4, Succeed participants: 4, Training loss: 4.599358620310677\n", + "(10-05) 10:58:02 INFO [client_manager.py:202] Wall clock time: 1019540, 4 clients online, 0 clients offline\n", + "(10-05) 10:58:02 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 10:58:03 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 10:58:03 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 10:58:04 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:58:04 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:58:11 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:58:11 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:58:18 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:58:18 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:58:25 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:58:25 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:58:32 INFO [aggregator.py:529] Wall clock: 1359387 s, round: 5, Planned participants: 4, Succeed participants: 4, Training loss: 4.436900660524809\n", + "(10-05) 10:58:32 INFO [client_manager.py:202] Wall clock time: 1359387, 4 clients online, 0 clients offline\n", + "(10-05) 10:58:32 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 10:58:33 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'model_test'])\n", + "(10-05) 10:58:33 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 10:58:34 INFO [aggregator.py:804] ====event queue 1, deque(['model_test'])\n", + "(10-05) 10:58:34 INFO [aggregator.py:827] Issue EVENT (model_test) to EXECUTOR (1)\n", + "(10-05) 10:58:53 INFO [aggragation.py:64] FL Testing in round: 5, virtual_clock: 1359386.95, top_1: 10.0 %, top_5: 51.42 %, test loss: 2.4887, test len: 10000\n", + "(10-05) 10:58:54 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:58:54 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:59:01 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:59:01 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:59:08 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:59:08 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:59:15 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:59:15 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:59:23 INFO [aggregator.py:529] Wall clock: 1699234 s, round: 6, Planned participants: 4, Succeed participants: 4, Training loss: 3.8854766698601377\n", + "(10-05) 10:59:23 INFO [client_manager.py:202] Wall clock time: 1699234, 4 clients online, 0 clients offline\n", + "(10-05) 10:59:23 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 10:59:23 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 10:59:23 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 10:59:25 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:59:25 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:59:32 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:59:32 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:59:39 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:59:39 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:59:46 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:59:46 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 10:59:53 INFO [aggregator.py:529] Wall clock: 2039080 s, round: 7, Planned participants: 4, Succeed participants: 4, Training loss: 3.4019230657444686\n", + "(10-05) 10:59:53 INFO [client_manager.py:202] Wall clock time: 2039080, 4 clients online, 0 clients offline\n", + "(10-05) 10:59:53 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 10:59:54 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 10:59:54 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 10:59:56 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 10:59:56 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:00:03 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:00:03 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:00:10 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:00:10 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:00:17 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:00:17 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:00:24 INFO [aggregator.py:529] Wall clock: 2378927 s, round: 8, Planned participants: 4, Succeed participants: 4, Training loss: 3.0577747210462976\n", + "(10-05) 11:00:24 INFO [client_manager.py:202] Wall clock time: 2378927, 4 clients online, 0 clients offline\n", + "(10-05) 11:00:24 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:00:25 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:00:25 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:00:26 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:00:26 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:00:33 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:00:33 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:00:40 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:00:40 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:00:47 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:00:47 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:00:54 INFO [aggregator.py:529] Wall clock: 2718774 s, round: 9, Planned participants: 4, Succeed participants: 4, Training loss: 2.8637842447127393\n", + "(10-05) 11:00:54 INFO [client_manager.py:202] Wall clock time: 2718774, 4 clients online, 0 clients offline\n", + "(10-05) 11:00:54 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:00:55 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:00:55 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:00:56 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:00:56 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:01:04 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:01:04 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:01:11 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:01:11 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:01:17 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:01:17 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:01:25 INFO [aggregator.py:529] Wall clock: 3058621 s, round: 10, Planned participants: 4, Succeed participants: 4, Training loss: 2.832259889787557\n", + "(10-05) 11:01:25 INFO [client_manager.py:202] Wall clock time: 3058621, 4 clients online, 0 clients offline\n", + "(10-05) 11:01:25 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:01:25 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'model_test'])\n", + "(10-05) 11:01:26 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:01:27 INFO [aggregator.py:804] ====event queue 1, deque(['model_test'])\n", + "(10-05) 11:01:27 INFO [aggregator.py:827] Issue EVENT (model_test) to EXECUTOR (1)\n", + "(10-05) 11:01:45 INFO [aggragation.py:64] FL Testing in round: 10, virtual_clock: 3058620.6374999997, top_1: 17.66 %, top_5: 67.99 %, test loss: 2.3363, test len: 10000\n", + "(10-05) 11:01:46 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:01:46 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:01:53 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:01:53 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:02:00 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:02:00 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:02:07 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:02:07 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:02:15 INFO [aggregator.py:529] Wall clock: 3398467 s, round: 11, Planned participants: 4, Succeed participants: 4, Training loss: 2.4575965823961834\n", + "(10-05) 11:02:15 INFO [client_manager.py:202] Wall clock time: 3398467, 4 clients online, 0 clients offline\n", + "(10-05) 11:02:15 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:02:16 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:02:16 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:02:17 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:02:17 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:02:24 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:02:24 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:02:31 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:02:31 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:02:38 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:02:38 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:02:45 INFO [aggregator.py:529] Wall clock: 3738314 s, round: 12, Planned participants: 4, Succeed participants: 4, Training loss: 2.3794690852301428\n", + "(10-05) 11:02:45 INFO [client_manager.py:202] Wall clock time: 3738314, 4 clients online, 0 clients offline\n", + "(10-05) 11:02:45 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:02:46 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:02:46 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:02:47 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:02:47 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:02:54 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:02:54 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:03:01 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:03:01 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:03:09 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:03:09 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:03:16 INFO [aggregator.py:529] Wall clock: 4078161 s, round: 13, Planned participants: 4, Succeed participants: 4, Training loss: 2.376686896891561\n", + "(10-05) 11:03:16 INFO [client_manager.py:202] Wall clock time: 4078161, 4 clients online, 0 clients offline\n", + "(10-05) 11:03:16 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:03:17 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:03:17 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:03:18 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:03:18 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:03:25 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:03:25 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:03:32 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:03:32 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:03:39 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:03:39 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:03:46 INFO [aggregator.py:529] Wall clock: 4418008 s, round: 14, Planned participants: 4, Succeed participants: 4, Training loss: 2.4138368855547325\n", + "(10-05) 11:03:46 INFO [client_manager.py:202] Wall clock time: 4418008, 4 clients online, 0 clients offline\n", + "(10-05) 11:03:46 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:03:47 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:03:47 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:03:49 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:03:49 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:03:56 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:03:56 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:04:02 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:04:02 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:04:10 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:04:10 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:04:17 INFO [aggregator.py:529] Wall clock: 4757854 s, round: 15, Planned participants: 4, Succeed participants: 4, Training loss: 2.3423781394464704\n", + "(10-05) 11:04:17 INFO [client_manager.py:202] Wall clock time: 4757854, 4 clients online, 0 clients offline\n", + "(10-05) 11:04:17 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:04:18 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'model_test'])\n", + "(10-05) 11:04:18 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:04:19 INFO [aggregator.py:804] ====event queue 1, deque(['model_test'])\n", + "(10-05) 11:04:19 INFO [aggregator.py:827] Issue EVENT (model_test) to EXECUTOR (1)\n", + "(10-05) 11:04:38 INFO [aggragation.py:64] FL Testing in round: 15, virtual_clock: 4757854.324999999, top_1: 24.31 %, top_5: 76.41 %, test loss: 2.2923, test len: 10000\n", + "(10-05) 11:04:39 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:04:39 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:04:46 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:04:46 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:04:53 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:04:53 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:05:00 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:05:00 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:05:07 INFO [aggregator.py:529] Wall clock: 5097701 s, round: 16, Planned participants: 4, Succeed participants: 4, Training loss: 2.2786644983594107\n", + "(10-05) 11:05:08 INFO [client_manager.py:202] Wall clock time: 5097701, 4 clients online, 0 clients offline\n", + "(10-05) 11:05:08 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:05:08 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:05:08 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:05:10 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:05:10 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:05:16 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:05:16 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:05:23 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:05:23 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:05:30 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:05:30 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:05:37 INFO [aggregator.py:529] Wall clock: 5437548 s, round: 17, Planned participants: 4, Succeed participants: 4, Training loss: 2.2587813556717866\n", + "(10-05) 11:05:37 INFO [client_manager.py:202] Wall clock time: 5437548, 4 clients online, 0 clients offline\n", + "(10-05) 11:05:37 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:05:38 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:05:38 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:05:40 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:05:40 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:05:46 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:05:46 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:05:54 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:05:54 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:06:01 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:06:01 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:06:08 INFO [aggregator.py:529] Wall clock: 5777395 s, round: 18, Planned participants: 4, Succeed participants: 4, Training loss: 2.1881389614638316\n", + "(10-05) 11:06:08 INFO [client_manager.py:202] Wall clock time: 5777395, 4 clients online, 0 clients offline\n", + "(10-05) 11:06:08 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:06:09 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:06:09 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:06:10 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:06:10 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:06:17 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:06:17 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:06:25 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:06:25 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:06:31 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:06:31 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:06:39 INFO [aggregator.py:529] Wall clock: 6117241 s, round: 19, Planned participants: 4, Succeed participants: 4, Training loss: 2.093780582858183\n", + "(10-05) 11:06:39 INFO [client_manager.py:202] Wall clock time: 6117241, 4 clients online, 0 clients offline\n", + "(10-05) 11:06:39 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n", + "(10-05) 11:06:39 INFO [aggregator.py:804] ====event queue 1, deque(['update_model', 'client_train'])\n", + "(10-05) 11:06:40 INFO [aggregator.py:827] Issue EVENT (update_model) to EXECUTOR (1)\n", + "(10-05) 11:06:41 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:06:41 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:06:48 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:06:48 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:06:55 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:06:55 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:07:02 INFO [aggregator.py:804] ====event queue 1, deque(['client_train'])\n", + "(10-05) 11:07:02 INFO [aggregator.py:827] Issue EVENT (client_train) to EXECUTOR (1)\n", + "(10-05) 11:07:09 INFO [aggregator.py:529] Wall clock: 6457088 s, round: 20, Planned participants: 4, Succeed participants: 4, Training loss: 2.1000436564237734\n", + "(10-05) 11:07:09 INFO [client_manager.py:202] Wall clock time: 6457088, 4 clients online, 0 clients offline\n", + "(10-05) 11:07:09 INFO [aggregator.py:541] Selected participants to run: [1, 2, 3, 4]\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "(03-05) 19:54:33 INFO [aggregator.py:307] Wall clock: 3058299 s, Epoch: 10, Planned participants: 4, Succeed participants: 4, Training loss: 3.9886572551584814\n", - "(03-05) 19:54:33 INFO [client_manager.py:163] Wall clock time: 3058299, 4 clients online, 0 clients offline\n", - "(03-05) 19:54:33 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:54:38 INFO [aggregator.py:391] FL Testing in epoch: 10, virtual_clock: 3058298.6062499997, top_1: 15.01 %, top_5: 64.02 %, test loss: 2.2961, test len: 10000\n", - "(03-05) 19:54:55 INFO [aggregator.py:307] Wall clock: 3398110 s, Epoch: 11, Planned participants: 4, Succeed participants: 4, Training loss: 3.823266688265342\n", - "(03-05) 19:54:55 INFO [client_manager.py:163] Wall clock time: 3398110, 4 clients online, 0 clients offline\n", - "(03-05) 19:54:55 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:55:11 INFO [aggregator.py:307] Wall clock: 3737921 s, Epoch: 12, Planned participants: 4, Succeed participants: 4, Training loss: 3.689186009494256\n", - "(03-05) 19:55:11 INFO [client_manager.py:163] Wall clock time: 3737921, 4 clients online, 0 clients offline\n", - "(03-05) 19:55:11 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:55:27 INFO [aggregator.py:307] Wall clock: 4077731 s, Epoch: 13, Planned participants: 4, Succeed participants: 4, Training loss: 3.570574830265214\n", - "(03-05) 19:55:27 INFO [client_manager.py:163] Wall clock time: 4077731, 4 clients online, 0 clients offline\n", - "(03-05) 19:55:27 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:55:44 INFO [aggregator.py:307] Wall clock: 4417542 s, Epoch: 14, Planned participants: 4, Succeed participants: 4, Training loss: 3.4716025625141187\n", - "(03-05) 19:55:44 INFO [client_manager.py:163] Wall clock time: 4417542, 4 clients online, 0 clients offline\n", - "(03-05) 19:55:44 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:56:01 INFO [aggregator.py:307] Wall clock: 4757353 s, Epoch: 15, Planned participants: 4, Succeed participants: 4, Training loss: 3.386468715312305\n", - "(03-05) 19:56:01 INFO [client_manager.py:163] Wall clock time: 4757353, 4 clients online, 0 clients offline\n", - "(03-05) 19:56:01 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:56:06 INFO [aggregator.py:391] FL Testing in epoch: 15, virtual_clock: 4757353.387499999, top_1: 19.13 %, top_5: 66.51 %, test loss: 2.2084, test len: 10000\n", - "(03-05) 19:56:22 INFO [aggregator.py:307] Wall clock: 5097164 s, Epoch: 16, Planned participants: 4, Succeed participants: 4, Training loss: 3.305582370973795\n", - "(03-05) 19:56:22 INFO [client_manager.py:163] Wall clock time: 5097164, 4 clients online, 0 clients offline\n", - "(03-05) 19:56:22 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:56:38 INFO [aggregator.py:307] Wall clock: 5436975 s, Epoch: 17, Planned participants: 4, Succeed participants: 4, Training loss: 3.2367431873404424\n", - "(03-05) 19:56:38 INFO [client_manager.py:163] Wall clock time: 5436975, 4 clients online, 0 clients offline\n", - "(03-05) 19:56:38 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:56:55 INFO [aggregator.py:307] Wall clock: 5776786 s, Epoch: 18, Planned participants: 4, Succeed participants: 4, Training loss: 3.1735540846295542\n", - "(03-05) 19:56:55 INFO [client_manager.py:163] Wall clock time: 5776786, 4 clients online, 0 clients offline\n", - "(03-05) 19:56:55 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:57:11 INFO [aggregator.py:307] Wall clock: 6116597 s, Epoch: 19, Planned participants: 4, Succeed participants: 4, Training loss: 3.1145161572941262\n", - "(03-05) 19:57:11 INFO [client_manager.py:163] Wall clock time: 6116597, 4 clients online, 0 clients offline\n", - "(03-05) 19:57:11 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:57:28 INFO [aggregator.py:307] Wall clock: 6456408 s, Epoch: 20, Planned participants: 4, Succeed participants: 4, Training loss: 3.061775009508827\n", - "(03-05) 19:57:28 INFO [client_manager.py:163] Wall clock time: 6456408, 4 clients online, 0 clients offline\n", - "(03-05) 19:57:28 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:57:32 INFO [aggregator.py:391] FL Testing in epoch: 20, virtual_clock: 6456408.168749998, top_1: 27.0 %, top_5: 79.4 %, test loss: 1.9998, test len: 10000\n", - "(03-05) 19:57:48 INFO [aggregator.py:307] Wall clock: 6796219 s, Epoch: 21, Planned participants: 4, Succeed participants: 4, Training loss: 3.013447777615297\n", - "(03-05) 19:57:48 INFO [client_manager.py:163] Wall clock time: 6796219, 4 clients online, 0 clients offline\n", - "(03-05) 19:57:48 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:58:05 INFO [aggregator.py:307] Wall clock: 7136030 s, Epoch: 22, Planned participants: 4, Succeed participants: 4, Training loss: 2.9698603141542357\n", - "(03-05) 19:58:05 INFO [client_manager.py:163] Wall clock time: 7136030, 4 clients online, 0 clients offline\n", - "(03-05) 19:58:05 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:58:21 INFO [aggregator.py:307] Wall clock: 7475841 s, Epoch: 23, Planned participants: 4, Succeed participants: 4, Training loss: 2.9299287277275288\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 19:58:21 INFO [client_manager.py:163] Wall clock time: 7475841, 4 clients online, 0 clients offline\n", - "(03-05) 19:58:21 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:58:37 INFO [aggregator.py:307] Wall clock: 7815652 s, Epoch: 24, Planned participants: 4, Succeed participants: 4, Training loss: 2.889732074043432\n", - "(03-05) 19:58:37 INFO [client_manager.py:163] Wall clock time: 7815652, 4 clients online, 0 clients offline\n", - "(03-05) 19:58:37 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:58:54 INFO [aggregator.py:307] Wall clock: 8155463 s, Epoch: 25, Planned participants: 4, Succeed participants: 4, Training loss: 2.8552758065776938\n", - "(03-05) 19:58:54 INFO [client_manager.py:163] Wall clock time: 8155463, 4 clients online, 0 clients offline\n", - "(03-05) 19:58:54 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:58:59 INFO [aggregator.py:391] FL Testing in epoch: 25, virtual_clock: 8155462.949999997, top_1: 23.86 %, top_5: 75.21 %, test loss: 2.0768, test len: 10000\n", - "(03-05) 19:59:15 INFO [aggregator.py:307] Wall clock: 8495274 s, Epoch: 26, Planned participants: 4, Succeed participants: 4, Training loss: 2.823712765442416\n", - "(03-05) 19:59:15 INFO [client_manager.py:163] Wall clock time: 8495274, 4 clients online, 0 clients offline\n", - "(03-05) 19:59:15 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:59:31 INFO [aggregator.py:307] Wall clock: 8835085 s, Epoch: 27, Planned participants: 4, Succeed participants: 4, Training loss: 2.794337058177189\n", - "(03-05) 19:59:31 INFO [client_manager.py:163] Wall clock time: 8835085, 4 clients online, 0 clients offline\n", - "(03-05) 19:59:31 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 19:59:47 INFO [aggregator.py:307] Wall clock: 9174896 s, Epoch: 28, Planned participants: 4, Succeed participants: 4, Training loss: 2.7667211220110177\n", - "(03-05) 19:59:47 INFO [client_manager.py:163] Wall clock time: 9174896, 4 clients online, 0 clients offline\n", - "(03-05) 19:59:47 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:00:03 INFO [aggregator.py:307] Wall clock: 9514707 s, Epoch: 29, Planned participants: 4, Succeed participants: 4, Training loss: 2.7399987090864397\n", - "(03-05) 20:00:03 INFO [client_manager.py:163] Wall clock time: 9514707, 4 clients online, 0 clients offline\n", - "(03-05) 20:00:03 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:00:19 INFO [aggregator.py:307] Wall clock: 9854518 s, Epoch: 30, Planned participants: 4, Succeed participants: 4, Training loss: 2.7155958749045332\n", - "(03-05) 20:00:19 INFO [client_manager.py:163] Wall clock time: 9854518, 4 clients online, 0 clients offline\n", - "(03-05) 20:00:19 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:00:24 INFO [aggregator.py:391] FL Testing in epoch: 30, virtual_clock: 9854517.731250001, top_1: 32.08 %, top_5: 83.72 %, test loss: 1.8732, test len: 10000\n", - "(03-05) 20:00:40 INFO [aggregator.py:307] Wall clock: 10194329 s, Epoch: 31, Planned participants: 4, Succeed participants: 4, Training loss: 2.691345205807315\n", - "(03-05) 20:00:40 INFO [client_manager.py:163] Wall clock time: 10194329, 4 clients online, 0 clients offline\n", - "(03-05) 20:00:40 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:00:57 INFO [aggregator.py:307] Wall clock: 10534140 s, Epoch: 32, Planned participants: 4, Succeed participants: 4, Training loss: 2.666735569342559\n", - "(03-05) 20:00:57 INFO [client_manager.py:163] Wall clock time: 10534140, 4 clients online, 0 clients offline\n", - "(03-05) 20:00:57 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:01:13 INFO [aggregator.py:307] Wall clock: 10873951 s, Epoch: 33, Planned participants: 4, Succeed participants: 4, Training loss: 2.643961913396924\n", - "(03-05) 20:01:13 INFO [client_manager.py:163] Wall clock time: 10873951, 4 clients online, 0 clients offline\n", - "(03-05) 20:01:13 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:01:29 INFO [aggregator.py:307] Wall clock: 11213762 s, Epoch: 34, Planned participants: 4, Succeed participants: 4, Training loss: 2.621859059915586\n", - "(03-05) 20:01:29 INFO [client_manager.py:163] Wall clock time: 11213762, 4 clients online, 0 clients offline\n", - "(03-05) 20:01:29 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:01:46 INFO [aggregator.py:307] Wall clock: 11553573 s, Epoch: 35, Planned participants: 4, Succeed participants: 4, Training loss: 2.601017124459975\n", - "(03-05) 20:01:46 INFO [client_manager.py:163] Wall clock time: 11553573, 4 clients online, 0 clients offline\n", - "(03-05) 20:01:46 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:01:50 INFO [aggregator.py:391] FL Testing in epoch: 35, virtual_clock: 11553572.512500005, top_1: 34.44 %, top_5: 84.85 %, test loss: 1.8117, test len: 10000\n", - "(03-05) 20:02:06 INFO [aggregator.py:307] Wall clock: 11893383 s, Epoch: 36, Planned participants: 4, Succeed participants: 4, Training loss: 2.5815619883196748\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 20:02:06 INFO [client_manager.py:163] Wall clock time: 11893383, 4 clients online, 0 clients offline\n", - "(03-05) 20:02:06 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:02:22 INFO [aggregator.py:307] Wall clock: 12233194 s, Epoch: 37, Planned participants: 4, Succeed participants: 4, Training loss: 2.5630652392238655\n", - "(03-05) 20:02:22 INFO [client_manager.py:163] Wall clock time: 12233194, 4 clients online, 0 clients offline\n", - "(03-05) 20:02:22 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:02:39 INFO [aggregator.py:307] Wall clock: 12573005 s, Epoch: 38, Planned participants: 4, Succeed participants: 4, Training loss: 2.54478694471509\n", - "(03-05) 20:02:39 INFO [client_manager.py:163] Wall clock time: 12573005, 4 clients online, 0 clients offline\n", - "(03-05) 20:02:39 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:02:55 INFO [aggregator.py:307] Wall clock: 12912816 s, Epoch: 39, Planned participants: 4, Succeed participants: 4, Training loss: 2.527968767072284\n", - "(03-05) 20:02:55 INFO [client_manager.py:163] Wall clock time: 12912816, 4 clients online, 0 clients offline\n", - "(03-05) 20:02:55 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:03:11 INFO [aggregator.py:307] Wall clock: 13252627 s, Epoch: 40, Planned participants: 4, Succeed participants: 4, Training loss: 2.5106288808545436\n", - "(03-05) 20:03:11 INFO [client_manager.py:163] Wall clock time: 13252627, 4 clients online, 0 clients offline\n", - "(03-05) 20:03:11 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:03:16 INFO [aggregator.py:391] FL Testing in epoch: 40, virtual_clock: 13252627.293750009, top_1: 36.87 %, top_5: 87.58 %, test loss: 1.7142, test len: 10000\n", - "(03-05) 20:03:32 INFO [aggregator.py:307] Wall clock: 13592438 s, Epoch: 41, Planned participants: 4, Succeed participants: 4, Training loss: 2.493924363558724\n", - "(03-05) 20:03:32 INFO [client_manager.py:163] Wall clock time: 13592438, 4 clients online, 0 clients offline\n", - "(03-05) 20:03:32 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:03:48 INFO [aggregator.py:307] Wall clock: 13932249 s, Epoch: 42, Planned participants: 4, Succeed participants: 4, Training loss: 2.478702486146707\n", - "(03-05) 20:03:48 INFO [client_manager.py:163] Wall clock time: 13932249, 4 clients online, 0 clients offline\n", - "(03-05) 20:03:48 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:04:04 INFO [aggregator.py:307] Wall clock: 14272060 s, Epoch: 43, Planned participants: 4, Succeed participants: 4, Training loss: 2.4632626396039212\n", - "(03-05) 20:04:04 INFO [client_manager.py:163] Wall clock time: 14272060, 4 clients online, 0 clients offline\n", - "(03-05) 20:04:04 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:04:20 INFO [aggregator.py:307] Wall clock: 14611871 s, Epoch: 44, Planned participants: 4, Succeed participants: 4, Training loss: 2.4479816367204665\n", - "(03-05) 20:04:20 INFO [client_manager.py:163] Wall clock time: 14611871, 4 clients online, 0 clients offline\n", - "(03-05) 20:04:20 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:04:36 INFO [aggregator.py:307] Wall clock: 14951682 s, Epoch: 45, Planned participants: 4, Succeed participants: 4, Training loss: 2.432726749043546\n", - "(03-05) 20:04:36 INFO [client_manager.py:163] Wall clock time: 14951682, 4 clients online, 0 clients offline\n", - "(03-05) 20:04:36 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:04:41 INFO [aggregator.py:391] FL Testing in epoch: 45, virtual_clock: 14951682.075000012, top_1: 37.32 %, top_5: 87.26 %, test loss: 1.7289, test len: 10000\n", - "(03-05) 20:04:57 INFO [aggregator.py:307] Wall clock: 15291493 s, Epoch: 46, Planned participants: 4, Succeed participants: 4, Training loss: 2.4174913718646134\n", - "(03-05) 20:04:57 INFO [client_manager.py:163] Wall clock time: 15291493, 4 clients online, 0 clients offline\n", - "(03-05) 20:04:57 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:05:13 INFO [aggregator.py:307] Wall clock: 15631304 s, Epoch: 47, Planned participants: 4, Succeed participants: 4, Training loss: 2.4040398512344963\n", - "(03-05) 20:05:13 INFO [client_manager.py:163] Wall clock time: 15631304, 4 clients online, 0 clients offline\n", - "(03-05) 20:05:13 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:05:30 INFO [aggregator.py:307] Wall clock: 15971115 s, Epoch: 48, Planned participants: 4, Succeed participants: 4, Training loss: 2.390771513237743\n", - "(03-05) 20:05:30 INFO [client_manager.py:163] Wall clock time: 15971115, 4 clients online, 0 clients offline\n", - "(03-05) 20:05:30 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:05:46 INFO [aggregator.py:307] Wall clock: 16310926 s, Epoch: 49, Planned participants: 4, Succeed participants: 4, Training loss: 2.3781074780177813\n", - "(03-05) 20:05:46 INFO [client_manager.py:163] Wall clock time: 16310926, 4 clients online, 0 clients offline\n", - "(03-05) 20:05:46 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "(03-05) 20:06:03 INFO [aggregator.py:307] Wall clock: 16650737 s, Epoch: 50, Planned participants: 4, Succeed participants: 4, Training loss: 2.366829282342857\n", - "(03-05) 20:06:03 INFO [client_manager.py:163] Wall clock time: 16650737, 4 clients online, 0 clients offline\n", - "(03-05) 20:06:03 INFO [aggregator.py:323] Selected participants to run: [1, 2, 3, 4]:\n", - "{1: {'computation': 1.8, 'communication': 339809.15625}, 2: {'computation': 1.8, 'communication': 339809.15625}, 3: {'computation': 1.8, 'communication': 339809.15625}, 4: {'computation': 1.8, 'communication': 339809.15625}}\n", - "(03-05) 20:06:03 INFO [aggregator.py:517] Terminating the aggregator ...\n", - "(03-05) 20:06:08 INFO [channelcontext.py:51] %%%%%%%%%% Closing grpc connection with 1 %%%%%%%%%%\n" + "(10-05) 11:07:10 INFO [aggregator.py:804] ====event queue 1, deque(['terminate_executor'])\n", + "(10-05) 11:07:10 INFO [aggregator.py:827] Issue EVENT (terminate_executor) to EXECUTOR (1)\n" ] } ], "source": [ "import sys, os\n", "\n", + "import fedscale.core.config_parser as parser\n", "from fedscale.core.execution.client import Client\n", "from fedscale.core.aggregation.aggregator import Aggregator\n", - "from fedscale.core.logger.execution import args\n", - "Demo_Aggregator = Aggregator(args)\n", + "Demo_Aggregator = Aggregator(parser.args)\n", "### On CPU\n", - "args.use_cuda = \"False\"\n", + "parser.args.use_cuda = \"False\"\n", "Demo_Aggregator.run()" ] }, @@ -312,9 +349,9 @@ ], "metadata": { "kernelspec": { - "display_name": "fedscale", + "display_name": "Python 3.7.13 ('fedscale': conda)", "language": "python", - "name": "fedscale" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -326,7 +363,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.7.13" + }, + "vscode": { + "interpreter": { + "hash": "c61f092c149070809c0e629644d57c0873f394886eddd38a7b619aa7d2ef8d75" + } } }, "nbformat": 4, diff --git a/fedscale/utils/model_test_module.py b/fedscale/utils/model_test_module.py index 5783c5bc..0edb8d3e 100755 --- a/fedscale/utils/model_test_module.py +++ b/fedscale/utils/model_test_module.py @@ -8,10 +8,10 @@ from torch.autograd import Variable # libs from fedscale -from fedscale.core.config_parser import args +import fedscale.core.config_parser as parser from fedscale.dataloaders.nlp import mask_tokens -if args.task == "detection": +if parser.args.task == "detection": import numpy as np import torch.nn as nn import torch.optim as optim @@ -69,18 +69,18 @@ def test_model(rank, model, test_data, device='cpu', criterion=nn.NLLLoss(), tok decoder = None - if args.task == 'voice': + if parser.args.task == 'voice': decoder = GreedyDecoder( model.labels, blank_index=model.labels.index('_')) with torch.no_grad(): - if args.task == 'detection': + if parser.args.task == 'detection': imdb, _, _, _ = combined_roidb( - "voc_2007_test", ['DATA_DIR', args.data_dir], server=True) + "voc_2007_test", ['DATA_DIR', parser.args.data_dir], server=True) data_iter = iter(test_data) num_images = len(test_data.dataset) - num_classes = len(readClass(args.data_dir + "/class.txt")) + num_classes = len(readClass(parser.args.data_dir + "/class.txt")) all_boxes = [[[] for _ in range(num_images)] for _ in range(num_classes)] @@ -156,23 +156,23 @@ def test_model(rank, model, test_data, device='cpu', criterion=nn.NLLLoss(), tok all_boxes[j][i] = all_boxes[j][i][keep, :] imdb._reset_index(test_data.dataset.index) - output_dir = args.test_output_dir + \ - "/learner/" + str(args.this_rank) + output_dir = parser.args.test_output_dir + \ + "/learner/" + str(parser.args.this_rank) _, mean_ap = imdb.evaluate_detections( - all_boxes, output_dir, args.this_rank) + all_boxes, output_dir, parser.args.this_rank) return 0, mean_ap, mean_ap, {'top_1': mean_ap, 'top_5': mean_ap, 'test_loss': 0, 'test_len': num_images} for data, target in test_data: try: - if args.task == 'nlp': + if parser.args.task == 'nlp': - # if args.mlm else (data, data) + # if parser.args.mlm else (data, data) data, target = mask_tokens( - data, tokenizer, args, device=device) + data, tokenizer, parser.args, device=device) data, target = Variable(data).to( device=device), Variable(target).to(device=device) - # if args.mlm else model(data, labels=target) + # if parser.args.mlm else model(data, labels=target) outputs = model(data, labels=target) loss = outputs[0] @@ -185,7 +185,7 @@ def test_model(rank, model, test_data, device='cpu', criterion=nn.NLLLoss(), tok correct += acc[0].item() top_5 += acc[1].item() - elif args.task == 'tag': + elif parser.args.task == 'tag': data, target = Variable(data).to( device=device), Variable(target).to(device=device) output = model(data) @@ -201,7 +201,7 @@ def test_model(rank, model, test_data, device='cpu', criterion=nn.NLLLoss(), tok test_loss += loss.data.item() - elif args.task == 'speech': + elif parser.args.task == 'speech': data, target = Variable(data).to( device=device), Variable(target).to(device=device) data = torch.unsqueeze(data, 1) @@ -215,7 +215,7 @@ def test_model(rank, model, test_data, device='cpu', criterion=nn.NLLLoss(), tok correct += acc[0].item() top_5 += acc[1].item() - elif args.task == 'text_clf' and args.model == 'albert-base-v2': + elif parser.args.task == 'text_clf' and parser.args.model == 'albert-base-v2': (inputs, masks) = data (inputs, masks, target) = (Variable(inputs).to(device=device), Variable(masks).to(device=device), Variable(target).to(device=device)) @@ -231,7 +231,7 @@ def test_model(rank, model, test_data, device='cpu', criterion=nn.NLLLoss(), tok correct += acc[0].item() top_5 += acc[1].item() - elif args.task == 'voice': + elif parser.args.task == 'voice': (inputs, target, input_percentages, target_sizes) = data input_sizes = input_percentages.mul_( @@ -282,7 +282,7 @@ def test_model(rank, model, test_data, device='cpu', criterion=nn.NLLLoss(), tok break test_len += len(target) - if args.task == 'voice': + if parser.args.task == 'voice': correct, top_5, test_len = float( total_wer), float(total_cer), float(num_tokens) @@ -298,7 +298,7 @@ def test_model(rank, model, test_data, device='cpu', criterion=nn.NLLLoss(), tok acc_5 = round(top_5 / test_len, 4) test_loss = round(test_loss, 4) - if args.task == 'tag': + if parser.args.task == 'tag': # precision, recall, f1, sup = precision_recall_fscore_support(targets_list, preds, average='samples') top_5, correct, test_len = cal_accuracy(targets_list, preds)