Skip to content
GitLab
Commit d437e33e authored by Aristeidis Tsaris's avatar Aristeidis Tsaris
Browse files

adding option for data loading only

parent a280ec34
Expand all Hide whitespace changes
Inline Side-by-side
ascent/ascent_logs/benchmark.data.nodd.opt.15GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633368580.678657", "datetime": "2021-10-04 13:29:40.678657", "elapsedtime": "1e-05", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": true, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.nodd.opt.15GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": true, "local_rank": 0, "gpu": 0, "world_size": 15}}
DLLL {"timestamp": "1633368580.678969", "datetime": "2021-10-04 13:29:40.678969", "elapsedtime": "0.000322", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633368580.683555", "elapsedtime": "0.004908", "datetime": "2021-10-04 13:29:40.683555", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633368580.684055", "elapsedtime": "0.005408", "datetime": "2021-10-04 13:29:40.684055", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633368580.684095", "elapsedtime": "0.005448", "datetime": "2021-10-04 13:29:40.684095", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368580.684128", "elapsedtime": "0.005481", "datetime": "2021-10-04 13:29:40.684128", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368580.68416", "elapsedtime": "0.005513", "datetime": "2021-10-04 13:29:40.684160", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368580.684191", "elapsedtime": "0.005544", "datetime": "2021-10-04 13:29:40.684191", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368646.944581", "datetime": "2021-10-04 13:30:46.944581", "elapsedtime": "66.265934", "type": "LOG", "step": [0, 10], "data": {"train.loss": 8.905070209503174, "train.total_ips": 6389.44978951381}}
DLLL {"timestamp": "1633368646.945175", "datetime": "2021-10-04 13:30:46.945175", "elapsedtime": "66.266528", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 6473.6486731910245, "train.data_time": 2.1612624883651734, "train.compute_time": 4.464561533927918}}
DLLL {"timestamp": "1633368651.25147", "datetime": "2021-10-04 13:30:51.251470", "elapsedtime": "70.572823", "type": "LOG", "step": [0, 20], "data": {"train.loss": 9.109191417694092, "train.total_ips": 4596.433817520007}}
DLLL {"timestamp": "1633368651.251646", "datetime": "2021-10-04 13:30:51.251646", "elapsedtime": "70.572999", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 4772.982603551252, "train.data_time": 0.014628076553344726, "train.compute_time": 0.41589579582214353}}
DLLL {"timestamp": "1633368657.95684", "datetime": "2021-10-04 13:30:57.956840", "elapsedtime": "77.278193", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.8272237300872805, "train.total_ips": 3037.2251743161896}}
DLLL {"timestamp": "1633368657.957079", "datetime": "2021-10-04 13:30:57.957079", "elapsedtime": "77.278432", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 3107.286167126355, "train.data_time": 0.012641692161560058, "train.compute_time": 0.6577740907669067}}
DLLL {"timestamp": "1633368665.573776", "datetime": "2021-10-04 13:31:05.573776", "elapsedtime": "84.895129", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.28309211730957, "train.total_ips": 2658.9515256612744}}
DLLL {"timestamp": "1633368665.574003", "datetime": "2021-10-04 13:31:05.574003", "elapsedtime": "84.895356", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 2727.3296430647606, "train.data_time": 0.018487381935119628, "train.compute_time": 0.7430117845535278}}
DLLL {"timestamp": "1633368673.122152", "datetime": "2021-10-04 13:31:13.122152", "elapsedtime": "92.443505", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.059678316116333, "train.total_ips": 2602.605282874574}}
DLLL {"timestamp": "1633368673.122362", "datetime": "2021-10-04 13:31:13.122362", "elapsedtime": "92.443715", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 2693.2395674618074, "train.data_time": 0.021999835968017578, "train.compute_time": 0.73272545337677}}
DLLL {"timestamp": "1633368680.073372", "datetime": "2021-10-04 13:31:20.073372", "elapsedtime": "99.394725", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.024379920959473, "train.total_ips": 2917.289312323478}}
DLLL {"timestamp": "1633368680.07364", "datetime": "2021-10-04 13:31:20.073640", "elapsedtime": "99.394993", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 2980.049593595777, "train.data_time": 0.013703417778015137, "train.compute_time": 0.6812222957611084}}
DLLL {"timestamp": "1633368686.796575", "datetime": "2021-10-04 13:31:26.796575", "elapsedtime": "106.117928", "type": "LOG", "step": [0, 70], "data": {"train.loss": 6.959270429611206, "train.total_ips": 3010.9175657984183}}
DLLL {"timestamp": "1633368686.796827", "datetime": "2021-10-04 13:31:26.796827", "elapsedtime": "106.11818", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 3114.4988392527257, "train.data_time": 0.014000844955444337, "train.compute_time": 0.658202075958252}}
DLLL {"timestamp": "1633368693.77067", "datetime": "2021-10-04 13:31:33.770670", "elapsedtime": "113.092023", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.941778516769409, "train.total_ips": 3429.7081059958764}}
DLLL {"timestamp": "1633368693.771014", "datetime": "2021-10-04 13:31:33.771014", "elapsedtime": "113.092367", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 3522.729058906014, "train.data_time": 0.014573359489440918, "train.compute_time": 0.682635235786438}}
DLLL {"timestamp": "1633368701.257512", "datetime": "2021-10-04 13:31:41.257512", "elapsedtime": "120.578865", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.930657529830933, "train.total_ips": 2693.1620197000298}}
DLLL {"timestamp": "1633368701.257712", "datetime": "2021-10-04 13:31:41.257712", "elapsedtime": "120.579065", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 2786.7778926332608, "train.data_time": 0.02726137638092041, "train.compute_time": 0.7207160472869873}}
DLLL {"timestamp": "1633368712.696503", "datetime": "2021-10-04 13:31:52.696503", "elapsedtime": "132.017856", "type": "LOG", "step": [0], "data": {"train.loss": 7.496806492805481, "train.total_ips": 3444.7074426028926}}
DLLL {"timestamp": "1633368712.696743", "datetime": "2021-10-04 13:31:52.696743", "elapsedtime": "132.018096", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 3532.706753183429, "train.data_time": 0.23064370155334474, "train.compute_time": 1.0392441320419312}}
DLLL {"timestamp": "1633368712.697045", "datetime": "2021-10-04 13:31:52.697045", "elapsedtime": "132.018398", "type": "LOG", "step": [], "data": {"train.loss": 7.496806492805481, "train.total_ips": 3444.7074426028926}}
DLLL {"timestamp": "1633368712.697195", "datetime": "2021-10-04 13:31:52.697195", "elapsedtime": "132.018548", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 3532.706753183429, "train.data_time": 0.23064370155334474, "train.compute_time": 1.0392441320419312}}
ascent/ascent_logs/benchmark.data.nodd.opt.20GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633368583.030069", "datetime": "2021-10-04 13:29:43.030069", "elapsedtime": "9e-06", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": true, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.nodd.opt.20GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": true, "local_rank": 0, "gpu": 0, "world_size": 20}}
DLLL {"timestamp": "1633368583.030395", "datetime": "2021-10-04 13:29:43.030395", "elapsedtime": "0.000335", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633368583.035312", "elapsedtime": "0.005252", "datetime": "2021-10-04 13:29:43.035312", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633368583.035896", "elapsedtime": "0.005836", "datetime": "2021-10-04 13:29:43.035896", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633368583.035937", "elapsedtime": "0.005877", "datetime": "2021-10-04 13:29:43.035937", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368583.035972", "elapsedtime": "0.005912", "datetime": "2021-10-04 13:29:43.035972", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368583.036004", "elapsedtime": "0.005944", "datetime": "2021-10-04 13:29:43.036004", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368583.036036", "elapsedtime": "0.005976", "datetime": "2021-10-04 13:29:43.036036", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368631.447573", "datetime": "2021-10-04 13:30:31.447573", "elapsedtime": "48.417513", "type": "LOG", "step": [0, 10], "data": {"train.loss": 8.915813684463501, "train.total_ips": 8584.1370181202}}
DLLL {"timestamp": "1633368631.448053", "datetime": "2021-10-04 13:30:31.448053", "elapsedtime": "48.417993", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 8645.534205403947, "train.data_time": 2.366662883758545, "train.compute_time": 2.4742587566375733}}
DLLL {"timestamp": "1633368635.910159", "datetime": "2021-10-04 13:30:35.910159", "elapsedtime": "52.880099", "type": "LOG", "step": [0, 20], "data": {"train.loss": 9.109982299804688, "train.total_ips": 6108.881292394909}}
DLLL {"timestamp": "1633368635.910449", "datetime": "2021-10-04 13:30:35.910449", "elapsedtime": "52.880389", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 6383.591078915261, "train.data_time": 0.01201937198638916, "train.compute_time": 0.4340439558029175}}
DLLL {"timestamp": "1633368642.303787", "datetime": "2021-10-04 13:30:42.303787", "elapsedtime": "59.273727", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.7820264339447025, "train.total_ips": 4148.383195620336}}
DLLL {"timestamp": "1633368642.304073", "datetime": "2021-10-04 13:30:42.304073", "elapsedtime": "59.274013", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 4325.249343010184, "train.data_time": 0.025083017349243165, "train.compute_time": 0.6141943454742431}}
DLLL {"timestamp": "1633368649.939104", "datetime": "2021-10-04 13:30:49.939104", "elapsedtime": "66.909044", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.401272773742676, "train.total_ips": 3487.4779059437615}}
DLLL {"timestamp": "1633368649.939377", "datetime": "2021-10-04 13:30:49.939377", "elapsedtime": "66.909317", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 3589.816356171057, "train.data_time": 0.019240164756774904, "train.compute_time": 0.7441050052642822}}
DLLL {"timestamp": "1633368657.323201", "datetime": "2021-10-04 13:30:57.323201", "elapsedtime": "74.293141", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.060693359375, "train.total_ips": 3569.469967955943}}
DLLL {"timestamp": "1633368657.323492", "datetime": "2021-10-04 13:30:57.323492", "elapsedtime": "74.293432", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 3657.0591110887517, "train.data_time": 0.014231324195861816, "train.compute_time": 0.7240637063980102}}
DLLL {"timestamp": "1633368665.425086", "datetime": "2021-10-04 13:31:05.425086", "elapsedtime": "82.395026", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.027244567871094, "train.total_ips": 3242.979710037537}}
DLLL {"timestamp": "1633368665.425391", "datetime": "2021-10-04 13:31:05.425391", "elapsedtime": "82.395331", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 3304.2447259878295, "train.data_time": 0.0157520055770874, "train.compute_time": 0.794244909286499}}
DLLL {"timestamp": "1633368672.425366", "datetime": "2021-10-04 13:31:12.425366", "elapsedtime": "89.395306", "type": "LOG", "step": [0, 70], "data": {"train.loss": 6.967746877670288, "train.total_ips": 3751.2963648558434}}
DLLL {"timestamp": "1633368672.425656", "datetime": "2021-10-04 13:31:12.425656", "elapsedtime": "89.395596", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 3845.544488742157, "train.data_time": 0.013252568244934083, "train.compute_time": 0.686657166481018}}
DLLL {"timestamp": "1633368679.860262", "datetime": "2021-10-04 13:31:19.860262", "elapsedtime": "96.830202", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.936493682861328, "train.total_ips": 3528.3532434775175}}
DLLL {"timestamp": "1633368679.860557", "datetime": "2021-10-04 13:31:19.860557", "elapsedtime": "96.830497", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 3560.794835174045, "train.data_time": 0.007741785049438477, "train.compute_time": 0.7355400085449219}}
DLLL {"timestamp": "1633368687.547554", "datetime": "2021-10-04 13:31:27.547554", "elapsedtime": "104.517494", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.935410881042481, "train.total_ips": 3621.6625334724095}}
DLLL {"timestamp": "1633368687.547852", "datetime": "2021-10-04 13:31:27.547852", "elapsedtime": "104.517792", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 3680.9443631909708, "train.data_time": 0.012058782577514648, "train.compute_time": 0.747539758682251}}
DLLL {"timestamp": "1633368700.058416", "datetime": "2021-10-04 13:31:40.058416", "elapsedtime": "117.028356", "type": "LOG", "step": [0], "data": {"train.loss": 7.506624412536621, "train.total_ips": 4372.890397927424}}
DLLL {"timestamp": "1633368700.058694", "datetime": "2021-10-04 13:31:40.058694", "elapsedtime": "117.028634", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 4482.959172389081, "train.data_time": 0.25047002792358397, "train.compute_time": 0.8686380028724671}}
DLLL {"timestamp": "1633368700.058993", "datetime": "2021-10-04 13:31:40.058993", "elapsedtime": "117.028933", "type": "LOG", "step": [], "data": {"train.loss": 7.506624412536621, "train.total_ips": 4372.890397927424}}
DLLL {"timestamp": "1633368700.059119", "datetime": "2021-10-04 13:31:40.059119", "elapsedtime": "117.029059", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 4482.959172389081, "train.data_time": 0.25047002792358397, "train.compute_time": 0.8686380028724671}}
ascent/ascent_logs/benchmark.data.noddp.opt.10GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633368765.528052", "datetime": "2021-10-04 13:32:45.528052", "elapsedtime": "1e-05", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": true, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.noddp.opt.10GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": true, "local_rank": 0, "gpu": 0, "world_size": 10}}
DLLL {"timestamp": "1633368765.528379", "datetime": "2021-10-04 13:32:45.528379", "elapsedtime": "0.000337", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633368765.533024", "elapsedtime": "0.004982", "datetime": "2021-10-04 13:32:45.533024", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633368765.533573", "elapsedtime": "0.005531", "datetime": "2021-10-04 13:32:45.533573", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633368765.533612", "elapsedtime": "0.00557", "datetime": "2021-10-04 13:32:45.533612", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368765.533646", "elapsedtime": "0.005604", "datetime": "2021-10-04 13:32:45.533646", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368765.533678", "elapsedtime": "0.005636", "datetime": "2021-10-04 13:32:45.533678", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368765.533709", "elapsedtime": "0.005667", "datetime": "2021-10-04 13:32:45.533709", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368812.372868", "datetime": "2021-10-04 13:33:32.372868", "elapsedtime": "46.844826", "type": "LOG", "step": [0, 10], "data": {"train.loss": 9.080923652648925, "train.total_ips": 2979.7606552116313}}
DLLL {"timestamp": "1633368812.373358", "datetime": "2021-10-04 13:33:32.373358", "elapsedtime": "46.845316", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 3095.0063886102052, "train.data_time": 2.2562681436538696, "train.compute_time": 2.4273983001708985}}
DLLL {"timestamp": "1633368817.196774", "datetime": "2021-10-04 13:33:37.196774", "elapsedtime": "51.668732", "type": "LOG", "step": [0, 20], "data": {"train.loss": 8.959366512298583, "train.total_ips": 2822.8469948685533}}
DLLL {"timestamp": "1633368817.197036", "datetime": "2021-10-04 13:33:37.197036", "elapsedtime": "51.668994", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 3002.7498468942094, "train.data_time": 0.020646333694458008, "train.compute_time": 0.4615530252456665}}
DLLL {"timestamp": "1633368823.178228", "datetime": "2021-10-04 13:33:43.178228", "elapsedtime": "57.650186", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.835245847702026, "train.total_ips": 2322.862417672294}}
DLLL {"timestamp": "1633368823.178515", "datetime": "2021-10-04 13:33:43.178515", "elapsedtime": "57.650473", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 2343.902633392147, "train.data_time": 0.006995439529418945, "train.compute_time": 0.5910444974899292}}
DLLL {"timestamp": "1633368828.876149", "datetime": "2021-10-04 13:33:48.876149", "elapsedtime": "63.348107", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.4600077152252195, "train.total_ips": 2616.4481985408847}}
DLLL {"timestamp": "1633368828.876438", "datetime": "2021-10-04 13:33:48.876438", "elapsedtime": "63.348396", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 2724.3758081607602, "train.data_time": 0.014265203475952148, "train.compute_time": 0.5553366899490356}}
DLLL {"timestamp": "1633368836.451182", "datetime": "2021-10-04 13:33:56.451182", "elapsedtime": "70.92314", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.154645395278931, "train.total_ips": 1885.6756840638488}}
DLLL {"timestamp": "1633368836.451502", "datetime": "2021-10-04 13:33:56.451502", "elapsedtime": "70.92346", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 1927.7501845770403, "train.data_time": 0.017845940589904786, "train.compute_time": 0.7395466327667236}}
DLLL {"timestamp": "1633368843.038569", "datetime": "2021-10-04 13:34:03.038569", "elapsedtime": "77.510527", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.013648128509521, "train.total_ips": 2047.3767037989855}}
DLLL {"timestamp": "1633368843.038866", "datetime": "2021-10-04 13:34:03.038866", "elapsedtime": "77.510824", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 2082.946929288299, "train.data_time": 0.011336278915405274, "train.compute_time": 0.6471736192703247}}
DLLL {"timestamp": "1633368850.467254", "datetime": "2021-10-04 13:34:10.467254", "elapsedtime": "84.939212", "type": "LOG", "step": [0, 70], "data": {"train.loss": 6.965586090087891, "train.total_ips": 1775.187506251436}}
DLLL {"timestamp": "1633368850.467558", "datetime": "2021-10-04 13:34:10.467558", "elapsedtime": "84.939516", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 1815.4518896012262, "train.data_time": 0.01747784614562988, "train.compute_time": 0.7252664804458618}}
DLLL {"timestamp": "1633368857.175593", "datetime": "2021-10-04 13:34:17.175593", "elapsedtime": "91.647551", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.953998708724976, "train.total_ips": 2026.6332015424384}}
DLLL {"timestamp": "1633368857.175887", "datetime": "2021-10-04 13:34:17.175887", "elapsedtime": "91.647845", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 2156.2297010687394, "train.data_time": 0.021082234382629395, "train.compute_time": 0.6495411634445191}}
DLLL {"timestamp": "1633368863.808228", "datetime": "2021-10-04 13:34:23.808228", "elapsedtime": "98.280186", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.932726669311523, "train.total_ips": 2077.382848391921}}
DLLL {"timestamp": "1633368863.808531", "datetime": "2021-10-04 13:34:23.808531", "elapsedtime": "98.280489", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 2117.5868467127802, "train.data_time": 0.010343337059020996, "train.compute_time": 0.652817177772522}}
DLLL {"timestamp": "1633368875.861681", "datetime": "2021-10-04 13:34:35.861681", "elapsedtime": "110.333639", "type": "LOG", "step": [0], "data": {"train.loss": 7.527999348640442, "train.total_ips": 2251.999142540437}}
DLLL {"timestamp": "1633368875.86195", "datetime": "2021-10-04 13:34:35.861950", "elapsedtime": "110.333908", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 2327.060536243199, "train.data_time": 0.23912926197052, "train.compute_time": 0.8139605975151062}}
DLLL {"timestamp": "1633368875.862317", "datetime": "2021-10-04 13:34:35.862317", "elapsedtime": "110.334275", "type": "LOG", "step": [], "data": {"train.loss": 7.527999348640442, "train.total_ips": 2251.999142540437}}
DLLL {"timestamp": "1633368875.862475", "datetime": "2021-10-04 13:34:35.862475", "elapsedtime": "110.334433", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 2327.060536243199, "train.data_time": 0.23912926197052, "train.compute_time": 0.8139605975151062}}
ascent/ascent_logs/benchmark.data.noddp.opt.1GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633368405.500884", "datetime": "2021-10-04 13:26:45.500884", "elapsedtime": "1.6e-05", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": true, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.noddp.opt.1GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": false, "local_rank": 0, "gpu": 0, "world_size": 1}}
DLLL {"timestamp": "1633368405.50119", "datetime": "2021-10-04 13:26:45.501190", "elapsedtime": "0.000322", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633368405.505703", "elapsedtime": "0.004835", "datetime": "2021-10-04 13:26:45.505703", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633368405.506234", "elapsedtime": "0.005366", "datetime": "2021-10-04 13:26:45.506234", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633368405.506272", "elapsedtime": "0.005404", "datetime": "2021-10-04 13:26:45.506272", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368405.506305", "elapsedtime": "0.005437", "datetime": "2021-10-04 13:26:45.506305", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368405.506336", "elapsedtime": "0.005468", "datetime": "2021-10-04 13:26:45.506336", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368405.506367", "elapsedtime": "0.005499", "datetime": "2021-10-04 13:26:45.506367", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368449.916711", "datetime": "2021-10-04 13:27:29.916711", "elapsedtime": "44.415843", "type": "LOG", "step": [0, 10], "data": {"train.loss": 8.757450485229493, "train.total_ips": 630.9194472479643}}
DLLL {"timestamp": "1633368449.917255", "datetime": "2021-10-04 13:27:29.917255", "elapsedtime": "44.416387", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 636.8346217996366, "train.data_time": 2.1605786800384523, "train.compute_time": 2.2785415649414062}}
DLLL {"timestamp": "1633368453.101424", "datetime": "2021-10-04 13:27:33.101424", "elapsedtime": "47.600556", "type": "LOG", "step": [0, 20], "data": {"train.loss": 8.875391435623168, "train.total_ips": 441.90853255296435}}
DLLL {"timestamp": "1633368453.101646", "datetime": "2021-10-04 13:27:33.101646", "elapsedtime": "47.600778", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 453.1970674215848, "train.data_time": 0.006869006156921387, "train.compute_time": 0.3132224798202515}}
DLLL {"timestamp": "1633368460.125396", "datetime": "2021-10-04 13:27:40.125396", "elapsedtime": "54.624528", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.979853343963623, "train.total_ips": 184.28384482608507}}
DLLL {"timestamp": "1633368460.125629", "datetime": "2021-10-04 13:27:40.125629", "elapsedtime": "54.624761", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 189.89309349043654, "train.data_time": 0.01830112934112549, "train.compute_time": 0.6839617729187012}}
DLLL {"timestamp": "1633368468.448415", "datetime": "2021-10-04 13:27:48.448415", "elapsedtime": "62.947547", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.601072311401367, "train.total_ips": 154.65418023425883}}
DLLL {"timestamp": "1633368468.44866", "datetime": "2021-10-04 13:27:48.448660", "elapsedtime": "62.947792", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 159.14389557797375, "train.data_time": 0.02395319938659668, "train.compute_time": 0.8082155466079712}}
DLLL {"timestamp": "1633368476.01458", "datetime": "2021-10-04 13:27:56.014580", "elapsedtime": "70.513712", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.023895359039306, "train.total_ips": 170.70273293089093}}
DLLL {"timestamp": "1633368476.014802", "datetime": "2021-10-04 13:27:56.014802", "elapsedtime": "70.513934", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 174.61551766301403, "train.data_time": 0.017736053466796874, "train.compute_time": 0.7387631416320801}}
DLLL {"timestamp": "1633368483.768266", "datetime": "2021-10-04 13:28:03.768266", "elapsedtime": "78.267398", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.191830253601074, "train.total_ips": 167.31962914451577}}
DLLL {"timestamp": "1633368483.768491", "datetime": "2021-10-04 13:28:03.768491", "elapsedtime": "78.267623", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 171.60100176234033, "train.data_time": 0.0215883731842041, "train.compute_time": 0.7536546230316162}}
DLLL {"timestamp": "1633368491.454542", "datetime": "2021-10-04 13:28:11.454542", "elapsedtime": "85.953674", "type": "LOG", "step": [0, 70], "data": {"train.loss": 6.963419342041016, "train.total_ips": 168.71357916588218}}
DLLL {"timestamp": "1633368491.454777", "datetime": "2021-10-04 13:28:11.454777", "elapsedtime": "85.953909", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 170.0064446649561, "train.data_time": 0.006293892860412598, "train.compute_time": 0.7622143268585205}}
DLLL {"timestamp": "1633368498.392758", "datetime": "2021-10-04 13:28:18.392758", "elapsedtime": "92.89189", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.972329759597779, "train.total_ips": 185.35538837916403}}
DLLL {"timestamp": "1633368498.392981", "datetime": "2021-10-04 13:28:18.392981", "elapsedtime": "92.892113", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 193.43761605615094, "train.data_time": 0.029880762100219727, "train.compute_time": 0.6638155937194824}}
DLLL {"timestamp": "1633368506.252961", "datetime": "2021-10-04 13:28:26.252961", "elapsedtime": "100.752093", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.918896007537842, "train.total_ips": 163.733317701019}}
DLLL {"timestamp": "1633368506.253213", "datetime": "2021-10-04 13:28:26.253213", "elapsedtime": "100.752345", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 168.77023394141332, "train.data_time": 0.0228287935256958, "train.compute_time": 0.7630669355392456}}
DLLL {"timestamp": "1633368518.9579", "datetime": "2021-10-04 13:28:38.957900", "elapsedtime": "113.457032", "type": "LOG", "step": [0], "data": {"train.loss": 7.5195167541503904, "train.total_ips": 243.4898967507523}}
DLLL {"timestamp": "1633368518.958153", "datetime": "2021-10-04 13:28:38.958153", "elapsedtime": "113.457285", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 248.90466786048736, "train.data_time": 0.2327361798286438, "train.compute_time": 0.8516455054283142}}
DLLL {"timestamp": "1633368518.9585", "datetime": "2021-10-04 13:28:38.958500", "elapsedtime": "113.457632", "type": "LOG", "step": [], "data": {"train.loss": 7.5195167541503904, "train.total_ips": 243.4898967507523}}
DLLL {"timestamp": "1633368518.958622", "datetime": "2021-10-04 13:28:38.958622", "elapsedtime": "113.457754", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 248.90466786048736, "train.data_time": 0.2327361798286438, "train.compute_time": 0.8516455054283142}}
ascent/ascent_logs/benchmark.data.noddp.opt.5GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633368411.373175", "datetime": "2021-10-04 13:26:51.373175", "elapsedtime": "1e-05", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": true, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.noddp.opt.5GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": true, "local_rank": 0, "gpu": 0, "world_size": 5}}
DLLL {"timestamp": "1633368411.373534", "datetime": "2021-10-04 13:26:51.373534", "elapsedtime": "0.000369", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633368411.37853", "elapsedtime": "0.005365", "datetime": "2021-10-04 13:26:51.378530", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633368411.37917", "elapsedtime": "0.006005", "datetime": "2021-10-04 13:26:51.379170", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633368411.379215", "elapsedtime": "0.00605", "datetime": "2021-10-04 13:26:51.379215", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368411.379251", "elapsedtime": "0.006086", "datetime": "2021-10-04 13:26:51.379251", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633368411.379283", "elapsedtime": "0.006118", "datetime": "2021-10-04 13:26:51.379283", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368411.379315", "elapsedtime": "0.00615", "datetime": "2021-10-04 13:26:51.379315", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633368457.958417", "datetime": "2021-10-04 13:27:37.958417", "elapsedtime": "46.585252", "type": "LOG", "step": [0, 10], "data": {"train.loss": 9.100788831710815, "train.total_ips": 1023.0338108736727}}
DLLL {"timestamp": "1633368457.959024", "datetime": "2021-10-04 13:27:37.959024", "elapsedtime": "46.585859", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 1050.3562699499917, "train.data_time": 2.165387678146362, "train.compute_time": 2.492262363433838}}
DLLL {"timestamp": "1633368462.585771", "datetime": "2021-10-04 13:27:42.585771", "elapsedtime": "51.212606", "type": "LOG", "step": [0, 20], "data": {"train.loss": 8.842906188964843, "train.total_ips": 1476.4623941364266}}
DLLL {"timestamp": "1633368462.586061", "datetime": "2021-10-04 13:27:42.586061", "elapsedtime": "51.212896", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 1509.5452264044384, "train.data_time": 0.009298229217529297, "train.compute_time": 0.44155519008636473}}
DLLL {"timestamp": "1633368468.073163", "datetime": "2021-10-04 13:27:48.073163", "elapsedtime": "56.699998", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.894550085067749, "train.total_ips": 1309.7409478911486}}
DLLL {"timestamp": "1633368468.073443", "datetime": "2021-10-04 13:27:48.073443", "elapsedtime": "56.700278", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 1407.5782143279978, "train.data_time": 0.02994515895843506, "train.compute_time": 0.5303663730621337}}
DLLL {"timestamp": "1633368471.629804", "datetime": "2021-10-04 13:27:51.629804", "elapsedtime": "60.256639", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.286630153656006, "train.total_ips": 2168.4869080236517}}
DLLL {"timestamp": "1633368471.630072", "datetime": "2021-10-04 13:27:51.630072", "elapsedtime": "60.256907", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 2181.421337732894, "train.data_time": 0.00236966609954834, "train.compute_time": 0.3531148672103882}}
DLLL {"timestamp": "1633368478.082938", "datetime": "2021-10-04 13:27:58.082938", "elapsedtime": "66.709773", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.09285979270935, "train.total_ips": 1160.865363382763}}
DLLL {"timestamp": "1633368478.083216", "datetime": "2021-10-04 13:27:58.083216", "elapsedtime": "66.710051", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 1255.3017728721406, "train.data_time": 0.026070141792297365, "train.compute_time": 0.6191290855407715}}
DLLL {"timestamp": "1633368483.509815", "datetime": "2021-10-04 13:28:03.509815", "elapsedtime": "72.13665", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.018149662017822, "train.total_ips": 1363.0384227114482}}
DLLL {"timestamp": "1633368483.510092", "datetime": "2021-10-04 13:28:03.510092", "elapsedtime": "72.136927", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 1389.6295251586498, "train.data_time": 0.01359412670135498, "train.compute_time": 0.5289002418518066}}
DLLL {"timestamp": "1633368488.55437", "datetime": "2021-10-04 13:28:08.554370", "elapsedtime": "77.181205", "type": "LOG", "step": [0, 70], "data": {"train.loss": 6.959124135971069, "train.total_ips": 1737.7102794707703}}
DLLL {"timestamp": "1633368488.554691", "datetime": "2021-10-04 13:28:08.554691", "elapsedtime": "77.181526", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 1786.1241223064808, "train.data_time": 0.02091536521911621, "train.compute_time": 0.4777300596237183}}
DLLL {"timestamp": "1633368494.104963", "datetime": "2021-10-04 13:28:14.104963", "elapsedtime": "82.731798", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.974686002731323, "train.total_ips": 1287.4322997681086}}
DLLL {"timestamp": "1633368494.105257", "datetime": "2021-10-04 13:28:14.105257", "elapsedtime": "82.732092", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 1324.3367900232347, "train.data_time": 0.017661690711975098, "train.compute_time": 0.5371885061264038}}
DLLL {"timestamp": "1633368501.893486", "datetime": "2021-10-04 13:28:21.893486", "elapsedtime": "90.520321", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.934035348892212, "train.total_ips": 1220.1601169422356}}
DLLL {"timestamp": "1633368501.893768", "datetime": "2021-10-04 13:28:21.893768", "elapsedtime": "90.520603", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 1250.327585304401, "train.data_time": 0.01345670223236084, "train.compute_time": 0.7639489650726319}}
DLLL {"timestamp": "1633368511.231701", "datetime": "2021-10-04 13:28:31.231701", "elapsedtime": "99.858536", "type": "LOG", "step": [0], "data": {"train.loss": 7.5033202600479125, "train.total_ips": 1443.6376695660338}}
DLLL {"timestamp": "1633368511.232077", "datetime": "2021-10-04 13:28:31.232077", "elapsedtime": "99.858912", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 1485.910316748399, "train.data_time": 0.230459041595459, "train.compute_time": 0.7171564745903015}}
DLLL {"timestamp": "1633368511.232506", "datetime": "2021-10-04 13:28:31.232506", "elapsedtime": "99.859341", "type": "LOG", "step": [], "data": {"train.loss": 7.5033202600479125, "train.total_ips": 1443.6376695660338}}
DLLL {"timestamp": "1633368511.232672", "datetime": "2021-10-04 13:28:31.232672", "elapsedtime": "99.859507", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 1485.910316748399, "train.data_time": 0.230459041595459, "train.compute_time": 0.7171564745903015}}
ascent/ascent_logs/benchmark.data.opt.10GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633367163.88397", "datetime": "2021-10-04 13:06:03.883970", "elapsedtime": "9e-06", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": false, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.10GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": true, "local_rank": 0, "gpu": 0, "world_size": 10}}
DLLL {"timestamp": "1633367163.884283", "datetime": "2021-10-04 13:06:03.884283", "elapsedtime": "0.000322", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633367163.88907", "elapsedtime": "0.005109", "datetime": "2021-10-04 13:06:03.889070", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633367164.373133", "elapsedtime": "0.489172", "datetime": "2021-10-04 13:06:04.373133", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633367164.373181", "elapsedtime": "0.48922", "datetime": "2021-10-04 13:06:04.373181", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633367164.373217", "elapsedtime": "0.489256", "datetime": "2021-10-04 13:06:04.373217", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633367164.373249", "elapsedtime": "0.489288", "datetime": "2021-10-04 13:06:04.373249", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633367164.373282", "elapsedtime": "0.489321", "datetime": "2021-10-04 13:06:04.373282", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633367214.118731", "datetime": "2021-10-04 13:06:54.118731", "elapsedtime": "50.23477", "type": "LOG", "step": [0, 10], "data": {"train.loss": 7.182714986801147, "train.total_ips": 2900.5382717389557}}
DLLL {"timestamp": "1633367214.119242", "datetime": "2021-10-04 13:06:54.119242", "elapsedtime": "50.235281", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 2943.1209908802944, "train.data_time": 2.1950519561767576, "train.compute_time": 2.779264545440674}}
DLLL {"timestamp": "1633367221.087661", "datetime": "2021-10-04 13:07:01.087661", "elapsedtime": "57.2037", "type": "LOG", "step": [0, 20], "data": {"train.loss": 7.5353974342346195, "train.total_ips": 1908.5698167164537}}
DLLL {"timestamp": "1633367221.087893", "datetime": "2021-10-04 13:07:01.087893", "elapsedtime": "57.203932", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 1934.8546291908547, "train.data_time": 0.008437204360961913, "train.compute_time": 0.6882588624954223}}
DLLL {"timestamp": "1633367229.563547", "datetime": "2021-10-04 13:07:09.563547", "elapsedtime": "65.679586", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.592690181732178, "train.total_ips": 1537.0231046989545}}
DLLL {"timestamp": "1633367229.563762", "datetime": "2021-10-04 13:07:09.563762", "elapsedtime": "65.679801", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 1576.0813896866366, "train.data_time": 0.019305205345153807, "train.compute_time": 0.8281798124313354}}
DLLL {"timestamp": "1633367237.949613", "datetime": "2021-10-04 13:07:17.949613", "elapsedtime": "74.065652", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.424406671524048, "train.total_ips": 1555.0286303925475}}
DLLL {"timestamp": "1633367237.949884", "datetime": "2021-10-04 13:07:17.949884", "elapsedtime": "74.065923", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 1567.836104385275, "train.data_time": 0.005643486976623535, "train.compute_time": 0.8297797203063965}}
DLLL {"timestamp": "1633367245.563657", "datetime": "2021-10-04 13:07:25.563657", "elapsedtime": "81.679696", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.1125153541564945, "train.total_ips": 1726.754814887116}}
DLLL {"timestamp": "1633367245.563918", "datetime": "2021-10-04 13:07:25.563918", "elapsedtime": "81.679957", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 1753.745676430381, "train.data_time": 0.011652684211730957, "train.compute_time": 0.7526200294494629}}
DLLL {"timestamp": "1633367253.928699", "datetime": "2021-10-04 13:07:33.928699", "elapsedtime": "90.044738", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.107547092437744, "train.total_ips": 1545.1452804115418}}
DLLL {"timestamp": "1633367253.928955", "datetime": "2021-10-04 13:07:33.928955", "elapsedtime": "90.044994", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 1568.8109196845762, "train.data_time": 0.011496329307556152, "train.compute_time": 0.8247853994369507}}
DLLL {"timestamp": "1633367262.146362", "datetime": "2021-10-04 13:07:42.146362", "elapsedtime": "98.262401", "type": "LOG", "step": [0, 70], "data": {"train.loss": 7.044234895706177, "train.total_ips": 1590.221454968975}}
DLLL {"timestamp": "1633367262.146623", "datetime": "2021-10-04 13:07:42.146623", "elapsedtime": "98.262662", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 1613.1519821642928, "train.data_time": 0.009932398796081543, "train.compute_time": 0.8117234230041503}}
DLLL {"timestamp": "1633367270.57467", "datetime": "2021-10-04 13:07:50.574670", "elapsedtime": "106.690709", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.961148262023926, "train.total_ips": 1529.0781388778603}}
DLLL {"timestamp": "1633367270.574924", "datetime": "2021-10-04 13:07:50.574924", "elapsedtime": "106.690963", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 1552.693158844095, "train.data_time": 0.012608814239501952, "train.compute_time": 0.8300262928009033}}
DLLL {"timestamp": "1633367278.738024", "datetime": "2021-10-04 13:07:58.738024", "elapsedtime": "114.854063", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.950328636169433, "train.total_ips": 1604.6865644901213}}
DLLL {"timestamp": "1633367278.738269", "datetime": "2021-10-04 13:07:58.738269", "elapsedtime": "114.854308", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 1623.1427877450085, "train.data_time": 0.008434033393859864, "train.compute_time": 0.807792067527771}}
DLLL {"timestamp": "1633367291.8264", "datetime": "2021-10-04 13:08:11.826400", "elapsedtime": "127.942439", "type": "LOG", "step": [0], "data": {"train.loss": 7.184423680305481, "train.total_ips": 1749.81881970693}}
DLLL {"timestamp": "1633367291.826617", "datetime": "2021-10-04 13:08:11.826617", "elapsedtime": "127.942656", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 1774.8908585713898, "train.data_time": 0.22897837162017823, "train.compute_time": 0.9953473091125489}}
DLLL {"timestamp": "1633367291.826819", "datetime": "2021-10-04 13:08:11.826819", "elapsedtime": "127.942858", "type": "LOG", "step": [], "data": {"train.loss": 7.184423680305481, "train.total_ips": 1749.81881970693}}
DLLL {"timestamp": "1633367291.826906", "datetime": "2021-10-04 13:08:11.826906", "elapsedtime": "127.942945", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 1774.8908585713898, "train.data_time": 0.22897837162017823, "train.compute_time": 0.9953473091125489}}
ascent/ascent_logs/benchmark.data.opt.15GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633366975.764423", "datetime": "2021-10-04 13:02:55.764423", "elapsedtime": "1.2e-05", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": false, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.15GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": true, "local_rank": 0, "gpu": 0, "world_size": 15}}
DLLL {"timestamp": "1633366975.76473", "datetime": "2021-10-04 13:02:55.764730", "elapsedtime": "0.000319", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633366975.769352", "elapsedtime": "0.004941", "datetime": "2021-10-04 13:02:55.769352", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633366975.81532", "elapsedtime": "0.050909", "datetime": "2021-10-04 13:02:55.815320", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633366975.815368", "elapsedtime": "0.050957", "datetime": "2021-10-04 13:02:55.815368", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633366975.815403", "elapsedtime": "0.050992", "datetime": "2021-10-04 13:02:55.815403", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633366975.815436", "elapsedtime": "0.051025", "datetime": "2021-10-04 13:02:55.815436", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633366975.815468", "elapsedtime": "0.051057", "datetime": "2021-10-04 13:02:55.815468", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633367026.009131", "datetime": "2021-10-04 13:03:46.009131", "elapsedtime": "50.24472", "type": "LOG", "step": [0, 10], "data": {"train.loss": 7.072188901901245, "train.total_ips": 2702.7503898395103}}
DLLL {"timestamp": "1633367026.009779", "datetime": "2021-10-04 13:03:46.009779", "elapsedtime": "50.245368", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 2789.203668346113, "train.data_time": 2.426859474182129, "train.compute_time": 2.5922511339187624}}
DLLL {"timestamp": "1633367034.701505", "datetime": "2021-10-04 13:03:54.701505", "elapsedtime": "58.937094", "type": "LOG", "step": [0, 20], "data": {"train.loss": 7.253705787658691, "train.total_ips": 2221.231557272754}}
DLLL {"timestamp": "1633367034.701726", "datetime": "2021-10-04 13:03:54.701726", "elapsedtime": "58.937315", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 2254.3114381548776, "train.data_time": 0.012134623527526856, "train.compute_time": 0.8569172143936157}}
DLLL {"timestamp": "1633367043.55764", "datetime": "2021-10-04 13:04:03.557640", "elapsedtime": "67.793229", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.446936082839966, "train.total_ips": 2192.936691771552}}
DLLL {"timestamp": "1633367043.55788", "datetime": "2021-10-04 13:04:03.557880", "elapsedtime": "67.793469", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 2239.505126407682, "train.data_time": 0.017815160751342773, "train.compute_time": 0.8676985740661621}}
DLLL {"timestamp": "1633367051.785041", "datetime": "2021-10-04 13:04:11.785041", "elapsedtime": "76.02063", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.209506225585938, "train.total_ips": 2357.3273998692466}}
DLLL {"timestamp": "1633367051.785285", "datetime": "2021-10-04 13:04:11.785285", "elapsedtime": "76.020874", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 2383.296374468251, "train.data_time": 0.008504319190979003, "train.compute_time": 0.8140435934066772}}
DLLL {"timestamp": "1633367059.844915", "datetime": "2021-10-04 13:04:19.844915", "elapsedtime": "84.080504", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.140084075927734, "train.total_ips": 2406.053376295948}}
DLLL {"timestamp": "1633367059.845175", "datetime": "2021-10-04 13:04:19.845175", "elapsedtime": "84.080764", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 2447.3460659358816, "train.data_time": 0.013176679611206055, "train.compute_time": 0.7927020072937012}}
DLLL {"timestamp": "1633367068.503738", "datetime": "2021-10-04 13:04:28.503738", "elapsedtime": "92.739327", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.1227202892303465, "train.total_ips": 2223.637787555887}}
DLLL {"timestamp": "1633367068.503984", "datetime": "2021-10-04 13:04:28.503984", "elapsedtime": "92.739573", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 2237.0888057334023, "train.data_time": 0.005039739608764649, "train.compute_time": 0.8606232166290283}}
DLLL {"timestamp": "1633367076.507854", "datetime": "2021-10-04 13:04:36.507854", "elapsedtime": "100.743443", "type": "LOG", "step": [0, 70], "data": {"train.loss": 7.04561824798584, "train.total_ips": 2414.5458255017843}}
DLLL {"timestamp": "1633367076.508085", "datetime": "2021-10-04 13:04:36.508085", "elapsedtime": "100.743674", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 2452.0596442682026, "train.data_time": 0.010829424858093262, "train.compute_time": 0.7894690752029419}}
DLLL {"timestamp": "1633367084.827289", "datetime": "2021-10-04 13:04:44.827289", "elapsedtime": "109.062878", "type": "LOG", "step": [0, 80], "data": {"train.loss": 7.005381536483765, "train.total_ips": 2325.503846533984}}
DLLL {"timestamp": "1633367084.827529", "datetime": "2021-10-04 13:04:44.827529", "elapsedtime": "109.063118", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 2369.591804672305, "train.data_time": 0.01459648609161377, "train.compute_time": 0.8171473026275635}}
DLLL {"timestamp": "1633367093.174972", "datetime": "2021-10-04 13:04:53.174972", "elapsedtime": "117.410561", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.984457397460938, "train.total_ips": 2313.403266571684}}
DLLL {"timestamp": "1633367093.175213", "datetime": "2021-10-04 13:04:53.175213", "elapsedtime": "117.410802", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 2334.6334916821675, "train.data_time": 0.007247233390808105, "train.compute_time": 0.8270395278930665}}
DLLL {"timestamp": "1633367106.323661", "datetime": "2021-10-04 13:05:06.323661", "elapsedtime": "130.55925", "type": "LOG", "step": [0], "data": {"train.loss": 7.121799411773682, "train.total_ips": 2356.3817591036977}}
DLLL {"timestamp": "1633367106.323829", "datetime": "2021-10-04 13:05:06.323829", "elapsedtime": "130.559418", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 2395.721183832494, "train.data_time": 0.25271469593048096, "train.compute_time": 1.0021142029762269}}
DLLL {"timestamp": "1633367106.324004", "datetime": "2021-10-04 13:05:06.324004", "elapsedtime": "130.559593", "type": "LOG", "step": [], "data": {"train.loss": 7.121799411773682, "train.total_ips": 2356.3817591036977}}
DLLL {"timestamp": "1633367106.324069", "datetime": "2021-10-04 13:05:06.324069", "elapsedtime": "130.559658", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 2395.721183832494, "train.data_time": 0.25271469593048096, "train.compute_time": 1.0021142029762269}}
ascent/ascent_logs/benchmark.data.opt.1GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633366782.005491", "datetime": "2021-10-04 12:59:42.005491", "elapsedtime": "1.4e-05", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": false, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.1GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": false, "local_rank": 0, "gpu": 0, "world_size": 1}}
DLLL {"timestamp": "1633366782.005798", "datetime": "2021-10-04 12:59:42.005798", "elapsedtime": "0.000321", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633366782.01028", "elapsedtime": "0.004803", "datetime": "2021-10-04 12:59:42.010280", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633366782.010801", "elapsedtime": "0.005324", "datetime": "2021-10-04 12:59:42.010801", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633366782.01084", "elapsedtime": "0.005363", "datetime": "2021-10-04 12:59:42.010840", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633366782.010873", "elapsedtime": "0.005396", "datetime": "2021-10-04 12:59:42.010873", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633366782.010905", "elapsedtime": "0.005428", "datetime": "2021-10-04 12:59:42.010905", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633366782.010937", "elapsedtime": "0.00546", "datetime": "2021-10-04 12:59:42.010937", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633366825.324186", "datetime": "2021-10-04 13:00:25.324186", "elapsedtime": "43.318709", "type": "LOG", "step": [0, 10], "data": {"train.loss": 9.143393945693969, "train.total_ips": 681.6669413468046}}
DLLL {"timestamp": "1633366825.324705", "datetime": "2021-10-04 13:00:25.324705", "elapsedtime": "43.319228", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 687.0915298032702, "train.data_time": 2.1264915466308594, "train.compute_time": 2.2046539545059205}}
DLLL {"timestamp": "1633366828.978403", "datetime": "2021-10-04 13:00:28.978403", "elapsedtime": "46.972926", "type": "LOG", "step": [0, 20], "data": {"train.loss": 9.324459075927734, "train.total_ips": 408.5973957368579}}
DLLL {"timestamp": "1633366828.978625", "datetime": "2021-10-04 13:00:28.978625", "elapsedtime": "46.973148", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 413.70488338069765, "train.data_time": 0.006155705451965332, "train.compute_time": 0.3591571807861328}}
DLLL {"timestamp": "1633366836.086278", "datetime": "2021-10-04 13:00:36.086278", "elapsedtime": "54.080801", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.754433965682983, "train.total_ips": 181.10683263170282}}
DLLL {"timestamp": "1633366836.086512", "datetime": "2021-10-04 13:00:36.086512", "elapsedtime": "54.081035", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 186.71396052363136, "train.data_time": 0.020975732803344728, "train.compute_time": 0.6896930456161499}}
DLLL {"timestamp": "1633366843.980084", "datetime": "2021-10-04 13:00:43.980084", "elapsedtime": "61.974607", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.532516384124756, "train.total_ips": 163.0120739762698}}
DLLL {"timestamp": "1633366843.980312", "datetime": "2021-10-04 13:00:43.980312", "elapsedtime": "61.974835", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 169.40591110620943, "train.data_time": 0.02857215404510498, "train.compute_time": 0.7606812953948975}}
DLLL {"timestamp": "1633366851.463074", "datetime": "2021-10-04 13:00:51.463074", "elapsedtime": "69.457597", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.2729172706604, "train.total_ips": 172.1038196851434}}
DLLL {"timestamp": "1633366851.463331", "datetime": "2021-10-04 13:00:51.463331", "elapsedtime": "69.457854", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 179.22825576359, "train.data_time": 0.0291060209274292, "train.compute_time": 0.7190677404403687}}
DLLL {"timestamp": "1633366858.976614", "datetime": "2021-10-04 13:00:58.976614", "elapsedtime": "76.971137", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.016049003601074, "train.total_ips": 172.354746650962}}
DLLL {"timestamp": "1633366858.976849", "datetime": "2021-10-04 13:00:58.976849", "elapsedtime": "76.971372", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 176.3413771256184, "train.data_time": 0.0176114559173584, "train.compute_time": 0.7336230039596557}}
DLLL {"timestamp": "1633366866.784137", "datetime": "2021-10-04 13:01:06.784137", "elapsedtime": "84.77866", "type": "LOG", "step": [0, 70], "data": {"train.loss": 6.960112810134888, "train.total_ips": 167.28117506353584}}
DLLL {"timestamp": "1633366866.807706", "datetime": "2021-10-04 13:01:06.807706", "elapsedtime": "84.802229", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 172.87520808495327, "train.data_time": 0.027497386932373045, "train.compute_time": 0.7488190174102783}}
DLLL {"timestamp": "1633366874.160927", "datetime": "2021-10-04 13:01:14.160927", "elapsedtime": "92.15545", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.94903883934021, "train.total_ips": 176.21874711460248}}
DLLL {"timestamp": "1633366874.161153", "datetime": "2021-10-04 13:01:14.161153", "elapsedtime": "92.155676", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 182.4007807941556, "train.data_time": 0.02353365421295166, "train.compute_time": 0.7140132904052734}}
DLLL {"timestamp": "1633366881.370676", "datetime": "2021-10-04 13:01:21.370676", "elapsedtime": "99.365199", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.917493391036987, "train.total_ips": 183.08413349211938}}
DLLL {"timestamp": "1633366881.370926", "datetime": "2021-10-04 13:01:21.370926", "elapsedtime": "99.365449", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 193.75800455772372, "train.data_time": 0.036658430099487306, "train.compute_time": 0.6768976211547851}}
DLLL {"timestamp": "1633366893.127849", "datetime": "2021-10-04 13:01:33.127849", "elapsedtime": "111.122372", "type": "LOG", "step": [0], "data": {"train.loss": 7.5794302225112915, "train.total_ips": 250.79697422674948}}
DLLL {"timestamp": "1633366893.128078", "datetime": "2021-10-04 13:01:33.128078", "elapsedtime": "111.122601", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 256.9279010434933, "train.data_time": 0.23351834535598756, "train.compute_time": 0.8262648439407348}}
DLLL {"timestamp": "1633366893.128394", "datetime": "2021-10-04 13:01:33.128394", "elapsedtime": "111.122917", "type": "LOG", "step": [], "data": {"train.loss": 7.5794302225112915, "train.total_ips": 250.79697422674948}}
DLLL {"timestamp": "1633366893.128518", "datetime": "2021-10-04 13:01:33.128518", "elapsedtime": "111.123041", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 256.9279010434933, "train.data_time": 0.23351834535598756, "train.compute_time": 0.8262648439407348}}
ascent/ascent_logs/benchmark.data.opt.20GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633366975.026215", "datetime": "2021-10-04 13:02:55.026215", "elapsedtime": "9e-06", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": false, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.20GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": true, "local_rank": 0, "gpu": 0, "world_size": 20}}
DLLL {"timestamp": "1633366975.026539", "datetime": "2021-10-04 13:02:55.026539", "elapsedtime": "0.000333", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633366975.031149", "elapsedtime": "0.004943", "datetime": "2021-10-04 13:02:55.031149", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633366975.098784", "elapsedtime": "0.072578", "datetime": "2021-10-04 13:02:55.098784", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633366975.098851", "elapsedtime": "0.072645", "datetime": "2021-10-04 13:02:55.098851", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633366975.098902", "elapsedtime": "0.072696", "datetime": "2021-10-04 13:02:55.098902", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633366975.09895", "elapsedtime": "0.072744", "datetime": "2021-10-04 13:02:55.098950", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633366975.098997", "elapsedtime": "0.072791", "datetime": "2021-10-04 13:02:55.098997", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633367046.03909", "datetime": "2021-10-04 13:04:06.039090", "elapsedtime": "71.012884", "type": "LOG", "step": [0, 10], "data": {"train.loss": 7.045035743713379, "train.total_ips": 5626.953493008938}}
DLLL {"timestamp": "1633367046.03964", "datetime": "2021-10-04 13:04:06.039640", "elapsedtime": "71.013434", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 5680.827602667565, "train.data_time": 2.125786066055298, "train.compute_time": 4.9679429769515995}}
DLLL {"timestamp": "1633367056.257319", "datetime": "2021-10-04 13:04:16.257319", "elapsedtime": "81.231113", "type": "LOG", "step": [0, 20], "data": {"train.loss": 7.174418258666992, "train.total_ips": 2529.850848037966}}
DLLL {"timestamp": "1633367056.257612", "datetime": "2021-10-04 13:04:16.257612", "elapsedtime": "81.231406", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 2567.090461807862, "train.data_time": 0.013248157501220704, "train.compute_time": 1.008377766609192}}
DLLL {"timestamp": "1633367065.438723", "datetime": "2021-10-04 13:04:25.438723", "elapsedtime": "90.412517", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.353969669342041, "train.total_ips": 2867.8258210773733}}
DLLL {"timestamp": "1633367065.438999", "datetime": "2021-10-04 13:04:25.438999", "elapsedtime": "90.412793", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 2899.7099127711745, "train.data_time": 0.010905313491821288, "train.compute_time": 0.906805419921875}}
DLLL {"timestamp": "1633367075.153855", "datetime": "2021-10-04 13:04:35.153855", "elapsedtime": "100.127649", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.220703220367431, "train.total_ips": 2665.520918334095}}
DLLL {"timestamp": "1633367075.154166", "datetime": "2021-10-04 13:04:35.154166", "elapsedtime": "100.12796", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 2683.422777068956, "train.data_time": 0.006829357147216797, "train.compute_time": 0.9644888162612915}}
DLLL {"timestamp": "1633367085.102168", "datetime": "2021-10-04 13:04:45.102168", "elapsedtime": "110.075962", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.118415880203247, "train.total_ips": 2613.0525717776677}}
DLLL {"timestamp": "1633367085.102494", "datetime": "2021-10-04 13:04:45.102494", "elapsedtime": "110.076288", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 2637.794069329174, "train.data_time": 0.010201644897460938, "train.compute_time": 0.9845112562179565}}
DLLL {"timestamp": "1633367095.264805", "datetime": "2021-10-04 13:04:55.264805", "elapsedtime": "120.238599", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.043138599395752, "train.total_ips": 2537.1542281634875}}
DLLL {"timestamp": "1633367095.265098", "datetime": "2021-10-04 13:04:55.265098", "elapsedtime": "120.238892", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 2562.5358148910545, "train.data_time": 0.010465145111083984, "train.compute_time": 1.005621862411499}}
DLLL {"timestamp": "1633367104.976593", "datetime": "2021-10-04 13:05:04.976593", "elapsedtime": "129.950387", "type": "LOG", "step": [0, 70], "data": {"train.loss": 7.019901466369629, "train.total_ips": 2654.5490773495812}}
DLLL {"timestamp": "1633367104.97686", "datetime": "2021-10-04 13:05:04.976860", "elapsedtime": "129.950654", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 2673.632711151256, "train.data_time": 0.007492280006408692, "train.compute_time": 0.9635673284530639}}
DLLL {"timestamp": "1633367115.039382", "datetime": "2021-10-04 13:05:15.039382", "elapsedtime": "140.013176", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.973300170898438, "train.total_ips": 2557.3187385538304}}
DLLL {"timestamp": "1633367115.039661", "datetime": "2021-10-04 13:05:15.039661", "elapsedtime": "140.013455", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 2568.1463085071487, "train.data_time": 0.003994441032409668, "train.compute_time": 1.0020900964736938}}
DLLL {"timestamp": "1633367124.861067", "datetime": "2021-10-04 13:05:24.861067", "elapsedtime": "149.834861", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.956139945983887, "train.total_ips": 2653.434155798601}}
DLLL {"timestamp": "1633367124.88142", "datetime": "2021-10-04 13:05:24.881420", "elapsedtime": "149.855214", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 2671.669342881167, "train.data_time": 0.005961227416992188, "train.compute_time": 0.9760968685150146}}
DLLL {"timestamp": "1633367140.074991", "datetime": "2021-10-04 13:05:40.074991", "elapsedtime": "165.048785", "type": "LOG", "step": [0], "data": {"train.loss": 7.084707942008972, "train.total_ips": 2923.739016411492}}
DLLL {"timestamp": "1633367140.07515", "datetime": "2021-10-04 13:05:40.075150", "elapsedtime": "165.048944", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 2948.5318411503954, "train.data_time": 0.21984706878662108, "train.compute_time": 1.3796652913093568}}
DLLL {"timestamp": "1633367140.075326", "datetime": "2021-10-04 13:05:40.075326", "elapsedtime": "165.04912", "type": "LOG", "step": [], "data": {"train.loss": 7.084707942008972, "train.total_ips": 2923.739016411492}}
DLLL {"timestamp": "1633367140.0754", "datetime": "2021-10-04 13:05:40.075400", "elapsedtime": "165.049194", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 2948.5318411503954, "train.data_time": 0.21984706878662108, "train.compute_time": 1.3796652913093568}}
ascent/ascent_logs/benchmark.data.opt.5GPU.json deleted 100644 → 0
View file @ a280ec34
DLLL {"timestamp": "1633366789.763861", "datetime": "2021-10-04 12:59:49.763861", "elapsedtime": "1.1e-05", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 28, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": false, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": true, "seed": null, "gather_checkpoints": false, "raport_file": "/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.5GPU.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nhwc", "use_ema": null, "augmentation": null, "num_classes": null, "distributed": true, "local_rank": 0, "gpu": 0, "world_size": 5}}
DLLL {"timestamp": "1633366789.764195", "datetime": "2021-10-04 12:59:49.764195", "elapsedtime": "0.000345", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1633366789.768848", "elapsedtime": "0.004998", "datetime": "2021-10-04 12:59:49.768848", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1633366790.253636", "elapsedtime": "0.489786", "datetime": "2021-10-04 12:59:50.253636", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1633366790.253694", "elapsedtime": "0.489844", "datetime": "2021-10-04 12:59:50.253694", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633366790.253731", "elapsedtime": "0.489881", "datetime": "2021-10-04 12:59:50.253731", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1633366790.253764", "elapsedtime": "0.489914", "datetime": "2021-10-04 12:59:50.253764", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633366790.253797", "elapsedtime": "0.489947", "datetime": "2021-10-04 12:59:50.253797", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1633366837.669666", "datetime": "2021-10-04 13:00:37.669666", "elapsedtime": "47.905816", "type": "LOG", "step": [0, 10], "data": {"train.loss": 7.436073589324951, "train.total_ips": 1558.367508961958}}
DLLL {"timestamp": "1633366837.670159", "datetime": "2021-10-04 13:00:37.670159", "elapsedtime": "47.906309", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 1588.4415661371518, "train.data_time": 2.352148246765137, "train.compute_time": 2.389200711250305}}
DLLL {"timestamp": "1633366842.943276", "datetime": "2021-10-04 13:00:42.943276", "elapsedtime": "53.179426", "type": "LOG", "step": [0, 20], "data": {"train.loss": 8.108831787109375, "train.total_ips": 1338.0017371429026}}
DLLL {"timestamp": "1633366842.943564", "datetime": "2021-10-04 13:00:42.943564", "elapsedtime": "53.179714", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 1352.1017431526247, "train.data_time": 0.0060283422470092775, "train.compute_time": 0.5211388826370239}}
DLLL {"timestamp": "1633366850.563941", "datetime": "2021-10-04 13:00:50.563941", "elapsedtime": "60.800091", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.379528522491455, "train.total_ips": 916.7487199272591}}
DLLL {"timestamp": "1633366850.564215", "datetime": "2021-10-04 13:00:50.564215", "elapsedtime": "60.800365", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 932.7611682083294, "train.data_time": 0.014114117622375489, "train.compute_time": 0.7408340692520141}}
DLLL {"timestamp": "1633366858.580591", "datetime": "2021-10-04 13:00:58.580591", "elapsedtime": "68.816741", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.248611783981323, "train.total_ips": 812.7829023048555}}
DLLL {"timestamp": "1633366858.58089", "datetime": "2021-10-04 13:00:58.580890", "elapsedtime": "68.81704", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 826.1597774475997, "train.data_time": 0.011307120323181152, "train.compute_time": 0.790150237083435}}
DLLL {"timestamp": "1633366864.199714", "datetime": "2021-10-04 13:01:04.199714", "elapsedtime": "74.435864", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.120347309112549, "train.total_ips": 1243.6476891065395}}
DLLL {"timestamp": "1633366864.200025", "datetime": "2021-10-04 13:01:04.200025", "elapsedtime": "74.436175", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 1278.8759358143914, "train.data_time": 0.010735273361206055, "train.compute_time": 0.5510577201843262}}
DLLL {"timestamp": "1633366869.678219", "datetime": "2021-10-04 13:01:09.678219", "elapsedtime": "79.914369", "type": "LOG", "step": [0, 60], "data": {"train.loss": 7.038975667953491,