Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Tsaris, Aristeidis (aris)
pytorch_tutorial
Commits
d437e33e
Commit
d437e33e
authored
Oct 05, 2021
by
Aristeidis Tsaris
Browse files
adding option for data loading only
parent
a280ec34
Changes
23
Expand all
Hide whitespace changes
Inline
Side-by-side
ascent/ascent_logs/benchmark.data.nodd.opt.15GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633368580.678657"
,
"datetime"
:
"2021-10-04 13:29:40.678657"
,
"elapsedtime"
:
"1e-05"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
true
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.nodd.opt.15GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
true
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
15
}}
DLLL
{
"timestamp"
:
"1633368580.678969"
,
"datetime"
:
"2021-10-04 13:29:40.678969"
,
"elapsedtime"
:
"0.000322"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633368580.683555"
,
"elapsedtime"
:
"0.004908"
,
"datetime"
:
"2021-10-04 13:29:40.683555"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633368580.684055"
,
"elapsedtime"
:
"0.005408"
,
"datetime"
:
"2021-10-04 13:29:40.684055"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368580.684095"
,
"elapsedtime"
:
"0.005448"
,
"datetime"
:
"2021-10-04 13:29:40.684095"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368580.684128"
,
"elapsedtime"
:
"0.005481"
,
"datetime"
:
"2021-10-04 13:29:40.684128"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368580.68416"
,
"elapsedtime"
:
"0.005513"
,
"datetime"
:
"2021-10-04 13:29:40.684160"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368580.684191"
,
"elapsedtime"
:
"0.005544"
,
"datetime"
:
"2021-10-04 13:29:40.684191"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368646.944581"
,
"datetime"
:
"2021-10-04 13:30:46.944581"
,
"elapsedtime"
:
"66.265934"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
8.905070209503174
,
"train.total_ips"
:
6389.44978951381
}}
DLLL
{
"timestamp"
:
"1633368646.945175"
,
"datetime"
:
"2021-10-04 13:30:46.945175"
,
"elapsedtime"
:
"66.266528"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
6473.6486731910245
,
"train.data_time"
:
2.1612624883651734
,
"train.compute_time"
:
4.464561533927918
}}
DLLL
{
"timestamp"
:
"1633368651.25147"
,
"datetime"
:
"2021-10-04 13:30:51.251470"
,
"elapsedtime"
:
"70.572823"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
9.109191417694092
,
"train.total_ips"
:
4596.433817520007
}}
DLLL
{
"timestamp"
:
"1633368651.251646"
,
"datetime"
:
"2021-10-04 13:30:51.251646"
,
"elapsedtime"
:
"70.572999"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
4772.982603551252
,
"train.data_time"
:
0.014628076553344726
,
"train.compute_time"
:
0.41589579582214353
}}
DLLL
{
"timestamp"
:
"1633368657.95684"
,
"datetime"
:
"2021-10-04 13:30:57.956840"
,
"elapsedtime"
:
"77.278193"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.8272237300872805
,
"train.total_ips"
:
3037.2251743161896
}}
DLLL
{
"timestamp"
:
"1633368657.957079"
,
"datetime"
:
"2021-10-04 13:30:57.957079"
,
"elapsedtime"
:
"77.278432"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
3107.286167126355
,
"train.data_time"
:
0.012641692161560058
,
"train.compute_time"
:
0.6577740907669067
}}
DLLL
{
"timestamp"
:
"1633368665.573776"
,
"datetime"
:
"2021-10-04 13:31:05.573776"
,
"elapsedtime"
:
"84.895129"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.28309211730957
,
"train.total_ips"
:
2658.9515256612744
}}
DLLL
{
"timestamp"
:
"1633368665.574003"
,
"datetime"
:
"2021-10-04 13:31:05.574003"
,
"elapsedtime"
:
"84.895356"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
2727.3296430647606
,
"train.data_time"
:
0.018487381935119628
,
"train.compute_time"
:
0.7430117845535278
}}
DLLL
{
"timestamp"
:
"1633368673.122152"
,
"datetime"
:
"2021-10-04 13:31:13.122152"
,
"elapsedtime"
:
"92.443505"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.059678316116333
,
"train.total_ips"
:
2602.605282874574
}}
DLLL
{
"timestamp"
:
"1633368673.122362"
,
"datetime"
:
"2021-10-04 13:31:13.122362"
,
"elapsedtime"
:
"92.443715"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
2693.2395674618074
,
"train.data_time"
:
0.021999835968017578
,
"train.compute_time"
:
0.73272545337677
}}
DLLL
{
"timestamp"
:
"1633368680.073372"
,
"datetime"
:
"2021-10-04 13:31:20.073372"
,
"elapsedtime"
:
"99.394725"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.024379920959473
,
"train.total_ips"
:
2917.289312323478
}}
DLLL
{
"timestamp"
:
"1633368680.07364"
,
"datetime"
:
"2021-10-04 13:31:20.073640"
,
"elapsedtime"
:
"99.394993"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.compute_ips"
:
2980.049593595777
,
"train.data_time"
:
0.013703417778015137
,
"train.compute_time"
:
0.6812222957611084
}}
DLLL
{
"timestamp"
:
"1633368686.796575"
,
"datetime"
:
"2021-10-04 13:31:26.796575"
,
"elapsedtime"
:
"106.117928"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.loss"
:
6.959270429611206
,
"train.total_ips"
:
3010.9175657984183
}}
DLLL
{
"timestamp"
:
"1633368686.796827"
,
"datetime"
:
"2021-10-04 13:31:26.796827"
,
"elapsedtime"
:
"106.11818"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.compute_ips"
:
3114.4988392527257
,
"train.data_time"
:
0.014000844955444337
,
"train.compute_time"
:
0.658202075958252
}}
DLLL
{
"timestamp"
:
"1633368693.77067"
,
"datetime"
:
"2021-10-04 13:31:33.770670"
,
"elapsedtime"
:
"113.092023"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.loss"
:
6.941778516769409
,
"train.total_ips"
:
3429.7081059958764
}}
DLLL
{
"timestamp"
:
"1633368693.771014"
,
"datetime"
:
"2021-10-04 13:31:33.771014"
,
"elapsedtime"
:
"113.092367"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.compute_ips"
:
3522.729058906014
,
"train.data_time"
:
0.014573359489440918
,
"train.compute_time"
:
0.682635235786438
}}
DLLL
{
"timestamp"
:
"1633368701.257512"
,
"datetime"
:
"2021-10-04 13:31:41.257512"
,
"elapsedtime"
:
"120.578865"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.loss"
:
6.930657529830933
,
"train.total_ips"
:
2693.1620197000298
}}
DLLL
{
"timestamp"
:
"1633368701.257712"
,
"datetime"
:
"2021-10-04 13:31:41.257712"
,
"elapsedtime"
:
"120.579065"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.compute_ips"
:
2786.7778926332608
,
"train.data_time"
:
0.02726137638092041
,
"train.compute_time"
:
0.7207160472869873
}}
DLLL
{
"timestamp"
:
"1633368712.696503"
,
"datetime"
:
"2021-10-04 13:31:52.696503"
,
"elapsedtime"
:
"132.017856"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"train.loss"
:
7.496806492805481
,
"train.total_ips"
:
3444.7074426028926
}}
DLLL
{
"timestamp"
:
"1633368712.696743"
,
"datetime"
:
"2021-10-04 13:31:52.696743"
,
"elapsedtime"
:
"132.018096"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
3532.706753183429
,
"train.data_time"
:
0.23064370155334474
,
"train.compute_time"
:
1.0392441320419312
}}
DLLL
{
"timestamp"
:
"1633368712.697045"
,
"datetime"
:
"2021-10-04 13:31:52.697045"
,
"elapsedtime"
:
"132.018398"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"train.loss"
:
7.496806492805481
,
"train.total_ips"
:
3444.7074426028926
}}
DLLL
{
"timestamp"
:
"1633368712.697195"
,
"datetime"
:
"2021-10-04 13:31:52.697195"
,
"elapsedtime"
:
"132.018548"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
3532.706753183429
,
"train.data_time"
:
0.23064370155334474
,
"train.compute_time"
:
1.0392441320419312
}}
ascent/ascent_logs/benchmark.data.nodd.opt.20GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633368583.030069"
,
"datetime"
:
"2021-10-04 13:29:43.030069"
,
"elapsedtime"
:
"9e-06"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
true
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.nodd.opt.20GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
true
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
20
}}
DLLL
{
"timestamp"
:
"1633368583.030395"
,
"datetime"
:
"2021-10-04 13:29:43.030395"
,
"elapsedtime"
:
"0.000335"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633368583.035312"
,
"elapsedtime"
:
"0.005252"
,
"datetime"
:
"2021-10-04 13:29:43.035312"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633368583.035896"
,
"elapsedtime"
:
"0.005836"
,
"datetime"
:
"2021-10-04 13:29:43.035896"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368583.035937"
,
"elapsedtime"
:
"0.005877"
,
"datetime"
:
"2021-10-04 13:29:43.035937"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368583.035972"
,
"elapsedtime"
:
"0.005912"
,
"datetime"
:
"2021-10-04 13:29:43.035972"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368583.036004"
,
"elapsedtime"
:
"0.005944"
,
"datetime"
:
"2021-10-04 13:29:43.036004"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368583.036036"
,
"elapsedtime"
:
"0.005976"
,
"datetime"
:
"2021-10-04 13:29:43.036036"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368631.447573"
,
"datetime"
:
"2021-10-04 13:30:31.447573"
,
"elapsedtime"
:
"48.417513"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
8.915813684463501
,
"train.total_ips"
:
8584.1370181202
}}
DLLL
{
"timestamp"
:
"1633368631.448053"
,
"datetime"
:
"2021-10-04 13:30:31.448053"
,
"elapsedtime"
:
"48.417993"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
8645.534205403947
,
"train.data_time"
:
2.366662883758545
,
"train.compute_time"
:
2.4742587566375733
}}
DLLL
{
"timestamp"
:
"1633368635.910159"
,
"datetime"
:
"2021-10-04 13:30:35.910159"
,
"elapsedtime"
:
"52.880099"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
9.109982299804688
,
"train.total_ips"
:
6108.881292394909
}}
DLLL
{
"timestamp"
:
"1633368635.910449"
,
"datetime"
:
"2021-10-04 13:30:35.910449"
,
"elapsedtime"
:
"52.880389"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
6383.591078915261
,
"train.data_time"
:
0.01201937198638916
,
"train.compute_time"
:
0.4340439558029175
}}
DLLL
{
"timestamp"
:
"1633368642.303787"
,
"datetime"
:
"2021-10-04 13:30:42.303787"
,
"elapsedtime"
:
"59.273727"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.7820264339447025
,
"train.total_ips"
:
4148.383195620336
}}
DLLL
{
"timestamp"
:
"1633368642.304073"
,
"datetime"
:
"2021-10-04 13:30:42.304073"
,
"elapsedtime"
:
"59.274013"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
4325.249343010184
,
"train.data_time"
:
0.025083017349243165
,
"train.compute_time"
:
0.6141943454742431
}}
DLLL
{
"timestamp"
:
"1633368649.939104"
,
"datetime"
:
"2021-10-04 13:30:49.939104"
,
"elapsedtime"
:
"66.909044"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.401272773742676
,
"train.total_ips"
:
3487.4779059437615
}}
DLLL
{
"timestamp"
:
"1633368649.939377"
,
"datetime"
:
"2021-10-04 13:30:49.939377"
,
"elapsedtime"
:
"66.909317"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
3589.816356171057
,
"train.data_time"
:
0.019240164756774904
,
"train.compute_time"
:
0.7441050052642822
}}
DLLL
{
"timestamp"
:
"1633368657.323201"
,
"datetime"
:
"2021-10-04 13:30:57.323201"
,
"elapsedtime"
:
"74.293141"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.060693359375
,
"train.total_ips"
:
3569.469967955943
}}
DLLL
{
"timestamp"
:
"1633368657.323492"
,
"datetime"
:
"2021-10-04 13:30:57.323492"
,
"elapsedtime"
:
"74.293432"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
3657.0591110887517
,
"train.data_time"
:
0.014231324195861816
,
"train.compute_time"
:
0.7240637063980102
}}
DLLL
{
"timestamp"
:
"1633368665.425086"
,
"datetime"
:
"2021-10-04 13:31:05.425086"
,
"elapsedtime"
:
"82.395026"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.027244567871094
,
"train.total_ips"
:
3242.979710037537
}}
DLLL
{
"timestamp"
:
"1633368665.425391"
,
"datetime"
:
"2021-10-04 13:31:05.425391"
,
"elapsedtime"
:
"82.395331"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.compute_ips"
:
3304.2447259878295
,
"train.data_time"
:
0.0157520055770874
,
"train.compute_time"
:
0.794244909286499
}}
DLLL
{
"timestamp"
:
"1633368672.425366"
,
"datetime"
:
"2021-10-04 13:31:12.425366"
,
"elapsedtime"
:
"89.395306"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.loss"
:
6.967746877670288
,
"train.total_ips"
:
3751.2963648558434
}}
DLLL
{
"timestamp"
:
"1633368672.425656"
,
"datetime"
:
"2021-10-04 13:31:12.425656"
,
"elapsedtime"
:
"89.395596"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.compute_ips"
:
3845.544488742157
,
"train.data_time"
:
0.013252568244934083
,
"train.compute_time"
:
0.686657166481018
}}
DLLL
{
"timestamp"
:
"1633368679.860262"
,
"datetime"
:
"2021-10-04 13:31:19.860262"
,
"elapsedtime"
:
"96.830202"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.loss"
:
6.936493682861328
,
"train.total_ips"
:
3528.3532434775175
}}
DLLL
{
"timestamp"
:
"1633368679.860557"
,
"datetime"
:
"2021-10-04 13:31:19.860557"
,
"elapsedtime"
:
"96.830497"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.compute_ips"
:
3560.794835174045
,
"train.data_time"
:
0.007741785049438477
,
"train.compute_time"
:
0.7355400085449219
}}
DLLL
{
"timestamp"
:
"1633368687.547554"
,
"datetime"
:
"2021-10-04 13:31:27.547554"
,
"elapsedtime"
:
"104.517494"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.loss"
:
6.935410881042481
,
"train.total_ips"
:
3621.6625334724095
}}
DLLL
{
"timestamp"
:
"1633368687.547852"
,
"datetime"
:
"2021-10-04 13:31:27.547852"
,
"elapsedtime"
:
"104.517792"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.compute_ips"
:
3680.9443631909708
,
"train.data_time"
:
0.012058782577514648
,
"train.compute_time"
:
0.747539758682251
}}
DLLL
{
"timestamp"
:
"1633368700.058416"
,
"datetime"
:
"2021-10-04 13:31:40.058416"
,
"elapsedtime"
:
"117.028356"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"train.loss"
:
7.506624412536621
,
"train.total_ips"
:
4372.890397927424
}}
DLLL
{
"timestamp"
:
"1633368700.058694"
,
"datetime"
:
"2021-10-04 13:31:40.058694"
,
"elapsedtime"
:
"117.028634"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
4482.959172389081
,
"train.data_time"
:
0.25047002792358397
,
"train.compute_time"
:
0.8686380028724671
}}
DLLL
{
"timestamp"
:
"1633368700.058993"
,
"datetime"
:
"2021-10-04 13:31:40.058993"
,
"elapsedtime"
:
"117.028933"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"train.loss"
:
7.506624412536621
,
"train.total_ips"
:
4372.890397927424
}}
DLLL
{
"timestamp"
:
"1633368700.059119"
,
"datetime"
:
"2021-10-04 13:31:40.059119"
,
"elapsedtime"
:
"117.029059"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
4482.959172389081
,
"train.data_time"
:
0.25047002792358397
,
"train.compute_time"
:
0.8686380028724671
}}
ascent/ascent_logs/benchmark.data.noddp.opt.10GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633368765.528052"
,
"datetime"
:
"2021-10-04 13:32:45.528052"
,
"elapsedtime"
:
"1e-05"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
true
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.noddp.opt.10GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
true
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
10
}}
DLLL
{
"timestamp"
:
"1633368765.528379"
,
"datetime"
:
"2021-10-04 13:32:45.528379"
,
"elapsedtime"
:
"0.000337"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633368765.533024"
,
"elapsedtime"
:
"0.004982"
,
"datetime"
:
"2021-10-04 13:32:45.533024"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633368765.533573"
,
"elapsedtime"
:
"0.005531"
,
"datetime"
:
"2021-10-04 13:32:45.533573"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368765.533612"
,
"elapsedtime"
:
"0.00557"
,
"datetime"
:
"2021-10-04 13:32:45.533612"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368765.533646"
,
"elapsedtime"
:
"0.005604"
,
"datetime"
:
"2021-10-04 13:32:45.533646"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368765.533678"
,
"elapsedtime"
:
"0.005636"
,
"datetime"
:
"2021-10-04 13:32:45.533678"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368765.533709"
,
"elapsedtime"
:
"0.005667"
,
"datetime"
:
"2021-10-04 13:32:45.533709"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368812.372868"
,
"datetime"
:
"2021-10-04 13:33:32.372868"
,
"elapsedtime"
:
"46.844826"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
9.080923652648925
,
"train.total_ips"
:
2979.7606552116313
}}
DLLL
{
"timestamp"
:
"1633368812.373358"
,
"datetime"
:
"2021-10-04 13:33:32.373358"
,
"elapsedtime"
:
"46.845316"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
3095.0063886102052
,
"train.data_time"
:
2.2562681436538696
,
"train.compute_time"
:
2.4273983001708985
}}
DLLL
{
"timestamp"
:
"1633368817.196774"
,
"datetime"
:
"2021-10-04 13:33:37.196774"
,
"elapsedtime"
:
"51.668732"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
8.959366512298583
,
"train.total_ips"
:
2822.8469948685533
}}
DLLL
{
"timestamp"
:
"1633368817.197036"
,
"datetime"
:
"2021-10-04 13:33:37.197036"
,
"elapsedtime"
:
"51.668994"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
3002.7498468942094
,
"train.data_time"
:
0.020646333694458008
,
"train.compute_time"
:
0.4615530252456665
}}
DLLL
{
"timestamp"
:
"1633368823.178228"
,
"datetime"
:
"2021-10-04 13:33:43.178228"
,
"elapsedtime"
:
"57.650186"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.835245847702026
,
"train.total_ips"
:
2322.862417672294
}}
DLLL
{
"timestamp"
:
"1633368823.178515"
,
"datetime"
:
"2021-10-04 13:33:43.178515"
,
"elapsedtime"
:
"57.650473"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
2343.902633392147
,
"train.data_time"
:
0.006995439529418945
,
"train.compute_time"
:
0.5910444974899292
}}
DLLL
{
"timestamp"
:
"1633368828.876149"
,
"datetime"
:
"2021-10-04 13:33:48.876149"
,
"elapsedtime"
:
"63.348107"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.4600077152252195
,
"train.total_ips"
:
2616.4481985408847
}}
DLLL
{
"timestamp"
:
"1633368828.876438"
,
"datetime"
:
"2021-10-04 13:33:48.876438"
,
"elapsedtime"
:
"63.348396"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
2724.3758081607602
,
"train.data_time"
:
0.014265203475952148
,
"train.compute_time"
:
0.5553366899490356
}}
DLLL
{
"timestamp"
:
"1633368836.451182"
,
"datetime"
:
"2021-10-04 13:33:56.451182"
,
"elapsedtime"
:
"70.92314"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.154645395278931
,
"train.total_ips"
:
1885.6756840638488
}}
DLLL
{
"timestamp"
:
"1633368836.451502"
,
"datetime"
:
"2021-10-04 13:33:56.451502"
,
"elapsedtime"
:
"70.92346"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
1927.7501845770403
,
"train.data_time"
:
0.017845940589904786
,
"train.compute_time"
:
0.7395466327667236
}}
DLLL
{
"timestamp"
:
"1633368843.038569"
,
"datetime"
:
"2021-10-04 13:34:03.038569"
,
"elapsedtime"
:
"77.510527"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.013648128509521
,
"train.total_ips"
:
2047.3767037989855
}}
DLLL
{
"timestamp"
:
"1633368843.038866"
,
"datetime"
:
"2021-10-04 13:34:03.038866"
,
"elapsedtime"
:
"77.510824"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.compute_ips"
:
2082.946929288299
,
"train.data_time"
:
0.011336278915405274
,
"train.compute_time"
:
0.6471736192703247
}}
DLLL
{
"timestamp"
:
"1633368850.467254"
,
"datetime"
:
"2021-10-04 13:34:10.467254"
,
"elapsedtime"
:
"84.939212"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.loss"
:
6.965586090087891
,
"train.total_ips"
:
1775.187506251436
}}
DLLL
{
"timestamp"
:
"1633368850.467558"
,
"datetime"
:
"2021-10-04 13:34:10.467558"
,
"elapsedtime"
:
"84.939516"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.compute_ips"
:
1815.4518896012262
,
"train.data_time"
:
0.01747784614562988
,
"train.compute_time"
:
0.7252664804458618
}}
DLLL
{
"timestamp"
:
"1633368857.175593"
,
"datetime"
:
"2021-10-04 13:34:17.175593"
,
"elapsedtime"
:
"91.647551"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.loss"
:
6.953998708724976
,
"train.total_ips"
:
2026.6332015424384
}}
DLLL
{
"timestamp"
:
"1633368857.175887"
,
"datetime"
:
"2021-10-04 13:34:17.175887"
,
"elapsedtime"
:
"91.647845"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.compute_ips"
:
2156.2297010687394
,
"train.data_time"
:
0.021082234382629395
,
"train.compute_time"
:
0.6495411634445191
}}
DLLL
{
"timestamp"
:
"1633368863.808228"
,
"datetime"
:
"2021-10-04 13:34:23.808228"
,
"elapsedtime"
:
"98.280186"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.loss"
:
6.932726669311523
,
"train.total_ips"
:
2077.382848391921
}}
DLLL
{
"timestamp"
:
"1633368863.808531"
,
"datetime"
:
"2021-10-04 13:34:23.808531"
,
"elapsedtime"
:
"98.280489"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.compute_ips"
:
2117.5868467127802
,
"train.data_time"
:
0.010343337059020996
,
"train.compute_time"
:
0.652817177772522
}}
DLLL
{
"timestamp"
:
"1633368875.861681"
,
"datetime"
:
"2021-10-04 13:34:35.861681"
,
"elapsedtime"
:
"110.333639"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"train.loss"
:
7.527999348640442
,
"train.total_ips"
:
2251.999142540437
}}
DLLL
{
"timestamp"
:
"1633368875.86195"
,
"datetime"
:
"2021-10-04 13:34:35.861950"
,
"elapsedtime"
:
"110.333908"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
2327.060536243199
,
"train.data_time"
:
0.23912926197052
,
"train.compute_time"
:
0.8139605975151062
}}
DLLL
{
"timestamp"
:
"1633368875.862317"
,
"datetime"
:
"2021-10-04 13:34:35.862317"
,
"elapsedtime"
:
"110.334275"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"train.loss"
:
7.527999348640442
,
"train.total_ips"
:
2251.999142540437
}}
DLLL
{
"timestamp"
:
"1633368875.862475"
,
"datetime"
:
"2021-10-04 13:34:35.862475"
,
"elapsedtime"
:
"110.334433"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
2327.060536243199
,
"train.data_time"
:
0.23912926197052
,
"train.compute_time"
:
0.8139605975151062
}}
ascent/ascent_logs/benchmark.data.noddp.opt.1GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633368405.500884"
,
"datetime"
:
"2021-10-04 13:26:45.500884"
,
"elapsedtime"
:
"1.6e-05"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
true
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.noddp.opt.1GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
false
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
1
}}
DLLL
{
"timestamp"
:
"1633368405.50119"
,
"datetime"
:
"2021-10-04 13:26:45.501190"
,
"elapsedtime"
:
"0.000322"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633368405.505703"
,
"elapsedtime"
:
"0.004835"
,
"datetime"
:
"2021-10-04 13:26:45.505703"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633368405.506234"
,
"elapsedtime"
:
"0.005366"
,
"datetime"
:
"2021-10-04 13:26:45.506234"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368405.506272"
,
"elapsedtime"
:
"0.005404"
,
"datetime"
:
"2021-10-04 13:26:45.506272"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368405.506305"
,
"elapsedtime"
:
"0.005437"
,
"datetime"
:
"2021-10-04 13:26:45.506305"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368405.506336"
,
"elapsedtime"
:
"0.005468"
,
"datetime"
:
"2021-10-04 13:26:45.506336"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368405.506367"
,
"elapsedtime"
:
"0.005499"
,
"datetime"
:
"2021-10-04 13:26:45.506367"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368449.916711"
,
"datetime"
:
"2021-10-04 13:27:29.916711"
,
"elapsedtime"
:
"44.415843"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
8.757450485229493
,
"train.total_ips"
:
630.9194472479643
}}
DLLL
{
"timestamp"
:
"1633368449.917255"
,
"datetime"
:
"2021-10-04 13:27:29.917255"
,
"elapsedtime"
:
"44.416387"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
636.8346217996366
,
"train.data_time"
:
2.1605786800384523
,
"train.compute_time"
:
2.2785415649414062
}}
DLLL
{
"timestamp"
:
"1633368453.101424"
,
"datetime"
:
"2021-10-04 13:27:33.101424"
,
"elapsedtime"
:
"47.600556"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
8.875391435623168
,
"train.total_ips"
:
441.90853255296435
}}
DLLL
{
"timestamp"
:
"1633368453.101646"
,
"datetime"
:
"2021-10-04 13:27:33.101646"
,
"elapsedtime"
:
"47.600778"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
453.1970674215848
,
"train.data_time"
:
0.006869006156921387
,
"train.compute_time"
:
0.3132224798202515
}}
DLLL
{
"timestamp"
:
"1633368460.125396"
,
"datetime"
:
"2021-10-04 13:27:40.125396"
,
"elapsedtime"
:
"54.624528"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.979853343963623
,
"train.total_ips"
:
184.28384482608507
}}
DLLL
{
"timestamp"
:
"1633368460.125629"
,
"datetime"
:
"2021-10-04 13:27:40.125629"
,
"elapsedtime"
:
"54.624761"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
189.89309349043654
,
"train.data_time"
:
0.01830112934112549
,
"train.compute_time"
:
0.6839617729187012
}}
DLLL
{
"timestamp"
:
"1633368468.448415"
,
"datetime"
:
"2021-10-04 13:27:48.448415"
,
"elapsedtime"
:
"62.947547"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.601072311401367
,
"train.total_ips"
:
154.65418023425883
}}
DLLL
{
"timestamp"
:
"1633368468.44866"
,
"datetime"
:
"2021-10-04 13:27:48.448660"
,
"elapsedtime"
:
"62.947792"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
159.14389557797375
,
"train.data_time"
:
0.02395319938659668
,
"train.compute_time"
:
0.8082155466079712
}}
DLLL
{
"timestamp"
:
"1633368476.01458"
,
"datetime"
:
"2021-10-04 13:27:56.014580"
,
"elapsedtime"
:
"70.513712"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.023895359039306
,
"train.total_ips"
:
170.70273293089093
}}
DLLL
{
"timestamp"
:
"1633368476.014802"
,
"datetime"
:
"2021-10-04 13:27:56.014802"
,
"elapsedtime"
:
"70.513934"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
174.61551766301403
,
"train.data_time"
:
0.017736053466796874
,
"train.compute_time"
:
0.7387631416320801
}}
DLLL
{
"timestamp"
:
"1633368483.768266"
,
"datetime"
:
"2021-10-04 13:28:03.768266"
,
"elapsedtime"
:
"78.267398"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.191830253601074
,
"train.total_ips"
:
167.31962914451577
}}
DLLL
{
"timestamp"
:
"1633368483.768491"
,
"datetime"
:
"2021-10-04 13:28:03.768491"
,
"elapsedtime"
:
"78.267623"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.compute_ips"
:
171.60100176234033
,
"train.data_time"
:
0.0215883731842041
,
"train.compute_time"
:
0.7536546230316162
}}
DLLL
{
"timestamp"
:
"1633368491.454542"
,
"datetime"
:
"2021-10-04 13:28:11.454542"
,
"elapsedtime"
:
"85.953674"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.loss"
:
6.963419342041016
,
"train.total_ips"
:
168.71357916588218
}}
DLLL
{
"timestamp"
:
"1633368491.454777"
,
"datetime"
:
"2021-10-04 13:28:11.454777"
,
"elapsedtime"
:
"85.953909"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.compute_ips"
:
170.0064446649561
,
"train.data_time"
:
0.006293892860412598
,
"train.compute_time"
:
0.7622143268585205
}}
DLLL
{
"timestamp"
:
"1633368498.392758"
,
"datetime"
:
"2021-10-04 13:28:18.392758"
,
"elapsedtime"
:
"92.89189"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.loss"
:
6.972329759597779
,
"train.total_ips"
:
185.35538837916403
}}
DLLL
{
"timestamp"
:
"1633368498.392981"
,
"datetime"
:
"2021-10-04 13:28:18.392981"
,
"elapsedtime"
:
"92.892113"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.compute_ips"
:
193.43761605615094
,
"train.data_time"
:
0.029880762100219727
,
"train.compute_time"
:
0.6638155937194824
}}
DLLL
{
"timestamp"
:
"1633368506.252961"
,
"datetime"
:
"2021-10-04 13:28:26.252961"
,
"elapsedtime"
:
"100.752093"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.loss"
:
6.918896007537842
,
"train.total_ips"
:
163.733317701019
}}
DLLL
{
"timestamp"
:
"1633368506.253213"
,
"datetime"
:
"2021-10-04 13:28:26.253213"
,
"elapsedtime"
:
"100.752345"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.compute_ips"
:
168.77023394141332
,
"train.data_time"
:
0.0228287935256958
,
"train.compute_time"
:
0.7630669355392456
}}
DLLL
{
"timestamp"
:
"1633368518.9579"
,
"datetime"
:
"2021-10-04 13:28:38.957900"
,
"elapsedtime"
:
"113.457032"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"train.loss"
:
7.5195167541503904
,
"train.total_ips"
:
243.4898967507523
}}
DLLL
{
"timestamp"
:
"1633368518.958153"
,
"datetime"
:
"2021-10-04 13:28:38.958153"
,
"elapsedtime"
:
"113.457285"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
248.90466786048736
,
"train.data_time"
:
0.2327361798286438
,
"train.compute_time"
:
0.8516455054283142
}}
DLLL
{
"timestamp"
:
"1633368518.9585"
,
"datetime"
:
"2021-10-04 13:28:38.958500"
,
"elapsedtime"
:
"113.457632"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"train.loss"
:
7.5195167541503904
,
"train.total_ips"
:
243.4898967507523
}}
DLLL
{
"timestamp"
:
"1633368518.958622"
,
"datetime"
:
"2021-10-04 13:28:38.958622"
,
"elapsedtime"
:
"113.457754"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
248.90466786048736
,
"train.data_time"
:
0.2327361798286438
,
"train.compute_time"
:
0.8516455054283142
}}
ascent/ascent_logs/benchmark.data.noddp.opt.5GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633368411.373175"
,
"datetime"
:
"2021-10-04 13:26:51.373175"
,
"elapsedtime"
:
"1e-05"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
true
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.noddp.opt.5GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
true
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
5
}}
DLLL
{
"timestamp"
:
"1633368411.373534"
,
"datetime"
:
"2021-10-04 13:26:51.373534"
,
"elapsedtime"
:
"0.000369"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633368411.37853"
,
"elapsedtime"
:
"0.005365"
,
"datetime"
:
"2021-10-04 13:26:51.378530"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633368411.37917"
,
"elapsedtime"
:
"0.006005"
,
"datetime"
:
"2021-10-04 13:26:51.379170"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368411.379215"
,
"elapsedtime"
:
"0.00605"
,
"datetime"
:
"2021-10-04 13:26:51.379215"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368411.379251"
,
"elapsedtime"
:
"0.006086"
,
"datetime"
:
"2021-10-04 13:26:51.379251"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633368411.379283"
,
"elapsedtime"
:
"0.006118"
,
"datetime"
:
"2021-10-04 13:26:51.379283"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368411.379315"
,
"elapsedtime"
:
"0.00615"
,
"datetime"
:
"2021-10-04 13:26:51.379315"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633368457.958417"
,
"datetime"
:
"2021-10-04 13:27:37.958417"
,
"elapsedtime"
:
"46.585252"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
9.100788831710815
,
"train.total_ips"
:
1023.0338108736727
}}
DLLL
{
"timestamp"
:
"1633368457.959024"
,
"datetime"
:
"2021-10-04 13:27:37.959024"
,
"elapsedtime"
:
"46.585859"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
1050.3562699499917
,
"train.data_time"
:
2.165387678146362
,
"train.compute_time"
:
2.492262363433838
}}
DLLL
{
"timestamp"
:
"1633368462.585771"
,
"datetime"
:
"2021-10-04 13:27:42.585771"
,
"elapsedtime"
:
"51.212606"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
8.842906188964843
,
"train.total_ips"
:
1476.4623941364266
}}
DLLL
{
"timestamp"
:
"1633368462.586061"
,
"datetime"
:
"2021-10-04 13:27:42.586061"
,
"elapsedtime"
:
"51.212896"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
1509.5452264044384
,
"train.data_time"
:
0.009298229217529297
,
"train.compute_time"
:
0.44155519008636473
}}
DLLL
{
"timestamp"
:
"1633368468.073163"
,
"datetime"
:
"2021-10-04 13:27:48.073163"
,
"elapsedtime"
:
"56.699998"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.894550085067749
,
"train.total_ips"
:
1309.7409478911486
}}
DLLL
{
"timestamp"
:
"1633368468.073443"
,
"datetime"
:
"2021-10-04 13:27:48.073443"
,
"elapsedtime"
:
"56.700278"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
1407.5782143279978
,
"train.data_time"
:
0.02994515895843506
,
"train.compute_time"
:
0.5303663730621337
}}
DLLL
{
"timestamp"
:
"1633368471.629804"
,
"datetime"
:
"2021-10-04 13:27:51.629804"
,
"elapsedtime"
:
"60.256639"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.286630153656006
,
"train.total_ips"
:
2168.4869080236517
}}
DLLL
{
"timestamp"
:
"1633368471.630072"
,
"datetime"
:
"2021-10-04 13:27:51.630072"
,
"elapsedtime"
:
"60.256907"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
2181.421337732894
,
"train.data_time"
:
0.00236966609954834
,
"train.compute_time"
:
0.3531148672103882
}}
DLLL
{
"timestamp"
:
"1633368478.082938"
,
"datetime"
:
"2021-10-04 13:27:58.082938"
,
"elapsedtime"
:
"66.709773"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.09285979270935
,
"train.total_ips"
:
1160.865363382763
}}
DLLL
{
"timestamp"
:
"1633368478.083216"
,
"datetime"
:
"2021-10-04 13:27:58.083216"
,
"elapsedtime"
:
"66.710051"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
1255.3017728721406
,
"train.data_time"
:
0.026070141792297365
,
"train.compute_time"
:
0.6191290855407715
}}
DLLL
{
"timestamp"
:
"1633368483.509815"
,
"datetime"
:
"2021-10-04 13:28:03.509815"
,
"elapsedtime"
:
"72.13665"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.018149662017822
,
"train.total_ips"
:
1363.0384227114482
}}
DLLL
{
"timestamp"
:
"1633368483.510092"
,
"datetime"
:
"2021-10-04 13:28:03.510092"
,
"elapsedtime"
:
"72.136927"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.compute_ips"
:
1389.6295251586498
,
"train.data_time"
:
0.01359412670135498
,
"train.compute_time"
:
0.5289002418518066
}}
DLLL
{
"timestamp"
:
"1633368488.55437"
,
"datetime"
:
"2021-10-04 13:28:08.554370"
,
"elapsedtime"
:
"77.181205"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.loss"
:
6.959124135971069
,
"train.total_ips"
:
1737.7102794707703
}}
DLLL
{
"timestamp"
:
"1633368488.554691"
,
"datetime"
:
"2021-10-04 13:28:08.554691"
,
"elapsedtime"
:
"77.181526"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.compute_ips"
:
1786.1241223064808
,
"train.data_time"
:
0.02091536521911621
,
"train.compute_time"
:
0.4777300596237183
}}
DLLL
{
"timestamp"
:
"1633368494.104963"
,
"datetime"
:
"2021-10-04 13:28:14.104963"
,
"elapsedtime"
:
"82.731798"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.loss"
:
6.974686002731323
,
"train.total_ips"
:
1287.4322997681086
}}
DLLL
{
"timestamp"
:
"1633368494.105257"
,
"datetime"
:
"2021-10-04 13:28:14.105257"
,
"elapsedtime"
:
"82.732092"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.compute_ips"
:
1324.3367900232347
,
"train.data_time"
:
0.017661690711975098
,
"train.compute_time"
:
0.5371885061264038
}}
DLLL
{
"timestamp"
:
"1633368501.893486"
,
"datetime"
:
"2021-10-04 13:28:21.893486"
,
"elapsedtime"
:
"90.520321"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.loss"
:
6.934035348892212
,
"train.total_ips"
:
1220.1601169422356
}}
DLLL
{
"timestamp"
:
"1633368501.893768"
,
"datetime"
:
"2021-10-04 13:28:21.893768"
,
"elapsedtime"
:
"90.520603"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.compute_ips"
:
1250.327585304401
,
"train.data_time"
:
0.01345670223236084
,
"train.compute_time"
:
0.7639489650726319
}}
DLLL
{
"timestamp"
:
"1633368511.231701"
,
"datetime"
:
"2021-10-04 13:28:31.231701"
,
"elapsedtime"
:
"99.858536"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"train.loss"
:
7.5033202600479125
,
"train.total_ips"
:
1443.6376695660338
}}
DLLL
{
"timestamp"
:
"1633368511.232077"
,
"datetime"
:
"2021-10-04 13:28:31.232077"
,
"elapsedtime"
:
"99.858912"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
1485.910316748399
,
"train.data_time"
:
0.230459041595459
,
"train.compute_time"
:
0.7171564745903015
}}
DLLL
{
"timestamp"
:
"1633368511.232506"
,
"datetime"
:
"2021-10-04 13:28:31.232506"
,
"elapsedtime"
:
"99.859341"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"train.loss"
:
7.5033202600479125
,
"train.total_ips"
:
1443.6376695660338
}}
DLLL
{
"timestamp"
:
"1633368511.232672"
,
"datetime"
:
"2021-10-04 13:28:31.232672"
,
"elapsedtime"
:
"99.859507"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
1485.910316748399
,
"train.data_time"
:
0.230459041595459
,
"train.compute_time"
:
0.7171564745903015
}}
ascent/ascent_logs/benchmark.data.opt.10GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633367163.88397"
,
"datetime"
:
"2021-10-04 13:06:03.883970"
,
"elapsedtime"
:
"9e-06"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
false
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.10GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
true
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
10
}}
DLLL
{
"timestamp"
:
"1633367163.884283"
,
"datetime"
:
"2021-10-04 13:06:03.884283"
,
"elapsedtime"
:
"0.000322"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633367163.88907"
,
"elapsedtime"
:
"0.005109"
,
"datetime"
:
"2021-10-04 13:06:03.889070"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633367164.373133"
,
"elapsedtime"
:
"0.489172"
,
"datetime"
:
"2021-10-04 13:06:04.373133"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633367164.373181"
,
"elapsedtime"
:
"0.48922"
,
"datetime"
:
"2021-10-04 13:06:04.373181"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633367164.373217"
,
"elapsedtime"
:
"0.489256"
,
"datetime"
:
"2021-10-04 13:06:04.373217"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633367164.373249"
,
"elapsedtime"
:
"0.489288"
,
"datetime"
:
"2021-10-04 13:06:04.373249"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633367164.373282"
,
"elapsedtime"
:
"0.489321"
,
"datetime"
:
"2021-10-04 13:06:04.373282"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633367214.118731"
,
"datetime"
:
"2021-10-04 13:06:54.118731"
,
"elapsedtime"
:
"50.23477"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
7.182714986801147
,
"train.total_ips"
:
2900.5382717389557
}}
DLLL
{
"timestamp"
:
"1633367214.119242"
,
"datetime"
:
"2021-10-04 13:06:54.119242"
,
"elapsedtime"
:
"50.235281"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
2943.1209908802944
,
"train.data_time"
:
2.1950519561767576
,
"train.compute_time"
:
2.779264545440674
}}
DLLL
{
"timestamp"
:
"1633367221.087661"
,
"datetime"
:
"2021-10-04 13:07:01.087661"
,
"elapsedtime"
:
"57.2037"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
7.5353974342346195
,
"train.total_ips"
:
1908.5698167164537
}}
DLLL
{
"timestamp"
:
"1633367221.087893"
,
"datetime"
:
"2021-10-04 13:07:01.087893"
,
"elapsedtime"
:
"57.203932"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
1934.8546291908547
,
"train.data_time"
:
0.008437204360961913
,
"train.compute_time"
:
0.6882588624954223
}}
DLLL
{
"timestamp"
:
"1633367229.563547"
,
"datetime"
:
"2021-10-04 13:07:09.563547"
,
"elapsedtime"
:
"65.679586"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.592690181732178
,
"train.total_ips"
:
1537.0231046989545
}}
DLLL
{
"timestamp"
:
"1633367229.563762"
,
"datetime"
:
"2021-10-04 13:07:09.563762"
,
"elapsedtime"
:
"65.679801"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
1576.0813896866366
,
"train.data_time"
:
0.019305205345153807
,
"train.compute_time"
:
0.8281798124313354
}}
DLLL
{
"timestamp"
:
"1633367237.949613"
,
"datetime"
:
"2021-10-04 13:07:17.949613"
,
"elapsedtime"
:
"74.065652"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.424406671524048
,
"train.total_ips"
:
1555.0286303925475
}}
DLLL
{
"timestamp"
:
"1633367237.949884"
,
"datetime"
:
"2021-10-04 13:07:17.949884"
,
"elapsedtime"
:
"74.065923"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
1567.836104385275
,
"train.data_time"
:
0.005643486976623535
,
"train.compute_time"
:
0.8297797203063965
}}
DLLL
{
"timestamp"
:
"1633367245.563657"
,
"datetime"
:
"2021-10-04 13:07:25.563657"
,
"elapsedtime"
:
"81.679696"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.1125153541564945
,
"train.total_ips"
:
1726.754814887116
}}
DLLL
{
"timestamp"
:
"1633367245.563918"
,
"datetime"
:
"2021-10-04 13:07:25.563918"
,
"elapsedtime"
:
"81.679957"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
1753.745676430381
,
"train.data_time"
:
0.011652684211730957
,
"train.compute_time"
:
0.7526200294494629
}}
DLLL
{
"timestamp"
:
"1633367253.928699"
,
"datetime"
:
"2021-10-04 13:07:33.928699"
,
"elapsedtime"
:
"90.044738"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.107547092437744
,
"train.total_ips"
:
1545.1452804115418
}}
DLLL
{
"timestamp"
:
"1633367253.928955"
,
"datetime"
:
"2021-10-04 13:07:33.928955"
,
"elapsedtime"
:
"90.044994"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.compute_ips"
:
1568.8109196845762
,
"train.data_time"
:
0.011496329307556152
,
"train.compute_time"
:
0.8247853994369507
}}
DLLL
{
"timestamp"
:
"1633367262.146362"
,
"datetime"
:
"2021-10-04 13:07:42.146362"
,
"elapsedtime"
:
"98.262401"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.loss"
:
7.044234895706177
,
"train.total_ips"
:
1590.221454968975
}}
DLLL
{
"timestamp"
:
"1633367262.146623"
,
"datetime"
:
"2021-10-04 13:07:42.146623"
,
"elapsedtime"
:
"98.262662"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.compute_ips"
:
1613.1519821642928
,
"train.data_time"
:
0.009932398796081543
,
"train.compute_time"
:
0.8117234230041503
}}
DLLL
{
"timestamp"
:
"1633367270.57467"
,
"datetime"
:
"2021-10-04 13:07:50.574670"
,
"elapsedtime"
:
"106.690709"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.loss"
:
6.961148262023926
,
"train.total_ips"
:
1529.0781388778603
}}
DLLL
{
"timestamp"
:
"1633367270.574924"
,
"datetime"
:
"2021-10-04 13:07:50.574924"
,
"elapsedtime"
:
"106.690963"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.compute_ips"
:
1552.693158844095
,
"train.data_time"
:
0.012608814239501952
,
"train.compute_time"
:
0.8300262928009033
}}
DLLL
{
"timestamp"
:
"1633367278.738024"
,
"datetime"
:
"2021-10-04 13:07:58.738024"
,
"elapsedtime"
:
"114.854063"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.loss"
:
6.950328636169433
,
"train.total_ips"
:
1604.6865644901213
}}
DLLL
{
"timestamp"
:
"1633367278.738269"
,
"datetime"
:
"2021-10-04 13:07:58.738269"
,
"elapsedtime"
:
"114.854308"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.compute_ips"
:
1623.1427877450085
,
"train.data_time"
:
0.008434033393859864
,
"train.compute_time"
:
0.807792067527771
}}
DLLL
{
"timestamp"
:
"1633367291.8264"
,
"datetime"
:
"2021-10-04 13:08:11.826400"
,
"elapsedtime"
:
"127.942439"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"train.loss"
:
7.184423680305481
,
"train.total_ips"
:
1749.81881970693
}}
DLLL
{
"timestamp"
:
"1633367291.826617"
,
"datetime"
:
"2021-10-04 13:08:11.826617"
,
"elapsedtime"
:
"127.942656"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
1774.8908585713898
,
"train.data_time"
:
0.22897837162017823
,
"train.compute_time"
:
0.9953473091125489
}}
DLLL
{
"timestamp"
:
"1633367291.826819"
,
"datetime"
:
"2021-10-04 13:08:11.826819"
,
"elapsedtime"
:
"127.942858"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"train.loss"
:
7.184423680305481
,
"train.total_ips"
:
1749.81881970693
}}
DLLL
{
"timestamp"
:
"1633367291.826906"
,
"datetime"
:
"2021-10-04 13:08:11.826906"
,
"elapsedtime"
:
"127.942945"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
1774.8908585713898
,
"train.data_time"
:
0.22897837162017823
,
"train.compute_time"
:
0.9953473091125489
}}
ascent/ascent_logs/benchmark.data.opt.15GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633366975.764423"
,
"datetime"
:
"2021-10-04 13:02:55.764423"
,
"elapsedtime"
:
"1.2e-05"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
false
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.15GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
true
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
15
}}
DLLL
{
"timestamp"
:
"1633366975.76473"
,
"datetime"
:
"2021-10-04 13:02:55.764730"
,
"elapsedtime"
:
"0.000319"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633366975.769352"
,
"elapsedtime"
:
"0.004941"
,
"datetime"
:
"2021-10-04 13:02:55.769352"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633366975.81532"
,
"elapsedtime"
:
"0.050909"
,
"datetime"
:
"2021-10-04 13:02:55.815320"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366975.815368"
,
"elapsedtime"
:
"0.050957"
,
"datetime"
:
"2021-10-04 13:02:55.815368"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633366975.815403"
,
"elapsedtime"
:
"0.050992"
,
"datetime"
:
"2021-10-04 13:02:55.815403"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633366975.815436"
,
"elapsedtime"
:
"0.051025"
,
"datetime"
:
"2021-10-04 13:02:55.815436"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366975.815468"
,
"elapsedtime"
:
"0.051057"
,
"datetime"
:
"2021-10-04 13:02:55.815468"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633367026.009131"
,
"datetime"
:
"2021-10-04 13:03:46.009131"
,
"elapsedtime"
:
"50.24472"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
7.072188901901245
,
"train.total_ips"
:
2702.7503898395103
}}
DLLL
{
"timestamp"
:
"1633367026.009779"
,
"datetime"
:
"2021-10-04 13:03:46.009779"
,
"elapsedtime"
:
"50.245368"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
2789.203668346113
,
"train.data_time"
:
2.426859474182129
,
"train.compute_time"
:
2.5922511339187624
}}
DLLL
{
"timestamp"
:
"1633367034.701505"
,
"datetime"
:
"2021-10-04 13:03:54.701505"
,
"elapsedtime"
:
"58.937094"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
7.253705787658691
,
"train.total_ips"
:
2221.231557272754
}}
DLLL
{
"timestamp"
:
"1633367034.701726"
,
"datetime"
:
"2021-10-04 13:03:54.701726"
,
"elapsedtime"
:
"58.937315"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
2254.3114381548776
,
"train.data_time"
:
0.012134623527526856
,
"train.compute_time"
:
0.8569172143936157
}}
DLLL
{
"timestamp"
:
"1633367043.55764"
,
"datetime"
:
"2021-10-04 13:04:03.557640"
,
"elapsedtime"
:
"67.793229"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.446936082839966
,
"train.total_ips"
:
2192.936691771552
}}
DLLL
{
"timestamp"
:
"1633367043.55788"
,
"datetime"
:
"2021-10-04 13:04:03.557880"
,
"elapsedtime"
:
"67.793469"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
2239.505126407682
,
"train.data_time"
:
0.017815160751342773
,
"train.compute_time"
:
0.8676985740661621
}}
DLLL
{
"timestamp"
:
"1633367051.785041"
,
"datetime"
:
"2021-10-04 13:04:11.785041"
,
"elapsedtime"
:
"76.02063"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.209506225585938
,
"train.total_ips"
:
2357.3273998692466
}}
DLLL
{
"timestamp"
:
"1633367051.785285"
,
"datetime"
:
"2021-10-04 13:04:11.785285"
,
"elapsedtime"
:
"76.020874"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
2383.296374468251
,
"train.data_time"
:
0.008504319190979003
,
"train.compute_time"
:
0.8140435934066772
}}
DLLL
{
"timestamp"
:
"1633367059.844915"
,
"datetime"
:
"2021-10-04 13:04:19.844915"
,
"elapsedtime"
:
"84.080504"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.140084075927734
,
"train.total_ips"
:
2406.053376295948
}}
DLLL
{
"timestamp"
:
"1633367059.845175"
,
"datetime"
:
"2021-10-04 13:04:19.845175"
,
"elapsedtime"
:
"84.080764"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
2447.3460659358816
,
"train.data_time"
:
0.013176679611206055
,
"train.compute_time"
:
0.7927020072937012
}}
DLLL
{
"timestamp"
:
"1633367068.503738"
,
"datetime"
:
"2021-10-04 13:04:28.503738"
,
"elapsedtime"
:
"92.739327"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.1227202892303465
,
"train.total_ips"
:
2223.637787555887
}}
DLLL
{
"timestamp"
:
"1633367068.503984"
,
"datetime"
:
"2021-10-04 13:04:28.503984"
,
"elapsedtime"
:
"92.739573"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.compute_ips"
:
2237.0888057334023
,
"train.data_time"
:
0.005039739608764649
,
"train.compute_time"
:
0.8606232166290283
}}
DLLL
{
"timestamp"
:
"1633367076.507854"
,
"datetime"
:
"2021-10-04 13:04:36.507854"
,
"elapsedtime"
:
"100.743443"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.loss"
:
7.04561824798584
,
"train.total_ips"
:
2414.5458255017843
}}
DLLL
{
"timestamp"
:
"1633367076.508085"
,
"datetime"
:
"2021-10-04 13:04:36.508085"
,
"elapsedtime"
:
"100.743674"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.compute_ips"
:
2452.0596442682026
,
"train.data_time"
:
0.010829424858093262
,
"train.compute_time"
:
0.7894690752029419
}}
DLLL
{
"timestamp"
:
"1633367084.827289"
,
"datetime"
:
"2021-10-04 13:04:44.827289"
,
"elapsedtime"
:
"109.062878"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.loss"
:
7.005381536483765
,
"train.total_ips"
:
2325.503846533984
}}
DLLL
{
"timestamp"
:
"1633367084.827529"
,
"datetime"
:
"2021-10-04 13:04:44.827529"
,
"elapsedtime"
:
"109.063118"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.compute_ips"
:
2369.591804672305
,
"train.data_time"
:
0.01459648609161377
,
"train.compute_time"
:
0.8171473026275635
}}
DLLL
{
"timestamp"
:
"1633367093.174972"
,
"datetime"
:
"2021-10-04 13:04:53.174972"
,
"elapsedtime"
:
"117.410561"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.loss"
:
6.984457397460938
,
"train.total_ips"
:
2313.403266571684
}}
DLLL
{
"timestamp"
:
"1633367093.175213"
,
"datetime"
:
"2021-10-04 13:04:53.175213"
,
"elapsedtime"
:
"117.410802"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.compute_ips"
:
2334.6334916821675
,
"train.data_time"
:
0.007247233390808105
,
"train.compute_time"
:
0.8270395278930665
}}
DLLL
{
"timestamp"
:
"1633367106.323661"
,
"datetime"
:
"2021-10-04 13:05:06.323661"
,
"elapsedtime"
:
"130.55925"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"train.loss"
:
7.121799411773682
,
"train.total_ips"
:
2356.3817591036977
}}
DLLL
{
"timestamp"
:
"1633367106.323829"
,
"datetime"
:
"2021-10-04 13:05:06.323829"
,
"elapsedtime"
:
"130.559418"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
2395.721183832494
,
"train.data_time"
:
0.25271469593048096
,
"train.compute_time"
:
1.0021142029762269
}}
DLLL
{
"timestamp"
:
"1633367106.324004"
,
"datetime"
:
"2021-10-04 13:05:06.324004"
,
"elapsedtime"
:
"130.559593"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"train.loss"
:
7.121799411773682
,
"train.total_ips"
:
2356.3817591036977
}}
DLLL
{
"timestamp"
:
"1633367106.324069"
,
"datetime"
:
"2021-10-04 13:05:06.324069"
,
"elapsedtime"
:
"130.559658"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
2395.721183832494
,
"train.data_time"
:
0.25271469593048096
,
"train.compute_time"
:
1.0021142029762269
}}
ascent/ascent_logs/benchmark.data.opt.1GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633366782.005491"
,
"datetime"
:
"2021-10-04 12:59:42.005491"
,
"elapsedtime"
:
"1.4e-05"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
false
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.1GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
false
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
1
}}
DLLL
{
"timestamp"
:
"1633366782.005798"
,
"datetime"
:
"2021-10-04 12:59:42.005798"
,
"elapsedtime"
:
"0.000321"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633366782.01028"
,
"elapsedtime"
:
"0.004803"
,
"datetime"
:
"2021-10-04 12:59:42.010280"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633366782.010801"
,
"elapsedtime"
:
"0.005324"
,
"datetime"
:
"2021-10-04 12:59:42.010801"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366782.01084"
,
"elapsedtime"
:
"0.005363"
,
"datetime"
:
"2021-10-04 12:59:42.010840"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633366782.010873"
,
"elapsedtime"
:
"0.005396"
,
"datetime"
:
"2021-10-04 12:59:42.010873"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633366782.010905"
,
"elapsedtime"
:
"0.005428"
,
"datetime"
:
"2021-10-04 12:59:42.010905"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366782.010937"
,
"elapsedtime"
:
"0.00546"
,
"datetime"
:
"2021-10-04 12:59:42.010937"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366825.324186"
,
"datetime"
:
"2021-10-04 13:00:25.324186"
,
"elapsedtime"
:
"43.318709"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
9.143393945693969
,
"train.total_ips"
:
681.6669413468046
}}
DLLL
{
"timestamp"
:
"1633366825.324705"
,
"datetime"
:
"2021-10-04 13:00:25.324705"
,
"elapsedtime"
:
"43.319228"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
687.0915298032702
,
"train.data_time"
:
2.1264915466308594
,
"train.compute_time"
:
2.2046539545059205
}}
DLLL
{
"timestamp"
:
"1633366828.978403"
,
"datetime"
:
"2021-10-04 13:00:28.978403"
,
"elapsedtime"
:
"46.972926"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
9.324459075927734
,
"train.total_ips"
:
408.5973957368579
}}
DLLL
{
"timestamp"
:
"1633366828.978625"
,
"datetime"
:
"2021-10-04 13:00:28.978625"
,
"elapsedtime"
:
"46.973148"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
413.70488338069765
,
"train.data_time"
:
0.006155705451965332
,
"train.compute_time"
:
0.3591571807861328
}}
DLLL
{
"timestamp"
:
"1633366836.086278"
,
"datetime"
:
"2021-10-04 13:00:36.086278"
,
"elapsedtime"
:
"54.080801"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.754433965682983
,
"train.total_ips"
:
181.10683263170282
}}
DLLL
{
"timestamp"
:
"1633366836.086512"
,
"datetime"
:
"2021-10-04 13:00:36.086512"
,
"elapsedtime"
:
"54.081035"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
186.71396052363136
,
"train.data_time"
:
0.020975732803344728
,
"train.compute_time"
:
0.6896930456161499
}}
DLLL
{
"timestamp"
:
"1633366843.980084"
,
"datetime"
:
"2021-10-04 13:00:43.980084"
,
"elapsedtime"
:
"61.974607"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.532516384124756
,
"train.total_ips"
:
163.0120739762698
}}
DLLL
{
"timestamp"
:
"1633366843.980312"
,
"datetime"
:
"2021-10-04 13:00:43.980312"
,
"elapsedtime"
:
"61.974835"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
169.40591110620943
,
"train.data_time"
:
0.02857215404510498
,
"train.compute_time"
:
0.7606812953948975
}}
DLLL
{
"timestamp"
:
"1633366851.463074"
,
"datetime"
:
"2021-10-04 13:00:51.463074"
,
"elapsedtime"
:
"69.457597"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.2729172706604
,
"train.total_ips"
:
172.1038196851434
}}
DLLL
{
"timestamp"
:
"1633366851.463331"
,
"datetime"
:
"2021-10-04 13:00:51.463331"
,
"elapsedtime"
:
"69.457854"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
179.22825576359
,
"train.data_time"
:
0.0291060209274292
,
"train.compute_time"
:
0.7190677404403687
}}
DLLL
{
"timestamp"
:
"1633366858.976614"
,
"datetime"
:
"2021-10-04 13:00:58.976614"
,
"elapsedtime"
:
"76.971137"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.016049003601074
,
"train.total_ips"
:
172.354746650962
}}
DLLL
{
"timestamp"
:
"1633366858.976849"
,
"datetime"
:
"2021-10-04 13:00:58.976849"
,
"elapsedtime"
:
"76.971372"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.compute_ips"
:
176.3413771256184
,
"train.data_time"
:
0.0176114559173584
,
"train.compute_time"
:
0.7336230039596557
}}
DLLL
{
"timestamp"
:
"1633366866.784137"
,
"datetime"
:
"2021-10-04 13:01:06.784137"
,
"elapsedtime"
:
"84.77866"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.loss"
:
6.960112810134888
,
"train.total_ips"
:
167.28117506353584
}}
DLLL
{
"timestamp"
:
"1633366866.807706"
,
"datetime"
:
"2021-10-04 13:01:06.807706"
,
"elapsedtime"
:
"84.802229"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.compute_ips"
:
172.87520808495327
,
"train.data_time"
:
0.027497386932373045
,
"train.compute_time"
:
0.7488190174102783
}}
DLLL
{
"timestamp"
:
"1633366874.160927"
,
"datetime"
:
"2021-10-04 13:01:14.160927"
,
"elapsedtime"
:
"92.15545"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.loss"
:
6.94903883934021
,
"train.total_ips"
:
176.21874711460248
}}
DLLL
{
"timestamp"
:
"1633366874.161153"
,
"datetime"
:
"2021-10-04 13:01:14.161153"
,
"elapsedtime"
:
"92.155676"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.compute_ips"
:
182.4007807941556
,
"train.data_time"
:
0.02353365421295166
,
"train.compute_time"
:
0.7140132904052734
}}
DLLL
{
"timestamp"
:
"1633366881.370676"
,
"datetime"
:
"2021-10-04 13:01:21.370676"
,
"elapsedtime"
:
"99.365199"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.loss"
:
6.917493391036987
,
"train.total_ips"
:
183.08413349211938
}}
DLLL
{
"timestamp"
:
"1633366881.370926"
,
"datetime"
:
"2021-10-04 13:01:21.370926"
,
"elapsedtime"
:
"99.365449"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.compute_ips"
:
193.75800455772372
,
"train.data_time"
:
0.036658430099487306
,
"train.compute_time"
:
0.6768976211547851
}}
DLLL
{
"timestamp"
:
"1633366893.127849"
,
"datetime"
:
"2021-10-04 13:01:33.127849"
,
"elapsedtime"
:
"111.122372"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"train.loss"
:
7.5794302225112915
,
"train.total_ips"
:
250.79697422674948
}}
DLLL
{
"timestamp"
:
"1633366893.128078"
,
"datetime"
:
"2021-10-04 13:01:33.128078"
,
"elapsedtime"
:
"111.122601"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
256.9279010434933
,
"train.data_time"
:
0.23351834535598756
,
"train.compute_time"
:
0.8262648439407348
}}
DLLL
{
"timestamp"
:
"1633366893.128394"
,
"datetime"
:
"2021-10-04 13:01:33.128394"
,
"elapsedtime"
:
"111.122917"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"train.loss"
:
7.5794302225112915
,
"train.total_ips"
:
250.79697422674948
}}
DLLL
{
"timestamp"
:
"1633366893.128518"
,
"datetime"
:
"2021-10-04 13:01:33.128518"
,
"elapsedtime"
:
"111.123041"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
256.9279010434933
,
"train.data_time"
:
0.23351834535598756
,
"train.compute_time"
:
0.8262648439407348
}}
ascent/ascent_logs/benchmark.data.opt.20GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633366975.026215"
,
"datetime"
:
"2021-10-04 13:02:55.026215"
,
"elapsedtime"
:
"9e-06"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
false
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.20GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
true
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
20
}}
DLLL
{
"timestamp"
:
"1633366975.026539"
,
"datetime"
:
"2021-10-04 13:02:55.026539"
,
"elapsedtime"
:
"0.000333"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633366975.031149"
,
"elapsedtime"
:
"0.004943"
,
"datetime"
:
"2021-10-04 13:02:55.031149"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633366975.098784"
,
"elapsedtime"
:
"0.072578"
,
"datetime"
:
"2021-10-04 13:02:55.098784"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366975.098851"
,
"elapsedtime"
:
"0.072645"
,
"datetime"
:
"2021-10-04 13:02:55.098851"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633366975.098902"
,
"elapsedtime"
:
"0.072696"
,
"datetime"
:
"2021-10-04 13:02:55.098902"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633366975.09895"
,
"elapsedtime"
:
"0.072744"
,
"datetime"
:
"2021-10-04 13:02:55.098950"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366975.098997"
,
"elapsedtime"
:
"0.072791"
,
"datetime"
:
"2021-10-04 13:02:55.098997"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633367046.03909"
,
"datetime"
:
"2021-10-04 13:04:06.039090"
,
"elapsedtime"
:
"71.012884"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
7.045035743713379
,
"train.total_ips"
:
5626.953493008938
}}
DLLL
{
"timestamp"
:
"1633367046.03964"
,
"datetime"
:
"2021-10-04 13:04:06.039640"
,
"elapsedtime"
:
"71.013434"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
5680.827602667565
,
"train.data_time"
:
2.125786066055298
,
"train.compute_time"
:
4.9679429769515995
}}
DLLL
{
"timestamp"
:
"1633367056.257319"
,
"datetime"
:
"2021-10-04 13:04:16.257319"
,
"elapsedtime"
:
"81.231113"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
7.174418258666992
,
"train.total_ips"
:
2529.850848037966
}}
DLLL
{
"timestamp"
:
"1633367056.257612"
,
"datetime"
:
"2021-10-04 13:04:16.257612"
,
"elapsedtime"
:
"81.231406"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
2567.090461807862
,
"train.data_time"
:
0.013248157501220704
,
"train.compute_time"
:
1.008377766609192
}}
DLLL
{
"timestamp"
:
"1633367065.438723"
,
"datetime"
:
"2021-10-04 13:04:25.438723"
,
"elapsedtime"
:
"90.412517"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.353969669342041
,
"train.total_ips"
:
2867.8258210773733
}}
DLLL
{
"timestamp"
:
"1633367065.438999"
,
"datetime"
:
"2021-10-04 13:04:25.438999"
,
"elapsedtime"
:
"90.412793"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
2899.7099127711745
,
"train.data_time"
:
0.010905313491821288
,
"train.compute_time"
:
0.906805419921875
}}
DLLL
{
"timestamp"
:
"1633367075.153855"
,
"datetime"
:
"2021-10-04 13:04:35.153855"
,
"elapsedtime"
:
"100.127649"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.220703220367431
,
"train.total_ips"
:
2665.520918334095
}}
DLLL
{
"timestamp"
:
"1633367075.154166"
,
"datetime"
:
"2021-10-04 13:04:35.154166"
,
"elapsedtime"
:
"100.12796"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
2683.422777068956
,
"train.data_time"
:
0.006829357147216797
,
"train.compute_time"
:
0.9644888162612915
}}
DLLL
{
"timestamp"
:
"1633367085.102168"
,
"datetime"
:
"2021-10-04 13:04:45.102168"
,
"elapsedtime"
:
"110.075962"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.118415880203247
,
"train.total_ips"
:
2613.0525717776677
}}
DLLL
{
"timestamp"
:
"1633367085.102494"
,
"datetime"
:
"2021-10-04 13:04:45.102494"
,
"elapsedtime"
:
"110.076288"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
2637.794069329174
,
"train.data_time"
:
0.010201644897460938
,
"train.compute_time"
:
0.9845112562179565
}}
DLLL
{
"timestamp"
:
"1633367095.264805"
,
"datetime"
:
"2021-10-04 13:04:55.264805"
,
"elapsedtime"
:
"120.238599"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.043138599395752
,
"train.total_ips"
:
2537.1542281634875
}}
DLLL
{
"timestamp"
:
"1633367095.265098"
,
"datetime"
:
"2021-10-04 13:04:55.265098"
,
"elapsedtime"
:
"120.238892"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.compute_ips"
:
2562.5358148910545
,
"train.data_time"
:
0.010465145111083984
,
"train.compute_time"
:
1.005621862411499
}}
DLLL
{
"timestamp"
:
"1633367104.976593"
,
"datetime"
:
"2021-10-04 13:05:04.976593"
,
"elapsedtime"
:
"129.950387"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.loss"
:
7.019901466369629
,
"train.total_ips"
:
2654.5490773495812
}}
DLLL
{
"timestamp"
:
"1633367104.97686"
,
"datetime"
:
"2021-10-04 13:05:04.976860"
,
"elapsedtime"
:
"129.950654"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
70
],
"data"
:
{
"train.compute_ips"
:
2673.632711151256
,
"train.data_time"
:
0.007492280006408692
,
"train.compute_time"
:
0.9635673284530639
}}
DLLL
{
"timestamp"
:
"1633367115.039382"
,
"datetime"
:
"2021-10-04 13:05:15.039382"
,
"elapsedtime"
:
"140.013176"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.loss"
:
6.973300170898438
,
"train.total_ips"
:
2557.3187385538304
}}
DLLL
{
"timestamp"
:
"1633367115.039661"
,
"datetime"
:
"2021-10-04 13:05:15.039661"
,
"elapsedtime"
:
"140.013455"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
80
],
"data"
:
{
"train.compute_ips"
:
2568.1463085071487
,
"train.data_time"
:
0.003994441032409668
,
"train.compute_time"
:
1.0020900964736938
}}
DLLL
{
"timestamp"
:
"1633367124.861067"
,
"datetime"
:
"2021-10-04 13:05:24.861067"
,
"elapsedtime"
:
"149.834861"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.loss"
:
6.956139945983887
,
"train.total_ips"
:
2653.434155798601
}}
DLLL
{
"timestamp"
:
"1633367124.88142"
,
"datetime"
:
"2021-10-04 13:05:24.881420"
,
"elapsedtime"
:
"149.855214"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
90
],
"data"
:
{
"train.compute_ips"
:
2671.669342881167
,
"train.data_time"
:
0.005961227416992188
,
"train.compute_time"
:
0.9760968685150146
}}
DLLL
{
"timestamp"
:
"1633367140.074991"
,
"datetime"
:
"2021-10-04 13:05:40.074991"
,
"elapsedtime"
:
"165.048785"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"train.loss"
:
7.084707942008972
,
"train.total_ips"
:
2923.739016411492
}}
DLLL
{
"timestamp"
:
"1633367140.07515"
,
"datetime"
:
"2021-10-04 13:05:40.075150"
,
"elapsedtime"
:
"165.048944"
,
"type"
:
"LOG"
,
"step"
:
[
0
],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
2948.5318411503954
,
"train.data_time"
:
0.21984706878662108
,
"train.compute_time"
:
1.3796652913093568
}}
DLLL
{
"timestamp"
:
"1633367140.075326"
,
"datetime"
:
"2021-10-04 13:05:40.075326"
,
"elapsedtime"
:
"165.04912"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"train.loss"
:
7.084707942008972
,
"train.total_ips"
:
2923.739016411492
}}
DLLL
{
"timestamp"
:
"1633367140.0754"
,
"datetime"
:
"2021-10-04 13:05:40.075400"
,
"elapsedtime"
:
"165.049194"
,
"type"
:
"LOG"
,
"step"
:
[],
"data"
:
{
"lr"
:
0.1
,
"train.compute_ips"
:
2948.5318411503954
,
"train.data_time"
:
0.21984706878662108
,
"train.compute_time"
:
1.3796652913093568
}}
ascent/ascent_logs/benchmark.data.opt.5GPU.json
deleted
100644 → 0
View file @
a280ec34
DLLL
{
"timestamp"
:
"1633366789.763861"
,
"datetime"
:
"2021-10-04 12:59:49.763861"
,
"elapsedtime"
:
"1.1e-05"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"data"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/imagenet/data/"
,
"data_backend"
:
"pytorch"
,
"interpolation"
:
"bilinear"
,
"arch"
:
"resnet50"
,
"workers"
:
28
,
"epochs"
:
1
,
"run_epochs"
:
-1
,
"early_stopping_patience"
:
-1
,
"image_size"
:
null
,
"batch_size"
:
128
,
"optimizer_batch_size"
:
-1
,
"lr"
:
0.1
,
"lr_schedule"
:
"step"
,
"end_lr"
:
0
,
"warmup"
:
0
,
"label_smoothing"
:
0.0
,
"mixup"
:
0.0
,
"optimizer"
:
"sgd"
,
"momentum"
:
0.9
,
"weight_decay"
:
0.0001
,
"bn_weight_decay"
:
false
,
"noDDP"
:
false
,
"rmsprop_alpha"
:
0.9
,
"rmsprop_eps"
:
0.001
,
"nesterov"
:
false
,
"print_freq"
:
10
,
"resume"
:
null
,
"static_loss_scale"
:
1
,
"dynamic_loss_scale"
:
false
,
"prof"
:
100
,
"amp"
:
true
,
"seed"
:
null
,
"gather_checkpoints"
:
false
,
"raport_file"
:
"/gpfs/wolf/gen166/proj-shared/atsaris/pytorch_tutorial/ascent/ascent_logs//benchmark.data.opt.5GPU.json"
,
"evaluate"
:
false
,
"training_only"
:
true
,
"save_checkpoints"
:
false
,
"checkpoint_filename"
:
"checkpoint.pth.tar"
,
"workspace"
:
"./"
,
"memory_format"
:
"nhwc"
,
"use_ema"
:
null
,
"augmentation"
:
null
,
"num_classes"
:
null
,
"distributed"
:
true
,
"local_rank"
:
0
,
"gpu"
:
0
,
"world_size"
:
5
}}
DLLL
{
"timestamp"
:
"1633366789.764195"
,
"datetime"
:
"2021-10-04 12:59:49.764195"
,
"elapsedtime"
:
"0.000345"
,
"type"
:
"LOG"
,
"step"
:
"PARAMETER"
,
"data"
:
{
"model.num_classes"
:
1000
,
"model.last_bn_0_init"
:
false
,
"model.conv_init"
:
"fan_in"
,
"model.trt"
:
false
,
"model.pretrained_from_file"
:
null
,
"model.pretrained"
:
false
}}
DLLL
{
"timestamp"
:
"1633366789.768848"
,
"elapsedtime"
:
"0.004998"
,
"datetime"
:
"2021-10-04 12:59:49.768848"
,
"type"
:
"METADATA"
,
"metric"
:
"lr"
,
"metadata"
:
{}}
DLLL
{
"timestamp"
:
"1633366790.253636"
,
"elapsedtime"
:
"0.489786"
,
"datetime"
:
"2021-10-04 12:59:50.253636"
,
"type"
:
"METADATA"
,
"metric"
:
"train.loss"
,
"metadata"
:
{
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366790.253694"
,
"elapsedtime"
:
"0.489844"
,
"datetime"
:
"2021-10-04 12:59:50.253694"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633366790.253731"
,
"elapsedtime"
:
"0.489881"
,
"datetime"
:
"2021-10-04 12:59:50.253731"
,
"type"
:
"METADATA"
,
"metric"
:
"train.total_ips"
,
"metadata"
:
{
"unit"
:
"img/s"
,
"format"
:
":.2f"
}}
DLLL
{
"timestamp"
:
"1633366790.253764"
,
"elapsedtime"
:
"0.489914"
,
"datetime"
:
"2021-10-04 12:59:50.253764"
,
"type"
:
"METADATA"
,
"metric"
:
"train.data_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366790.253797"
,
"elapsedtime"
:
"0.489947"
,
"datetime"
:
"2021-10-04 12:59:50.253797"
,
"type"
:
"METADATA"
,
"metric"
:
"train.compute_time"
,
"metadata"
:
{
"unit"
:
"s"
,
"format"
:
":.5f"
}}
DLLL
{
"timestamp"
:
"1633366837.669666"
,
"datetime"
:
"2021-10-04 13:00:37.669666"
,
"elapsedtime"
:
"47.905816"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.loss"
:
7.436073589324951
,
"train.total_ips"
:
1558.367508961958
}}
DLLL
{
"timestamp"
:
"1633366837.670159"
,
"datetime"
:
"2021-10-04 13:00:37.670159"
,
"elapsedtime"
:
"47.906309"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
10
],
"data"
:
{
"train.compute_ips"
:
1588.4415661371518
,
"train.data_time"
:
2.352148246765137
,
"train.compute_time"
:
2.389200711250305
}}
DLLL
{
"timestamp"
:
"1633366842.943276"
,
"datetime"
:
"2021-10-04 13:00:42.943276"
,
"elapsedtime"
:
"53.179426"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.loss"
:
8.108831787109375
,
"train.total_ips"
:
1338.0017371429026
}}
DLLL
{
"timestamp"
:
"1633366842.943564"
,
"datetime"
:
"2021-10-04 13:00:42.943564"
,
"elapsedtime"
:
"53.179714"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
20
],
"data"
:
{
"train.compute_ips"
:
1352.1017431526247
,
"train.data_time"
:
0.0060283422470092775
,
"train.compute_time"
:
0.5211388826370239
}}
DLLL
{
"timestamp"
:
"1633366850.563941"
,
"datetime"
:
"2021-10-04 13:00:50.563941"
,
"elapsedtime"
:
"60.800091"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.loss"
:
7.379528522491455
,
"train.total_ips"
:
916.7487199272591
}}
DLLL
{
"timestamp"
:
"1633366850.564215"
,
"datetime"
:
"2021-10-04 13:00:50.564215"
,
"elapsedtime"
:
"60.800365"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
30
],
"data"
:
{
"train.compute_ips"
:
932.7611682083294
,
"train.data_time"
:
0.014114117622375489
,
"train.compute_time"
:
0.7408340692520141
}}
DLLL
{
"timestamp"
:
"1633366858.580591"
,
"datetime"
:
"2021-10-04 13:00:58.580591"
,
"elapsedtime"
:
"68.816741"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.loss"
:
7.248611783981323
,
"train.total_ips"
:
812.7829023048555
}}
DLLL
{
"timestamp"
:
"1633366858.58089"
,
"datetime"
:
"2021-10-04 13:00:58.580890"
,
"elapsedtime"
:
"68.81704"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
40
],
"data"
:
{
"train.compute_ips"
:
826.1597774475997
,
"train.data_time"
:
0.011307120323181152
,
"train.compute_time"
:
0.790150237083435
}}
DLLL
{
"timestamp"
:
"1633366864.199714"
,
"datetime"
:
"2021-10-04 13:01:04.199714"
,
"elapsedtime"
:
"74.435864"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.loss"
:
7.120347309112549
,
"train.total_ips"
:
1243.6476891065395
}}
DLLL
{
"timestamp"
:
"1633366864.200025"
,
"datetime"
:
"2021-10-04 13:01:04.200025"
,
"elapsedtime"
:
"74.436175"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
50
],
"data"
:
{
"train.compute_ips"
:
1278.8759358143914
,
"train.data_time"
:
0.010735273361206055
,
"train.compute_time"
:
0.5510577201843262
}}
DLLL
{
"timestamp"
:
"1633366869.678219"
,
"datetime"
:
"2021-10-04 13:01:09.678219"
,
"elapsedtime"
:
"79.914369"
,
"type"
:
"LOG"
,
"step"
:
[
0
,
60
],
"data"
:
{
"train.loss"
:
7.038975667953491
,