Commit ca6d4d40 authored by Tsaris, Aristeidis's avatar Tsaris, Aristeidis
Browse files

change allocation time

parent d33bd65c
#!/bin/bash
# Begin LSF directives
#BSUB -P stf011
#BSUB -J sc21
#BSUB -o logs/sc21.o%J
#BSUB -W 0:10
#BSUB -J olcfTut21
#BSUB -o olcfTut21.o%J
#BSUB -W 0:20
#BSUB -nnodes 1
#BSUB -alloc_flags "nvme smt4"
#BSUB -N
......
DLLL {"timestamp": "1635187398.219428", "datetime": "2021-10-25 14:43:18.219428", "elapsedtime": "1.2e-05", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/alpine/world-shared/stf011/junqi/choco_env/dl_code/data/ILSVRC", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 1, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": false, "dtLdTime": false, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": false, "seed": null, "gather_checkpoints": false, "raport_file": "summit_logs//benchmark.baseline.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nchw", "use_ema": null, "augmentation": null, "num_classes": null, "use_benchy": false, "distributed": false, "local_rank": 0, "gpu": 0, "world_size": 1}}
DLLL {"timestamp": "1635187398.219732", "datetime": "2021-10-25 14:43:18.219732", "elapsedtime": "0.000316", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1635187398.22118", "elapsedtime": "0.001764", "datetime": "2021-10-25 14:43:18.221180", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1635187398.221762", "elapsedtime": "0.002346", "datetime": "2021-10-25 14:43:18.221762", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1635187398.221802", "elapsedtime": "0.002386", "datetime": "2021-10-25 14:43:18.221802", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1635187398.221835", "elapsedtime": "0.002419", "datetime": "2021-10-25 14:43:18.221835", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1635187398.221868", "elapsedtime": "0.002452", "datetime": "2021-10-25 14:43:18.221868", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1635187398.221899", "elapsedtime": "0.002483", "datetime": "2021-10-25 14:43:18.221899", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1635187460.969696", "datetime": "2021-10-25 14:44:20.969696", "elapsedtime": "62.75028", "type": "LOG", "step": [0, 10], "data": {"train.loss": 9.938906335830689, "train.total_ips": 91.33423744306337}}
DLLL {"timestamp": "1635187460.969978", "datetime": "2021-10-25 14:44:20.969978", "elapsedtime": "62.750562", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 338.35426455969156, "train.data_time": 4.777253007888794, "train.compute_time": 1.49733145236969}}
DLLL {"timestamp": "1635187515.167427", "datetime": "2021-10-25 14:45:15.167427", "elapsedtime": "116.948011", "type": "LOG", "step": [0, 20], "data": {"train.loss": 8.735642766952514, "train.total_ips": 23.63165075066589}}
DLLL {"timestamp": "1635187515.167547", "datetime": "2021-10-25 14:45:15.167547", "elapsedtime": "116.948131", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 376.09835061722686, "train.data_time": 5.079348874092102, "train.compute_time": 0.34034578800201415}}
DLLL {"timestamp": "1635187570.299267", "datetime": "2021-10-25 14:46:10.299267", "elapsedtime": "172.079851", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.688778400421143, "train.total_ips": 23.228842477326445}}
DLLL {"timestamp": "1635187570.299395", "datetime": "2021-10-25 14:46:10.299395", "elapsedtime": "172.079979", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 373.4979819612004, "train.data_time": 5.170261263847351, "train.compute_time": 0.3428434133529663}}
DLLL {"timestamp": "1635187625.539204", "datetime": "2021-10-25 14:47:05.539204", "elapsedtime": "227.319788", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.2341704845428465, "train.total_ips": 23.20199407078189}}
DLLL {"timestamp": "1635187625.53933", "datetime": "2021-10-25 14:47:05.539330", "elapsedtime": "227.319914", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 374.65857423944647, "train.data_time": 5.18212661743164, "train.compute_time": 0.34178938865661623}}
DLLL {"timestamp": "1635187680.195564", "datetime": "2021-10-25 14:48:00.195564", "elapsedtime": "281.976148", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.157378911972046, "train.total_ips": 23.44387929898201}}
DLLL {"timestamp": "1635187680.195776", "datetime": "2021-10-25 14:48:00.195776", "elapsedtime": "281.97636", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 374.857966929642, "train.data_time": 5.123978781700134, "train.compute_time": 0.3415658712387085}}
DLLL {"timestamp": "1635187734.936847", "datetime": "2021-10-25 14:48:54.936847", "elapsedtime": "336.717431", "type": "LOG", "step": [0, 60], "data": {"train.loss": 6.985260725021362, "train.total_ips": 23.394995807926023}}
DLLL {"timestamp": "1635187734.936975", "datetime": "2021-10-25 14:48:54.936975", "elapsedtime": "336.717559", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 375.0498110822012, "train.data_time": 5.1326531887054445, "train.compute_time": 0.3414011001586914}}
DLLL {"timestamp": "1635187789.092916", "datetime": "2021-10-25 14:49:49.092916", "elapsedtime": "390.8735", "type": "LOG", "step": [0, 70], "data": {"train.loss": 7.015526151657104, "train.total_ips": 23.662577790282633}}
DLLL {"timestamp": "1635187789.093088", "datetime": "2021-10-25 14:49:49.093088", "elapsedtime": "390.873672", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 374.54180322790324, "train.data_time": 5.073618197441101, "train.compute_time": 0.34190080165863035}}
DLLL {"timestamp": "1635187844.594655", "datetime": "2021-10-25 14:50:44.594655", "elapsedtime": "446.375239", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.929033088684082, "train.total_ips": 23.0782494094607}}
DLLL {"timestamp": "1635187844.594778", "datetime": "2021-10-25 14:50:44.594778", "elapsedtime": "446.375362", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 374.7601883866349, "train.data_time": 5.208436131477356, "train.compute_time": 0.34166531562805175}}
DLLL {"timestamp": "1635187899.380499", "datetime": "2021-10-25 14:51:39.380499", "elapsedtime": "501.161083", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.9077019691467285, "train.total_ips": 23.39098139411821}}
DLLL {"timestamp": "1635187899.380626", "datetime": "2021-10-25 14:51:39.380626", "elapsedtime": "501.16121", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 372.5128894553009, "train.data_time": 5.134687542915344, "train.compute_time": 0.34381809234619143}}
DLLL {"timestamp": "1635187957.939023", "datetime": "2021-10-25 14:52:37.939023", "elapsedtime": "559.719607", "type": "LOG", "step": [0], "data": {"train.loss": 7.551360754966736, "train.total_ips": 30.404459913457057}}
DLLL {"timestamp": "1635187957.939162", "datetime": "2021-10-25 14:52:37.939162", "elapsedtime": "559.719746", "type": "LOG", "step": [0], "data": {"lr": 0.1, "train.compute_ips": 370.9942612152928, "train.data_time": 5.055179784297943, "train.compute_time": 0.45734944820404055}}
DLLL {"timestamp": "1635187957.939312", "datetime": "2021-10-25 14:52:37.939312", "elapsedtime": "559.719896", "type": "LOG", "step": [], "data": {"train.loss": 7.551360754966736, "train.total_ips": 30.404459913457057}}
DLLL {"timestamp": "1635187957.939378", "datetime": "2021-10-25 14:52:37.939378", "elapsedtime": "559.719962", "type": "LOG", "step": [], "data": {"lr": 0.1, "train.compute_ips": 370.9942612152928, "train.data_time": 5.055179784297943, "train.compute_time": 0.45734944820404055}}
DLLL {"timestamp": "1635352051.824312", "datetime": "2021-10-27 12:27:31.824312", "elapsedtime": "1.4e-05", "type": "LOG", "step": "PARAMETER", "data": {"data": "/gpfs/alpine/world-shared/stf011/junqi/choco_env/dl_code/data/ILSVRC", "data_backend": "pytorch", "interpolation": "bilinear", "arch": "resnet50", "workers": 1, "epochs": 1, "run_epochs": -1, "early_stopping_patience": -1, "image_size": null, "batch_size": 128, "optimizer_batch_size": -1, "lr": 0.1, "lr_schedule": "step", "end_lr": 0, "warmup": 0, "label_smoothing": 0.0, "mixup": 0.0, "optimizer": "sgd", "momentum": 0.9, "weight_decay": 0.0001, "bn_weight_decay": false, "noDDP": false, "dtLdTime": false, "rmsprop_alpha": 0.9, "rmsprop_eps": 0.001, "nesterov": false, "print_freq": 10, "resume": null, "static_loss_scale": 1, "dynamic_loss_scale": false, "prof": 100, "amp": false, "seed": null, "gather_checkpoints": false, "raport_file": "summit_logs//benchmark.baseline.json", "evaluate": false, "training_only": true, "save_checkpoints": false, "checkpoint_filename": "checkpoint.pth.tar", "workspace": "./", "memory_format": "nchw", "use_ema": null, "augmentation": null, "num_classes": null, "use_benchy": false, "distributed": false, "local_rank": 0, "gpu": 0, "world_size": 1}}
DLLL {"timestamp": "1635352051.824614", "datetime": "2021-10-27 12:27:31.824614", "elapsedtime": "0.000316", "type": "LOG", "step": "PARAMETER", "data": {"model.num_classes": 1000, "model.last_bn_0_init": false, "model.conv_init": "fan_in", "model.trt": false, "model.pretrained_from_file": null, "model.pretrained": false}}
DLLL {"timestamp": "1635352051.826026", "elapsedtime": "0.001728", "datetime": "2021-10-27 12:27:31.826026", "type": "METADATA", "metric": "lr", "metadata": {}}
DLLL {"timestamp": "1635352051.826604", "elapsedtime": "0.002306", "datetime": "2021-10-27 12:27:31.826604", "type": "METADATA", "metric": "train.loss", "metadata": {"format": ":.5f"}}
DLLL {"timestamp": "1635352051.826645", "elapsedtime": "0.002347", "datetime": "2021-10-27 12:27:31.826645", "type": "METADATA", "metric": "train.compute_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1635352051.826679", "elapsedtime": "0.002381", "datetime": "2021-10-27 12:27:31.826679", "type": "METADATA", "metric": "train.total_ips", "metadata": {"unit": "img/s", "format": ":.2f"}}
DLLL {"timestamp": "1635352051.826711", "elapsedtime": "0.002413", "datetime": "2021-10-27 12:27:31.826711", "type": "METADATA", "metric": "train.data_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1635352051.826743", "elapsedtime": "0.002445", "datetime": "2021-10-27 12:27:31.826743", "type": "METADATA", "metric": "train.compute_time", "metadata": {"unit": "s", "format": ":.5f"}}
DLLL {"timestamp": "1635352133.525475", "datetime": "2021-10-27 12:28:53.525475", "elapsedtime": "81.701177", "type": "LOG", "step": [0, 10], "data": {"train.loss": 8.533158206939698, "train.total_ips": 89.5058865781402}}
DLLL {"timestamp": "1635352133.526428", "datetime": "2021-10-27 12:28:53.526428", "elapsedtime": "81.70213", "type": "LOG", "step": [0, 10], "data": {"train.compute_ips": 338.67888063592784, "train.data_time": 5.34410514831543, "train.compute_time": 2.8255025863647463}}
DLLL {"timestamp": "1635352187.480028", "datetime": "2021-10-27 12:29:47.480028", "elapsedtime": "135.65573", "type": "LOG", "step": [0, 20], "data": {"train.loss": 8.851646089553833, "train.total_ips": 23.906113363575386}}
DLLL {"timestamp": "1635352187.480156", "datetime": "2021-10-27 12:29:47.480156", "elapsedtime": "135.655858", "type": "LOG", "step": [0, 20], "data": {"train.compute_ips": 372.2747187707016, "train.data_time": 5.051431369781494, "train.compute_time": 0.34401004314422606}}
DLLL {"timestamp": "1635352244.731567", "datetime": "2021-10-27 12:30:44.731567", "elapsedtime": "192.907269", "type": "LOG", "step": [0, 30], "data": {"train.loss": 7.961615085601807, "train.total_ips": 22.508505559385885}}
DLLL {"timestamp": "1635352244.731754", "datetime": "2021-10-27 12:30:44.731754", "elapsedtime": "192.907456", "type": "LOG", "step": [0, 30], "data": {"train.compute_ips": 375.0279140509979, "train.data_time": 5.383709049224853, "train.compute_time": 0.34135634899139405}}
DLLL {"timestamp": "1635352291.517006", "datetime": "2021-10-27 12:31:31.517006", "elapsedtime": "239.692708", "type": "LOG", "step": [0, 40], "data": {"train.loss": 7.203387498855591, "train.total_ips": 27.43991979540195}}
DLLL {"timestamp": "1635352291.517133", "datetime": "2021-10-27 12:31:31.517133", "elapsedtime": "239.692835", "type": "LOG", "step": [0, 40], "data": {"train.compute_ips": 374.24969153358404, "train.data_time": 4.336400508880615, "train.compute_time": 0.3420675754547119}}
DLLL {"timestamp": "1635352345.224286", "datetime": "2021-10-27 12:32:25.224286", "elapsedtime": "293.399988", "type": "LOG", "step": [0, 50], "data": {"train.loss": 7.059615850448608, "train.total_ips": 23.952376806275353}}
DLLL {"timestamp": "1635352345.224414", "datetime": "2021-10-27 12:32:25.224414", "elapsedtime": "293.400116", "type": "LOG", "step": [0, 50], "data": {"train.compute_ips": 374.32099579349114, "train.data_time": 5.028588891029358, "train.compute_time": 0.3420623779296875}}
DLLL {"timestamp": "1635352398.738479", "datetime": "2021-10-27 12:33:18.738479", "elapsedtime": "346.914181", "type": "LOG", "step": [0, 60], "data": {"train.loss": 6.9575042724609375, "train.total_ips": 23.959202117484082}}
DLLL {"timestamp": "1635352398.738613", "datetime": "2021-10-27 12:33:18.738613", "elapsedtime": "346.914315", "type": "LOG", "step": [0, 60], "data": {"train.compute_ips": 375.2904960114342, "train.data_time": 5.010231518745423, "train.compute_time": 0.3411083459854126}}
DLLL {"timestamp": "1635352452.081702", "datetime": "2021-10-27 12:34:12.081702", "elapsedtime": "400.257404", "type": "LOG", "step": [0, 70], "data": {"train.loss": 6.942108392715454, "train.total_ips": 24.022426783243894}}
DLLL {"timestamp": "1635352452.081919", "datetime": "2021-10-27 12:34:12.081919", "elapsedtime": "400.257621", "type": "LOG", "step": [0, 70], "data": {"train.compute_ips": 375.0738312523796, "train.data_time": 4.992926907539368, "train.compute_time": 0.34130685329437255}}
DLLL {"timestamp": "1635352504.604788", "datetime": "2021-10-27 12:35:04.604788", "elapsedtime": "452.78049", "type": "LOG", "step": [0, 80], "data": {"train.loss": 6.943663215637207, "train.total_ips": 24.440133473626297}}
DLLL {"timestamp": "1635352504.60492", "datetime": "2021-10-27 12:35:04.604920", "elapsedtime": "452.780622", "type": "LOG", "step": [0, 80], "data": {"train.compute_ips": 374.2360943099309, "train.data_time": 4.910101318359375, "train.compute_time": 0.34213335514068605}}
DLLL {"timestamp": "1635352560.256526", "datetime": "2021-10-27 12:36:00.256526", "elapsedtime": "508.432228", "type": "LOG", "step": [0, 90], "data": {"train.loss": 6.920790481567383, "train.total_ips": 23.056404983762377}}
DLLL {"timestamp": "1635352560.256654", "datetime": "2021-10-27 12:36:00.256654", "elapsedtime": "508.432356", "type": "LOG", "step": [0, 90], "data": {"train.compute_ips": 374.41869291846933, "train.data_time": 5.223108530044556, "train.compute_time": 0.34198181629180907}}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment