Commit 5a26746c authored by Yin, Junqi's avatar Yin, Junqi

add wikitext103

parent 7eb85c6d
#!/bin/bash
#BSUB -P stf011
#BSUB -W 8:00
#BSUB -nnodes 16
#BSUB -W 1:00
#BSUB -nnodes 1
#BSUB -alloc_flags "nvme gpumps"
#BSUB -J chocosgd
#BSUB -o logs/chocosgd.o%J
#BSUB -e logs/chocosgd.e%J
#BSUB -q batch-hm
##BSUB -q batch-hm
##BSUB -q killable
NNODES=$(cat ${LSB_DJOB_HOSTFILE} | sort | uniq | grep -v login | grep -v batch | wc -l)
source choco_env.sh
CHECKPOINT_DIR="data/checkpoint/imagenet/resnet50/test/16-nodes_ring_DDP-False_SHUFFLE_GRAPH-False_FREQ-10_HYBRID-False_FREQ-5_620280"
CHECKPOINT_DIR=None #"data/checkpoint/imagenet/resnet50/test/16-nodes_ring_DDP-False_SHUFFLE_GRAPH-False_FREQ-10_HYBRID-False_FREQ-5_620280"
RESUME_DIR=$(printf '%s\n' "$CHECKPOINT_DIR" | sed -e 's/[\/&]/\\&/g')
single_rank_per_node=false
#resnet20, lstm, resnet50, densenet100
EXPERIMENT=resnet50
EXPERIMENT=lstm
PRINT_GRAD=False
# centralized: complete; decentralized: ring, torus, expander, margulis_expander, social
TOPOLOGY=ring
TOPOLOGY=complete
# ddp only supports complete topology
DDP=False
# shuffle graph
SHUFFLE_GRAPH=False
FREQ_SHUFFLE=10
# hybrid mode
HYBRID=True
HYBRID=False
FREQ_HYBRID=5
# stage data to nvme
......
......@@ -149,6 +149,10 @@ def _get_nlp_lm_dataset(name, datasets_path, batch_first):
train, valid, test = torchtext.datasets.WikiText2.splits(
TEXT, root=datasets_path
)
elif "wikitext103" in name:
train, valid, test = torchtext.datasets.WikiText103.splits(
TEXT, root=datasets_path
)
elif "ptb" in name:
train, valid, test = torchtext.datasets.PennTreebank.splits(
TEXT, root=datasets_path
......@@ -187,7 +191,7 @@ def get_dataset(
return _get_epsilon_or_rcv1(root, name, split)
elif name == "rcv1":
return _get_epsilon_or_rcv1(root, name, split)
elif name == "wikitext2" or name == "ptb":
elif "wikitext" in name or name == "ptb":
return _get_nlp_lm_dataset(name, datasets_path, batch_first=False)
else:
raise NotImplementedError
......@@ -2,7 +2,7 @@
python -u main.py \
--work_dir $(pwd) \
--remote_exec False \
--data wikitext2 \
--data wikitext103 \
--use_lmdb_data False \
--partition_data random \
--pin_memory True \
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment