mirror of
https://github.com/vale981/ray
synced 2025-03-06 10:31:39 -05:00
56d2cf6479
* Fix SC2001: See if you can use ${variable//search/replace} instead.
* Fix SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames.
* Fix SC2012: Use find instead of ls to better handle non-alphanumeric filenames.
* Fix SC2015: Note that A && B || C is not if-then-else. C may run when A is true.
* Fix SC2028: echo may not expand escape sequences. Use printf.
* Fix SC2034: variable appears unused. Verify use (or export if used externally).
* Fix SC2035: Use ./*glob* or -- *glob* so names with dashes won't become options.
* Fix SC2071: > is for string comparisons. Use -gt instead.
* Fix SC2154: variable is referenced but not assigned
* Fix SC2164: Use 'cd ... || exit' or 'cd ... || return' in case cd fails.
* Fix SC2188: This redirection doesn't have a command. Move to its command (or use 'true' as no-op).
* Fix SC2236: Use -n instead of ! -z.
* Fix SC2242: Can only exit with status 0-255. Other data should be written to stdout/stderr.
* Fix SC2086: Double quote to prevent globbing and word splitting.
Co-authored-by: Mehrdad <noreply@github.com>
26 lines
1.4 KiB
Bash
26 lines
1.4 KiB
Bash
#!/bin/bash
|
|
|
|
TOTAL_UPDATES=125000 # Total number of training steps
|
|
WARMUP_UPDATES=10000 # Warmup the learning rate over this many updates
|
|
PEAK_LR=0.0005 # Peak learning rate, adjust as needed
|
|
TOKENS_PER_SAMPLE=512 # Max sequence length
|
|
#MAX_POSITIONS=512 # Num. positional embeddings (usually same as above)
|
|
MAX_SENTENCES=8 # Number of sequences per batch on one GPU (batch size)
|
|
FIX_BATCH_SIZE=2048 # Number of batch size in total (max_sentences * update_freq * n_gpus)
|
|
SAVE_INTERVAL_UPDATES=1000 # save a checkpoint every N updates
|
|
|
|
LOG_DIR=$HOME/efs/lm/log/
|
|
DATA_DIR=$HOME/efs/lm/data-bin/wikitext-103/
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
python "$HOME"/efs/lm/ray_train.py --fp16 "$DATA_DIR" \
|
|
--task masked_lm --criterion masked_lm \
|
|
--arch roberta_base --sample-break-mode complete --tokens-per-sample $TOKENS_PER_SAMPLE \
|
|
--optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-6 --clip-norm 0.0 \
|
|
--lr-scheduler polynomial_decay --lr $PEAK_LR --warmup-updates $WARMUP_UPDATES --total-num-update $TOTAL_UPDATES \
|
|
--dropout 0.1 --attention-dropout 0.1 --weight-decay 0.01 \
|
|
--max-sentences $MAX_SENTENCES \
|
|
--fix-batch-size $FIX_BATCH_SIZE \
|
|
--max-update $TOTAL_UPDATES --log-format simple --log-interval 1 \
|
|
--save-interval-updates $SAVE_INTERVAL_UPDATES \
|
|
--save-dir "$LOG_DIR" --ddp-backend=no_c10d
|