Kilonova LodeRunner

applications/evaluation/TandV.input

@@ -1,13 +1,13 @@
 --basedir
-/usr/projects/artimis/mpmm/hickmank/chicoma/ch_yoke/applications/harnesses/ch_DDP_loderunner/ddp_lrstudy1/
+/net/sescratch1/atoivonen/projects/yoke_runs/KN_loderunner_fine_tune/runs
 --IDX
-10
+5
 --Nsamps_per_trn_pt
 200
 --Nsamps_per_val_pt
 100
 --ylim
 0.25
 --savedir
-./ch_ddp_lrstudy1/
+./KN_se_fine_tune/
 --savefig

applications/evaluation/TandVplot.py

@@ -153,8 +153,8 @@
 # trn_idxlist = trn_DF.index.values
 # val_idxlist = val_DF.index.values
 
-trn_csv_list = ["train.csv"]
-val_csv_list = ["valcsv"]
+#trn_csv_list = [] #["train.csv"]
+#val_csv_list = [] #["valcsv"]
 
 # Plot loss for training over all steps and epochs
 fig1 = plt.figure(num=1, figsize=(6, 6))
@@ -275,5 +275,5 @@
     plt.figure(fig1.number)
     filenameA = f"{savedir}/study{IDX:03d}_TandV_curve.png"
     plt.savefig(filenameA, bbox_inches="tight")
-else:
+    #else:
     plt.show()

applications/evaluation/img_compare_loderunner.input

@@ -9,7 +9,7 @@ design_lsc240420_MASTER.csv
 --LSC_NPZ_DIR
 /data2/lsc240420/
 --checkpoint
-study011_modelState_epoch0090.hdf5
+/net/sescratch1/atoivonen/projects/yoke_runs/KN_loderunner_fine_tune/runs/study_005/study005_modelState_epoch0100.pth
 --sampIDX
-3914
+10
 -S

applications/harnesses/KN_loderunner/ddp_production.csv

@@ -0,0 +1,62 @@
+studyIDX,YOKE_TORCH_ENV,KNODES,NGPUS,EMBED_DIM,B0,B1,B2,B3,NUM_WORKERS,BATCH_SIZE,NTRN_BATCH,NVAL_BATCH,ANCHOR_LR,NUM_CYCLES,MIN_FRACTION,TERMINAL_STEPS,WARMUP_STEPS,NOISE_SCALE,train_script
+# This is a longer epoch production run after tuning.
+# Single epoch with validation should be ~30 mins
+#1,torch_ch_gpu_241112,4,4,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#2,torch_ch_gpu_241112,1,4,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#3,torch_ch_gpu_241112,1,4,128,1,1,9,1,2,10,200,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#4,yoke311,1,4,128,1,1,9,1,2,10,200,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#5,yoke311,1,4,128,1,1,9,1,2,10,200,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#6,yoke311,1,4,128,1,1,9,1,2,10,200,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#7,yoke311,1,4,128,1,1,9,1,2,10,200,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#8,yoke311,1,4,128,1,1,9,1,2,10,200,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#9,yoke311,1,4,128,1,1,9,1,1,1,5,0,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.#py
+#10,yoke311,1,1,128,1,1,9,1,2,10,200,50,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#11,yoke311,1,4,128,1,1,9,1,2,10,1000,500,1.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#12,yoke311,1,4,128,1,1,9,1,2,10,1000,500,1.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#13,yoke311,1,4,128,1,1,9,1,2,10,1000,500,1.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#14,yoke311,1,4,128,1,1,9,1,2,10,1000,500,1.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#15,yoke311,1,4,128,1,1,9,1,2,10,1000,500,2.0e-4,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#16,yoke311,1,4,128,1,1,9,1,2,10,1000,500,2.0e-4,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#17,yoke311,1,4,128,1,1,9,1,2,10,1000,500,2.0e-4,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#18,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-4,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#19,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#20,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#21,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#22,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#23,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#24,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#25,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#26,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#27,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#28,yoke311,1,1,128,1,1,9,1,2,10,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#29,yoke311,1,8,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py 
+#30,yoke311,1,8,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#31,yoke311,1,8,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#32,yoke311,1,8,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#33,yoke311,1,8,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#34,yoke311,1,1,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#35,yoke311,1,1,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#36,yoke311,1,4,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#37,yoke311,1,1,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#38,yoke311,1,1,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#39,yoke311,1,1,128,1,1,9,1,2,10,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#40,yoke311,1,2,128,1,1,9,1,2,5,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#41,yoke311,1,1,128,1,1,9,1,2,5,1000,500,2.0e-3,0.5,0.5,1000,500,0.0,train_LodeRunner_ddp.py
+#42,yoke311,1,2,128,1,1,9,1,2,5,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#43,yoke311,1,8,128,1,1,9,1,2,5,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#44,yoke311,2,6,128,1,1,9,1,2,5,84,42,2.0e-3,0.5,0.5,84,42,0.0,train_LodeRunner_ddp.py
+#45,yoke311,2,6,128,1,1,9,1,2,5,84,42,2.0e-3,0.5,0.5,84,42,0.0,train_LodeRunner_ddp.py
+#46,yoke311,2,6,128,1,1,9,1,2,5,84,42,2.0e-3,0.5,0.5,84,42,0.0,train_LodeRunner_ddp.py
+#47,yoke311,2,6,128,1,1,9,1,2,5,84,42,2.0e-3,0.5,0.5,84,42,0.0,train_LodeRunner_ddp.py
+#48,yoke311,1,8,128,1,1,9,1,2,5,125,63,2.0e-3,0.5,0.5,125,63,0.0,train_LodeRunner_ddp.py
+#49,yoke311,1,1,128,1,1,9,1,2,5,125,63,2.0e-3,0.5,0.5,100,50,0.0,train_LodeRunner_ddp.py
+#50,yoke311,1,1,128,1,1,9,1,2,5,125,63,2.0e-3,0.5,0.5,100,50,0.0,train_LodeRunner_ddp.py
+#51,yoke311,1,1,128,1,1,9,1,2,5,125,63,2.0e-3,0.5,0.5,100,50,0.0,train_LodeRunner_ddp.py
+#52,yoke311,1,1,128,1,1,9,1,2,5,125,63,2.0e-3,0.5,0.5,100,50,0.0,train_LodeRunner_ddp.py
+# Study 56: reproduce study 54's DESCENDING schedule on current KN code.
+# 1 node x 1 GPU (no NVSwitch fabric -> no Error 802), NTRN_BATCH=84, and the
+# long schedule TERMINAL_STEPS=500/WARMUP=100 that let study 54's validation
+# descend 1.0->0.3 (study 55 at 84/42 never left warmup and looked flat).
+# Model config matches study 44: PREDICT_DELTA=True, TREND_DECAY_ANCHOR=False,
+# Huber delta=0.1. Once this descends, layer features back one at a time.
+56,yoke311,1,1,128,1,1,9,1,2,5,84,42,2.0e-3,0.5,0.5,500,100,0.0,train_LodeRunner_ddp.py