diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/START_study.py b/applications/harnesses/ch_DDP_loderunner_cylex/START_study.py new file mode 100644 index 00000000..846b445d --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/START_study.py @@ -0,0 +1 @@ +../START_study.py diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/cp_files.txt b/applications/harnesses/ch_DDP_loderunner_cylex/cp_files.txt new file mode 100644 index 00000000..ad5fbb3d --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/cp_files.txt @@ -0,0 +1 @@ +train_LodeRunner_ddp_cylex.py diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/ddp_test.csv b/applications/harnesses/ch_DDP_loderunner_cylex/ddp_test.csv new file mode 100644 index 00000000..0fe2c760 --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/ddp_test.csv @@ -0,0 +1,4 @@ +studyIDX,YOKE_TORCH_ENV,KNODES,NGPUS,EMBED_DIM,B0,B1,B2,B3,NUM_WORKERS,BATCH_SIZE,NTRN_BATCH,NVAL_BATCH,ANCHOR_LR,NUM_CYCLES,MIN_FRACTION,TERMINAL_STEPS,WARMUP_STEPS,train_script +# This is a longer epoch production run after tuning. +# Single epoch with validation should be ~30 mins +1,torch_se_gpu_120226,1,4,128,1,1,9,1,2,1,1000,500,5.0e-4,0.5,0.5,1000,500,train_LodeRunner_ddp_cylex.py diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/eval/avg_eval_csv.py b/applications/harnesses/ch_DDP_loderunner_cylex/eval/avg_eval_csv.py new file mode 100644 index 00000000..cdd09a90 --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/eval/avg_eval_csv.py @@ -0,0 +1,40 @@ +"""Compute the statistics for MSE over evaluation CSV.""" + +import argparse +import pandas as pd + + +def main(): + """Compute and print.""" + # Parser CLI CSV-filename + parser = argparse.ArgumentParser( + description="Compute summary statistics of evaluation CSV." + ) + parser.add_argument( + '--csv', + type=str, + default='./testing_evaluation.csv', + help="Path to evaluation CSV." + ) + + args = parser.parse_args() + + # Read the CSV file + df = pd.read_csv(args.csv, header=None) + + # Extract the third column + col = df[2] + + # Compute statistics of the third column + mean_MSE = col.mean() + p2_5, p50, p97_5 = col.quantile([0.025, 0.5, 0.975]) + + # Print results + print(f"Average MSE: {mean_MSE}") + print(f"2.5-percentile: {p2_5}") + print(f"50-percentile: {p50}") + print(f"97.5-percentile: {p97_5}") + + +if __name__ == "__main__": + main() diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/eval/eval_LodeRunner_cylex.py b/applications/harnesses/ch_DDP_loderunner_cylex/eval/eval_LodeRunner_cylex.py new file mode 100644 index 00000000..5b0587c0 --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/eval/eval_LodeRunner_cylex.py @@ -0,0 +1,235 @@ +"""Evaluate trained model on test set. + +Model is evaluated on `cycle_epoch` number of epochs with `test_batches` number of +batches each of a set `batch_size`. + +""" + +import argparse +import os +import time + +import torch +import torch.nn as nn +from torch.utils.data import DataLoader + +from yoke.models.vit.swin.bomberman import LodeRunner +from yoke.datasets.load_npz_dataset import TemporalDataSet +from yoke.utils.checkpointing import load_model_and_optimizer +from yoke.utils.training.epoch.loderunner import eval_loderunner_epoch +from yoke.helpers import cli + + +descr_str = ( + "Single-GPU evaluation for a saved Yoke LodeRunner checkpoint on real dataset " + "batches. Mirrors train_LodeRunner_ddp.py structure but without DDP." +) +parser = argparse.ArgumentParser( + prog="LodeRunner Evaluation", description=descr_str, fromfile_prefix_chars="@" +) + +# Reuse the same CLI arg groups as training so you can pass the same @argfiles. +parser = cli.add_default_args(parser=parser) +parser = cli.add_filepath_args(parser=parser) +parser = cli.add_computing_args(parser=parser) +parser = cli.add_training_args(parser=parser) + +# Keep the same default filelists as train_LodeRunner_ddp.py +parser.set_defaults( + train_filelist="cx241203_prefixes_train_80pct_noBe_noVoid_truncated.txt", + validation_filelist="cx241203_prefixes_val_10pct_noBe_noVoid_truncated.txt", + test_filelist="cx241203_prefixes_test_10pct_noBe_noVoid_truncated.txt", +) + +def main(args: argparse.Namespace) -> None: + """Main evaluation function.""" + ############################################# + # Process Inputs + ############################################# + # Device (single GPU) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Paths + filelist_dir = args.FILELIST_DIR + filelist_path = os.path.join(filelist_dir, args.test_filelist) + + # Dataloader params + batch_size = args.batch_size + num_workers = args.num_workers + test_batches = args.test_batches + batch_size = args.batch_size + cycle_epochs = args.cycle_epochs + test_rcrd_filename = args.test_rcrd_filename + + model_args = { + "default_vars": [ + "Rcoord", # 4 kinematic variable fields + "Zcoord", + "Uvelocity", + "Wvelocity", + "density_Air", # 39 thermodynamic variable fields + "energy_Air", + "pressure_Air", + "density_Al", + "energy_Al", + "pressure_Al", + "density_Be", + "energy_Be", + "pressure_Be", + "density_booster", + "energy_booster", + "pressure_booster", + "density_Cu", + "energy_Cu", + "pressure_Cu", + "density_U.DU", + "energy_U.DU", + "pressure_U.DU", + "density_maincharge", + "energy_maincharge", + "pressure_maincharge", + "density_N", + "energy_N", + "pressure_N", + "density_Sn", + "energy_Sn", + "pressure_Sn", + "density_Steel.alloySS304L", + "energy_Steel.alloySS304L", + "pressure_Steel.alloySS304L", + "density_Polymer.Sylgard", + "energy_Polymer.Sylgard", + "pressure_Polymer.Sylgard", + "density_Ta", + "energy_Ta", + "pressure_Ta", + "density_Void", + "energy_Void", + "pressure_Void", + "density_Water", + "energy_Water", + "pressure_Water", + ], + "image_size": (1120, 400), + "patch_size": (10, 5), + "embed_dim": 128, + "emb_factor": 2, + "num_heads": 8, + "block_structure": (1, 1, 9, 1), + "window_sizes": [(8, 8), (8, 8), (4, 4), (2, 2)], + "patch_merge_scales": [(2, 2), (2, 2), (2, 2)], + } + + model = LodeRunner(**model_args) + + ############################################# + # Load Model Checkpoint + ############################################# + available_models = {"LodeRunner": LodeRunner} + + # NOTE: optimizer args are required by load_model_and_optimizer, even for eval. + model, _optimizer, starting_epoch = load_model_and_optimizer( + args.pretrained_model, + optimizer_class=torch.optim.AdamW, + optimizer_kwargs={ + "lr": 1e-6, + "betas": (0.9, 0.999), + "eps": 1e-08, + "weight_decay": 0.01, + }, + available_models=available_models, + device=device, + ) + starting_epoch = 0 + model.to(device) + model.eval() + + # load_and_eval_YokePth.py prints these; keep similar behavior here + print(f"Loaded checkpoint: {args.pretrained_model}", flush=True) + print(f"Checkpoint starting_epoch: {starting_epoch}", flush=True) + if hasattr(model, "default_vars"): + print("Default LodeRunner fields:", model.default_vars, flush=True) + if hasattr(model, "image_size"): + print("LodeRunner image size:", model.image_size, flush=True) + + ############################################# + # Dataset / Dataloader (non-distributed) + ############################################# + testing_dataset = TemporalDataSet( + args.NPZ_DIR, + args.CSV_FILEPATH, + file_prefix_list=filelist_path, + max_timeIDX_offset=2, + max_file_checks=10, + half_image=True, + ) + + from torch.utils.data.dataloader import default_collate + + def collate_skip_none(batch): + batch = [b for b in batch if b is not None] + if len(batch) == 0: + return None + return default_collate(batch) + + test_dataloader = DataLoader( + dataset=testing_dataset, + batch_size=batch_size, + shuffle=False, + num_workers=num_workers, + pin_memory=torch.cuda.is_available(), + #drop_last=False, + collate_fn=collate_skip_none, + #prefetch_factor=2, + ) + + ############################################# + # Loss + Evaluation Loop + ############################################# + # Match train_LodeRunner_ddp.py: use per-element MSE so we can reduce ourselves + loss_fn = nn.MSELoss(reduction="none") + + + ############################################# + # Testing Loop + ############################################# + # Train Model + print("Testing Model . . .") + starting_epoch += 1 + ending_epoch = starting_epoch + cycle_epochs + + for epochIDX in range(starting_epoch, ending_epoch): + # Time each epoch and print to stdout + startTime = time.time() + + # Testing epoch + # for cylex channel_map changes per sample. So pass None here + # & calculate channel_map later in the datastep function. + eval_loderunner_epoch( + testing_data=test_dataloader, + num_test_batches=test_batches, + model=model, + dataset='cylex', + channel_map=None, + loss_fn=loss_fn, + epochIDX=epochIDX, + test_rcrd_filename=test_rcrd_filename, + device=device, + ) + + # Time each epoch and print to stdout + endTime = time.time() + + epoch_time = (endTime - startTime) / 60 + + # Print Summary Results + print(f"Completed epoch {epochIDX}...", flush=True) + print(f"Epoch time (minutes): {epoch_time:.2f}", flush=True) + + +if __name__ == "__main__": + """Parse arguments and run main evaluation function.""" + + args = parser.parse_args() + + main(args) diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/eval/eval_START.input b/applications/harnesses/ch_DDP_loderunner_cylex/eval/eval_START.input new file mode 100644 index 00000000..7b4a3ccb --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/eval/eval_START.input @@ -0,0 +1,22 @@ +--pretrain_checkpoint +/net/sescratch1/exempt/artimis/soumide/projects/yoke_runs/cylex_full_run_lr5e-4_w_val/runs/study_001/study001_modelState_epoch0100.pth +--FILELIST_DIR +/usr/projects/artimis/mpmm/hickmank/github_yoke/applications/filelists/ +--NPZ_DIR +/net/sescratch1/exempt/artimis/data/cx241203/ +--CSV_FILEPATH +/net/sescratch1/exempt/artimis/mpmm/design_cx241203_MASTER.csv +--test_filelist +cx241203_prefixes_test_10pct_noBe_noVoid.txt +--test_rcrd_filename +./testing_evaluation.csv +--batch_size +1 +--num_workers +2 +--total_epochs +10 +--cycle_epochs +10 +--test_batches +1000 diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/eval/eval_START.slurm b/applications/harnesses/ch_DDP_loderunner_cylex/eval/eval_START.slurm new file mode 100644 index 00000000..4add9f9d --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/eval/eval_START.slurm @@ -0,0 +1,79 @@ +#!/bin/bash + +# This is a setup for GPU training on Selene. Find out how much +# memory per node, number of CPUs/node. + +# NOTE: Number of CPUs per GPU must be an even number since there are +# 2 threads per core. If an odd number is requested the next higher +# even number gets used. + +# The following are one set of SBATCH options for the Selene GPU +# partition. There are optional other constraints. + +#SBATCH --job-name=cylfull_eval +#SBATCH --account=y26_artimis-fmod_g +#SBATCH --partition=standard +#SBATCH --time=8:00:00 +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=4 +#SBATCH --gpus-per-node=4 +#SBATCH --cpus-per-task=8 +#SBATCH --mem-per-gpu=120G +#SBATCH --output=eval_test.out +#SBATCH --error=eval_test.err +#SBATCH -vvv + +# Set the master node's address and port +MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1) +MASTER_PORT=$(shuf -i 1024-65535 -n 1) # Choose a random port +export MASTER_ADDR +export MASTER_PORT + +# Enable shell debugging: Debugging selene slurm +set -xv + +# Check available GPUs +sinfo -o "%P %.24G %N" +#srun -vv --cpu-bind=verbose /usr/bin/echo $CUDA_AVAILABLE_DEVICES +nvidia-smi + +# Specify NCCL communication +export NCCL_SOCKET_IFNAME=ib0 # Check possible interfaces with `ip link show` + +# for multi-node training +export NCCL_IB_HCA=mlx5_0 +export NCCL_IB_GID_INDEX=3 # sometimes 0 or 3 depending on subnet manager config + +# Debugging distributed data parallel +# export NCCL_DEBUG=INFO +# export NCCL_DEBUG_SUBSYS=INIT + +# Debugging selene slurm +export SLURM_CPU_BIND=verbose + +# Load correct conda environment +module load anaconda/3.12 +source activate +conda activate torch_se_gpu_120226 + +# Set number of threads per GPU +export OMP_NUM_THREADS=10 + +# Get start time +export date00=`date` + +# Start the Code +# Explicitly set TCP environment for the following... +#srun -vv --cpu-bind=verbose python eval_LodeRunner.py @eval_START.input +python eval_LodeRunner.py @eval_START.input + +# Start the Code +#python eval_LodeRunner.py @eval_START.input + +# Get end time and print to stdout +export date01=`date` + +echo "===================TIME STARTED===================" +echo $date00 +echo "===================TIME FINISHED===================" +echo $date01 diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/train_LodeRunner_ddp_cylex.py b/applications/harnesses/ch_DDP_loderunner_cylex/train_LodeRunner_ddp_cylex.py new file mode 100644 index 00000000..3a0b2ee3 --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/train_LodeRunner_ddp_cylex.py @@ -0,0 +1,403 @@ +import os +import time +import argparse +import numpy as np +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel as DDP + +from yoke.models.vit.swin.bomberman import LodeRunner +from yoke.datasets.load_npz_dataset import TemporalDataSet + +from yoke.utils.training.epoch.loderunner import train_DDP_loderunner_epoch +from yoke.utils.restart import continuation_setup +from yoke.utils.dataload import make_distributed_dataloader +from yoke.utils.checkpointing import load_model_and_optimizer +from yoke.utils.checkpointing import save_model_and_optimizer + +from yoke.lr_schedulers import CosineWithWarmupScheduler +from yoke.helpers import cli + + +############################################# +# Inputs +############################################# +descr_str = ( + "Uses DDP to train LodeRunner architecture on single-timstep input and output " + "of the lsc240420 per-material density fields." +) +parser = argparse.ArgumentParser( + prog="DDP LodeRunner Training", description=descr_str, fromfile_prefix_chars="@" +) +parser = cli.add_default_args(parser=parser) +parser = cli.add_filepath_args(parser=parser) +parser = cli.add_computing_args(parser=parser) +parser = cli.add_model_args(parser=parser) +parser = cli.add_training_args(parser=parser) +parser = cli.add_cosine_lr_scheduler_args(parser=parser) + +# Change some default filepaths. +parser.set_defaults( + train_filelist="cx241203_prefixes_train_80pct.txt", + validation_filelist="cx241203_prefixes_val_10pct.txt", + test_filelist="cx241203_prefixes_test_10pct.txt", +) + + +def setup_distributed(): + # ----- 1) Basic setup & environment variables ----- + # Rely on Slurm variables: SLURM_PROCID, SLURM_NTASKS, SLURM_LOCALID, etc. + rank = int(os.environ["SLURM_PROCID"]) # global rank + world_size = int(os.environ["SLURM_NTASKS"]) # total number of processes + local_rank = int(os.environ["SLURM_LOCALID"]) # local rank (GPU index on this node) + + master_addr = os.environ["MASTER_ADDR"] + master_port = os.environ["MASTER_PORT"] + + print("============================", flush=True) + print(f"[Rank {rank}] DDP setup, master_addr: {master_addr}", flush=True) + print(f"[Rank {rank}] DDP setup, master_port: {master_port}", flush=True) + print(f"[Rank {rank}] DDP setup, rank: {rank}", flush=True) + print(f"[Rank {rank}] DDP setup, local_rank: {local_rank}", flush=True) + print(f"[Rank {rank}] DDP setup, world_size: {world_size}", flush=True) + print("============================", flush=True) + + # ----- 2) Set the current GPU device for this process ----- + torch.cuda.set_device(local_rank) + device = torch.device(f"cuda:{local_rank}") + + # ----- 3) Initialize the process group ----- + dist.init_process_group( + backend="nccl", + init_method=f"tcp://{master_addr}:{master_port}", + world_size=world_size, + rank=rank, + ) + + return rank, world_size, local_rank, device + + +def cleanup_distributed(): + # ----- 8) Clean up (optional) ----- + dist.destroy_process_group() + + +def main(args, rank, world_size, local_rank, device): + ############################################# + # Process Inputs + ############################################# + # Study ID + studyIDX = args.studyIDX + + # Resources + Ngpus = args.Ngpus + Knodes = args.Knodes + + # Data Paths + train_filelist = args.FILELIST_DIR + args.train_filelist + validation_filelist = args.FILELIST_DIR + args.validation_filelist + test_filelist = args.FILELIST_DIR + args.test_filelist + + # Model Parameters + embed_dim = args.embed_dim + block_structure = tuple(args.block_structure) + + # Training Parameters + anchor_lr = args.anchor_lr + num_cycles = args.num_cycles + min_fraction = args.min_fraction + terminal_steps = args.terminal_steps + warmup_steps = args.warmup_steps + + # Number of workers controls how batches of data are prefetched and, + # possibly, pre-loaded onto GPUs. If the number of workers is large they + # will swamp memory and jobs will fail. + num_workers = args.num_workers + + # Epoch Parameters + batch_size = args.batch_size + total_epochs = args.total_epochs + cycle_epochs = args.cycle_epochs + train_batches = args.train_batches + val_batches = args.val_batches + train_per_val = args.TRAIN_PER_VAL + trn_rcrd_filename = args.trn_rcrd_filename + val_rcrd_filename = args.val_rcrd_filename + CONTINUATION = args.continuation + START = not CONTINUATION + checkpoint = args.checkpoint + + # Dictionary of available models. + available_models = { + "LodeRunner": LodeRunner + } + + ############################################# + # Model Arguments for Dynamic Reconstruction + ############################################# + model_args = { + "default_vars": [ + "Rcoord", # 4 kinematic variable fields + "Zcoord", + "Uvelocity", + "Wvelocity", + "density_Air", # 39 thermodynamic variable fields + "energy_Air", + "pressure_Air", + "density_Al", + "energy_Al", + "pressure_Al", + "density_Be", + "energy_Be", + "pressure_Be", + "density_booster", + "energy_booster", + "pressure_booster", + "density_Cu", + "energy_Cu", + "pressure_Cu", + "density_U.DU", + "energy_U.DU", + "pressure_U.DU", + "density_maincharge", + "energy_maincharge", + "pressure_maincharge", + "density_N", + "energy_N", + "pressure_N", + "density_Sn", + "energy_Sn", + "pressure_Sn", + "density_Steel.alloySS304L", + "energy_Steel.alloySS304L", + "pressure_Steel.alloySS304L", + "density_Polymer.Sylgard", + "energy_Polymer.Sylgard", + "pressure_Polymer.Sylgard", + "density_Ta", + "energy_Ta", + "pressure_Ta", + "density_Void", + "energy_Void", + "pressure_Void", + "density_Water", + "energy_Water", + "pressure_Water", + ], + "image_size": (1120, 400), + "patch_size": (10, 5), + "embed_dim": embed_dim, + "emb_factor": 2, + "num_heads": 8, + "block_structure": block_structure, + "window_sizes": [(8, 8), (8, 8), (4, 4), (2, 2)], + "patch_merge_scales": [(2, 2), (2, 2), (2, 2)], + } + + ############################################# + # Load Model for Continuation (Rank 0 only) + ############################################# + # Wait to move model to GPU until after the checkpoint load. Then + # explicitly move model and optimizer state to GPU. + if CONTINUATION: + model, optimizer, starting_epoch = load_model_and_optimizer( + checkpoint, + optimizer_class=torch.optim.AdamW, + optimizer_kwargs={ + "lr": 1e-6, + "betas": (0.9, 0.999), + "eps": 1e-08, + "weight_decay": 0.01, + }, + available_models=available_models, + device=device, + ) + print("Model state loaded for continuation.") + else: + # Initialize model and optimizer state. + # If not continuing, set starting_epoch to 0. + starting_epoch = 0 + model = LodeRunner(**model_args) + # Move model to GPU before instantiating optimizer and DDP. + model.to(device) + + # Instantiate optimizer and move state to GPU. + optimizer = torch.optim.AdamW( + model.parameters(), + lr=1e-6, + betas=(0.9, 0.999), + eps=1e-08, + weight_decay=0.01 + ) + + for state in optimizer.state.values(): + for key, value in state.items(): + if isinstance(value, torch.Tensor): + state[key] = value.to(device) + + ############################################# + # Initialize Loss + ############################################# + # Use `reduction='none'` so loss on each sample in batch can be recorded. + loss_fn = nn.MSELoss(reduction="none") + + ############################################# + # Move Model to DistributedDataParallel + ############################################# + model = DDP(model, device_ids=[local_rank], output_device=local_rank) + + ############################################# + # Learning Rate Scheduler + ############################################# + if starting_epoch == 0: + last_epoch = -1 + else: + last_epoch = train_batches * (starting_epoch - 1) + + # Scale the anchor LR by global batchsize + # + # # For multi-node + lr_scale = np.sqrt(float(Ngpus) * float(Knodes) * float(batch_size)) + original_batchsize = 40.0 # 1 node, 4 gpus, 10 samples/gpu + ddp_anchor_lr = anchor_lr * lr_scale / original_batchsize + # + # For single node + # ddp_anchor_lr = anchor_lr + + LRsched = CosineWithWarmupScheduler( + optimizer, + anchor_lr=ddp_anchor_lr, + terminal_steps=terminal_steps, + warmup_steps=warmup_steps, + num_cycles=num_cycles, + min_fraction=min_fraction, + last_epoch=last_epoch, + ) + + ############################################# + # Data Initialization (Distributed Dataloader) + ############################################# + train_dataset = TemporalDataSet( + args.NPZ_DIR, + args.CSV_FILEPATH, + file_prefix_list=train_filelist, + max_timeIDX_offset=2, + max_file_checks=10, + half_image=True, + ) + val_dataset = TemporalDataSet( + args.NPZ_DIR, + args.CSV_FILEPATH, + file_prefix_list=validation_filelist, + max_timeIDX_offset=2, + max_file_checks=10, + half_image=True, + ) + + # NOTE: For DDP the batch_size is the per-GPU batch_size!!! + train_dataloader = make_distributed_dataloader( + train_dataset, + batch_size, + shuffle=True, + num_workers=num_workers, + rank=rank, + world_size=world_size, + ) + val_dataloader = make_distributed_dataloader( + val_dataset, + batch_size, + shuffle=False, + num_workers=num_workers, + rank=rank, + world_size=world_size, + ) + + ############################################# + # Training Loop (Modified for DDP) + ############################################# + # Train Model + print("Training Model . . .") + starting_epoch += 1 + ending_epoch = min(starting_epoch + cycle_epochs, total_epochs + 1) + + TIME_EPOCH = True + for epochIDX in range(starting_epoch, ending_epoch): + train_sampler = train_dataloader.sampler + train_sampler.set_epoch(epochIDX) + + # For timing epochs + if TIME_EPOCH: + # Synchronize before starting the timer + dist.barrier() # Ensure that all nodes sync + torch.cuda.synchronize(device) # Ensure GPUs on each node sync + # Time each epoch and print to stdout + startTime = time.time() + + # Train and Validate + train_DDP_loderunner_epoch( + training_data=train_dataloader, + validation_data=val_dataloader, + dataset='cylex', + num_train_batches=train_batches, + num_val_batches=val_batches, + model=model, + optimizer=optimizer, + loss_fn=loss_fn, + LRsched=LRsched, + epochIDX=epochIDX, + train_per_val=train_per_val, + train_rcrd_filename=trn_rcrd_filename, + val_rcrd_filename=val_rcrd_filename, + device=device, + rank=rank, + world_size=world_size, + ) + + if TIME_EPOCH: + # Synchronize before stopping the timer + torch.cuda.synchronize(device) # Ensure GPUs on each node sync + dist.barrier() # Ensure that all nodes sync + # Time each epoch and print to stdout + endTime = time.time() + + epoch_time = (endTime - startTime) / 60 + + # Print Summary Results + if rank == 0: + print(f"Completed epoch {epochIDX}...", flush=True) + print(f"Epoch time (minutes): {epoch_time:.2f}", flush=True) + + # Save model and optimizer state in hdf5 + chkpt_name_str = "study{0:03d}_modelState_epoch{1:04d}.pth" + new_chkpt_path = os.path.join("./", chkpt_name_str.format(studyIDX, epochIDX)) + + save_model_and_optimizer( + model, + optimizer, + epochIDX, + new_chkpt_path, + model_class=LodeRunner, + model_args=model_args + ) + + if rank == 0: + ############################################# + # Continue if Necessary + ############################################# + FINISHED_TRAINING = epochIDX + 1 > total_epochs + if not FINISHED_TRAINING: + new_slurm_file = continuation_setup( + new_chkpt_path, studyIDX, last_epoch=epochIDX + ) + os.system(f"sbatch {new_slurm_file}") + + +if __name__ == "__main__": + args = parser.parse_args() + + rank, world_size, local_rank, device = setup_distributed() + + main(args, rank, world_size, local_rank, device) + + cleanup_distributed() diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/training_START.input b/applications/harnesses/ch_DDP_loderunner_cylex/training_START.input new file mode 100644 index 00000000..c796aac4 --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/training_START.input @@ -0,0 +1,53 @@ +--studyIDX + +--FILELIST_DIR +/usr/projects/artimis/mpmm/hickmank/github_yoke/applications/filelists/ +--NPZ_DIR +/lustre/scratch5/exempt/artimis/data/cx241203_copy/cx241203/ +--CSV_FILEPATH +/lustre/scratch5/exempt/artimis/mpmm/design_cx241203_MASTER.csv +--train_filelist +cx241203_prefixes_train_80pct_noBe_noVoid_truncated.txt +--validation_filelist +cx241203_prefixes_val_10pct_noBe_noVoid_truncated.txt +--test_filelist +cx241203_prefixes_test_10pct_noBe_noVoid_truncated.txt +--block_structure + + + + +--embed_dim + +--anchor_lr + +--num_cycles + +--min_fraction + +--terminal_steps + +--warmup_steps + +--trn_rcrd_filename +./training_study_epoch.csv +--val_rcrd_filename +./validation_study_epoch.csv +--batch_size + +--num_workers + +--Ngpus + +--Knodes + +--total_epochs +100 +--cycle_epochs +1 +--train_batches + +--val_batches + +--TRAIN_PER_VAL +5 diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/training_START.slurm b/applications/harnesses/ch_DDP_loderunner_cylex/training_START.slurm new file mode 100644 index 00000000..ed5eede8 --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/training_START.slurm @@ -0,0 +1,75 @@ +#!/bin/bash + +# This is a setup for GPU training on Venado. Find out how much +# memory per node, number of CPUs/node. + +# NOTE: Number of CPUs per GPU must be an even number since there are +# 2 threads per core. If an odd number is requested the next higher +# even number gets used. + +# The following are one set of SBATCH options for the Venado GPU +# partition. There are optional other constraints. + +#SBATCH --job-name=ddp_s_e0001 +#SBATCH --account=y26_artimis-fmod_g +#SBATCH --partition=standard +#SBATCH --time=1:30:00 +#SBATCH --nodes= +#SBATCH --ntasks-per-node= +#SBATCH --gpus-per-node= +#SBATCH --cpus-per-task=8 +#SBATCH --mem-per-gpu=120G +#SBATCH --output=study_epoch0001.out +#SBATCH --error=study_epoch0001.err +#SBATCH -vvv + +# Set the master node's address and port +MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1) +MASTER_PORT=$(shuf -i 1024-65535 -n 1) # Choose a random port +export MASTER_ADDR +export MASTER_PORT + +# Enable shell debugging: Debugging selene slurm +set -xv + +# Check available GPUs +sinfo -o "%P %.24G %N" +srun -vv --cpu-bind=verbose /usr/bin/echo $CUDA_AVAILABLE_DEVICES +nvidia-smi + +# Specify NCCL communication +export NCCL_SOCKET_IFNAME=ib0 # Check possible interfaces with `ip link show` + +# for multi-node training +export NCCL_IB_HCA=mlx5_0 +export NCCL_IB_GID_INDEX=3 # sometimes 0 or 3 depending on subnet manager config + +# Debugging distributed data parallel +# export NCCL_DEBUG=INFO +# export NCCL_DEBUG_SUBSYS=INIT + +# Debugging selene slurm +export SLURM_CPU_BIND=verbose + +# Load correct conda environment +module load anaconda/3.12 +source activate +conda activate + +# Set number of threads per GPU +export OMP_NUM_THREADS=10 + +# Get start time +export date00=`date` + +# Start the Code +# Explicitly set TCP environment for the following... +srun -vv --cpu-bind=verbose python -u @study_START.input + +# Get end time and print to stdout +export date01=`date` + +echo "===================TIME STARTED===================" +echo $date00 +echo "===================TIME FINISHED===================" +echo $date01 diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/training_input.tmpl b/applications/harnesses/ch_DDP_loderunner_cylex/training_input.tmpl new file mode 100644 index 00000000..bb857811 --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/training_input.tmpl @@ -0,0 +1,56 @@ +--studyIDX + +--FILELIST_DIR +/usr/projects/artimis/mpmm/hickmank/github_yoke/applications/filelists/ +--NPZ_DIR +/lustre/scratch5/exempt/artimis/data/cx241203_copy/cx241203/ +--CSV_FILEPATH +/lustre/scratch5/exempt/artimis/mpmm/design_cx241203_MASTER.csv +--train_filelist +cx241203_prefixes_train_80pct_noBe_noVoid_truncated.txt +--validation_filelist +cx241203_prefixes_val_10pct_noBe_noVoid_truncated.txt +--test_filelist +cx241203_prefixes_test_10pct_noBe_noVoid_truncated.txt +--block_structure + + + + +--embed_dim + +--anchor_lr + +--num_cycles + +--min_fraction + +--terminal_steps + +--warmup_steps + +--trn_rcrd_filename +./training_study_epoch.csv +--val_rcrd_filename +./validation_study_epoch.csv +--batch_size + +--num_workers + +--Ngpus + +--Knodes + +--total_epochs +100 +--cycle_epochs +1 +--train_batches + +--val_batches + +--TRAIN_PER_VAL +5 +--continuation +--checkpoint + diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/training_slurm.tmpl b/applications/harnesses/ch_DDP_loderunner_cylex/training_slurm.tmpl new file mode 100644 index 00000000..4708024c --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/training_slurm.tmpl @@ -0,0 +1,74 @@ +#!/bin/bash + +# This is a setup for GPU training on Venado. Find out how much +# memory per node, number of CPUs/node. + +# NOTE: Number of CPUs per GPU must be an even number since there are +# 2 threads per core. If an odd number is requested the next higher +# even number gets used. + +# The following are one set of SBATCH options for the Venado GPU +# partition. There are optional other constraints. + +#SBATCH --job-name=ddp_s_e +#SBATCH --account=y26_artimis-fmod_g +#SBATCH --time=1:30:00 +#SBATCH --nodes= +#SBATCH --ntasks-per-node= +#SBATCH --gpus-per-node= +#SBATCH --cpus-per-task=8 +#SBATCH --mem-per-gpu=120G +#SBATCH --output=study_epoch.out +#SBATCH --error=study_epoch.err +#SBATCH -vvv + +# Set the master node's address and port +MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1) +MASTER_PORT=$(shuf -i 1024-65535 -n 1) # Choose a random port +export MASTER_ADDR +export MASTER_PORT + +# Enable shell debugging: Debugging selene slurm +set -xv + +# Check available GPUs +sinfo -o "%P %.24G %N" +srun -vv --cpu-bind=verbose /usr/bin/echo $CUDA_AVAILABLE_DEVICES +nvidia-smi + +# Specify NCCL communication +export NCCL_SOCKET_IFNAME=ib0 # Check possible interfaces with `ip link show` + +# for multi-node training +export NCCL_IB_HCA=mlx5_0 +export NCCL_IB_GID_INDEX=3 # sometimes 0 or 3 depending on subnet manager config + +# Debugging distributed data parallel +# export NCCL_DEBUG=INFO +# export NCCL_DEBUG_SUBSYS=INIT + +# Debugging selene slurm +export SLURM_CPU_BIND=verbose + +# Load correct conda environment +module load anaconda/3.12 +source activate +conda activate + +# Set number of threads per GPU +export OMP_NUM_THREADS=10 + +# Get start time +export date00=`date` + +# Start the Code +# Explicitly set TCP environment for the following... +srun -vv --cpu-bind=verbose python -u @ + +# Get end time and print to stdout +export date01=`date` + +echo "===================TIME STARTED===================" +echo $date00 +echo "===================TIME FINISHED===================" +echo $date01 diff --git a/applications/harnesses/ch_DDP_loderunner_cylex/training_study001_epoch0097.csv b/applications/harnesses/ch_DDP_loderunner_cylex/training_study001_epoch0097.csv new file mode 100644 index 00000000..010559e9 --- /dev/null +++ b/applications/harnesses/ch_DDP_loderunner_cylex/training_study001_epoch0097.csv @@ -0,0 +1,4000 @@ +97, 0, 0.00138704 +97, 0, 0.00187118 +97, 0, 0.12831871 +97, 0, 0.09132817 +97, 1, 0.00341067 +97, 1, 0.00973844 +97, 1, 0.03463279 +97, 1, 0.00307773 +97, 2, 0.00160005 +97, 2, 1.47929108 +97, 2, 0.00366840 +97, 2, 0.03825688 +97, 3, 0.00124751 +97, 3, 0.04842843 +97, 3, 0.00324528 +97, 3, 0.51072836 +97, 4, 0.00377124 +97, 4, 0.00478902 +97, 4, 0.00059686 +97, 4, 0.00502141 +97, 5, 0.00243590 +97, 5, 0.05922880 +97, 5, 0.00090193 +97, 5, 0.12478821 +97, 6, 0.01236468 +97, 6, 0.00134289 +97, 6, 0.00201886 +97, 6, 0.00099749 +97, 7, 0.00627874 +97, 7, 0.00353818 +97, 7, 0.00216271 +97, 7, 0.01273601 +97, 8, 0.00112185 +97, 8, 0.00296771 +97, 8, 0.02323138 +97, 8, 0.01227100 +97, 9, 0.01832628 +97, 9, 0.00683710 +97, 9, 0.00338213 +97, 9, 0.00212001 +97, 10, 0.00295842 +97, 10, 0.01289389 +97, 10, 0.03652410 +97, 10, 0.00440789 +97, 11, 0.00242169 +97, 11, 0.00447490 +97, 11, 0.00510039 +97, 11, 0.00816081 +97, 12, 0.00099395 +97, 12, 0.00271469 +97, 12, 0.01348200 +97, 12, 0.00445979 +97, 13, 0.00198571 +97, 13, 0.00055898 +97, 13, 0.00300501 +97, 13, 0.00281190 +97, 14, 0.01287823 +97, 14, 0.00740782 +97, 14, 0.00471384 +97, 14, 0.01340465 +97, 15, 0.01372931 +97, 15, 0.00124695 +97, 15, 0.00162567 +97, 15, 0.00195554 +97, 16, 0.20997116 +97, 16, 0.00094052 +97, 16, 0.00210519 +97, 16, 0.01190222 +97, 17, 0.00831347 +97, 17, 0.02166200 +97, 17, 0.00640325 +97, 17, 0.02025850 +97, 18, 0.00559053 +97, 18, 0.00146869 +97, 18, 0.05464288 +97, 18, 0.07333051 +97, 19, 0.00466724 +97, 19, 0.00198821 +97, 19, 0.00620908 +97, 19, 0.00081216 +97, 20, 0.00769165 +97, 20, 0.00207101 +97, 20, 0.00175323 +97, 20, 0.00364310 +97, 21, 0.00152954 +97, 21, 0.00020618 +97, 21, 0.00796673 +97, 21, 0.00409786 +97, 22, 0.00481769 +97, 22, 0.00123574 +97, 22, 0.00566311 +97, 22, 0.00415522 +97, 23, 0.00644463 +97, 23, 0.09821115 +97, 23, 0.01124552 +97, 23, 0.00131165 +97, 24, 0.04818678 +97, 24, 0.00174133 +97, 24, 0.00963174 +97, 24, 0.00591894 +97, 25, 0.00203236 +97, 25, 0.01100728 +97, 25, 0.00121845 +97, 25, 0.01229567 +97, 26, 0.00050973 +97, 26, 0.00073598 +97, 26, 0.00975992 +97, 26, 0.00170262 +97, 27, 0.32101256 +97, 27, 0.00320262 +97, 27, 0.06566099 +97, 27, 0.00045629 +97, 28, 0.00476971 +97, 28, 0.00122896 +97, 28, 0.14791198 +97, 28, 0.14265524 +97, 29, 0.00197441 +97, 29, 0.00899804 +97, 29, 0.01216650 +97, 29, 0.00182389 +97, 30, 0.00351131 +97, 30, 0.00203429 +97, 30, 0.12421282 +97, 30, 0.00336529 +97, 31, 0.01630091 +97, 31, 0.00127290 +97, 31, 0.01400237 +97, 31, 0.00579812 +97, 32, 0.01524202 +97, 32, 0.00210274 +97, 32, 0.00812363 +97, 32, 0.00932711 +97, 33, 0.00459249 +97, 33, 0.01080283 +97, 33, 0.00186762 +97, 33, 0.11537913 +97, 34, 0.00499517 +97, 34, 0.01093933 +97, 34, 0.00319297 +97, 34, 0.45486179 +97, 35, 0.00123261 +97, 35, 0.19650283 +97, 35, 0.00258075 +97, 35, 0.00516129 +97, 36, 0.00156418 +97, 36, 0.00360955 +97, 36, 0.01363563 +97, 36, 0.00084921 +97, 37, 0.00474745 +97, 37, 0.00156069 +97, 37, 0.00493454 +97, 37, 0.09601478 +97, 38, 0.02592089 +97, 38, 0.00459460 +97, 38, 0.00108170 +97, 38, 0.00284984 +97, 39, 0.01728824 +97, 39, 0.01251310 +97, 39, 0.00349603 +97, 39, 0.00632802 +97, 40, 0.00148149 +97, 40, 0.20465527 +97, 40, 0.00958897 +97, 40, 0.77901357 +97, 41, 0.00141050 +97, 41, 0.00050879 +97, 41, 0.00878266 +97, 41, 0.00093125 +97, 42, 0.00165780 +97, 42, 0.00055128 +97, 42, 0.00772097 +97, 42, 0.00271560 +97, 43, 0.05177592 +97, 43, 0.00384261 +97, 43, 0.00203657 +97, 43, 0.00095938 +97, 44, 0.00762163 +97, 44, 0.00161398 +97, 44, 0.05808929 +97, 44, 0.33038214 +97, 45, 0.00297912 +97, 45, 0.00220213 +97, 45, 0.00440348 +97, 45, 0.00274795 +97, 46, 0.02113124 +97, 46, 0.01986947 +97, 46, 0.15042894 +97, 46, 0.04951159 +97, 47, 0.00751123 +97, 47, 0.00497973 +97, 47, 0.05297387 +97, 47, 0.01242832 +97, 48, 0.00185441 +97, 48, 0.00146549 +97, 48, 0.35493448 +97, 48, 0.01147550 +97, 49, 0.00440521 +97, 49, 0.00387315 +97, 49, 0.00266289 +97, 49, 0.00065790 +97, 50, 0.00134014 +97, 50, 0.00548704 +97, 50, 0.00542653 +97, 50, 0.02241835 +97, 51, 0.00163826 +97, 51, 0.00460088 +97, 51, 0.02626622 +97, 51, 0.02208221 +97, 52, 0.00184740 +97, 52, 0.00339555 +97, 52, 0.00305812 +97, 52, 0.00154607 +97, 53, 0.00059519 +97, 53, 0.00057682 +97, 53, 0.00392831 +97, 53, 0.00093947 +97, 54, 0.00098101 +97, 54, 0.03125067 +97, 54, 0.00335431 +97, 54, 0.00854041 +97, 55, 0.00662978 +97, 55, 0.00054013 +97, 55, 0.00499445 +97, 55, 0.07696079 +97, 56, 0.00322055 +97, 56, 0.00059368 +97, 56, 0.00546559 +97, 56, 0.00191254 +97, 57, 0.00160886 +97, 57, 0.00358464 +97, 57, 0.02532303 +97, 57, 0.00102626 +97, 58, 0.00196085 +97, 58, 0.00336882 +97, 58, 0.00066421 +97, 58, 0.06991021 +97, 59, 0.00185956 +97, 59, 0.00192814 +97, 59, 0.00683830 +97, 59, 0.00069857 +97, 60, 0.00045274 +97, 60, 0.01500234 +97, 60, 0.00219405 +97, 60, 0.00201460 +97, 61, 0.00141779 +97, 61, 0.01503989 +97, 61, 0.00063959 +97, 61, 0.00692737 +97, 62, 0.00133590 +97, 62, 0.00191814 +97, 62, 0.45136043 +97, 62, 0.01263395 +97, 63, 0.00133574 +97, 63, 0.00092917 +97, 63, 0.00716440 +97, 63, 0.00365700 +97, 64, 0.00082666 +97, 64, 0.70111489 +97, 64, 0.00107586 +97, 64, 0.00217376 +97, 65, 0.00220581 +97, 65, 0.00698323 +97, 65, 0.01028163 +97, 65, 0.00187872 +97, 66, 0.00686769 +97, 66, 0.00062597 +97, 66, 0.01929699 +97, 66, 0.09103273 +97, 67, 0.00142809 +97, 67, 0.00374764 +97, 67, 0.00052572 +97, 67, 0.00456156 +97, 68, 0.00246824 +97, 68, 0.00252320 +97, 68, 0.69246626 +97, 68, 0.04798173 +97, 69, 0.02455747 +97, 69, 0.00290780 +97, 69, 0.00082135 +97, 69, 0.01086328 +97, 70, 0.00139896 +97, 70, 0.00203997 +97, 70, 0.00778477 +97, 70, 0.00162053 +97, 71, 0.00545950 +97, 71, 0.03425789 +97, 71, 0.00172073 +97, 71, 0.12114760 +97, 72, 0.00672966 +97, 72, 0.00510649 +97, 72, 0.00445986 +97, 72, 0.00642701 +97, 73, 0.07058020 +97, 73, 0.00163807 +97, 73, 0.00072506 +97, 73, 0.00085739 +97, 74, 0.09092809 +97, 74, 0.04619924 +97, 74, 0.00075120 +97, 74, 0.00156715 +97, 75, 0.00067425 +97, 75, 1.07353568 +97, 75, 0.00354575 +97, 75, 0.00543173 +97, 76, 0.00070144 +97, 76, 0.02067754 +97, 76, 0.00353815 +97, 76, 0.00410097 +97, 77, 0.00471628 +97, 77, 0.00368598 +97, 77, 0.00140456 +97, 77, 0.01082986 +97, 78, 0.00290284 +97, 78, 0.00718487 +97, 78, 0.02433351 +97, 78, 0.00175013 +97, 79, 0.00200246 +97, 79, 0.00093487 +97, 79, 0.00568711 +97, 79, 0.00068066 +97, 80, 0.00181206 +97, 80, 0.00126520 +97, 80, 0.00280522 +97, 80, 0.00166907 +97, 81, 0.01130680 +97, 81, 0.00169928 +97, 81, 0.00418402 +97, 81, 0.00058339 +97, 82, 0.00452305 +97, 82, 0.00993008 +97, 82, 0.00249777 +97, 82, 0.01777920 +97, 83, 0.00325067 +97, 83, 0.00535506 +97, 83, 0.02216361 +97, 83, 0.00234731 +97, 84, 0.66337359 +97, 84, 0.00237600 +97, 84, 0.00431139 +97, 84, 0.00123089 +97, 85, 0.05868255 +97, 85, 0.01404199 +97, 85, 0.05091449 +97, 85, 0.00161727 +97, 86, 0.00565564 +97, 86, 0.00198978 +97, 86, 0.00131504 +97, 86, 0.00513995 +97, 87, 0.00125841 +97, 87, 0.00135340 +97, 87, 0.02287394 +97, 87, 0.15282518 +97, 88, 0.00054940 +97, 88, 0.00881693 +97, 88, 0.19902195 +97, 88, 0.01541402 +97, 89, 0.00285384 +97, 89, 0.03375267 +97, 89, 0.00631707 +97, 89, 0.00827394 +97, 90, 0.00511812 +97, 90, 0.00058706 +97, 90, 0.00119661 +97, 90, 0.00292828 +97, 91, 0.00076518 +97, 91, 0.00342100 +97, 91, 0.05774121 +97, 91, 0.00162302 +97, 92, 0.00192116 +97, 92, 0.00210883 +97, 92, 0.00196495 +97, 92, 0.00927284 +97, 93, 0.00494668 +97, 93, 0.00163369 +97, 93, 0.02705766 +97, 93, 0.00083322 +97, 94, 0.00882862 +97, 94, 0.00077294 +97, 94, 0.00293670 +97, 94, 0.00496623 +97, 95, 0.00613514 +97, 95, 0.00308040 +97, 95, 0.02561595 +97, 95, 0.04104795 +97, 96, 0.00047588 +97, 96, 0.00104495 +97, 96, 0.00321559 +97, 96, 0.11443437 +97, 97, 0.00270132 +97, 97, 0.00388000 +97, 97, 0.00682107 +97, 97, 0.00226376 +97, 98, 0.00962659 +97, 98, 0.00076301 +97, 98, 0.00196930 +97, 98, 0.00484580 +97, 99, 0.01373423 +97, 99, 0.14087932 +97, 99, 0.01416024 +97, 99, 0.00399137 +97, 100, 0.03148878 +97, 100, 0.01586194 +97, 100, 0.00451959 +97, 100, 0.00092745 +97, 101, 0.00082600 +97, 101, 0.00553973 +97, 101, 0.00215312 +97, 101, 0.00108809 +97, 102, 0.00093481 +97, 102, 0.00191089 +97, 102, 0.00369378 +97, 102, 0.00064606 +97, 103, 0.00767678 +97, 103, 0.00375549 +97, 103, 0.00194656 +97, 103, 0.00126741 +97, 104, 0.00055296 +97, 104, 0.19179517 +97, 104, 0.01020192 +97, 104, 0.00425650 +97, 105, 0.03835954 +97, 105, 0.29078060 +97, 105, 0.09847819 +97, 105, 0.05101645 +97, 106, 0.01381631 +97, 106, 0.02580642 +97, 106, 0.00582682 +97, 106, 0.00377495 +97, 107, 0.00279448 +97, 107, 0.00687739 +97, 107, 0.00201294 +97, 107, 0.00162334 +97, 108, 0.00179346 +97, 108, 0.00075471 +97, 108, 0.30688003 +97, 108, 0.00327423 +97, 109, 0.00083591 +97, 109, 0.00083233 +97, 109, 0.00522682 +97, 109, 0.08032886 +97, 110, 0.01499438 +97, 110, 0.00309975 +97, 110, 0.00212395 +97, 110, 0.00177763 +97, 111, 0.00141968 +97, 111, 0.00079415 +97, 111, 0.00098234 +97, 111, 0.00079909 +97, 112, 0.00109395 +97, 112, 0.00103456 +97, 112, 0.01866337 +97, 112, 0.00261977 +97, 113, 0.00262124 +97, 113, 0.00048448 +97, 113, 0.00234418 +97, 113, 0.00151659 +97, 114, 0.00625085 +97, 114, 0.00137057 +97, 114, 0.00758781 +97, 114, 0.00262799 +97, 115, 0.05179161 +97, 115, 0.03331302 +97, 115, 0.00097277 +97, 115, 0.00115751 +97, 116, 0.00699307 +97, 116, 0.01582550 +97, 116, 0.00108085 +97, 116, 0.11054599 +97, 117, 0.00242750 +97, 117, 0.00176546 +97, 117, 0.00172569 +97, 117, 0.00218967 +97, 118, 0.00136484 +97, 118, 0.00186236 +97, 118, 0.00105471 +97, 118, 0.00190903 +97, 119, 0.00490497 +97, 119, 0.00114482 +97, 119, 0.00210839 +97, 119, 0.00128930 +97, 120, 0.01251960 +97, 120, 0.00157206 +97, 120, 0.00517593 +97, 120, 0.00154531 +97, 121, 0.00159247 +97, 121, 0.00534736 +97, 121, 0.01211502 +97, 121, 0.00372903 +97, 122, 0.00079057 +97, 122, 0.00816816 +97, 122, 0.00353119 +97, 122, 0.01767105 +97, 123, 0.00259618 +97, 123, 0.00400324 +97, 123, 0.00308319 +97, 123, 0.07703348 +97, 124, 0.00528419 +97, 124, 0.00315628 +97, 124, 0.00484809 +97, 124, 0.00074193 +97, 125, 0.00622933 +97, 125, 0.00028605 +97, 125, 0.02674712 +97, 125, 0.00120662 +97, 126, 0.00300210 +97, 126, 0.00130569 +97, 126, 0.00293335 +97, 126, 0.00457840 +97, 127, 0.00164628 +97, 127, 0.00130896 +97, 127, 0.00117377 +97, 127, 0.11058138 +97, 128, 0.00103449 +97, 128, 0.00564319 +97, 128, 0.00185808 +97, 128, 0.01467188 +97, 129, 0.00338019 +97, 129, 0.01616732 +97, 129, 0.02240480 +97, 129, 0.00487446 +97, 130, 0.00096527 +97, 130, 0.00210065 +97, 130, 0.00078953 +97, 130, 0.00096710 +97, 131, 0.00914075 +97, 131, 0.00159377 +97, 131, 0.03008811 +97, 131, 0.00167772 +97, 132, 0.00118443 +97, 132, 0.33513999 +97, 132, 0.00193763 +97, 132, 0.03322690 +97, 133, 0.01956887 +97, 133, 0.00335156 +97, 133, 0.01656854 +97, 133, 0.00168359 +97, 134, 0.00122603 +97, 134, 0.00056999 +97, 134, 0.01261880 +97, 134, 0.00133608 +97, 135, 0.00411039 +97, 135, 0.00163548 +97, 135, 0.00859537 +97, 135, 0.07580241 +97, 136, 0.00104689 +97, 136, 0.00298241 +97, 136, 0.00139008 +97, 136, 0.00082478 +97, 137, 0.23623954 +97, 137, 0.00069603 +97, 137, 0.00879002 +97, 137, 0.00154334 +97, 138, 0.00133373 +97, 138, 0.00090653 +97, 138, 0.10442207 +97, 138, 0.00136910 +97, 139, 0.00138882 +97, 139, 0.00132039 +97, 139, 0.00323526 +97, 139, 0.00217025 +97, 140, 0.01260056 +97, 140, 0.00466620 +97, 140, 0.00681434 +97, 140, 0.01320468 +97, 141, 0.01158144 +97, 141, 0.00693942 +97, 141, 0.00115573 +97, 141, 0.00171780 +97, 142, 0.00577861 +97, 142, 0.01121369 +97, 142, 0.00402752 +97, 142, 0.00198731 +97, 143, 0.16917317 +97, 143, 0.00342823 +97, 143, 0.03353523 +97, 143, 0.01834162 +97, 144, 0.01527327 +97, 144, 0.00588543 +97, 144, 0.01067145 +97, 144, 0.00456801 +97, 145, 0.04779301 +97, 145, 0.25862208 +97, 145, 0.01832331 +97, 145, 0.00185926 +97, 146, 0.00602877 +97, 146, 0.00073631 +97, 146, 0.00064315 +97, 146, 0.16222267 +97, 147, 0.06392961 +97, 147, 0.00120675 +97, 147, 0.00420197 +97, 147, 0.00229202 +97, 148, 0.00295260 +97, 148, 0.00074402 +97, 148, 0.01811028 +97, 148, 0.01703569 +97, 149, 0.00312177 +97, 149, 0.00158871 +97, 149, 0.00606179 +97, 149, 0.00174748 +97, 150, 0.14060064 +97, 150, 0.00190085 +97, 150, 0.00146668 +97, 150, 0.00454497 +97, 151, 0.00131427 +97, 151, 0.00505239 +97, 151, 0.00134723 +97, 151, 0.06383763 +97, 152, 0.00095593 +97, 152, 0.00588030 +97, 152, 0.04234222 +97, 152, 0.00458733 +97, 153, 0.00110683 +97, 153, 0.00376927 +97, 153, 0.00138518 +97, 153, 0.00049923 +97, 154, 0.01519455 +97, 154, 0.00626225 +97, 154, 0.00463390 +97, 154, 0.00523398 +97, 155, 0.00151104 +97, 155, 0.00126669 +97, 155, 0.02639066 +97, 155, 0.00133667 +97, 156, 0.00237436 +97, 156, 0.01831001 +97, 156, 0.00384842 +97, 156, 0.00509325 +97, 157, 0.00570063 +97, 157, 0.00915853 +97, 157, 0.13650066 +97, 157, 0.00102086 +97, 158, 0.00307799 +97, 158, 0.03176594 +97, 158, 0.14888035 +97, 158, 0.00556868 +97, 159, 0.00093094 +97, 159, 0.00925693 +97, 159, 0.02165194 +97, 159, 0.00165873 +97, 160, 0.00091872 +97, 160, 0.00352531 +97, 160, 0.00084487 +97, 160, 0.00518961 +97, 161, 0.00035625 +97, 161, 0.00440504 +97, 161, 0.00889999 +97, 161, 0.02994526 +97, 162, 0.03861312 +97, 162, 0.00203587 +97, 162, 0.00139708 +97, 162, 0.00259291 +97, 163, 0.01052496 +97, 163, 0.00215965 +97, 163, 0.00107540 +97, 163, 0.04232432 +97, 164, 0.00088423 +97, 164, 0.00227117 +97, 164, 0.00316725 +97, 164, 0.00497414 +97, 165, 0.00157830 +97, 165, 0.00228348 +97, 165, 0.00176327 +97, 165, 0.00105406 +97, 166, 0.00445787 +97, 166, 0.00141990 +97, 166, 0.00303742 +97, 166, 0.00119822 +97, 167, 0.04925518 +97, 167, 0.15686460 +97, 167, 0.02449757 +97, 167, 0.02498479 +97, 168, 0.00285916 +97, 168, 0.00898542 +97, 168, 0.00444820 +97, 168, 0.00486112 +97, 169, 0.00812406 +97, 169, 0.17651887 +97, 169, 0.00072154 +97, 169, 0.00408875 +97, 170, 0.00461324 +97, 170, 0.00102101 +97, 170, 0.00214270 +97, 170, 0.00132058 +97, 171, 0.05380950 +97, 171, 0.83979946 +97, 171, 0.00152201 +97, 171, 0.00424545 +97, 172, 0.02291838 +97, 172, 0.00523431 +97, 172, 0.00970637 +97, 172, 0.00157365 +97, 173, 0.00164469 +97, 173, 0.00074225 +97, 173, 0.00566184 +97, 173, 0.00145155 +97, 174, 0.00261471 +97, 174, 0.00847311 +97, 174, 0.00365246 +97, 174, 0.00040770 +97, 175, 0.00377730 +97, 175, 0.00309265 +97, 175, 0.03628319 +97, 175, 0.01558669 +97, 176, 0.04569650 +97, 176, 0.09102929 +97, 176, 0.00221152 +97, 176, 0.25845447 +97, 177, 0.69074345 +97, 177, 0.03914826 +97, 177, 0.00315060 +97, 177, 0.13484588 +97, 178, 0.01026502 +97, 178, 0.00155679 +97, 178, 0.02485952 +97, 178, 0.00701621 +97, 179, 0.03142494 +97, 179, 0.00071122 +97, 179, 0.00176401 +97, 179, 0.00298372 +97, 180, 0.00141663 +97, 180, 0.00076740 +97, 180, 0.15410696 +97, 180, 0.00188354 +97, 181, 0.00381048 +97, 181, 0.00418666 +97, 181, 0.00561473 +97, 181, 0.00118715 +97, 182, 0.01592189 +97, 182, 0.52316719 +97, 182, 0.00125719 +97, 182, 0.00209630 +97, 183, 0.01494312 +97, 183, 0.00077946 +97, 183, 0.47670117 +97, 183, 0.01630422 +97, 184, 0.00270305 +97, 184, 0.00124529 +97, 184, 0.00360276 +97, 184, 0.00300963 +97, 185, 0.00067056 +97, 185, 0.00220381 +97, 185, 0.00759886 +97, 185, 0.00509607 +97, 186, 0.00161169 +97, 186, 0.00661676 +97, 186, 0.00207279 +97, 186, 0.10482449 +97, 187, 0.00442427 +97, 187, 0.00387436 +97, 187, 0.00215270 +97, 187, 0.00049404 +97, 188, 0.00575677 +97, 188, 0.00077291 +97, 188, 0.05980590 +97, 188, 0.06330167 +97, 189, 0.00054013 +97, 189, 0.00320240 +97, 189, 0.00203306 +97, 189, 0.01421317 +97, 190, 0.00113048 +97, 190, 0.00058105 +97, 190, 0.00138733 +97, 190, 0.00075240 +97, 191, 0.00151393 +97, 191, 0.00230656 +97, 191, 0.02213009 +97, 191, 0.00619339 +97, 192, 0.00120600 +97, 192, 0.00433176 +97, 192, 0.00136569 +97, 192, 0.00060127 +97, 193, 0.00543204 +97, 193, 0.00210085 +97, 193, 0.05316222 +97, 193, 0.00979398 +97, 194, 0.28508946 +97, 194, 0.01257334 +97, 194, 0.01073588 +97, 194, 0.01766741 +97, 195, 0.01914377 +97, 195, 0.00086990 +97, 195, 0.00160798 +97, 195, 0.00405642 +97, 196, 0.07002625 +97, 196, 0.00131051 +97, 196, 0.00261783 +97, 196, 0.00797306 +97, 197, 0.17626332 +97, 197, 0.00993870 +97, 197, 0.00230646 +97, 197, 0.00492594 +97, 198, 0.00501878 +97, 198, 0.00469836 +97, 198, 0.01346245 +97, 198, 0.00570283 +97, 199, 0.00164242 +97, 199, 0.00187379 +97, 199, 0.01312959 +97, 199, 0.00094542 +97, 200, 0.03490132 +97, 200, 0.00061543 +97, 200, 0.00167971 +97, 200, 0.00299453 +97, 201, 0.00160143 +97, 201, 0.00536059 +97, 201, 0.00051501 +97, 201, 0.00076505 +97, 202, 0.00443152 +97, 202, 0.00487141 +97, 202, 0.01019670 +97, 202, 0.01119152 +97, 203, 0.09560776 +97, 203, 0.08395532 +97, 203, 0.03919809 +97, 203, 0.00079514 +97, 204, 0.00097274 +97, 204, 0.00240487 +97, 204, 0.00663346 +97, 204, 0.00115731 +97, 205, 0.17289275 +97, 205, 0.15500095 +97, 205, 0.00317200 +97, 205, 0.00209759 +97, 206, 0.00297980 +97, 206, 0.00223747 +97, 206, 0.00167558 +97, 206, 0.01118736 +97, 207, 0.00687936 +97, 207, 0.00440801 +97, 207, 0.14273787 +97, 207, 0.00077537 +97, 208, 0.00639445 +97, 208, 0.12431326 +97, 208, 0.00985643 +97, 208, 0.00288938 +97, 209, 0.09092502 +97, 209, 0.00184109 +97, 209, 0.00773309 +97, 209, 0.00081712 +97, 210, 0.00412923 +97, 210, 0.00153123 +97, 210, 0.00170819 +97, 210, 0.00097037 +97, 211, 0.00629669 +97, 211, 2.36008334 +97, 211, 0.00198968 +97, 211, 0.00067067 +97, 212, 0.01606919 +97, 212, 0.01954398 +97, 212, 0.03501168 +97, 212, 0.00756100 +97, 213, 0.00251400 +97, 213, 0.00437479 +97, 213, 0.00064624 +97, 213, 0.01076545 +97, 214, 0.00843939 +97, 214, 0.06260486 +97, 214, 0.01319144 +97, 214, 0.00228267 +97, 215, 0.00091075 +97, 215, 0.01110797 +97, 215, 0.00486534 +97, 215, 0.00100490 +97, 216, 0.00166273 +97, 216, 0.00385526 +97, 216, 0.01390772 +97, 216, 0.00142099 +97, 217, 0.22300301 +97, 217, 0.00651219 +97, 217, 0.01253117 +97, 217, 0.08213416 +97, 218, 0.00171667 +97, 218, 0.00578519 +97, 218, 0.00890369 +97, 218, 0.01962249 +97, 219, 0.00056111 +97, 219, 0.00328260 +97, 219, 0.00368510 +97, 219, 0.00046530 +97, 220, 0.28900573 +97, 220, 0.00139786 +97, 220, 0.00605594 +97, 220, 0.00230285 +97, 221, 0.27126449 +97, 221, 0.00608586 +97, 221, 0.00047127 +97, 221, 0.00374225 +97, 222, 0.00299874 +97, 222, 0.00599294 +97, 222, 0.23565139 +97, 222, 0.00382743 +97, 223, 0.00508072 +97, 223, 0.00067148 +97, 223, 0.00244153 +97, 223, 0.01399064 +97, 224, 0.00847410 +97, 224, 0.01110451 +97, 224, 0.00359239 +97, 224, 0.00345407 +97, 225, 0.00288930 +97, 225, 0.00553438 +97, 225, 0.00477742 +97, 225, 0.01695670 +97, 226, 0.00170961 +97, 226, 0.00096768 +97, 226, 0.00069811 +97, 226, 0.00208474 +97, 227, 0.01022111 +97, 227, 0.00200149 +97, 227, 0.02556382 +97, 227, 0.00236693 +97, 228, 0.01885182 +97, 228, 0.59514850 +97, 228, 0.04004624 +97, 228, 0.00224020 +97, 229, 0.00199627 +97, 229, 0.01422147 +97, 229, 0.00928219 +97, 229, 0.16535054 +97, 230, 0.00244469 +97, 230, 0.00217515 +97, 230, 0.00935262 +97, 230, 0.00134822 +97, 231, 0.00608895 +97, 231, 0.00441768 +97, 231, 0.00330097 +97, 231, 0.00159187 +97, 232, 0.00127390 +97, 232, 0.00107638 +97, 232, 0.19143011 +97, 232, 0.00267655 +97, 233, 0.00082270 +97, 233, 0.01554641 +97, 233, 0.02220439 +97, 233, 0.01045496 +97, 234, 0.00073255 +97, 234, 0.00065242 +97, 234, 0.01441715 +97, 234, 0.00698934 +97, 235, 0.00724732 +97, 235, 0.00144626 +97, 235, 0.00155120 +97, 235, 0.01064437 +97, 236, 0.00094379 +97, 236, 0.01111775 +97, 236, 0.00545614 +97, 236, 0.00078280 +97, 237, 0.00146523 +97, 237, 0.00142032 +97, 237, 0.00666956 +97, 237, 0.00095738 +97, 238, 0.08278388 +97, 238, 0.00352829 +97, 238, 0.01141787 +97, 238, 0.00538958 +97, 239, 0.00621340 +97, 239, 0.00829932 +97, 239, 0.04520747 +97, 239, 0.00220436 +97, 240, 0.00083742 +97, 240, 0.21236674 +97, 240, 0.00368782 +97, 240, 0.01568631 +97, 241, 0.02048260 +97, 241, 0.00156394 +97, 241, 0.00727176 +97, 241, 0.00287563 +97, 242, 0.00535070 +97, 242, 0.20743562 +97, 242, 0.00190011 +97, 242, 0.02759740 +97, 243, 0.01340402 +97, 243, 0.00564593 +97, 243, 0.01005483 +97, 243, 0.00141530 +97, 244, 0.00882794 +97, 244, 0.00421975 +97, 244, 0.00120827 +97, 244, 0.00371414 +97, 245, 0.00194971 +97, 245, 0.00686370 +97, 245, 0.00151025 +97, 245, 4.34730005 +97, 246, 0.09025075 +97, 246, 0.00049890 +97, 246, 0.00255116 +97, 246, 0.00336565 +97, 247, 0.66073024 +97, 247, 0.00110083 +97, 247, 0.03479118 +97, 247, 0.00662437 +97, 248, 0.00578450 +97, 248, 0.01198697 +97, 248, 0.05057829 +97, 248, 0.01411550 +97, 249, 0.00288103 +97, 249, 0.00229048 +97, 249, 0.01098945 +97, 249, 0.00165105 +97, 250, 0.00580684 +97, 250, 0.00214980 +97, 250, 0.00099967 +97, 250, 0.00341900 +97, 251, 0.00190962 +97, 251, 0.00092458 +97, 251, 0.00216595 +97, 251, 0.10233930 +97, 252, 0.00866012 +97, 252, 0.00454942 +97, 252, 0.00339367 +97, 252, 0.00072598 +97, 253, 0.00210787 +97, 253, 0.00254516 +97, 253, 0.08600250 +97, 253, 0.00132241 +97, 254, 0.00845220 +97, 254, 0.09280775 +97, 254, 0.00080982 +97, 254, 0.20701027 +97, 255, 0.03055969 +97, 255, 0.00291934 +97, 255, 0.00852103 +97, 255, 0.00543408 +97, 256, 0.04718194 +97, 256, 0.00186335 +97, 256, 0.00105264 +97, 256, 0.01009975 +97, 257, 0.00542712 +97, 257, 0.00759808 +97, 257, 0.00542429 +97, 257, 0.00053944 +97, 258, 0.00236937 +97, 258, 0.00365195 +97, 258, 0.00296817 +97, 258, 0.00327673 +97, 259, 0.00966891 +97, 259, 0.00210309 +97, 259, 0.00350189 +97, 259, 0.00060412 +97, 260, 0.00205459 +97, 260, 0.00172887 +97, 260, 0.01253973 +97, 260, 0.00203906 +97, 261, 0.00221677 +97, 261, 0.03025921 +97, 261, 0.64223325 +97, 261, 0.00301432 +97, 262, 0.00269373 +97, 262, 0.00418486 +97, 262, 0.00639511 +97, 262, 0.00314595 +97, 263, 0.17071931 +97, 263, 0.00176913 +97, 263, 0.00165126 +97, 263, 0.00415185 +97, 264, 0.00844641 +97, 264, 0.00331238 +97, 264, 0.00419636 +97, 264, 0.00139028 +97, 265, 0.02664947 +97, 265, 0.02681110 +97, 265, 0.00110216 +97, 265, 0.00488459 +97, 266, 0.00061596 +97, 266, 0.00217920 +97, 266, 0.00328433 +97, 266, 0.00483485 +97, 267, 0.00133393 +97, 267, 0.00496551 +97, 267, 0.00087214 +97, 267, 0.00962989 +97, 268, 0.02987745 +97, 268, 0.00332626 +97, 268, 0.62502396 +97, 268, 0.00608802 +97, 269, 0.00114425 +97, 269, 0.00223710 +97, 269, 0.00122306 +97, 269, 0.00069852 +97, 270, 0.00587990 +97, 270, 0.00812135 +97, 270, 0.00138357 +97, 270, 0.00567243 +97, 271, 0.00331158 +97, 271, 0.00035477 +97, 271, 0.00886868 +97, 271, 0.00239839 +97, 272, 0.00248585 +97, 272, 0.00223157 +97, 272, 0.00733429 +97, 272, 0.00409663 +97, 273, 0.00689021 +97, 273, 0.00174941 +97, 273, 0.00556886 +97, 273, 0.00240842 +97, 274, 0.00432177 +97, 274, 0.07002567 +97, 274, 0.03083517 +97, 274, 0.00500003 +97, 275, 0.00224960 +97, 275, 0.00078520 +97, 275, 0.00967628 +97, 275, 0.00756225 +97, 276, 0.00163530 +97, 276, 0.00397933 +97, 276, 0.00234517 +97, 276, 0.00266923 +97, 277, 0.00114358 +97, 277, 0.00502358 +97, 277, 0.01720134 +97, 277, 0.00784186 +97, 278, 0.16193809 +97, 278, 0.23271079 +97, 278, 0.00097778 +97, 278, 0.00510396 +97, 279, 0.00978909 +97, 279, 0.00070514 +97, 279, 0.00589693 +97, 279, 0.00066115 +97, 280, 0.31585482 +97, 280, 0.03352172 +97, 280, 0.00409111 +97, 280, 0.00466134 +97, 281, 0.00692380 +97, 281, 0.00147248 +97, 281, 0.00381268 +97, 281, 0.00085290 +97, 282, 0.00457799 +97, 282, 0.01525175 +97, 282, 0.00368125 +97, 282, 0.00470994 +97, 283, 0.00192396 +97, 283, 0.15446226 +97, 283, 0.00534300 +97, 283, 0.00144400 +97, 284, 0.00523632 +97, 284, 0.58167154 +97, 284, 0.00108780 +97, 284, 0.00114638 +97, 285, 0.00270139 +97, 285, 0.00270001 +97, 285, 0.00314767 +97, 285, 0.00193594 +97, 286, 0.00151115 +97, 286, 0.00486717 +97, 286, 0.00341055 +97, 286, 0.00730540 +97, 287, 0.22351021 +97, 287, 0.02524529 +97, 287, 0.02766232 +97, 287, 0.00127015 +97, 288, 0.00251079 +97, 288, 0.02966944 +97, 288, 0.00286851 +97, 288, 0.00220362 +97, 289, 0.01571706 +97, 289, 0.00223013 +97, 289, 0.00215105 +97, 289, 0.00891459 +97, 290, 0.01241263 +97, 290, 0.01223359 +97, 290, 0.00372697 +97, 290, 0.00177736 +97, 291, 0.08077065 +97, 291, 0.00415931 +97, 291, 0.00366917 +97, 291, 0.00312781 +97, 292, 0.00162979 +97, 292, 0.00175629 +97, 292, 0.00053464 +97, 292, 0.00068811 +97, 293, 0.02814868 +97, 293, 0.00129540 +97, 293, 0.00470430 +97, 293, 6.42227077 +97, 294, 0.09970160 +97, 294, 0.01127869 +97, 294, 0.00105272 +97, 294, 0.01134062 +97, 295, 0.00646745 +97, 295, 0.00265868 +97, 295, 0.00656181 +97, 295, 0.00274374 +97, 296, 0.00442405 +97, 296, 0.00108203 +97, 296, 0.00246877 +97, 296, 0.00123237 +97, 297, 0.00378067 +97, 297, 0.00221112 +97, 297, 0.01806898 +97, 297, 0.00779646 +97, 298, 0.00061354 +97, 298, 0.00621086 +97, 298, 0.00144983 +97, 298, 0.01057670 +97, 299, 0.00255236 +97, 299, 0.00121006 +97, 299, 0.08357212 +97, 299, 0.00073095 +97, 300, 0.00423274 +97, 300, 0.00177658 +97, 300, 0.00288260 +97, 300, 0.00379711 +97, 301, 0.00543963 +97, 301, 0.00090319 +97, 301, 0.01053236 +97, 301, 0.00056174 +97, 302, 0.00097360 +97, 302, 0.00112886 +97, 302, 0.00311093 +97, 302, 0.00466403 +97, 303, 0.00073351 +97, 303, 0.06352162 +97, 303, 0.00383147 +97, 303, 0.00267679 +97, 304, 0.00028921 +97, 304, 0.00305792 +97, 304, 0.00407213 +97, 304, 0.00129605 +97, 305, 0.00379600 +97, 305, 0.00128316 +97, 305, 0.03988281 +97, 305, 0.00333955 +97, 306, 0.00172449 +97, 306, 0.00492703 +97, 306, 0.00291579 +97, 306, 0.00083177 +97, 307, 0.00931827 +97, 307, 0.00474295 +97, 307, 0.01297074 +97, 307, 0.00123133 +97, 308, 0.01337774 +97, 308, 0.00090302 +97, 308, 0.01310366 +97, 308, 0.00753910 +97, 309, 0.00401970 +97, 309, 0.01249370 +97, 309, 0.00042692 +97, 309, 0.00109035 +97, 310, 0.00356108 +97, 310, 0.00247416 +97, 310, 0.01852674 +97, 310, 0.00508489 +97, 311, 0.00539229 +97, 311, 0.00590748 +97, 311, 0.00138392 +97, 311, 0.00271757 +97, 312, 0.01910914 +97, 312, 0.00176351 +97, 312, 0.00270881 +97, 312, 0.01727448 +97, 313, 0.01929162 +97, 313, 3.19622755 +97, 313, 0.00603453 +97, 313, 0.02771251 +97, 314, 0.00113958 +97, 314, 0.00253995 +97, 314, 0.00330275 +97, 314, 0.00539197 +97, 315, 0.00092902 +97, 315, 0.00052650 +97, 315, 0.00068831 +97, 315, 0.00549477 +97, 316, 0.00330578 +97, 316, 0.00268785 +97, 316, 0.00233375 +97, 316, 0.00320303 +97, 317, 0.00172400 +97, 317, 0.01082724 +97, 317, 0.00084249 +97, 317, 0.05818665 +97, 318, 0.00337643 +97, 318, 0.00149757 +97, 318, 0.00772818 +97, 318, 0.00639430 +97, 319, 0.01473238 +97, 319, 0.00275000 +97, 319, 0.00156895 +97, 319, 0.00397323 +97, 320, 0.00079077 +97, 320, 0.00472576 +97, 320, 0.00175368 +97, 320, 0.00168598 +97, 321, 0.00132023 +97, 321, 0.02080295 +97, 321, 0.00127793 +97, 321, 0.00055059 +97, 322, 0.00965681 +97, 322, 0.00049960 +97, 322, 0.00108359 +97, 322, 0.00182168 +97, 323, 0.00679758 +97, 323, 0.00332290 +97, 323, 0.00673487 +97, 323, 0.01259383 +97, 324, 0.00172134 +97, 324, 0.05752983 +97, 324, 0.14894450 +97, 324, 0.00622699 +97, 325, 0.00135463 +97, 325, 0.00214036 +97, 325, 0.04158906 +97, 325, 0.00134339 +97, 326, 0.11322694 +97, 326, 0.01426034 +97, 326, 0.01804757 +97, 326, 0.01483848 +97, 327, 0.00103389 +97, 327, 0.00331500 +97, 327, 0.00310153 +97, 327, 0.01535286 +97, 328, 0.04445244 +97, 328, 0.01151528 +97, 328, 0.00211109 +97, 328, 0.00166712 +97, 329, 1.16643083 +97, 329, 0.01020279 +97, 329, 0.00523485 +97, 329, 0.00156531 +97, 330, 0.00288430 +97, 330, 0.04156959 +97, 330, 0.00881829 +97, 330, 0.00140073 +97, 331, 0.00160343 +97, 331, 0.00726046 +97, 331, 0.00360348 +97, 331, 0.00066095 +97, 332, 0.00375055 +97, 332, 0.00765146 +97, 332, 0.00052782 +97, 332, 0.00782018 +97, 333, 0.00086196 +97, 333, 0.00148518 +97, 333, 0.00116255 +97, 333, 0.00401565 +97, 334, 0.00214669 +97, 334, 0.00812402 +97, 334, 0.00275861 +97, 334, 0.10453800 +97, 335, 0.00552315 +97, 335, 0.00996438 +97, 335, 0.00082938 +97, 335, 0.00076726 +97, 336, 0.00077031 +97, 336, 0.00120070 +97, 336, 0.00107806 +97, 336, 0.39601585 +97, 337, 0.00358274 +97, 337, 0.00045581 +97, 337, 0.00355227 +97, 337, 0.01453981 +97, 338, 0.00105251 +97, 338, 0.00312765 +97, 338, 0.00198039 +97, 338, 0.00591695 +97, 339, 0.05946956 +97, 339, 0.00471269 +97, 339, 0.00285707 +97, 339, 0.00217084 +97, 340, 0.00352198 +97, 340, 0.00261398 +97, 340, 0.00125945 +97, 340, 0.00129435 +97, 341, 0.00105101 +97, 341, 0.00348131 +97, 341, 0.00583651 +97, 341, 0.02127189 +97, 342, 0.00212077 +97, 342, 0.03535269 +97, 342, 0.28297037 +97, 342, 0.00291846 +97, 343, 0.00278823 +97, 343, 0.00038183 +97, 343, 0.01156736 +97, 343, 0.22384560 +97, 344, 0.80508965 +97, 344, 0.00161466 +97, 344, 0.00626494 +97, 344, 0.00154619 +97, 345, 0.00561426 +97, 345, 0.00292815 +97, 345, 0.00185746 +97, 345, 0.00138818 +97, 346, 0.01038664 +97, 346, 0.65007395 +97, 346, 0.00329942 +97, 346, 0.00448374 +97, 347, 0.00175920 +97, 347, 0.06807887 +97, 347, 0.01083292 +97, 347, 0.00838218 +97, 348, 0.00151696 +97, 348, 0.00747740 +97, 348, 0.00126336 +97, 348, 0.00072399 +97, 349, 0.00887160 +97, 349, 0.00342228 +97, 349, 0.02756083 +97, 349, 0.01030746 +97, 350, 0.01582640 +97, 350, 0.00902029 +97, 350, 0.00079556 +97, 350, 0.00408464 +97, 351, 0.00089995 +97, 351, 0.00561405 +97, 351, 0.01136839 +97, 351, 0.69519562 +97, 352, 0.00513555 +97, 352, 0.06864209 +97, 352, 0.00534973 +97, 352, 0.25978631 +97, 353, 0.00493175 +97, 353, 0.00148067 +97, 353, 0.01120990 +97, 353, 0.00054948 +97, 354, 0.00115123 +97, 354, 0.00241079 +97, 354, 0.00238333 +97, 354, 0.00257986 +97, 355, 0.01136596 +97, 355, 0.02886483 +97, 355, 0.01136596 +97, 355, 0.00145865 +97, 356, 0.01018397 +97, 356, 0.00108358 +97, 356, 0.00535841 +97, 356, 0.00150788 +97, 357, 0.00299497 +97, 357, 0.00619991 +97, 357, 0.00088367 +97, 357, 0.00401917 +97, 358, 0.02243535 +97, 358, 0.00421871 +97, 358, 0.00942591 +97, 358, 0.01093320 +97, 359, 0.00237196 +97, 359, 0.00216036 +97, 359, 0.01917954 +97, 359, 0.00416905 +97, 360, 0.00076028 +97, 360, 0.00080214 +97, 360, 0.00222475 +97, 360, 0.00118188 +97, 361, 0.12724838 +97, 361, 0.00312057 +97, 361, 0.02225892 +97, 361, 0.00473026 +97, 362, 0.00419062 +97, 362, 0.00108302 +97, 362, 0.00544477 +97, 362, 0.06080534 +97, 363, 0.04764083 +97, 363, 0.00119869 +97, 363, 0.00254494 +97, 363, 0.00319486 +97, 364, 0.00057767 +97, 364, 0.02236918 +97, 364, 0.00294907 +97, 364, 0.00108821 +97, 365, 0.00899823 +97, 365, 0.01687640 +97, 365, 0.00116069 +97, 365, 0.00188505 +97, 366, 0.02410601 +97, 366, 4.03790092 +97, 366, 0.07455339 +97, 366, 0.00193854 +97, 367, 0.03379572 +97, 367, 0.00402180 +97, 367, 0.00123011 +97, 367, 0.00408477 +97, 368, 0.00772284 +97, 368, 0.00189982 +97, 368, 0.00063777 +97, 368, 0.01489154 +97, 369, 0.00155738 +97, 369, 0.00136679 +97, 369, 0.01152973 +97, 369, 0.00541471 +97, 370, 0.00284329 +97, 370, 0.00184507 +97, 370, 0.01671560 +97, 370, 0.00707093 +97, 371, 0.00279852 +97, 371, 0.00349966 +97, 371, 0.00679071 +97, 371, 0.00144783 +97, 372, 0.00415709 +97, 372, 0.04810737 +97, 372, 0.00127543 +97, 372, 0.09711013 +97, 373, 0.00081692 +97, 373, 0.00526783 +97, 373, 0.00263824 +97, 373, 0.02851065 +97, 374, 0.00071756 +97, 374, 0.00519929 +97, 374, 0.00150600 +97, 374, 0.00375055 +97, 375, 0.17607386 +97, 375, 0.00621712 +97, 375, 0.00373105 +97, 375, 0.21780434 +97, 376, 0.00232161 +97, 376, 0.21584329 +97, 376, 0.00405910 +97, 376, 0.01782076 +97, 377, 0.14567943 +97, 377, 0.00163119 +97, 377, 0.00218140 +97, 377, 0.00625254 +97, 378, 0.00781732 +97, 378, 0.00620677 +97, 378, 0.00138743 +97, 378, 0.00043012 +97, 379, 0.00093691 +97, 379, 0.00114015 +97, 379, 0.00107436 +97, 379, 0.00265441 +97, 380, 0.04489894 +97, 380, 0.00198516 +97, 380, 0.00406410 +97, 380, 0.00274565 +97, 381, 0.00339261 +97, 381, 0.01500210 +97, 381, 0.00779001 +97, 381, 0.00742602 +97, 382, 0.00300303 +97, 382, 0.00722099 +97, 382, 0.00100169 +97, 382, 0.02190871 +97, 383, 0.24263909 +97, 383, 0.05451502 +97, 383, 0.00382209 +97, 383, 0.00388869 +97, 384, 0.00969456 +97, 384, 0.00067811 +97, 384, 0.02211169 +97, 384, 0.00343484 +97, 385, 0.00496154 +97, 385, 0.01046286 +97, 385, 0.00807617 +97, 385, 0.00771424 +97, 386, 0.01509704 +97, 386, 0.29547736 +97, 386, 0.00930090 +97, 386, 0.00111541 +97, 387, 0.00406100 +97, 387, 0.00180394 +97, 387, 0.00077032 +97, 387, 0.00684308 +97, 388, 0.06975015 +97, 388, 0.00149732 +97, 388, 0.00258331 +97, 388, 0.00169957 +97, 389, 0.00460066 +97, 389, 0.00309567 +97, 389, 0.05133144 +97, 389, 0.02191734 +97, 390, 0.00915392 +97, 390, 0.00905741 +97, 390, 0.00172967 +97, 390, 0.00089731 +97, 391, 0.00366615 +97, 391, 0.00190518 +97, 391, 0.00062598 +97, 391, 0.00058213 +97, 392, 0.02134240 +97, 392, 0.00516606 +97, 392, 0.00150405 +97, 392, 0.00134090 +97, 393, 0.00178324 +97, 393, 0.00086308 +97, 393, 0.16848622 +97, 393, 0.02602166 +97, 394, 0.00309687 +97, 394, 0.00150857 +97, 394, 0.00156990 +97, 394, 0.05943120 +97, 395, 0.00406691 +97, 395, 0.11392274 +97, 395, 0.00619496 +97, 395, 0.07607772 +97, 396, 0.00193484 +97, 396, 0.01519131 +97, 396, 0.01067748 +97, 396, 0.00214096 +97, 397, 0.00406916 +97, 397, 0.00114712 +97, 397, 0.01899087 +97, 397, 0.00123743 +97, 398, 0.00512190 +97, 398, 0.02274730 +97, 398, 0.00536512 +97, 398, 0.00432406 +97, 399, 0.02114036 +97, 399, 0.01595471 +97, 399, 0.00428640 +97, 399, 0.00115917 +97, 400, 0.00062598 +97, 400, 0.00837845 +97, 400, 0.00171381 +97, 400, 0.03372965 +97, 401, 0.00463061 +97, 401, 0.01054305 +97, 401, 0.00539550 +97, 401, 0.00684137 +97, 402, 0.00046087 +97, 402, 0.00109417 +97, 402, 0.00391546 +97, 402, 0.00179861 +97, 403, 0.02442618 +97, 403, 0.00361190 +97, 403, 0.04327831 +97, 403, 0.00676859 +97, 404, 0.00335901 +97, 404, 0.20145713 +97, 404, 0.00758915 +97, 404, 0.01299924 +97, 405, 0.00259488 +97, 405, 0.01746873 +97, 405, 0.00405729 +97, 405, 0.00187693 +97, 406, 0.00197099 +97, 406, 0.00148384 +97, 406, 0.17597070 +97, 406, 0.00049325 +97, 407, 0.02694014 +97, 407, 0.11308116 +97, 407, 0.00382690 +97, 407, 0.02651240 +97, 408, 0.00066370 +97, 408, 0.00129488 +97, 408, 0.00431376 +97, 408, 0.00135120 +97, 409, 0.00059958 +97, 409, 0.00798769 +97, 409, 0.00147822 +97, 409, 0.00595211 +97, 410, 0.01321342 +97, 410, 0.00078516 +97, 410, 0.03474789 +97, 410, 0.01523703 +97, 411, 0.01192958 +97, 411, 0.00172654 +97, 411, 0.01253002 +97, 411, 0.02717606 +97, 412, 0.04377988 +97, 412, 0.00629779 +97, 412, 0.00089641 +97, 412, 0.00149790 +97, 413, 0.03151838 +97, 413, 0.05091457 +97, 413, 0.02123274 +97, 413, 0.00056430 +97, 414, 0.00275223 +97, 414, 0.00801924 +97, 414, 0.00880885 +97, 414, 0.00057995 +97, 415, 0.00083312 +97, 415, 0.04905458 +97, 415, 0.00055574 +97, 415, 0.01638683 +97, 416, 0.04202475 +97, 416, 0.00804706 +97, 416, 0.00350763 +97, 416, 0.00304469 +97, 417, 0.01419226 +97, 417, 0.00052936 +97, 417, 0.00260564 +97, 417, 0.00560990 +97, 418, 0.09360162 +97, 418, 0.00341539 +97, 418, 0.02491770 +97, 418, 0.00251250 +97, 419, 0.00110626 +97, 419, 0.00628589 +97, 419, 0.00436898 +97, 419, 0.00520338 +97, 420, 0.00410931 +97, 420, 0.00324489 +97, 420, 0.12426465 +97, 420, 0.02365283 +97, 421, 0.00075865 +97, 421, 0.00272997 +97, 421, 0.00190958 +97, 421, 0.53307498 +97, 422, 0.00206430 +97, 422, 0.00172588 +97, 422, 0.00359035 +97, 422, 0.00318562 +97, 423, 0.00240751 +97, 423, 0.00176847 +97, 423, 0.00106193 +97, 423, 0.01566222 +97, 424, 0.00169185 +97, 424, 0.01504220 +97, 424, 0.00065171 +97, 424, 0.00565233 +97, 425, 0.00233369 +97, 425, 0.01654939 +97, 425, 0.01722015 +97, 425, 0.00169458 +97, 426, 0.00192984 +97, 426, 0.00562207 +97, 426, 0.00356511 +97, 426, 0.00302754 +97, 427, 0.00080886 +97, 427, 0.00184792 +97, 427, 0.00824570 +97, 427, 0.01571701 +97, 428, 0.00115707 +97, 428, 0.00198868 +97, 428, 0.00201348 +97, 428, 0.00361404 +97, 429, 0.04991493 +97, 429, 0.03502303 +97, 429, 0.00435321 +97, 429, 0.17567706 +97, 430, 0.00115470 +97, 430, 0.02075192 +97, 430, 0.03775342 +97, 430, 0.00489391 +97, 431, 0.00725686 +97, 431, 0.00310447 +97, 431, 0.00259151 +97, 431, 0.02561234 +97, 432, 0.01858702 +97, 432, 0.52894294 +97, 432, 0.02295860 +97, 432, 0.39000496 +97, 433, 0.00106414 +97, 433, 0.00903211 +97, 433, 0.00302803 +97, 433, 0.00845553 +97, 434, 0.00068647 +97, 434, 0.00166464 +97, 434, 0.00224668 +97, 434, 0.01446923 +97, 435, 0.00732218 +97, 435, 0.18335125 +97, 435, 0.02472256 +97, 435, 0.00309979 +97, 436, 0.00101855 +97, 436, 0.05482881 +97, 436, 0.00066068 +97, 436, 0.00496957 +97, 437, 0.00839071 +97, 437, 0.00543882 +97, 437, 0.00083195 +97, 437, 0.06630323 +97, 438, 0.00136949 +97, 438, 0.00368951 +97, 438, 0.00229771 +97, 438, 0.01737507 +97, 439, 0.00381316 +97, 439, 0.00865320 +97, 439, 0.00295042 +97, 439, 0.00879702 +97, 440, 0.01320077 +97, 440, 0.00646820 +97, 440, 0.00223877 +97, 440, 0.00797398 +97, 441, 0.00215477 +97, 441, 0.00808423 +97, 441, 0.00077134 +97, 441, 0.00680222 +97, 442, 0.00564026 +97, 442, 0.00276679 +97, 442, 0.00170266 +97, 442, 0.00442488 +97, 443, 0.00145578 +97, 443, 0.01314187 +97, 443, 0.04585422 +97, 443, 0.01101432 +97, 444, 0.00349759 +97, 444, 0.00033601 +97, 444, 0.00863617 +97, 444, 0.00160200 +97, 445, 0.03173821 +97, 445, 0.00098815 +97, 445, 0.02741568 +97, 445, 0.00623956 +97, 446, 0.00138983 +97, 446, 0.02131183 +97, 446, 0.00094029 +97, 446, 0.00219711 +97, 447, 0.00161039 +97, 447, 0.00150259 +97, 447, 0.01417274 +97, 447, 0.00270848 +97, 448, 0.00076769 +97, 448, 0.00219696 +97, 448, 0.00214047 +97, 448, 0.00396050 +97, 449, 0.05186437 +97, 449, 0.00743135 +97, 449, 0.00200951 +97, 449, 0.77966237 +97, 450, 0.00243827 +97, 450, 0.00066345 +97, 450, 0.01708141 +97, 450, 0.00236241 +97, 451, 0.01604786 +97, 451, 0.01205051 +97, 451, 0.00081862 +97, 451, 0.00495462 +97, 452, 0.00826701 +97, 452, 0.00993112 +97, 452, 0.01791851 +97, 452, 0.00083984 +97, 453, 0.02088200 +97, 453, 0.00152997 +97, 453, 0.00513221 +97, 453, 0.00310213 +97, 454, 0.01993635 +97, 454, 0.00135930 +97, 454, 0.04677761 +97, 454, 0.00250223 +97, 455, 0.00368089 +97, 455, 0.00161578 +97, 455, 0.00123566 +97, 455, 0.01920848 +97, 456, 0.00131338 +97, 456, 0.00449414 +97, 456, 0.00546399 +97, 456, 0.00333952 +97, 457, 0.02942035 +97, 457, 0.01544899 +97, 457, 0.00095414 +97, 457, 0.00135081 +97, 458, 0.01063640 +97, 458, 0.00908451 +97, 458, 0.00395939 +97, 458, 0.00252509 +97, 459, 0.01397135 +97, 459, 0.06683090 +97, 459, 0.00163604 +97, 459, 0.01129621 +97, 460, 0.00222469 +97, 460, 0.20679748 +97, 460, 0.01512531 +97, 460, 0.01732171 +97, 461, 0.00171060 +97, 461, 0.01611547 +97, 461, 0.00414867 +97, 461, 0.00566861 +97, 462, 0.05943140 +97, 462, 0.00159004 +97, 462, 0.01171976 +97, 462, 0.11701215 +97, 463, 0.08560293 +97, 463, 0.00952016 +97, 463, 0.00346029 +97, 463, 0.00599375 +97, 464, 0.00227223 +97, 464, 0.00115415 +97, 464, 0.01366661 +97, 464, 0.05553505 +97, 465, 0.00082050 +97, 465, 0.00904025 +97, 465, 0.00177052 +97, 465, 0.00257506 +97, 466, 0.00216632 +97, 466, 0.00193457 +97, 466, 0.34486148 +97, 466, 0.01079715 +97, 467, 0.00667064 +97, 467, 0.01221076 +97, 467, 0.01731979 +97, 467, 0.02184353 +97, 468, 0.00988060 +97, 468, 0.00146108 +97, 468, 0.00526709 +97, 468, 0.00200785 +97, 469, 0.01035036 +97, 469, 0.01839576 +97, 469, 0.00266657 +97, 469, 0.00560793 +97, 470, 0.01281132 +97, 470, 0.00345592 +97, 470, 0.01788927 +97, 470, 0.03856343 +97, 471, 0.00090773 +97, 471, 0.29558176 +97, 471, 0.00136973 +97, 471, 0.00244587 +97, 472, 0.97646940 +97, 472, 0.00036796 +97, 472, 0.00402359 +97, 472, 0.00243510 +97, 473, 0.74902159 +97, 473, 0.00363373 +97, 473, 0.05293961 +97, 473, 0.00079326 +97, 474, 0.00302515 +97, 474, 0.00769475 +97, 474, 0.00267549 +97, 474, 0.02986027 +97, 475, 0.20683070 +97, 475, 0.00218626 +97, 475, 0.01685642 +97, 475, 0.00082768 +97, 476, 0.02985119 +97, 476, 0.01043731 +97, 476, 0.00095753 +97, 476, 0.14914577 +97, 477, 0.11891881 +97, 477, 0.00187946 +97, 477, 0.00785731 +97, 477, 0.00372136 +97, 478, 0.00535400 +97, 478, 0.01697589 +97, 478, 0.01068316 +97, 478, 0.27304393 +97, 479, 0.00069951 +97, 479, 0.01171692 +97, 479, 0.03056691 +97, 479, 0.00166225 +97, 480, 0.03035683 +97, 480, 0.00172440 +97, 480, 0.01000570 +97, 480, 0.07292078 +97, 481, 0.00771593 +97, 481, 0.04720951 +97, 481, 0.00331588 +97, 481, 0.00055321 +97, 482, 0.00535591 +97, 482, 0.00067944 +97, 482, 0.00097971 +97, 482, 0.00715621 +97, 483, 0.01786353 +97, 483, 0.00349729 +97, 483, 0.06440303 +97, 483, 0.00238026 +97, 484, 0.00610854 +97, 484, 0.01162738 +97, 484, 0.01730001 +97, 484, 0.00268207 +97, 485, 0.02485617 +97, 485, 0.00906016 +97, 485, 0.00185973 +97, 485, 0.00475415 +97, 486, 0.00231566 +97, 486, 0.00363458 +97, 486, 0.00554679 +97, 486, 0.04567207 +97, 487, 0.00264993 +97, 487, 0.01487370 +97, 487, 0.07038464 +97, 487, 0.10752248 +97, 488, 0.30102143 +97, 488, 0.01113478 +97, 488, 0.01709000 +97, 488, 0.00157307 +97, 489, 0.00550095 +97, 489, 0.07316007 +97, 489, 0.00820976 +97, 489, 0.00893259 +97, 490, 0.01274920 +97, 490, 0.00705231 +97, 490, 0.22382352 +97, 490, 0.01481524 +97, 491, 0.00108808 +97, 491, 0.00524035 +97, 491, 0.00169550 +97, 491, 0.00222114 +97, 492, 0.00201554 +97, 492, 0.01483669 +97, 492, 0.00166727 +97, 492, 0.07897276 +97, 493, 0.00191915 +97, 493, 0.00816476 +97, 493, 0.00719200 +97, 493, 0.05465085 +97, 494, 0.00900172 +97, 494, 0.12616968 +97, 494, 0.00293565 +97, 494, 0.01212236 +97, 495, 0.00397385 +97, 495, 0.00149719 +97, 495, 0.01020788 +97, 495, 0.00125516 +97, 496, 0.00153184 +97, 496, 0.00705783 +97, 496, 0.00331109 +97, 496, 0.00237190 +97, 497, 0.27484736 +97, 497, 0.00295373 +97, 497, 0.17158794 +97, 497, 0.00214064 +97, 498, 0.00187408 +97, 498, 0.01652004 +97, 498, 0.04435959 +97, 498, 0.00233121 +97, 499, 0.00100867 +97, 499, 0.00113306 +97, 499, 0.00531120 +97, 499, 0.05015353 +97, 500, 0.01193696 +97, 500, 0.01143783 +97, 500, 0.00504786 +97, 500, 0.00950395 +97, 501, 0.00090091 +97, 501, 0.01020739 +97, 501, 0.00493796 +97, 501, 0.00064639 +97, 502, 0.00172333 +97, 502, 0.00845669 +97, 502, 0.01750853 +97, 502, 0.00498023 +97, 503, 0.30223584 +97, 503, 0.15197389 +97, 503, 0.00322480 +97, 503, 0.01167289 +97, 504, 0.00487308 +97, 504, 0.00922822 +97, 504, 0.00247769 +97, 504, 0.01930996 +97, 505, 0.00233723 +97, 505, 0.00212657 +97, 505, 0.00212827 +97, 505, 0.00914177 +97, 506, 0.00736296 +97, 506, 0.00154520 +97, 506, 0.00205652 +97, 506, 0.01476392 +97, 507, 0.00549363 +97, 507, 0.00171336 +97, 507, 0.00081924 +97, 507, 0.02866458 +97, 508, 0.19365548 +97, 508, 0.01038419 +97, 508, 0.01101135 +97, 508, 0.63389111 +97, 509, 0.01040127 +97, 509, 0.00376031 +97, 509, 0.00396169 +97, 509, 0.00513091 +97, 510, 0.01433878 +97, 510, 0.08302347 +97, 510, 0.02513316 +97, 510, 0.00102754 +97, 511, 0.00819424 +97, 511, 0.06250760 +97, 511, 0.00381066 +97, 511, 0.00411418 +97, 512, 0.00098684 +97, 512, 0.00423190 +97, 512, 0.01258470 +97, 512, 0.00864843 +97, 513, 0.03955091 +97, 513, 0.00246862 +97, 513, 0.01589620 +97, 513, 0.01206146 +97, 514, 0.01482081 +97, 514, 0.00179680 +97, 514, 0.02484962 +97, 514, 0.00698658 +97, 515, 0.00663080 +97, 515, 0.00144592 +97, 515, 0.00092846 +97, 515, 0.00315052 +97, 516, 0.00125555 +97, 516, 0.02001781 +97, 516, 0.10666678 +97, 516, 0.00130349 +97, 517, 0.00150797 +97, 517, 0.00576847 +97, 517, 0.00578675 +97, 517, 0.01686968 +97, 518, 0.00270568 +97, 518, 0.04202911 +97, 518, 0.00289595 +97, 518, 0.00302476 +97, 519, 0.08286803 +97, 519, 0.01308082 +97, 519, 0.00142431 +97, 519, 0.00805598 +97, 520, 0.00360355 +97, 520, 0.00750054 +97, 520, 0.00411727 +97, 520, 0.00617065 +97, 521, 0.01212598 +97, 521, 0.00801363 +97, 521, 0.00104971 +97, 521, 0.00289717 +97, 522, 0.01185007 +97, 522, 0.00066195 +97, 522, 0.00645396 +97, 522, 0.00113225 +97, 523, 0.02300331 +97, 523, 0.00109031 +97, 523, 0.01578211 +97, 523, 0.00963764 +97, 524, 0.00248334 +97, 524, 0.01971507 +97, 524, 0.41469660 +97, 524, 0.00924102 +97, 525, 0.00104740 +97, 525, 0.00581034 +97, 525, 0.10333887 +97, 525, 0.01030853 +97, 526, 0.00094160 +97, 526, 0.00531243 +97, 526, 0.00584058 +97, 526, 0.08173888 +97, 527, 0.00613134 +97, 527, 0.00114092 +97, 527, 0.00041193 +97, 527, 0.02225145 +97, 528, 0.00049246 +97, 528, 0.07474732 +97, 528, 0.00178996 +97, 528, 0.00741296 +97, 529, 0.04453406 +97, 529, 0.00162020 +97, 529, 0.01360634 +97, 529, 0.00780826 +97, 530, 0.00225854 +97, 530, 0.09183758 +97, 530, 0.00711859 +97, 530, 0.05389418 +97, 531, 0.00669719 +97, 531, 0.02797407 +97, 531, 0.00767869 +97, 531, 0.00076241 +97, 532, 0.02656445 +97, 532, 0.00406730 +97, 532, 0.00676046 +97, 532, 0.00512524 +97, 533, 0.00621142 +97, 533, 0.00472122 +97, 533, 0.14265178 +97, 533, 0.00321210 +97, 534, 0.00037639 +97, 534, 0.00494971 +97, 534, 0.01916541 +97, 534, 0.03052082 +97, 535, 0.00347509 +97, 535, 0.00149397 +97, 535, 0.00060485 +97, 535, 0.00251161 +97, 536, 0.00293685 +97, 536, 0.00075643 +97, 536, 0.00549031 +97, 536, 0.00475394 +97, 537, 0.00443033 +97, 537, 0.00118627 +97, 537, 0.72819072 +97, 537, 0.00377466 +97, 538, 0.01472136 +97, 538, 0.01320256 +97, 538, 0.05809765 +97, 538, 0.00213511 +97, 539, 0.04720871 +97, 539, 0.00944399 +97, 539, 0.00413244 +97, 539, 0.13392654 +97, 540, 0.01078607 +97, 540, 0.00093583 +97, 540, 0.00092620 +97, 540, 0.08447280 +97, 541, 0.00315981 +97, 541, 0.00254008 +97, 541, 0.00150256 +97, 541, 0.00260768 +97, 542, 0.00340920 +97, 542, 0.00651357 +97, 542, 0.00147103 +97, 542, 0.00151137 +97, 543, 0.01198836 +97, 543, 0.00145252 +97, 543, 0.11948118 +97, 543, 0.00138028 +97, 544, 0.00159503 +97, 544, 0.00323930 +97, 544, 0.01192109 +97, 544, 0.00821380 +97, 545, 0.01173082 +97, 545, 0.00635954 +97, 545, 0.01630509 +97, 545, 0.05934070 +97, 546, 0.00440476 +97, 546, 0.00374523 +97, 546, 0.00255200 +97, 546, 0.00406621 +97, 547, 0.00277405 +97, 547, 0.00176785 +97, 547, 0.00144017 +97, 547, 0.02035961 +97, 548, 0.02390298 +97, 548, 0.01086057 +97, 548, 0.00231581 +97, 548, 0.02632446 +97, 549, 0.01509308 +97, 549, 0.00371923 +97, 549, 0.01788446 +97, 549, 0.00367982 +97, 550, 0.00076769 +97, 550, 0.00614960 +97, 550, 0.00061287 +97, 550, 0.00058073 +97, 551, 0.00128071 +97, 551, 0.00442794 +97, 551, 0.00346173 +97, 551, 0.00827789 +97, 552, 0.00268702 +97, 552, 0.02116809 +97, 552, 0.28811148 +97, 552, 0.06096853 +97, 553, 0.00157192 +97, 553, 0.01309413 +97, 553, 0.00380236 +97, 553, 0.00102685 +97, 554, 0.00362242 +97, 554, 0.00055375 +97, 554, 0.00640128 +97, 554, 0.00152676 +97, 555, 0.00087514 +97, 555, 0.01360128 +97, 555, 0.00625813 +97, 555, 0.00663635 +97, 556, 0.00248672 +97, 556, 0.00678969 +97, 556, 0.04529119 +97, 556, 0.01276387 +97, 557, 1.03831279 +97, 557, 0.00263778 +97, 557, 0.00309619 +97, 557, 0.00735453 +97, 558, 0.10941539 +97, 558, 0.00070725 +97, 558, 0.09636541 +97, 558, 0.05647038 +97, 559, 0.00056819 +97, 559, 0.00121868 +97, 559, 0.00235081 +97, 559, 0.00126538 +97, 560, 0.00064127 +97, 560, 0.09835660 +97, 560, 0.02256676 +97, 560, 0.00086728 +97, 561, 0.00684283 +97, 561, 0.00293660 +97, 561, 0.00504330 +97, 561, 0.00639535 +97, 562, 0.00483786 +97, 562, 0.00317781 +97, 562, 0.02936884 +97, 562, 0.00357917 +97, 563, 0.44156715 +97, 563, 0.00183720 +97, 563, 0.00463420 +97, 563, 0.00636623 +97, 564, 0.00406546 +97, 564, 0.00075793 +97, 564, 0.00639129 +97, 564, 0.01442866 +97, 565, 0.00208259 +97, 565, 0.01355279 +97, 565, 0.02706098 +97, 565, 0.00870042 +97, 566, 0.00324619 +97, 566, 0.00518198 +97, 566, 0.00206127 +97, 566, 0.00164171 +97, 567, 0.01725457 +97, 567, 0.19158944 +97, 567, 0.01285276 +97, 567, 0.59790057 +97, 568, 0.07484257 +97, 568, 0.00306505 +97, 568, 0.01609856 +97, 568, 0.01077887 +97, 569, 0.01396962 +97, 569, 0.00412741 +97, 569, 0.00120099 +97, 569, 0.00058482 +97, 570, 0.00187088 +97, 570, 0.00981066 +97, 570, 0.02088240 +97, 570, 0.00073934 +97, 571, 0.00347028 +97, 571, 0.00533610 +97, 571, 0.03224910 +97, 571, 0.01948521 +97, 572, 0.01214383 +97, 572, 0.13316895 +97, 572, 0.01702383 +97, 572, 0.00692750 +97, 573, 0.01205064 +97, 573, 0.05847093 +97, 573, 0.00181242 +97, 573, 0.01229402 +97, 574, 0.00081038 +97, 574, 0.00184132 +97, 574, 0.00181342 +97, 574, 0.00953373 +97, 575, 0.00287676 +97, 575, 0.00279184 +97, 575, 0.00050686 +97, 575, 0.00377844 +97, 576, 0.00557043 +97, 576, 0.00527209 +97, 576, 0.22374511 +97, 576, 0.02070968 +97, 577, 0.00162293 +97, 577, 0.01520916 +97, 577, 0.00094821 +97, 577, 0.00512370 +97, 578, 0.00232207 +97, 578, 0.00545260 +97, 578, 0.00962709 +97, 578, 0.03341166 +97, 579, 0.02705589 +97, 579, 0.00696407 +97, 579, 0.21256740 +97, 579, 0.00805090 +97, 580, 0.00992559 +97, 580, 0.02912954 +97, 580, 0.00533622 +97, 580, 0.03920272 +97, 581, 0.00176007 +97, 581, 0.00531721 +97, 581, 0.01491991 +97, 581, 0.00436777 +97, 582, 0.03413914 +97, 582, 0.04583627 +97, 582, 0.00310149 +97, 582, 0.02782998 +97, 583, 0.00241756 +97, 583, 0.01731246 +97, 583, 0.16828547 +97, 583, 0.38526282 +97, 584, 0.00471835 +97, 584, 0.04546878 +97, 584, 0.00712026 +97, 584, 0.00167144 +97, 585, 0.00447018 +97, 585, 0.00576364 +97, 585, 0.00807280 +97, 585, 0.00409667 +97, 586, 0.00341257 +97, 586, 0.00349289 +97, 586, 0.01273114 +97, 586, 0.00600646 +97, 587, 0.00306837 +97, 587, 0.00367507 +97, 587, 0.00291918 +97, 587, 0.01238875 +97, 588, 0.01649618 +97, 588, 0.00483183 +97, 588, 0.07158775 +97, 588, 0.00034254 +97, 589, 0.00686827 +97, 589, 0.00769106 +97, 589, 0.00816109 +97, 589, 0.00624034 +97, 590, 0.03198785 +97, 590, 0.01286056 +97, 590, 0.00115899 +97, 590, 0.00610672 +97, 591, 0.00168601 +97, 591, 0.00789841 +97, 591, 0.00279212 +97, 591, 0.00151463 +97, 592, 0.00279940 +97, 592, 0.00505313 +97, 592, 0.00423574 +97, 592, 0.00434982 +97, 593, 0.00155281 +97, 593, 0.00745181 +97, 593, 0.00877379 +97, 593, 0.00372658 +97, 594, 0.00342677 +97, 594, 0.02725964 +97, 594, 0.00314854 +97, 594, 0.00466142 +97, 595, 0.01426728 +97, 595, 0.02834840 +97, 595, 0.02365311 +97, 595, 0.00853380 +97, 596, 0.00276378 +97, 596, 0.01626255 +97, 596, 0.03262765 +97, 596, 1.06769180 +97, 597, 0.00833225 +97, 597, 0.00181391 +97, 597, 0.01021344 +97, 597, 0.00837419 +97, 598, 0.00677561 +97, 598, 0.00568262 +97, 598, 0.00519983 +97, 598, 0.01726126 +97, 599, 0.01026085 +97, 599, 0.92605084 +97, 599, 0.01119832 +97, 599, 0.00787828 +97, 600, 0.77560413 +97, 600, 0.00475408 +97, 600, 0.00377693 +97, 600, 0.04244280 +97, 601, 0.00628330 +97, 601, 0.00122757 +97, 601, 0.01419643 +97, 601, 0.00530781 +97, 602, 0.00189382 +97, 602, 0.00288146 +97, 602, 0.00410376 +97, 602, 0.01205891 +97, 603, 0.00069284 +97, 603, 0.64819789 +97, 603, 0.00047626 +97, 603, 0.02700382 +97, 604, 0.00530212 +97, 604, 0.00710123 +97, 604, 0.00054141 +97, 604, 0.00185329 +97, 605, 0.00091051 +97, 605, 0.04418967 +97, 605, 0.01193478 +97, 605, 0.01710228 +97, 606, 0.01252429 +97, 606, 0.00414736 +97, 606, 0.00438594 +97, 606, 0.00449668 +97, 607, 0.00263168 +97, 607, 0.00120201 +97, 607, 0.00287332 +97, 607, 0.00453570 +97, 608, 0.00367912 +97, 608, 0.00527007 +97, 608, 0.01178922 +97, 608, 0.00216149 +97, 609, 0.05362953 +97, 609, 0.02014115 +97, 609, 0.00174081 +97, 609, 0.00493148 +97, 610, 0.00168205 +97, 610, 0.00414727 +97, 610, 0.00684403 +97, 610, 0.05081680 +97, 611, 0.00141021 +97, 611, 0.00169156 +97, 611, 0.02081065 +97, 611, 0.00286861 +97, 612, 0.00244014 +97, 612, 0.00109728 +97, 612, 0.00456293 +97, 612, 0.00201073 +97, 613, 0.00437958 +97, 613, 0.00129713 +97, 613, 0.02232028 +97, 613, 0.00561395 +97, 614, 0.00955613 +97, 614, 0.00909098 +97, 614, 0.00447293 +97, 614, 0.00164899 +97, 615, 0.00810997 +97, 615, 0.02397875 +97, 615, 0.01782857 +97, 615, 0.00161582 +97, 616, 0.00127934 +97, 616, 0.01348056 +97, 616, 0.00757321 +97, 616, 0.01294460 +97, 617, 0.01630351 +97, 617, 0.00875042 +97, 617, 0.00037804 +97, 617, 0.00837033 +97, 618, 0.00313648 +97, 618, 0.00571856 +97, 618, 0.00727550 +97, 618, 0.00850813 +97, 619, 0.00175318 +97, 619, 0.01776921 +97, 619, 0.02242772 +97, 619, 0.00258772 +97, 620, 0.00408252 +97, 620, 0.00145715 +97, 620, 0.00125276 +97, 620, 0.01950087 +97, 621, 0.02770484 +97, 621, 0.01689953 +97, 621, 0.00478966 +97, 621, 0.00213953 +97, 622, 0.00071451 +97, 622, 0.00584297 +97, 622, 0.00148434 +97, 622, 0.01094494 +97, 623, 0.00804842 +97, 623, 0.16437899 +97, 623, 0.01500778 +97, 623, 0.00060228 +97, 624, 0.00387815 +97, 624, 0.00813036 +97, 624, 0.00070548 +97, 624, 0.00178711 +97, 625, 0.00110057 +97, 625, 0.01228166 +97, 625, 0.00148673 +97, 625, 0.01120604 +97, 626, 0.00311455 +97, 626, 0.00784269 +97, 626, 0.07658004 +97, 626, 0.00160257 +97, 627, 0.01788396 +97, 627, 0.02645715 +97, 627, 0.00088240 +97, 627, 0.00911297 +97, 628, 0.00132035 +97, 628, 0.00069971 +97, 628, 0.00071780 +97, 628, 0.00067116 +97, 629, 0.02166774 +97, 629, 0.00731511 +97, 629, 0.72234887 +97, 629, 0.01300851 +97, 630, 0.00100493 +97, 630, 0.00871135 +97, 630, 0.00067738 +97, 630, 0.04791214 +97, 631, 0.00416601 +97, 631, 0.01842307 +97, 631, 0.00516375 +97, 631, 0.00164141 +97, 632, 0.01045060 +97, 632, 0.00614755 +97, 632, 0.04480987 +97, 632, 0.01269577 +97, 633, 0.00928231 +97, 633, 0.00204127 +97, 633, 0.00217783 +97, 633, 0.01354858 +97, 634, 0.00566467 +97, 634, 0.00119694 +97, 634, 0.00113516 +97, 634, 0.00126015 +97, 635, 0.00976836 +97, 635, 0.00652769 +97, 635, 0.05914502 +97, 635, 0.09174644 +97, 636, 0.00150676 +97, 636, 0.00302088 +97, 636, 0.01292352 +97, 636, 0.02430602 +97, 637, 0.00057205 +97, 637, 0.00479403 +97, 637, 0.00297760 +97, 637, 0.01959052 +97, 638, 0.00171695 +97, 638, 0.00364077 +97, 638, 0.00779992 +97, 638, 0.00255104 +97, 639, 0.00504055 +97, 639, 0.00536331 +97, 639, 0.09237413 +97, 639, 0.00132360 +97, 640, 0.01579721 +97, 640, 0.01748917 +97, 640, 0.13017261 +97, 640, 0.00438614 +97, 641, 0.00274224 +97, 641, 0.00290009 +97, 641, 0.00137693 +97, 641, 0.00028845 +97, 642, 0.01945596 +97, 642, 0.00464392 +97, 642, 0.02120675 +97, 642, 0.01543375 +97, 643, 0.00263057 +97, 643, 0.00221529 +97, 643, 0.00559742 +97, 643, 0.00576887 +97, 644, 0.00574800 +97, 644, 0.00725324 +97, 644, 0.00196247 +97, 644, 0.00151624 +97, 645, 0.00512079 +97, 645, 0.00799359 +97, 645, 0.01055237 +97, 645, 0.00439833 +97, 646, 0.00114460 +97, 646, 0.00293877 +97, 646, 0.00831460 +97, 646, 0.00139585 +97, 647, 0.00290446 +97, 647, 0.03254590 +97, 647, 0.03938914 +97, 647, 0.01374461 +97, 648, 0.00226745 +97, 648, 0.03059208 +97, 648, 0.01139233 +97, 648, 0.00327264 +97, 649, 0.00136476 +97, 649, 0.00119899 +97, 649, 0.00292532 +97, 649, 0.00570563 +97, 650, 0.20914192 +97, 650, 0.04740012 +97, 650, 0.00403487 +97, 650, 0.00400615 +97, 651, 0.01677972 +97, 651, 0.02029539 +97, 651, 0.01210901 +97, 651, 0.12055869 +97, 652, 0.00306552 +97, 652, 0.00748631 +97, 652, 0.00656692 +97, 652, 0.00305977 +97, 653, 0.00696242 +97, 653, 0.00178569 +97, 653, 0.00970294 +97, 653, 0.00776893 +97, 654, 0.00181166 +97, 654, 0.00170058 +97, 654, 0.00368179 +97, 654, 0.00387295 +97, 655, 0.00992461 +97, 655, 0.00103154 +97, 655, 0.55138177 +97, 655, 0.01109678 +97, 656, 0.00254984 +97, 656, 0.00172591 +97, 656, 0.01546021 +97, 656, 0.00834682 +97, 657, 0.00127650 +97, 657, 0.00390944 +97, 657, 0.43309352 +97, 657, 0.02480137 +97, 658, 0.00602071 +97, 658, 0.02205604 +97, 658, 0.00330503 +97, 658, 0.01602532 +97, 659, 0.00297136 +97, 659, 0.00957573 +97, 659, 0.01622327 +97, 659, 0.00108761 +97, 660, 0.01113140 +97, 660, 0.01049940 +97, 660, 0.40453729 +97, 660, 0.01845511 +97, 661, 0.00059809 +97, 661, 0.13515703 +97, 661, 0.07181185 +97, 661, 0.00290504 +97, 662, 0.00100495 +97, 662, 0.00073576 +97, 662, 0.00133043 +97, 662, 0.00194318 +97, 663, 0.07449664 +97, 663, 0.04623475 +97, 663, 0.00591287 +97, 663, 0.00880235 +97, 664, 0.03521549 +97, 664, 0.00209527 +97, 664, 0.00826024 +97, 664, 0.00133978 +97, 665, 0.00076999 +97, 665, 0.00084440 +97, 665, 0.00350546 +97, 665, 0.00625533 +97, 666, 0.01084940 +97, 666, 0.04060486 +97, 666, 0.00753719 +97, 666, 0.00352112 +97, 667, 0.00423170 +97, 667, 0.00157067 +97, 667, 0.00880917 +97, 667, 0.00602520 +97, 668, 0.01647699 +97, 668, 0.00049952 +97, 668, 0.00479796 +97, 668, 0.00337209 +97, 669, 0.01419096 +97, 669, 0.00083097 +97, 669, 0.54952824 +97, 669, 0.00958044 +97, 670, 0.00211478 +97, 670, 0.00291041 +97, 670, 0.00037544 +97, 670, 0.00392898 +97, 671, 0.00109643 +97, 671, 0.00641638 +97, 671, 0.00679323 +97, 671, 0.03557387 +97, 672, 0.00386442 +97, 672, 0.00249376 +97, 672, 0.00264371 +97, 672, 0.59815288 +97, 673, 0.00041603 +97, 673, 0.52553988 +97, 673, 0.00235026 +97, 673, 0.00113310 +97, 674, 0.00203390 +97, 674, 0.09895388 +97, 674, 0.00393257 +97, 674, 0.00278424 +97, 675, 0.00107817 +97, 675, 0.00279665 +97, 675, 0.02331666 +97, 675, 0.00339937 +97, 676, 0.00270450 +97, 676, 0.01491054 +97, 676, 0.05004966 +97, 676, 0.01262310 +97, 677, 0.00990806 +97, 677, 0.00415776 +97, 677, 0.02687250 +97, 677, 0.16697820 +97, 678, 0.00106627 +97, 678, 0.00375428 +97, 678, 0.00720786 +97, 678, 0.01510796 +97, 679, 0.04421986 +97, 679, 0.00084663 +97, 679, 0.19820647 +97, 679, 0.00329097 +97, 680, 0.00141404 +97, 680, 0.00231230 +97, 680, 0.05603450 +97, 680, 0.00650671 +97, 681, 0.00093753 +97, 681, 0.00401778 +97, 681, 0.00446211 +97, 681, 0.00272667 +97, 682, 0.00578410 +97, 682, 0.01629739 +97, 682, 0.00226113 +97, 682, 0.04365673 +97, 683, 0.00287477 +97, 683, 0.00121248 +97, 683, 0.01431402 +97, 683, 0.00704335 +97, 684, 0.00427806 +97, 684, 0.26018307 +97, 684, 0.00938455 +97, 684, 0.00472374 +97, 685, 0.00400703 +97, 685, 0.00430081 +97, 685, 0.20088649 +97, 685, 0.00341935 +97, 686, 0.00198989 +97, 686, 0.00556986 +97, 686, 0.00466266 +97, 686, 0.01875117 +97, 687, 0.00217727 +97, 687, 0.00043638 +97, 687, 0.01275676 +97, 687, 0.00425451 +97, 688, 0.00245287 +97, 688, 0.00618222 +97, 688, 0.00299612 +97, 688, 0.00594660 +97, 689, 0.00562697 +97, 689, 0.01540586 +97, 689, 0.00052611 +97, 689, 0.00329305 +97, 690, 0.06319342 +97, 690, 0.00053817 +97, 690, 0.01023395 +97, 690, 0.00117737 +97, 691, 0.17321098 +97, 691, 0.00142671 +97, 691, 0.00271012 +97, 691, 0.01535768 +97, 692, 0.00179203 +97, 692, 0.00438973 +97, 692, 0.00669208 +97, 692, 0.00205356 +97, 693, 0.00315099 +97, 693, 0.55970705 +97, 693, 0.00145826 +97, 693, 0.00826139 +97, 694, 0.00068711 +97, 694, 0.00974586 +97, 694, 0.00619026 +97, 694, 0.00271464 +97, 695, 0.08602001 +97, 695, 0.00681228 +97, 695, 0.00299968 +97, 695, 0.01729822 +97, 696, 0.00999305 +97, 696, 0.00328851 +97, 696, 0.00166931 +97, 696, 0.05341894 +97, 697, 0.00660171 +97, 697, 0.01747192 +97, 697, 0.00782490 +97, 697, 0.00172313 +97, 698, 0.00069776 +97, 698, 0.00508297 +97, 698, 0.05532947 +97, 698, 0.00383225 +97, 699, 0.00428137 +97, 699, 0.00294917 +97, 699, 0.00204133 +97, 699, 0.00508739 +97, 700, 0.00270081 +97, 700, 0.00072684 +97, 700, 0.00321448 +97, 700, 0.00260392 +97, 701, 0.24612361 +97, 701, 0.00413160 +97, 701, 0.19517322 +97, 701, 0.00803474 +97, 702, 0.01134163 +97, 702, 0.00513021 +97, 702, 0.01470239 +97, 702, 0.00670906 +97, 703, 0.01162644 +97, 703, 0.00112957 +97, 703, 0.00847471 +97, 703, 0.00365139 +97, 704, 0.00941177 +97, 704, 0.06407770 +97, 704, 0.00390092 +97, 704, 0.02623998 +97, 705, 0.46481147 +97, 705, 0.00527053 +97, 705, 0.00473786 +97, 705, 0.02812951 +97, 706, 0.00722665 +97, 706, 0.00249634 +97, 706, 0.00140063 +97, 706, 0.00137289 +97, 707, 0.00437474 +97, 707, 0.00846100 +97, 707, 0.00311205 +97, 707, 0.01022648 +97, 708, 0.00376027 +97, 708, 0.02494865 +97, 708, 0.00483395 +97, 708, 0.00368452 +97, 709, 0.01217890 +97, 709, 0.00670365 +97, 709, 0.04485425 +97, 709, 0.00076758 +97, 710, 0.01103490 +97, 710, 0.00398511 +97, 710, 0.01081321 +97, 710, 0.01690177 +97, 711, 0.00062338 +97, 711, 0.00413560 +97, 711, 0.00155556 +97, 711, 0.00222095 +97, 712, 0.00331216 +97, 712, 0.03228199 +97, 712, 0.00491176 +97, 712, 0.00641091 +97, 713, 0.04117515 +97, 713, 0.00201468 +97, 713, 0.00339093 +97, 713, 0.00189220 +97, 714, 0.00793017 +97, 714, 0.00747757 +97, 714, 0.01732993 +97, 714, 0.00403456 +97, 715, 0.00158756 +97, 715, 0.02474437 +97, 715, 0.00734693 +97, 715, 0.00038191 +97, 716, 0.00465496 +97, 716, 0.00125664 +97, 716, 0.00634823 +97, 716, 0.00448292 +97, 717, 0.00071064 +97, 717, 0.00462768 +97, 717, 0.02730670 +97, 717, 0.00126888 +97, 718, 0.32232225 +97, 718, 0.00352536 +97, 718, 0.00379537 +97, 718, 0.11682484 +97, 719, 0.00104037 +97, 719, 0.00234538 +97, 719, 0.04283641 +97, 719, 0.00194041 +97, 720, 0.00283186 +97, 720, 0.00418705 +97, 720, 0.00535205 +97, 720, 0.01635211 +97, 721, 0.00646538 +97, 721, 0.00142489 +97, 721, 0.00492340 +97, 721, 0.01221530 +97, 722, 0.02534512 +97, 722, 0.00500287 +97, 722, 0.00419144 +97, 722, 0.00625903 +97, 723, 0.00793683 +97, 723, 0.00212361 +97, 723, 0.00419598 +97, 723, 0.00213495 +97, 724, 0.00306948 +97, 724, 0.02557596 +97, 724, 0.00194652 +97, 724, 0.00312152 +97, 725, 0.00386146 +97, 725, 0.00115645 +97, 725, 0.00285769 +97, 725, 0.00091957 +97, 726, 0.00105121 +97, 726, 0.00219481 +97, 726, 0.00150596 +97, 726, 0.00178873 +97, 727, 0.03634916 +97, 727, 0.00209426 +97, 727, 0.00208293 +97, 727, 0.00133072 +97, 728, 0.01244254 +97, 728, 0.00094356 +97, 728, 0.00630098 +97, 728, 0.00682665 +97, 729, 0.00103604 +97, 729, 0.00403519 +97, 729, 0.00614865 +97, 729, 0.00441737 +97, 730, 0.00696195 +97, 730, 0.00215580 +97, 730, 0.00168364 +97, 730, 0.00232529 +97, 731, 0.01010087 +97, 731, 0.00161273 +97, 731, 0.01550282 +97, 731, 0.00673759 +97, 732, 0.05883959 +97, 732, 0.61366135 +97, 732, 0.00153359 +97, 732, 0.00134020 +97, 733, 0.00858349 +97, 733, 0.00618844 +97, 733, 0.00108272 +97, 733, 0.00271644 +97, 734, 0.01078407 +97, 734, 0.00330728 +97, 734, 0.00437236 +97, 734, 0.00879930 +97, 735, 0.00160390 +97, 735, 0.01710939 +97, 735, 0.11348201 +97, 735, 0.00460990 +97, 736, 0.00282782 +97, 736, 0.00458836 +97, 736, 0.00310301 +97, 736, 0.00170286 +97, 737, 0.00105971 +97, 737, 0.00035597 +97, 737, 0.00210934 +97, 737, 0.00114005 +97, 738, 0.00083802 +97, 738, 0.00148479 +97, 738, 0.00372367 +97, 738, 0.00513143 +97, 739, 0.00420524 +97, 739, 0.00640177 +97, 739, 0.00126010 +97, 739, 0.00717053 +97, 740, 0.00277120 +97, 740, 0.00881773 +97, 740, 0.00120448 +97, 740, 0.02011279 +97, 741, 0.00255424 +97, 741, 0.19410969 +97, 741, 0.03282370 +97, 741, 0.00464471 +97, 742, 0.00253049 +97, 742, 0.00080179 +97, 742, 0.00890684 +97, 742, 0.39510733 +97, 743, 0.00327422 +97, 743, 0.00504917 +97, 743, 0.01947212 +97, 743, 0.00268017 +97, 744, 0.10330510 +97, 744, 0.22335199 +97, 744, 0.01244659 +97, 744, 0.18066095 +97, 745, 0.00487092 +97, 745, 0.00185145 +97, 745, 0.00797866 +97, 745, 0.00720438 +97, 746, 0.00084900 +97, 746, 0.01114500 +97, 746, 0.00652353 +97, 746, 0.00521012 +97, 747, 0.00377509 +97, 747, 0.01486171 +97, 747, 0.00236715 +97, 747, 0.00094737 +97, 748, 0.01360510 +97, 748, 0.00405719 +97, 748, 0.00445152 +97, 748, 0.00142717 +97, 749, 0.00071308 +97, 749, 0.01565802 +97, 749, 0.00477345 +97, 749, 0.00571650 +97, 750, 0.00105547 +97, 750, 0.01631250 +97, 750, 0.00212515 +97, 750, 0.00257640 +97, 751, 0.00172050 +97, 751, 0.00433675 +97, 751, 0.00589159 +97, 751, 0.00200587 +97, 752, 0.00033298 +97, 752, 0.00497419 +97, 752, 0.00243994 +97, 752, 0.00144957 +97, 753, 0.00158429 +97, 753, 0.00099126 +97, 753, 0.01029806 +97, 753, 0.03419976 +97, 754, 0.04122076 +97, 754, 0.00367249 +97, 754, 0.00400490 +97, 754, 0.00885487 +97, 755, 0.00221042 +97, 755, 0.00626051 +97, 755, 0.01987846 +97, 755, 0.00542216 +97, 756, 0.02688421 +97, 756, 0.01155028 +97, 756, 0.00198270 +97, 756, 0.00446876 +97, 757, 0.12402338 +97, 757, 0.00483877 +97, 757, 0.00290575 +97, 757, 0.00396899 +97, 758, 0.00068369 +97, 758, 0.01163381 +97, 758, 0.06827591 +97, 758, 0.00968862 +97, 759, 0.03313125 +97, 759, 0.00328326 +97, 759, 0.03333383 +97, 759, 0.00445279 +97, 760, 0.00158504 +97, 760, 0.00481131 +97, 760, 0.01084565 +97, 760, 0.00075892 +97, 761, 0.01157951 +97, 761, 0.00094830 +97, 761, 0.00665484 +97, 761, 0.00570226 +97, 762, 0.00160439 +97, 762, 0.09905706 +97, 762, 0.00147762 +97, 762, 0.03151090 +97, 763, 0.08401532 +97, 763, 0.00078951 +97, 763, 0.01489352 +97, 763, 0.30669734 +97, 764, 0.71147335 +97, 764, 0.15473980 +97, 764, 0.00072207 +97, 764, 0.09110860 +97, 765, 0.00645805 +97, 765, 0.00050995 +97, 765, 0.03148526 +97, 765, 0.00375763 +97, 766, 0.00632125 +97, 766, 0.00189536 +97, 766, 0.03141392 +97, 766, 0.00103918 +97, 767, 0.00190375 +97, 767, 0.40824264 +97, 767, 0.00730891 +97, 767, 0.00142315 +97, 768, 0.00262925 +97, 768, 0.09463432 +97, 768, 0.00082457 +97, 768, 0.02460309 +97, 769, 0.22455645 +97, 769, 0.00149656 +97, 769, 0.00088963 +97, 769, 0.02957292 +97, 770, 0.00394718 +97, 770, 0.00079288 +97, 770, 0.05924099 +97, 770, 0.01025307 +97, 771, 0.00916470 +97, 771, 0.05227059 +97, 771, 0.48106101 +97, 771, 0.00212856 +97, 772, 0.04573559 +97, 772, 0.01464430 +97, 772, 0.00518491 +97, 772, 0.03378227 +97, 773, 0.00172119 +97, 773, 0.01361568 +97, 773, 0.01021329 +97, 773, 0.00324961 +97, 774, 0.02058077 +97, 774, 0.00347228 +97, 774, 0.00600471 +97, 774, 0.00089423 +97, 775, 0.00093468 +97, 775, 0.25204557 +97, 775, 0.00994849 +97, 775, 0.00276211 +97, 776, 0.01087351 +97, 776, 0.01804184 +97, 776, 0.00114444 +97, 776, 0.00124332 +97, 777, 0.00823892 +97, 777, 0.01690230 +97, 777, 0.00174876 +97, 777, 0.00116221 +97, 778, 0.00090448 +97, 778, 0.02849279 +97, 778, 0.00478857 +97, 778, 0.00563424 +97, 779, 0.00084956 +97, 779, 0.00112610 +97, 779, 0.01121606 +97, 779, 0.00145796 +97, 780, 0.00566956 +97, 780, 0.00295934 +97, 780, 0.00271990 +97, 780, 0.00108443 +97, 781, 0.01899237 +97, 781, 0.00194308 +97, 781, 0.07751277 +97, 781, 0.01208801 +97, 782, 0.00755377 +97, 782, 0.00382714 +97, 782, 0.00439430 +97, 782, 0.00414691 +97, 783, 0.00200495 +97, 783, 0.00086858 +97, 783, 0.00096779 +97, 783, 0.02923443 +97, 784, 0.01334213 +97, 784, 0.00623661 +97, 784, 0.00426498 +97, 784, 0.00216748 +97, 785, 0.00860600 +97, 785, 0.01287496 +97, 785, 0.00743698 +97, 785, 0.00313247 +97, 786, 0.00121704 +97, 786, 0.00128699 +97, 786, 0.01086638 +97, 786, 0.00395305 +97, 787, 0.00197071 +97, 787, 0.00176658 +97, 787, 0.00494738 +97, 787, 0.13621615 +97, 788, 0.01361661 +97, 788, 0.00296717 +97, 788, 0.00196463 +97, 788, 0.00109964 +97, 789, 0.00427735 +97, 789, 0.00294901 +97, 789, 0.01462500 +97, 789, 0.02306714 +97, 790, 0.02480527 +97, 790, 0.09546313 +97, 790, 0.00285830 +97, 790, 0.00064161 +97, 791, 0.08885799 +97, 791, 0.01547696 +97, 791, 0.00403561 +97, 791, 0.00199520 +97, 792, 0.00505745 +97, 792, 0.00211716 +97, 792, 0.13351856 +97, 792, 0.00471355 +97, 793, 0.00196539 +97, 793, 0.08922037 +97, 793, 0.24758701 +97, 793, 0.00205697 +97, 794, 0.00108858 +97, 794, 0.06378251 +97, 794, 0.00861291 +97, 794, 0.00380650 +97, 795, 0.00331003 +97, 795, 0.00097994 +97, 795, 0.00562783 +97, 795, 0.00272413 +97, 796, 0.07851710 +97, 796, 0.00271429 +97, 796, 0.02418198 +97, 796, 0.00220464 +97, 797, 0.00253321 +97, 797, 0.08630626 +97, 797, 0.07630384 +97, 797, 0.00436741 +97, 798, 0.06238749 +97, 798, 0.00383301 +97, 798, 0.00079804 +97, 798, 0.00303865 +97, 799, 0.00067055 +97, 799, 0.00200143 +97, 799, 0.00397034 +97, 799, 0.00109347 +97, 800, 0.00119957 +97, 800, 0.00865449 +97, 800, 0.00094561 +97, 800, 0.06226772 +97, 801, 0.00337823 +97, 801, 0.00074253 +97, 801, 0.00162412 +97, 801, 0.00049005 +97, 802, 0.01557442 +97, 802, 0.00282167 +97, 802, 0.00292926 +97, 802, 0.00066550 +97, 803, 0.01015939 +97, 803, 0.30213419 +97, 803, 0.00169350 +97, 803, 0.00763939 +97, 804, 0.01799336 +97, 804, 0.00771408 +97, 804, 0.00362996 +97, 804, 0.04396406 +97, 805, 0.00050821 +97, 805, 0.01163508 +97, 805, 0.01962041 +97, 805, 0.00590167 +97, 806, 0.00350689 +97, 806, 0.00291893 +97, 806, 0.00449828 +97, 806, 0.00078207 +97, 807, 0.00747685 +97, 807, 0.00368763 +97, 807, 0.01386529 +97, 807, 0.00057765 +97, 808, 0.00223254 +97, 808, 0.00183425 +97, 808, 0.00631023 +97, 808, 0.00086766 +97, 809, 0.00286027 +97, 809, 0.01559289 +97, 809, 0.30269465 +97, 809, 0.00054829 +97, 810, 0.00389323 +97, 810, 0.03217750 +97, 810, 0.00066763 +97, 810, 0.11690643 +97, 811, 0.00313482 +97, 811, 0.01285781 +97, 811, 0.00100578 +97, 811, 0.09319371 +97, 812, 0.00065909 +97, 812, 0.00276993 +97, 812, 0.00235362 +97, 812, 0.00085269 +97, 813, 0.00222698 +97, 813, 0.00308005 +97, 813, 0.04375054 +97, 813, 0.01161181 +97, 814, 0.10989580 +97, 814, 0.00598375 +97, 814, 0.01579743 +97, 814, 0.00494573 +97, 815, 0.00531130 +97, 815, 0.00128686 +97, 815, 0.02916059 +97, 815, 0.02867989 +97, 816, 0.00302197 +97, 816, 0.00194981 +97, 816, 0.00120846 +97, 816, 0.00151607 +97, 817, 0.00384592 +97, 817, 0.00295693 +97, 817, 0.01632088 +97, 817, 0.00233411 +97, 818, 0.02161503 +97, 818, 0.01372119 +97, 818, 0.00686055 +97, 818, 0.01753005 +97, 819, 0.00217393 +97, 819, 0.05214477 +97, 819, 0.00153637 +97, 819, 0.10607578 +97, 820, 0.00130962 +97, 820, 0.00091120 +97, 820, 0.00205319 +97, 820, 0.00275435 +97, 821, 0.00115552 +97, 821, 0.01396280 +97, 821, 0.00651872 +97, 821, 0.00347574 +97, 822, 0.00869087 +97, 822, 0.00600213 +97, 822, 0.00195606 +97, 822, 0.01748060 +97, 823, 0.01478037 +97, 823, 0.00281026 +97, 823, 0.00078355 +97, 823, 0.01562526 +97, 824, 0.00113943 +97, 824, 0.01659769 +97, 824, 0.00038545 +97, 824, 0.00117400 +97, 825, 0.00063463 +97, 825, 0.04109856 +97, 825, 0.00183921 +97, 825, 0.00723277 +97, 826, 0.00283327 +97, 826, 0.00747229 +97, 826, 0.05123153 +97, 826, 0.01720105 +97, 827, 0.01451015 +97, 827, 0.00172354 +97, 827, 0.00906677 +97, 827, 0.00281025 +97, 828, 0.00407878 +97, 828, 0.00359513 +97, 828, 0.01322525 +97, 828, 0.02121917 +97, 829, 0.01143168 +97, 829, 4.08441687 +97, 829, 0.01810392 +97, 829, 0.00165001 +97, 830, 0.00066704 +97, 830, 0.01843488 +97, 830, 0.00207096 +97, 830, 0.46332413 +97, 831, 0.00364315 +97, 831, 0.03554603 +97, 831, 0.00213940 +97, 831, 0.00822337 +97, 832, 0.00166830 +97, 832, 0.00085060 +97, 832, 0.01004738 +97, 832, 0.01308764 +97, 833, 0.00543929 +97, 833, 0.02180243 +97, 833, 0.00425950 +97, 833, 0.00682878 +97, 834, 0.72604531 +97, 834, 0.00128048 +97, 834, 0.00717949 +97, 834, 0.00417303 +97, 835, 0.00216355 +97, 835, 0.00833463 +97, 835, 0.00245012 +97, 835, 0.06488359 +97, 836, 0.00553936 +97, 836, 0.00261917 +97, 836, 0.00398004 +97, 836, 0.00668307 +97, 837, 0.00162288 +97, 837, 0.01205931 +97, 837, 0.00933728 +97, 837, 0.03850842 +97, 838, 0.00863189 +97, 838, 0.00143129 +97, 838, 0.17239100 +97, 838, 0.00305988 +97, 839, 0.01482585 +97, 839, 0.00220790 +97, 839, 0.00466191 +97, 839, 0.00433327 +97, 840, 0.00106745 +97, 840, 0.00149735 +97, 840, 0.89798987 +97, 840, 0.00186360 +97, 841, 0.00353443 +97, 841, 0.00062544 +97, 841, 0.09039522 +97, 841, 0.00082661 +97, 842, 0.03159591 +97, 842, 0.00755217 +97, 842, 0.00099519 +97, 842, 0.00478172 +97, 843, 0.00336912 +97, 843, 0.00431829 +97, 843, 0.00173339 +97, 843, 0.00145234 +97, 844, 4.65415764 +97, 844, 0.02922538 +97, 844, 0.00138216 +97, 844, 0.01328904 +97, 845, 0.01156584 +97, 845, 0.01774993 +97, 845, 0.00488622 +97, 845, 0.10274617 +97, 846, 0.00199808 +97, 846, 0.00566116 +97, 846, 0.00152024 +97, 846, 0.02322581 +97, 847, 0.00247101 +97, 847, 0.00328448 +97, 847, 0.04660376 +97, 847, 0.01874360 +97, 848, 0.02953754 +97, 848, 0.00365114 +97, 848, 0.00456621 +97, 848, 0.00135808 +97, 849, 0.01177595 +97, 849, 0.00087238 +97, 849, 0.00375041 +97, 849, 0.00299370 +97, 850, 0.00179201 +97, 850, 0.00212122 +97, 850, 0.00136321 +97, 850, 0.00402676 +97, 851, 0.00544235 +97, 851, 0.02987510 +97, 851, 0.00096769 +97, 851, 0.00913267 +97, 852, 0.00684588 +97, 852, 0.00700161 +97, 852, 0.00923181 +97, 852, 0.04981855 +97, 853, 0.02019084 +97, 853, 0.00213500 +97, 853, 0.01062444 +97, 853, 0.00231517 +97, 854, 0.00089857 +97, 854, 0.00951522 +97, 854, 0.01078018 +97, 854, 0.00500229 +97, 855, 0.01007076 +97, 855, 0.00471271 +97, 855, 0.00123644 +97, 855, 0.17835261 +97, 856, 0.00171796 +97, 856, 0.01049644 +97, 856, 0.00061540 +97, 856, 0.00517402 +97, 857, 0.00797380 +97, 857, 0.01102834 +97, 857, 0.00228385 +97, 857, 0.00214666 +97, 858, 0.00337114 +97, 858, 0.00579336 +97, 858, 0.00076824 +97, 858, 0.00157863 +97, 859, 0.00092821 +97, 859, 0.00319564 +97, 859, 0.00121810 +97, 859, 0.02441673 +97, 860, 0.00111370 +97, 860, 0.02383371 +97, 860, 0.22067447 +97, 860, 0.01282876 +97, 861, 0.01044148 +97, 861, 0.00277025 +97, 861, 0.00253120 +97, 861, 0.02224559 +97, 862, 0.08126130 +97, 862, 0.00151980 +97, 862, 0.01212454 +97, 862, 0.01413164 +97, 863, 0.21019396 +97, 863, 0.00037545 +97, 863, 0.00101292 +97, 863, 0.03433817 +97, 864, 0.00373474 +97, 864, 0.01056041 +97, 864, 0.00391597 +97, 864, 0.00062857 +97, 865, 0.00705490 +97, 865, 0.00190445 +97, 865, 0.00085835 +97, 865, 0.00873933 +97, 866, 0.00536486 +97, 866, 0.00115497 +97, 866, 0.00159399 +97, 866, 0.00439461 +97, 867, 0.01249062 +97, 867, 0.02480008 +97, 867, 0.12328801 +97, 867, 0.00214443 +97, 868, 0.00127054 +97, 868, 0.00225780 +97, 868, 0.00236568 +97, 868, 0.00260641 +97, 869, 0.00087532 +97, 869, 0.00140816 +97, 869, 0.00156018 +97, 869, 0.01490689 +97, 870, 0.07537997 +97, 870, 0.00381067 +97, 870, 0.01111038 +97, 870, 0.00315343 +97, 871, 0.00258287 +97, 871, 0.00198317 +97, 871, 0.00130522 +97, 871, 0.00206607 +97, 872, 0.04263550 +97, 872, 0.00122179 +97, 872, 0.00661416 +97, 872, 0.10754775 +97, 873, 0.09505272 +97, 873, 0.01285205 +97, 873, 0.17034952 +97, 873, 0.00621870 +97, 874, 0.00901448 +97, 874, 0.00262532 +97, 874, 0.00537208 +97, 874, 0.00447226 +97, 875, 0.00101311 +97, 875, 0.02728073 +97, 875, 0.00135608 +97, 875, 0.01273451 +97, 876, 0.00800803 +97, 876, 0.00231230 +97, 876, 0.00125070 +97, 876, 0.00079471 +97, 877, 0.28350994 +97, 877, 0.50704497 +97, 877, 0.09765636 +97, 877, 0.00216874 +97, 878, 0.00202421 +97, 878, 0.00063581 +97, 878, 0.02209781 +97, 878, 0.00441250 +97, 879, 0.00463837 +97, 879, 0.00441075 +97, 879, 0.00309146 +97, 879, 0.00781560 +97, 880, 0.00363217 +97, 880, 0.00184681 +97, 880, 0.00373819 +97, 880, 0.00996157 +97, 881, 0.00264563 +97, 881, 0.00458098 +97, 881, 0.00517693 +97, 881, 0.00158451 +97, 882, 0.00049959 +97, 882, 0.06542551 +97, 882, 0.00898474 +97, 882, 0.00682432 +97, 883, 0.00309878 +97, 883, 0.03209050 +97, 883, 0.00507024 +97, 883, 0.00324999 +97, 884, 0.00058978 +97, 884, 0.00771928 +97, 884, 0.00867696 +97, 884, 0.00084886 +97, 885, 0.34468332 +97, 885, 0.00104415 +97, 885, 0.13186643 +97, 885, 0.00804148 +97, 886, 0.00489336 +97, 886, 0.00238721 +97, 886, 0.02459213 +97, 886, 0.00460734 +97, 887, 0.00425463 +97, 887, 0.00268149 +97, 887, 0.01188958 +97, 887, 0.00205614 +97, 888, 0.00140269 +97, 888, 0.01351638 +97, 888, 0.00098493 +97, 888, 0.00432382 +97, 889, 0.00192951 +97, 889, 0.00180529 +97, 889, 0.00164327 +97, 889, 0.00092966 +97, 890, 0.00784336 +97, 890, 0.00138460 +97, 890, 0.01423236 +97, 890, 0.04367150 +97, 891, 0.03334327 +97, 891, 0.00730544 +97, 891, 0.00146440 +97, 891, 0.01994694 +97, 892, 0.00596256 +97, 892, 0.00328280 +97, 892, 0.00154435 +97, 892, 0.00255545 +97, 893, 0.00231939 +97, 893, 0.00171345 +97, 893, 0.00054963 +97, 893, 0.00165554 +97, 894, 0.00685484 +97, 894, 0.01163922 +97, 894, 0.00137229 +97, 894, 0.00745532 +97, 895, 0.00148501 +97, 895, 0.00148816 +97, 895, 0.02445276 +97, 895, 0.00783736 +97, 896, 0.00106244 +97, 896, 0.00341721 +97, 896, 0.00180686 +97, 896, 0.28195438 +97, 897, 0.01980005 +97, 897, 0.00158279 +97, 897, 0.00016577 +97, 897, 0.00841456 +97, 898, 0.16980508 +97, 898, 0.00162450 +97, 898, 0.00113489 +97, 898, 0.07727598 +97, 899, 0.01147900 +97, 899, 0.00754197 +97, 899, 0.01084195 +97, 899, 0.00297962 +97, 900, 0.01221337 +97, 900, 0.00074762 +97, 900, 0.01176602 +97, 900, 0.02129076 +97, 901, 0.00508938 +97, 901, 0.00814837 +97, 901, 0.01496919 +97, 901, 0.00600691 +97, 902, 0.14114001 +97, 902, 0.00130343 +97, 902, 0.00157851 +97, 902, 0.00188820 +97, 903, 2.48595047 +97, 903, 0.00740395 +97, 903, 0.00947469 +97, 903, 0.01391274 +97, 904, 0.00298511 +97, 904, 0.00368151 +97, 904, 0.00125667 +97, 904, 0.00447158 +97, 905, 0.00279889 +97, 905, 0.00273434 +97, 905, 0.00112698 +97, 905, 0.00320735 +97, 906, 0.00427915 +97, 906, 0.00452199 +97, 906, 0.00145281 +97, 906, 0.00872767 +97, 907, 0.00965773 +97, 907, 0.00116567 +97, 907, 0.00180349 +97, 907, 0.01460780 +97, 908, 0.02638925 +97, 908, 0.00365360 +97, 908, 0.00112311 +97, 908, 0.01236773 +97, 909, 0.00358776 +97, 909, 0.02184822 +97, 909, 0.00438353 +97, 909, 0.00320165 +97, 910, 0.11883775 +97, 910, 0.00636490 +97, 910, 0.00420386 +97, 910, 0.00450999 +97, 911, 0.03382907 +97, 911, 0.00141712 +97, 911, 0.00086463 +97, 911, 0.00191104 +97, 912, 0.00068220 +97, 912, 0.00343061 +97, 912, 0.09602061 +97, 912, 0.01287428 +97, 913, 0.00261245 +97, 913, 0.00935899 +97, 913, 0.05128600 +97, 913, 0.00873801 +97, 914, 0.14467263 +97, 914, 0.21133940 +97, 914, 0.00071798 +97, 914, 0.00377781 +97, 915, 0.00471755 +97, 915, 0.00167079 +97, 915, 0.00256807 +97, 915, 0.06371378 +97, 916, 0.00703255 +97, 916, 0.42087454 +97, 916, 0.00371565 +97, 916, 0.04252050 +97, 917, 0.01889619 +97, 917, 0.01302692 +97, 917, 0.00381641 +97, 917, 0.00206172 +97, 918, 0.00390599 +97, 918, 0.00242760 +97, 918, 0.07317423 +97, 918, 0.00197168 +97, 919, 0.00121810 +97, 919, 0.00114853 +97, 919, 0.00382326 +97, 919, 0.00571352 +97, 920, 0.00155475 +97, 920, 0.00121991 +97, 920, 0.08828509 +97, 920, 0.02630284 +97, 921, 0.00075838 +97, 921, 0.00274044 +97, 921, 0.01050888 +97, 921, 0.00073239 +97, 922, 0.00682856 +97, 922, 0.00219782 +97, 922, 0.05482049 +97, 922, 0.00335560 +97, 923, 0.01127809 +97, 923, 0.00559710 +97, 923, 0.00133089 +97, 923, 0.11921301 +97, 924, 0.03365348 +97, 924, 0.01463120 +97, 924, 0.00299857 +97, 924, 0.00539352 +97, 925, 0.14841978 +97, 925, 0.00773426 +97, 925, 0.02891760 +97, 925, 0.00087112 +97, 926, 0.00301955 +97, 926, 0.00126787 +97, 926, 0.00090658 +97, 926, 0.00100352 +97, 927, 0.00069955 +97, 927, 0.00720768 +97, 927, 0.00597765 +97, 927, 0.00084145 +97, 928, 0.04262056 +97, 928, 0.02898032 +97, 928, 0.07358781 +97, 928, 0.00203219 +97, 929, 0.00291612 +97, 929, 0.48537311 +97, 929, 0.00207154 +97, 929, 0.00255542 +97, 930, 0.00853250 +97, 930, 0.00016893 +97, 930, 0.00701421 +97, 930, 0.01488066 +97, 931, 0.00373106 +97, 931, 0.00259902 +97, 931, 0.00210302 +97, 931, 0.00301050 +97, 932, 0.00520476 +97, 932, 0.01398258 +97, 932, 0.00291924 +97, 932, 0.01738336 +97, 933, 0.05783254 +97, 933, 0.00534084 +97, 933, 0.01909180 +97, 933, 0.00195653 +97, 934, 0.01515275 +97, 934, 0.01742635 +97, 934, 0.01367537 +97, 934, 0.04175452 +97, 935, 0.00260937 +97, 935, 0.03156637 +97, 935, 0.00631984 +97, 935, 0.00974362 +97, 936, 0.01011771 +97, 936, 0.00173066 +97, 936, 0.00432690 +97, 936, 0.03805338 +97, 937, 0.00209439 +97, 937, 0.00450347 +97, 937, 0.00463411 +97, 937, 0.00162309 +97, 938, 0.00623297 +97, 938, 0.03466664 +97, 938, 0.00675029 +97, 938, 0.00074578 +97, 939, 0.00305900 +97, 939, 0.00561394 +97, 939, 0.00439211 +97, 939, 0.00193843 +97, 940, 0.00197961 +97, 940, 0.01323610 +97, 940, 0.03520975 +97, 940, 0.00380338 +97, 941, 0.00127685 +97, 941, 0.00298964 +97, 941, 0.00165890 +97, 941, 0.00634040 +97, 942, 0.00575955 +97, 942, 0.00628426 +97, 942, 0.00038454 +97, 942, 0.01023422 +97, 943, 0.16227295 +97, 943, 0.00779964 +97, 943, 0.00114524 +97, 943, 0.05499278 +97, 944, 0.01199432 +97, 944, 0.07559940 +97, 944, 0.00541471 +97, 944, 0.00196052 +97, 945, 0.63562095 +97, 945, 0.04842107 +97, 945, 0.00136081 +97, 945, 0.00241510 +97, 946, 0.01157879 +97, 946, 0.01340608 +97, 946, 0.00269634 +97, 946, 0.00125632 +97, 947, 0.00562399 +97, 947, 0.00392503 +97, 947, 0.00741130 +97, 947, 0.00463435 +97, 948, 0.00369424 +97, 948, 0.00452606 +97, 948, 0.00191969 +97, 948, 0.02949647 +97, 949, 0.00158935 +97, 949, 0.00737355 +97, 949, 0.00611163 +97, 949, 0.00154843 +97, 950, 0.00139649 +97, 950, 0.00075013 +97, 950, 0.00540719 +97, 950, 0.00113777 +97, 951, 0.00907471 +97, 951, 0.00103566 +97, 951, 0.00597597 +97, 951, 0.00166409 +97, 952, 0.00108938 +97, 952, 0.02454032 +97, 952, 0.01940371 +97, 952, 0.00448642 +97, 953, 0.13788833 +97, 953, 0.05429479 +97, 953, 0.00526601 +97, 953, 0.00075303 +97, 954, 0.02878713 +97, 954, 0.00680576 +97, 954, 0.06683565 +97, 954, 0.01791453 +97, 955, 0.00523574 +97, 955, 0.00614917 +97, 955, 0.04372089 +97, 955, 0.05458012 +97, 956, 0.00122420 +97, 956, 0.00342484 +97, 956, 0.14948395 +97, 956, 0.01122390 +97, 957, 0.01347239 +97, 957, 0.07003888 +97, 957, 0.00224629 +97, 957, 0.00546850 +97, 958, 0.02734569 +97, 958, 0.00876783 +97, 958, 0.00142539 +97, 958, 0.00461080 +97, 959, 0.00577334 +97, 959, 0.02843091 +97, 959, 0.02933422 +97, 959, 0.00594337 +97, 960, 0.00627423 +97, 960, 0.00137924 +97, 960, 0.00084111 +97, 960, 0.00397357 +97, 961, 0.01484175 +97, 961, 0.01766224 +97, 961, 0.00842525 +97, 961, 0.00114888 +97, 962, 0.00658504 +97, 962, 0.00100485 +97, 962, 0.01020390 +97, 962, 0.00570276 +97, 963, 0.01068666 +97, 963, 0.00100679 +97, 963, 0.00286422 +97, 963, 0.00362948 +97, 964, 0.00131609 +97, 964, 0.44600534 +97, 964, 0.00217327 +97, 964, 0.03371754 +97, 965, 0.00310639 +97, 965, 0.00136407 +97, 965, 0.00054276 +97, 965, 0.00401266 +97, 966, 0.01053699 +97, 966, 0.00842164 +97, 966, 0.00452141 +97, 966, 0.24299021 +97, 967, 0.00996588 +97, 967, 0.00464918 +97, 967, 0.00142103 +97, 967, 0.00314139 +97, 968, 0.00303553 +97, 968, 0.63744551 +97, 968, 0.00356734 +97, 968, 0.00577862 +97, 969, 0.00491493 +97, 969, 0.00285904 +97, 969, 0.00159291 +97, 969, 0.00371533 +97, 970, 0.00338008 +97, 970, 0.01154121 +97, 970, 0.02799309 +97, 970, 0.00279694 +97, 971, 0.00053271 +97, 971, 0.25008658 +97, 971, 0.00107293 +97, 971, 0.00416232 +97, 972, 0.00134498 +97, 972, 0.00388716 +97, 972, 0.00152863 +97, 972, 0.00098353 +97, 973, 0.01373360 +97, 973, 0.00074333 +97, 973, 0.00541961 +97, 973, 0.00121336 +97, 974, 0.01331224 +97, 974, 0.00951859 +97, 974, 0.00244559 +97, 974, 0.01098684 +97, 975, 0.00204721 +97, 975, 0.00796058 +97, 975, 0.00600662 +97, 975, 0.01339086 +97, 976, 0.00063888 +97, 976, 0.03348644 +97, 976, 0.00304873 +97, 976, 0.00112037 +97, 977, 0.00328299 +97, 977, 0.00386477 +97, 977, 0.00289842 +97, 977, 0.00136752 +97, 978, 0.00611568 +97, 978, 0.00078717 +97, 978, 0.00153506 +97, 978, 0.02192712 +97, 979, 0.00093093 +97, 979, 0.00137645 +97, 979, 0.00541316 +97, 979, 0.00055927 +97, 980, 0.00225175 +97, 980, 0.00757056 +97, 980, 0.06722775 +97, 980, 0.00333898 +97, 981, 0.00160149 +97, 981, 0.00972550 +97, 981, 0.23000516 +97, 981, 0.00134525 +97, 982, 0.00160434 +97, 982, 0.00521334 +97, 982, 0.00281588 +97, 982, 0.03421547 +97, 983, 0.00063033 +97, 983, 0.00146553 +97, 983, 0.00607751 +97, 983, 0.00263343 +97, 984, 0.00222904 +97, 984, 0.00214270 +97, 984, 0.00621225 +97, 984, 0.00631357 +97, 985, 0.05627932 +97, 985, 0.00064158 +97, 985, 0.00885954 +97, 985, 0.00191247 +97, 986, 0.00867476 +97, 986, 0.12021582 +97, 986, 0.00954647 +97, 986, 0.00102437 +97, 987, 0.00436834 +97, 987, 0.00389007 +97, 987, 0.00521631 +97, 987, 0.00627852 +97, 988, 0.00169168 +97, 988, 0.00079706 +97, 988, 0.00105364 +97, 988, 0.00034110 +97, 989, 0.07729374 +97, 989, 0.00135164 +97, 989, 0.00096587 +97, 989, 0.13600352 +97, 990, 0.01635784 +97, 990, 0.00086832 +97, 990, 0.00109120 +97, 990, 0.03653363 +97, 991, 0.00132861 +97, 991, 0.00251371 +97, 991, 0.02895567 +97, 991, 0.00367546 +97, 992, 0.00363213 +97, 992, 0.00472645 +97, 992, 0.00427179 +97, 992, 0.01969523 +97, 993, 0.00553127 +97, 993, 0.00074491 +97, 993, 0.09563741 +97, 993, 0.00247775 +97, 994, 0.00290260 +97, 994, 0.00321233 +97, 994, 0.00737345 +97, 994, 0.00435079 +97, 995, 0.00361248 +97, 995, 0.00086571 +97, 995, 0.00871328 +97, 995, 0.00690827 +97, 996, 0.00334205 +97, 996, 0.00217056 +97, 996, 0.00233519 +97, 996, 0.01034915 +97, 997, 0.07802983 +97, 997, 0.01146265 +97, 997, 0.00125370 +97, 997, 0.00313468 +97, 998, 0.04085832 +97, 998, 0.04564139 +97, 998, 0.00204099 +97, 998, 0.00106480 +97, 999, 0.00298910 +97, 999, 0.00534509 +97, 999, 0.00112555 +97, 999, 0.00560922 diff --git a/src/yoke/helpers/cli.py b/src/yoke/helpers/cli.py index 30072f8b..8a999f3d 100644 --- a/src/yoke/helpers/cli.py +++ b/src/yoke/helpers/cli.py @@ -112,6 +112,20 @@ def add_filepath_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParse default=os.path.join(YOKE_PATH, "data_examples/lsc240420/"), help="Directory in which LSC *.npz files live.", ) + parser.add_argument( + "--NPZ_DIR", + action="store", + type=str, + default=os.path.join(YOKE_PATH, "data_examples/cx241203_fp16_half/"), + help="Directory in which CX *.npz files live.", + ) + parser.add_argument( + "--CSV_FILEPATH", + action="store", + type=str, + default=os.path.join(YOKE_PATH, "data_examples/design_cx241203_MASTER.csv"), + help="Filepath to CX csv file.", + ) parser.add_argument( "--NC_NPZ_DIR", action="store", @@ -289,6 +303,13 @@ def add_training_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParse default="./default_validation.csv", help="Filename for text file of validation loss and metrics on each batch", ) + parser.add_argument( + "--test_rcrd_filename", + action="store", + type=str, + default="./default_testing.csv", + help="Filename for text file of test evaluation loss and metrics on each batch", + ) parser.add_argument( "--continuation", action="store_true", diff --git a/src/yoke/utils/training/datastep/loderunner.py b/src/yoke/utils/training/datastep/loderunner.py index 600996ab..89bb444a 100644 --- a/src/yoke/utils/training/datastep/loderunner.py +++ b/src/yoke/utils/training/datastep/loderunner.py @@ -224,6 +224,71 @@ def train_DDP_loderunner_datastep( return end_img, pred_img, all_losses +def train_DDP_loderunner_datastep_cylex( + data: tuple, + model, + optimizer, + loss_fn, + device: torch.device, + rank: int, + world_size: int, +): + + """A DDP-compatible training step for multi-input, multi-output data. + + Args: + data (tuple): tuple of model input, corresponding ground truth, and lead time + model (loaded pytorch model): model to train + optimizer (torch.optim): optimizer for training set + loss_fn (torch.nn Loss Function): loss function for training set + device (torch.device): device index to select + rank (int): Rank of device + world_size (int): Number of total DDP processes + """ + # Extract data + start_img, channel_map, end_img, channel_map, Dt = data + print("In train_DDP_loderunner_datastep: channel_map =", channel_map) + + start_img = start_img.to(device, non_blocking=True) + Dt = Dt.to(device, non_blocking=True) + end_img = end_img.to(device, non_blocking=True) + + # Set model to train mode + model.train() + + # Map input and output variable indices according to channel map + in_vars = torch.tensor(channel_map).flatten().to(device, non_blocking=True) + out_vars = torch.tensor(channel_map).flatten().to(device, non_blocking=True) + + # Forward pass + pred_img = model(start_img, in_vars, out_vars, Dt) + + # Compute loss + loss = loss_fn(pred_img, end_img) + per_sample_loss = loss.mean(dim=[1, 2, 3]) # Per-sample loss + + # Backward pass and optimization + optimizer.zero_grad(set_to_none=True) + loss.mean().backward() + optimizer.step() + + # Gather per-sample losses from all processes + gathered_losses = [torch.zeros_like(per_sample_loss) for _ in range(world_size)] + dist.all_gather(gathered_losses, per_sample_loss) + + # Rank 0 concatenates and saves or returns all losses + if rank == 0: + all_losses = torch.cat(gathered_losses, dim=0) # Shape: (total_batch_size,) + + else: + all_losses = None + + # Free memory + del in_vars, out_vars + torch.cuda.empty_cache() + + return end_img, pred_img, all_losses + #################################### # Evaluating on a Datastep #################################### @@ -294,6 +359,58 @@ def eval_loderunner_datastep( return end_img, pred_img, per_sample_loss +def eval_loderunner_datastep_cylex( + data: tuple, + model: torch.nn.Module, + loss_fn: torch.nn.Module, + device: torch.device, + channel_map: None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """An evaluation step for which the data is of multi-input, multi-output type. + + This is currently a proto-type function to get the LodeRunner architecture + training on a non-variable set of channels. + + Args: + data (tuple): tuple of model input, corresponding ground truth, and lead time + model (torch.nn.Module): model to evaluate + loss_fn (torch.nn.Module): loss function for evaluation + device (torch.device): device index to select + channel_map (list[int]): list of channel indices to use + + Returns: + end_img (torch.Tensor): Ground truth end image + pred_img (torch.Tensor): Predicted end image + per_sample_loss (torch.Tensor): Per-sample loss for the batch + + """ + # Set model to train + model.eval() + + # Extract data + start_img, channel_map, end_img, channel_map, Dt = data + start_img = start_img.to(device, non_blocking=True) + Dt = Dt.to(device, non_blocking=True) + end_img = end_img.to(device, non_blocking=True) + + in_vars = torch.tensor(channel_map).flatten().to(device, non_blocking=True) + out_vars = torch.tensor(channel_map).flatten().to(device, non_blocking=True) + + # Perform a forward pass + # NOTE: If training on GPU model should have already been moved to GPU + # prior to initalizing optimizer. + pred_img = model(start_img, in_vars, out_vars, Dt) + + # Expecting to use a *reduction="none"* loss function so we can track loss + # between individual samples. However, this will make the loss be computed + # element-wise so we need to still average over the (channel, height, + # width) dimensions to get the per-sample loss. + loss = loss_fn(pred_img, end_img) + per_sample_loss = loss.mean(dim=[1, 2, 3]) # Shape: (batch_size,) + + return end_img, pred_img, per_sample_loss + + def eval_scheduled_loderunner_datastep( data: tuple, model: torch.nn.Module, @@ -419,3 +536,61 @@ def eval_DDP_loderunner_datastep( all_losses = None return end_img, pred_img, all_losses + + +def eval_DDP_loderunner_datastep_cylex( + data: tuple, + model, + loss_fn, + device: torch.device, + rank: int, + world_size: int, +): + """A DDP-compatible evaluation step. + + Args: + data (tuple): tuple of model input, corresponding ground truth, and lead time + model (loaded pytorch model): model to train + loss_fn (torch.nn Loss Function): loss function for training set + device (torch.device): device index to select + rank (int): Rank of device + world_size (int): Total number of DDP processes + + """ + print("starting in eval_DDP_loderunner_datastep") + # Set model to evaluation mode + model.eval() + + # Extract data + start_img, channel_map, end_img, channel_map, Dt = data + start_img = start_img.to(device, non_blocking=True) + Dt = Dt.to(device, non_blocking=True) + end_img = end_img.to(device, non_blocking=True) + + # Map input and output variable indices according to channel map + in_vars = torch.tensor(channel_map).flatten().to(device, non_blocking=True) + out_vars = torch.tensor(channel_map).flatten().to(device, non_blocking=True) + + # Forward pass + with torch.no_grad(): + pred_img = model(start_img, in_vars, out_vars, Dt) + + # Compute loss + loss = loss_fn(pred_img, end_img) + per_sample_loss = loss.mean(dim=[1, 2, 3]) # Per-sample loss + + # Gather per-sample losses from all processes + gathered_losses = [torch.zeros_like(per_sample_loss) for _ in range(world_size)] + dist.all_gather(gathered_losses, per_sample_loss) + + # Rank 0 concatenates and saves or returns all losses + if rank == 0: + all_losses = torch.cat(gathered_losses, dim=0) # Shape: (total_batch_size,) + else: + all_losses = None + + # Free memory + del in_vars, out_vars + torch.cuda.empty_cache() + + return end_img, pred_img, all_losses diff --git a/src/yoke/utils/training/epoch/loderunner.py b/src/yoke/utils/training/epoch/loderunner.py index 26629e4e..4b51d41c 100644 --- a/src/yoke/utils/training/epoch/loderunner.py +++ b/src/yoke/utils/training/epoch/loderunner.py @@ -8,12 +8,27 @@ from yoke.utils.training.datastep.loderunner import ( train_loderunner_datastep, eval_loderunner_datastep, + eval_loderunner_datastep_cylex, train_scheduled_loderunner_datastep, eval_scheduled_loderunner_datastep, train_DDP_loderunner_datastep, eval_DDP_loderunner_datastep, + train_DDP_loderunner_datastep_cylex, + eval_DDP_loderunner_datastep_cylex, ) +DATASTEP_FN = { + "pli": { + "train_ddp": train_DDP_loderunner_datastep, + "eval_ddp": eval_DDP_loderunner_datastep, + "eval": eval_loderunner_datastep + }, + "cylex": { + "train_ddp": train_DDP_loderunner_datastep_cylex, + "eval_ddp": eval_DDP_loderunner_datastep_cylex, + "eval": eval_loderunner_datastep_cylex + }, +} def train_simple_loderunner_epoch( channel_map: list, @@ -346,6 +361,7 @@ def train_DDP_loderunner_epoch( device: torch.device, rank: int, world_size: int, + dataset: str = "pli", ) -> None: """Distributed data-parallel LodeRunner Epoch. @@ -370,6 +386,7 @@ def train_DDP_loderunner_epoch( device (torch.device): device index to select rank (int): rank of process world_size (int): number of total processes + dataset (str): Name of dataset to train on. Options are "pli" and "cylex". """ # Initialize things to save @@ -386,9 +403,16 @@ def train_DDP_loderunner_epoch( # Stop when number of training batches is reached if trainbatch_ID >= num_train_batches: break + + # Get correct datastep function + dataset_fns = DATASTEP_FN.get(dataset) + if dataset_fns is None: + raise ValueError(f"Unsupported dataset: {dataset}") + + train_fn = dataset_fns["train_ddp"] # Perform a single training step - truth, pred, train_losses = train_DDP_loderunner_datastep( + truth, pred, train_losses = train_fn( traindata, model, optimizer, loss_fn, device, rank, world_size ) @@ -419,8 +443,9 @@ def train_DDP_loderunner_epoch( # Stop when number of training batches is reached if valbatch_ID >= num_val_batches: break - - end_img, pred_img, val_losses = eval_DDP_loderunner_datastep( + + eval_fn = dataset_fns["eval_ddp"] + end_img, pred_img, val_losses = eval_fn( valdata, model, loss_fn, @@ -439,3 +464,71 @@ def train_DDP_loderunner_epoch( ] ) np.savetxt(val_rcrd_file, batch_records, fmt="%d, %d, %.8f") + + +def eval_loderunner_epoch( + testing_data: torch.utils.data.DataLoader, + num_test_batches: int, + model: torch.nn.Module, + channel_map: list[int], + loss_fn: torch.nn.Module, + epochIDX: int, + test_rcrd_filename: str, + device: torch.device, + dataset: str = "pli", +) -> None: + """LodeRunner Evaluation-Only Epoch. + + Function to complete a testing epoch on the LodeRunner architecture with + fixed channels in the input and output. Testing information is saved to successive + CSV files. + + Args: + testing_data (torch.utils.data.DataLoader): testing dataloader + num_test_batches (int): Number of batches in training epoch + model (torch.nn.Module): model to train + channel_map (list[int]): list of channel indices to use + loss_fn (torch.nn.Module): loss function for training set + epochIDX (int): Index of current training epoch + test_rcrd_filename (str): Name of CSV file to save testing sample stats to + device (torch.device): device index to select + + """ + # Initialize things to save + testbatch_ID = 0 + + # Testing loop + model.eval() + test_rcrd_filename = test_rcrd_filename.replace("", f"{epochIDX:04d}") + + # Get correct datastep function + dataset_fns = DATASTEP_FN.get(dataset) + if dataset_fns is None: + raise ValueError(f"Unsupported dataset: {dataset}") + + eval_fn = dataset_fns["eval"] + + with open(test_rcrd_filename, "a") as test_rcrd_file: + for testbatch_ID, testdata in enumerate(testing_data): + # Stop when number of training batches is reached + if testbatch_ID >= num_test_batches: + break + + # Perform a single test step + end_img, pred_img, test_losses = eval_fn( + testdata, + model, + loss_fn, + device, + channel_map, + ) + + # Save testing record + batch_records = np.column_stack( + [ + np.full(len(test_losses), epochIDX), + np.full(len(test_losses), testbatch_ID), + test_losses.cpu().detach().numpy().flatten(), + ] + ) + np.savetxt(test_rcrd_file, batch_records, fmt="%d, %d, %.8f")