Archive sbatch script

beeflow/common/worker/slurm_worker.py

@@ -216,6 +216,7 @@ def submit_task(self, task):
             task.stdout = stdout
             task.stderr = stderr
             task_script = sbatch_script
+            self.save_script(task,task_script)
         else:
             task_script = self.write_script(task)
 

beeflow/common/worker/utils.py

@@ -63,9 +63,7 @@ def format_runtime(runtime):
 def calc_runtime(job_state,job_info):
     """Calculate the runtime of a task when using slurmrestd."""
     start_time = job_info['start_time']['number']
-    log.info(f"Start time: {start_time}")
     end_time = job_info['end_time']['number']
-    log.info(f"End time: {end_time}")
 
     cur_time = dt.now()
     cur_unix_time = int(cur_time.timestamp())

beeflow/common/worker/worker.py

@@ -5,6 +5,7 @@
 
 from abc import ABC, abstractmethod
 import os
+import shutil
 from beeflow.common import log as bee_logging
 from beeflow.common.crt_interface import ContainerRuntimeInterface
 
@@ -73,6 +74,21 @@ def task_save_path(self, task):
         """Return the task save path used for storing submission scripts output logs."""
         return f'{self.workdir}/workflows/{task.workflow_id}/{task.name}-{task.id[:4]}'
 
+    def save_script(self,task,sbatch_script):
+        """Save provided sbatch script into the appropriate archive and workflow directories"""
+        sbatch_script_name = os.path.basename(sbatch_script)
+
+        sbatch_archive_dir = self.task_save_path(task)
+        os.makedirs(sbatch_archive_dir,exist_ok=True)
+        sbatch_script_archive = f"{sbatch_archive_dir}/{sbatch_script_name}"
+
+        sbatch_script_dir = f"{task.workdir}/{task.name}-{task.id[:4]}"
+        os.makedirs(sbatch_script_dir, exist_ok=True)
+        sbatch_script_workdir = f"{sbatch_script_dir}/{sbatch_script_name}"
+
+        shutil.copy(sbatch_script, sbatch_script_archive)
+        shutil.copy(sbatch_script, sbatch_script_workdir)
+
     def write_script(self, task):
         """Build task script; returns filename of script."""
         # If the user has provided an sbatch script this will just return

beeflow/wf_manager/resources/wf_utils.py

@@ -271,7 +271,7 @@ def copy_task_output(task):
     # Need to get this from the worker
     task_save_path = pathlib.Path(bee_workdir) / "workflows" / task.workflow_id / task_name_id
     task_workdir = pathlib.Path(task.workdir)
-    task_metadata_path = task_workdir / task_name_id / "metadata.txt"
+    task_metadata_path = task_workdir / task_name_id / "metadata.yaml"
     if task.stdout is not None:
         stdout_path = _resolve_output_path(task_workdir, task.stdout)
     else:
@@ -286,7 +286,7 @@ def copy_task_output(task):
     shutil.copy(stderr_path, task_save_path / f"{task_name_id}.err")
 
     if task_metadata_path.exists():
-        shutil.copy(task_metadata_path, task_save_path / "metadata.txt")
+        shutil.copy(task_metadata_path, task_save_path / "metadata.yaml")
     else:
         log.warning("Task metadata file not found: %s", task_metadata_path)