diff --git a/_episodes/01-analysis_scripts_and_snakemake.md b/_episodes/01-analysis_scripts_and_snakemake.md index a9df5fa..8f38856 100644 --- a/_episodes/01-analysis_scripts_and_snakemake.md +++ b/_episodes/01-analysis_scripts_and_snakemake.md @@ -77,18 +77,9 @@ In order to demonstrate the advantages of using snakefiles, let's start using th First let's use snakemake to grab some simulation campaign files from the online storage space. In your `tutorial_directory/starting_script/` directory make a new file called `Snakefile`. Open the file and add these lines: ```python -import os - -# Set environment mode (local or eicweb) -ENV_MODE = os.getenv("ENV_MODE", "local") # Defaults to "local" if not set -# Output directory based on environment -OUTPUT_DIR = "../../sim_output/" if ENV_MODE == "eicweb" else "sim_output/" -# Benchmark directory based on environment -BENCH_DIR = "benchmarks/your_benchmark/" if ENV_MODE == "eicweb" else "./" - rule your_benchmark_campaign_reco_get: output: - f"{OUTPUT_DIR}rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", + f"sim_output/rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", shell: """ xrdcp root://dtn-eic.jlab.org//volatile/eic/EPIC/RECO/24.07.0/epic_craterlake/EXCLUSIVE/UCHANNEL_RHO/10x100/rho_10x100_uChannel_Q2of0to10_hiDiv.{wildcards.INDEX}.eicrecon.tree.edm4eic.root {output} """ @@ -96,8 +87,6 @@ xrdcp root://dtn-eic.jlab.org//volatile/eic/EPIC/RECO/24.07.0/epic_craterlake/EX If you're having trouble copying and pasting, you can also copy from [here](https://github.com/eic/tutorial-developing-benchmarks/blob/gh-pages/files/Snakefile). -Thinking ahead to when we want to put our benchmark on eicweb, we add this `ENV_MODE` variable which allows us to specify paths differently based on whether we're running locally or in GitLab's pipelines. - We also defined a new rule: `your_benchmark_campaign_reco_get`. This rule defines how to download a single file from the JLab servers to the location `sim_output`. After saving the Snakefile, let's try running it. @@ -120,10 +109,10 @@ But the benefits from using Snakemake become more apparent as the number of task ```python rule your_benchmark_analysis: input: - script=f"{BENCH_DIR}analysis/uchannelrho.cxx", - data=f"{OUTPUT_DIR}rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", + script=workflow.source_path("analysis/uchannelrho.cxx"), + data=f"sim_output/rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", output: - plots=f"{OUTPUT_DIR}campaign_24.07.0_{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", + plots=f"sim_output/campaign_24.07.0_{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", shell: """ mkdir -p $(dirname "{output.plots}") @@ -131,7 +120,7 @@ root -l -b -q '{input.script}+("{input.data}","{output.plots}")' """ ``` -This rule runs an analysis script to create ROOT files containing plots. The rule uses the simulation campaign file downloaded from JLab as input data, and it runs the analysis script `uchannelrho.cxx`. +This rule runs an analysis script to create ROOT files containing plots. The rule uses the simulation campaign file downloaded from JLab as input data, and it runs the analysis script `uchannelrho.cxx`. Note that we use `workflow.source_path()` to reference the script - this function returns the correct path to the script relative to the Snakefile's location in the benchmark directory. Now let's request the output file `"sim_output/campaign_24.07.0_0005.eicrecon.tree.edm4eic/plots.root"`. When we request this, Snakemake will identify that it needs to run the new `your_benchmark_analysis` rule. But in order to do this, it now needs a file we don't have: `sim_output/rho_10x100_uChannel_Q2of0to10_hiDiv.0005.eicrecon.tree.edm4eic.root` because we only downloaded the file with index `0000` already. What Snakemake will do automatically is recognize that in order to get that file, it first needs to run the `your_benchmark_campaign_reco_get` rule. It will do this first, and then circle back to the `your_benchmark_analysis` rule. @@ -154,13 +143,13 @@ That's still not very impressive. Snakemake gets more useful when we want to run rule your_benchmark_combine: input: lambda wildcards: expand( - f"{OUTPUT_DIR}campaign_24.07.0_{% raw %}{{INDEX:04d}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", + f"sim_output/campaign_24.07.0_{% raw %}{{INDEX:04d}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", INDEX=range(int(wildcards.N)), ), wildcard_constraints: N="\d+", output: - f"{OUTPUT_DIR}campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", + f"sim_output/campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", shell: """ hadd {output} {input} @@ -185,10 +174,10 @@ Now let's add one more rule to create benchmark plots: ```python rule your_benchmark_plots: input: - script=f"{BENCH_DIR}macros/plot_rho_physics_benchmark.C", - plots=f"{OUTPUT_DIR}campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", + script=workflow.source_path("macros/plot_rho_physics_benchmark.C"), + plots=f"sim_output/campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", output: - f"{OUTPUT_DIR}campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots_figures/benchmark_rho_mass.pdf", + f"sim_output/campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots_figures/benchmark_rho_mass.pdf", shell: """ if [ ! -d "{input.plots}_figures" ]; then @@ -227,28 +216,20 @@ If we want to scale up the plots to include 15 simulation campaign files instead The final Snakefile should look like this: ```python -import os - -# Set environment mode (local or eicweb) -ENV_MODE = os.getenv("ENV_MODE", "local") # Defaults to "local" if not set -# Output directory based on environment -OUTPUT_DIR = "../../sim_output/" if ENV_MODE == "eicweb" else "sim_output/" -# Benchmark directory based on environment -BENCH_DIR = "benchmarks/your_benchmark/" if ENV_MODE == "eicweb" else "./" - rule your_benchmark_campaign_reco_get: output: - f"{OUTPUT_DIR}rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", + f"sim_output/rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", + retries: 3 shell: """ xrdcp root://dtn-eic.jlab.org//volatile/eic/EPIC/RECO/24.07.0/epic_craterlake/EXCLUSIVE/UCHANNEL_RHO/10x100/rho_10x100_uChannel_Q2of0to10_hiDiv.{wildcards.INDEX}.eicrecon.tree.edm4eic.root {output} """ rule your_benchmark_analysis: input: - script=f"{BENCH_DIR}analysis/uchannelrho.cxx", - data=f"{OUTPUT_DIR}rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", + script=workflow.source_path("analysis/uchannelrho.cxx"), + data=f"sim_output/rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", output: - plots=f"{OUTPUT_DIR}campaign_24.07.0_{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", + plots=f"sim_output/campaign_24.07.0_{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", shell: """ mkdir -p $(dirname "{output.plots}") @@ -258,13 +239,13 @@ root -l -b -q '{input.script}+("{input.data}","{output.plots}")' rule your_benchmark_combine: input: lambda wildcards: expand( - f"{OUTPUT_DIR}campaign_24.07.0_{% raw %}{{INDEX:04d}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", + f"sim_output/campaign_24.07.0_{% raw %}{{INDEX:04d}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", INDEX=range(int(wildcards.N)), ), wildcard_constraints: N="\d+", output: - f"{OUTPUT_DIR}campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", + f"sim_output/campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", shell: """ hadd {output} {input} @@ -272,10 +253,10 @@ hadd {output} {input} rule your_benchmark_plots: input: - script=f"{BENCH_DIR}macros/plot_rho_physics_benchmark.C", - plots=f"{OUTPUT_DIR}campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", + script=workflow.source_path("macros/plot_rho_physics_benchmark.C"), + plots=f"sim_output/campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", output: - f"{OUTPUT_DIR}campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots_figures/benchmark_rho_mass.pdf", + f"sim_output/campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots_figures/benchmark_rho_mass.pdf", shell: """ if [ ! -d "{input.plots}_figures" ]; then diff --git a/_episodes/02-first_benchmark.md b/_episodes/02-first_benchmark.md index 3ac364a..98b91bf 100644 --- a/_episodes/02-first_benchmark.md +++ b/_episodes/02-first_benchmark.md @@ -52,7 +52,7 @@ mkdir benchmarks/your_benchmark The Continuous Integration system needs to know what steps it has to execute. This is specified using YAML files. Create a file `benchmarks/your_benchmark/config.yml`. -For a physics benchmark and to follow along with this tutorial, create a `config.yml` with the following contents: +For a physics benchmark, create a `config.yml` with the following contents: ~~~ your_benchmark:compile: extends: .phy_benchmark @@ -69,6 +69,8 @@ your_benchmark:simulate: your_benchmark:results: extends: .phy_benchmark stage: collect + needs: + - ["your_benchmark:simulate"] script: - echo "I will collect results here!" diff --git a/_episodes/03-filling_out_your_benchmark.md b/_episodes/03-filling_out_your_benchmark.md index 629ef50..8baf509 100644 --- a/_episodes/03-filling_out_your_benchmark.md +++ b/_episodes/03-filling_out_your_benchmark.md @@ -24,8 +24,6 @@ Create a new file: [`benchmarks/your_benchmark/setup.config`](https://github.com #!/bin/bash source strict-mode.sh -export ENV_MODE=eicweb - USE_SIMULATION_CAMPAIGN=true N_EVENTS=100 diff --git a/files/Snakefile b/files/Snakefile index df438ea..272893e 100644 --- a/files/Snakefile +++ b/files/Snakefile @@ -1,26 +1,19 @@ import os -from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider - -# Set environment mode (local or eicweb) -ENV_MODE = os.getenv("ENV_MODE", "local") # Defaults to "local" if not set -# Output directory based on environment -OUTPUT_DIR = "../../sim_output/" if ENV_MODE == "eicweb" else "sim_output/" -# Benchmark directory based on environment -BENCH_DIR = "benchmarks/your_benchmark/" if ENV_MODE == "eicweb" else "./" rule your_benchmark_campaign_reco_get: output: - f"{OUTPUT_DIR}rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", + f"sim_output/rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", + retries: 3 shell: """ xrdcp root://dtn-eic.jlab.org//volatile/eic/EPIC/RECO/24.07.0/epic_craterlake/EXCLUSIVE/UCHANNEL_RHO/10x100/rho_10x100_uChannel_Q2of0to10_hiDiv.{wildcards.INDEX}.eicrecon.tree.edm4eic.root {output} """ rule your_benchmark_analysis: input: - script=f"{BENCH_DIR}analysis/uchannelrho.cxx", - data=f"{OUTPUT_DIR}rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", + script=workflow.source_path("analysis/uchannelrho.cxx"), + data=f"sim_output/rho_10x100_uChannel_Q2of0to10_hiDiv.{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic.root", output: - plots=f"{OUTPUT_DIR}campaign_24.07.0_{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", + plots=f"sim_output/campaign_24.07.0_{% raw %}{{INDEX}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", shell: """ mkdir -p $(dirname "{output.plots}") @@ -30,13 +23,13 @@ root -l -b -q '{input.script}+("{input.data}","{output.plots}")' rule your_benchmark_combine: input: lambda wildcards: expand( - f"{OUTPUT_DIR}campaign_24.07.0_{% raw %}{{INDEX:04d}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", + f"sim_output/campaign_24.07.0_{% raw %}{{INDEX:04d}}{% endraw %}.eicrecon.tree.edm4eic/plots.root", INDEX=range(int(wildcards.N)), ), wildcard_constraints: N="\d+", output: - f"{OUTPUT_DIR}campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", + f"sim_output/campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", shell: """ hadd {output} {input} @@ -44,10 +37,10 @@ hadd {output} {input} rule your_benchmark_plots: input: - script=f"{BENCH_DIR}macros/plot_rho_physics_benchmark.C", - plots=f"{OUTPUT_DIR}campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", + script=workflow.source_path("macros/plot_rho_physics_benchmark.C"), + plots=f"sim_output/campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots.root", output: - f"{OUTPUT_DIR}campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots_figures/benchmark_rho_mass.pdf", + f"sim_output/campaign_24.07.0_combined_{% raw %}{{N}}{% endraw %}files.eicrecon.tree.edm4eic.plots_figures/benchmark_rho_mass.pdf", shell: """ if [ ! -d "{input.plots}_figures" ]; then diff --git a/files/setup.config b/files/setup.config index e3c8886..b0d6f2f 100644 --- a/files/setup.config +++ b/files/setup.config @@ -1,8 +1,6 @@ #!/bin/bash source strict-mode.sh -export ENV_MODE=eicweb - USE_SIMULATION_CAMPAIGN=true N_EVENTS=100