Offsets

docs/metadata.md

@@ -67,8 +67,10 @@ and `ts.metadata["SLiM"]` contains information about the simulation:
     * `index`: the index of the trait in SLiM
     * `name`: the name in SLiM for the trait
     * `type`: additive, multiplicative, or logistic
-    * `baselineOffset`, `baselineAccumulation`: a value added to all traits, and whether the effect of substitutions
-      accumulate in that value
+    * `baselineOffsetFromUser`: a value added to all traits
+    * `baselineOffsetFromSubstitutions`: the total effect of all substitutions on the traits
+      at the time the tree sequence was saved
+    * `baselineAccumulation`: whether the effect of substitutions accumulate in that value
     * `directFitnessEffect`: whether the trait has a direct effect on fitness
     * `individualOffsetMean`, `individualOffsetSD`: parameters governing the individual-level offsets
       (i.e., "environment" effects)
@@ -148,7 +150,7 @@ we might do:
 md = ts.metadata
 traits = md["SLiM"]["traits"]
 values = [
-    [x['phenotype'] - y["baselineOffset"] for x, y in zip(ind.metadata['per_trait'], traits)]
+    [x['phenotype'] - y["baselineOffsetFromUser"] for x, y in zip(ind.metadata['per_trait'], traits)]
     for ind in ts.individuals()
 ]
 ```

pyslim/slim_metadata.py

@@ -186,9 +186,13 @@ def is_vacant_num_bytes(num_chromosomes):
                                         "description": "Whether the baseline offset includes accumulated effects from fixed (substituted) mutations.",
                                         "type": "boolean",
                                     },
-                                    "baselineOffset": {
-                                        "description": "The baseline offset of the trait.",
+                                    "baselineOffsetFromUser": {
                                         "type": "number",
+                                        "description": "The from-user component of the baseline offset of the trait.",
+                                    },
+                                    "baselineOffsetFromSubstitutions": {
+                                        "type": "number",
+                                        "description": "The from-substitutions component of the baseline offset of the trait.",
                                     },
                                     "directFitnessEffect": {
                                         "description": "Whether the trait's effects are used directly as fitness effects.",

tests/recipe_specs.py

@@ -3,6 +3,7 @@
 # possible attributes to simulation scripts are
 #  (the script name itself, to specify a particular one)
 #  WF, nonWF
+#  minimal: a minimal set of recipes to test basic things with
 #  adds_mutations
 #  nucleotides, non-nucleotides: has the respective sorts of mutations
 #  mutation_spectrum: writes out info file on mutation spectrum
@@ -21,10 +22,15 @@
 #  (chromosome type)
 # All files are of the form `tests/test_recipes/{key}`
 recipe_specs = {
-    "recipe_nonWF.slim": {"nonWF": True, "pedigree": True, "record_mutations": True},
-    "recipe_nonWF_X.slim": {"nonWF": True, "pedigree": True, "X": True},
+    "recipe_nonWF.slim": {
+        "nonWF": True,
+        "pedigree": True,
+        "record_mutations": True,
+        "minimal": True,
+    },
+    "recipe_nonWF_X.slim": {"nonWF": True, "pedigree": True, "X": True, "minimal": True},
     "recipe_nonWF_Y.slim": {"nonWF": True, "pedigree": True, "Y": True},
-    "recipe_nonWF_H.slim": {"nonWF": True, "pedigree": True, "H": True},
+    "recipe_nonWF_H.slim": {"nonWF": True, "pedigree": True, "H": True, "minimal": True},
     "recipe_WF_X.slim": {
         "WF": True,
         "pedigree": True,
@@ -83,7 +89,12 @@
     },
     "recipe_long_nonWF.slim": {"nonWF": True, "long": True},
     "recipe_old_nonWF.slim": {"nonWF": True, "remembered_first": True},
-    "recipe_WF.slim": {"WF": True, "pedigree": True, "record_mutations": True},
+    "recipe_WF.slim": {
+        "WF": True,
+        "pedigree": True,
+        "record_mutations": True,
+        "minimal": True,
+    },
     "recipe_no_simplify.slim": {"WF": True, "no_simplify": True},
     "recipe_long_WF.slim": {"WF": True, "long": True},
     "recipe_WF_migration.slim": {"WF": True, "pedigree": True, "multipop": True},
@@ -93,6 +104,7 @@
         "nucleotides": True,
         "record_mutations": True,
         "refseq": True,
+        "minimal": True,
     },
     "recipe_nucleotides_nonWF.slim": {
         "nonWF": True,
@@ -183,6 +195,7 @@
         "HM": True,
         "ML": True,
         "Y-": True,
+        "minimal": True,
     },
     "recipe_chromosomes_adds_muts.slim": {
         "WF": True,
@@ -214,6 +227,7 @@
         "X": True,
         "Y": True,
         "H": True,
+        "minimal": True,
     },
 }
 

tests/test_annotation.py

@@ -42,7 +42,19 @@ def canonicalise_tables(tables):
     tables.canonicalise()
 
 
-def verify_slim_restart_equality(in_ts_dict, out_ts_dict, check_prov=True):
+def invalidate_phenotypes(tables):
+    indivs = tables.individuals.copy()
+    tables.individuals.clear()
+    for ind in indivs:
+        md = ind.metadata
+        for x in md["per_trait"]:
+            x["phenotype"] = np.nan
+        tables.individuals.append(ind.replace(metadata=md))
+
+
+def verify_slim_restart_equality(
+    in_ts_dict, out_ts_dict, check_prov=True, phenotypes=True
+):
     """
     Check for equality, in everything but the last provenance.
     """
@@ -56,6 +68,9 @@ def verify_slim_restart_equality(in_ts_dict, out_ts_dict, check_prov=True):
         canonicalise_tables(in_tables)
         out_tables = out_ts.dump_tables()
         canonicalise_tables(out_tables)
+        if not phenotypes:
+            invalidate_phenotypes(in_tables)
+            invalidate_phenotypes(out_tables)
         in_tables.assert_equals(out_tables, ignore_provenance=True)
 
 
@@ -585,6 +600,7 @@ def test_empty_populations(self, helper_functions, tmp_path):
                 WF=False,
                 subpop_map=subpop_map,
                 multichrom=False,
+                PHENOTYPES=False,
             )["default"]
             self.verify_remapping(ts, sts, subpop_map)
 
@@ -670,6 +686,7 @@ def test_many_species(self, helper_functions, tmp_path):
                     tmp_path,
                     WF=wf,
                     subpop_map=subpop_map,
+                    PHENOTYPES=False,
                 )
                 self.verify_remapping(fox_ts, tsd["fox"], subpop_map["fox"])
                 self.verify_remapping(mouse_ts, tsd["mouse"], subpop_map["mouse"])
@@ -797,6 +814,7 @@ def test_remapping_empty_pops(self, helper_functions, tmp_path):
                 multichrom=False,
                 WF=False,
                 subpop_map=subpop_map,
+                PHENOTYPES=False,
             )["default"]
             self.verify_remapping(ts, rts, subpop_map)
 
@@ -863,6 +881,7 @@ def test_remapping(self, helper_functions, tmp_path):
                 multichrom=False,
                 WF=True,
                 subpop_map=subpop_map,
+                PHENOTYPES=False,
             )["default"]
             self.verify_remapping(ts, rts, subpop_map)
 
@@ -934,6 +953,7 @@ def test_reload_recapitate(self, restart_name, recipe, helper_functions, tmp_pat
             tmp_path,
             "multichrom" in recipe,
             WF=False,
+            PHENOTYPES=True,
         )
         # check for equality, in everything but the last provenance
         verify_slim_restart_equality(in_ts, out_ts)
@@ -947,6 +967,7 @@ def test_reload_recapitate(self, restart_name, recipe, helper_functions, tmp_pat
                 tmp_path,
                 "multichrom" in recipe,
                 subpop_map=subpop_map,
+                PHENOTYPES=True,
             )
             assert in_ts.keys() == out_ts.keys()
             for k in in_ts:
@@ -982,10 +1003,13 @@ def test_reload_annotate(self, restart_name, recipe, helper_functions, tmp_path)
             in_ts[chrom] = tables.tree_sequence()
         # put it through SLiM (which just reads in and writes out)
         out_ts = helper_functions.run_slim_restart(
-            in_ts, restart_name, tmp_path, "multichrom" in recipe
+            in_ts,
+            restart_name,
+            tmp_path,
+            "multichrom" in recipe,
         )
         # check for equality, in everything but the last provenance
-        verify_slim_restart_equality(in_ts, out_ts)
+        verify_slim_restart_equality(in_ts, out_ts, phenotypes=False)
 
 
 class TestReload(tests.PyslimTestCase):

tests/test_metadata.py

@@ -2,6 +2,8 @@
 Test cases for the metadata reading/writing of pyslim.
 """
 
+import sys
+
 import numpy as np
 import pytest
 import tskit
@@ -91,6 +93,11 @@ def test_default_metadata(self):
                 decoded = schema.decode_row(encoded)
                 assert entry == decoded
 
+    @pytest.mark.skipif(
+        sys.platform.startswith("win"),
+        reason="failing because of dict and OrderedDict comparison?",
+    )
+    @pytest.mark.parametrize("recipe", recipe_eq("minimal"), indirect=True)
     def test_slim_metadata_schema_equality(self, recipe):
         num_chromosomes = len(recipe["ts"])
         for ts in recipe["ts"].values():

tests/test_recipes/recipe_WF.slim

@@ -52,6 +52,7 @@ mutation() {
 
 // OUTPUT/FINISH
 10 late() {
+    sim.demandPhenotype(NULL);
     sim.treeSeqOutput(TREES_FILE, metadata=MD);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/recipe_chromosomes_adds_muts.slim

@@ -9,7 +9,7 @@ initialize() {
 	initializeSex();
 	initializeMutationTypeNuc("m1", 0.5, "f", 0.0);
 	initializeMutationType("m2", 0.5, "n", 0.0, 0.1);
-	initializeGenomicElementType("g1", m1, 1.0, mmJukesCantor(4e-4));
+	initializeGenomicElementType("g1", m1, 1.0, mmJukesCantor(1e-4));
 	m1.convertToSubstitution = T;
 
 	types = c("A", "X", "Y", "H", "Z", "W", "HF", "FL", "HM", "ML", "-Y");

tests/test_recipes/recipe_nucleotides_WF.slim

@@ -62,6 +62,7 @@ mutation() {
 
 // OUTPUT/FINISH
 10 late() {
+    sim.demandPhenotype(NULL);
     sim.treeSeqOutput(TREES_FILE, metadata=MD);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/recipe_nucleotides_nonWF.slim

@@ -71,6 +71,7 @@ mutation() {
 
 // OUTPUT/FINISH
 10 late() {
+    sim.demandPhenotype(NULL);
     sim.treeSeqOutput(TREES_FILE, metadata=MD);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/recipe_record_everyone_WF_early.slim

@@ -23,7 +23,7 @@ early() {
     sim.treeSeqRememberIndividuals(p1.individuals);
 }
 
-10 early() {
+5 early() {
     sim.treeSeqOutput(TREES_FILE);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/recipe_record_everyone_WF_first.slim

@@ -23,7 +23,7 @@ initialize()
     sim.treeSeqRememberIndividuals(p1.individuals);
 }
 
-10 first() {
+5 first() {
     sim.treeSeqOutput(TREES_FILE);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/recipe_record_everyone_WF_late.slim

@@ -24,7 +24,7 @@ late() {
     sim.treeSeqRememberIndividuals(p1.individuals);
 }
 
-10 late() {
+5 late() {
     sim.treeSeqOutput(TREES_FILE);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/recipe_record_everyone_nonWF_early.slim

@@ -32,7 +32,7 @@ early() {
     sim.treeSeqRememberIndividuals(p1.individuals);
 }
 
-10 early() {
+5 early() {
     sim.treeSeqOutput(TREES_FILE);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/recipe_record_everyone_nonWF_first.slim

@@ -32,7 +32,7 @@ early() {
     sim.treeSeqRememberIndividuals(p1.individuals);
 }
 
-10 first() {
+5 first() {
     sim.treeSeqOutput(TREES_FILE);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/recipe_record_everyone_nonWF_late.slim

@@ -32,7 +32,7 @@ late() {
     sim.treeSeqRememberIndividuals(p1.individuals);
 }
 
-10 late() {
+5 late() {
     sim.treeSeqOutput(TREES_FILE);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/recipe_resettable_WF.slim

@@ -22,6 +22,7 @@ initialize()
 }
 
 10 late() {
+    sim.demandPhenotype(NULL);
     sim.treeSeqOutput(TREES_FILE, metadata=METADATA);
     catn("Done.");
     sim.simulationFinished();

tests/test_recipes/restart_WF.slim

@@ -4,6 +4,7 @@ initialize()
     if (!exists("TREES_FILE")) defineGlobal("TREES_FILE", "out.trees"); 
     // as well as any bespoke TREES_FILE for output, you need to define:
     //  TREES_IN = path to trees to initialise
+    if (!exists("PHENOTYPES")) defineGlobal("PHENOTYPES", T);
     initializeTreeSeq(timeUnit="generations");
     initializeMutationRate(1e-2);
     initializeMutationType("m1", 0.5, "f", -0.1);
@@ -18,13 +19,15 @@ initialize()
   if (STAGE == "early") {
     md = treeSeqMetadata(TREES_IN, userData=F).getValue("SLiM").getValue("user_metadata");
     sim.readFromPopulationFile(TREES_IN, subpopMap=SUBPOP_MAP);
+    if (PHENOTYPES) sim.demandPhenotype(NULL);
     sim.treeSeqOutput(TREES_FILE, simplify=F, metadata=md);
   }
 }
 1 late() { 
   if (STAGE == "late") {
     md = treeSeqMetadata(TREES_IN, userData=F).getValue("SLiM").getValue("user_metadata");
     sim.readFromPopulationFile(TREES_IN, subpopMap=SUBPOP_MAP);
+    if (PHENOTYPES) sim.demandPhenotype(NULL);
     sim.treeSeqOutput(TREES_FILE, simplify=F, metadata=md);
   }
 }

tests/test_recipes/restart_and_run_WF.slim

@@ -14,7 +14,6 @@ initialize()
     if (!exists("SUBPOP_MAP")) defineConstant("SUBPOP_MAP", NULL);
 }
 
-// output immediately! Nothing should have changed.
 1 early() { 
   if (STAGE == "early") {
     sim.readFromPopulationFile(TREES_IN, subpopMap=SUBPOP_MAP);
@@ -28,6 +27,7 @@ initialize()
 
 20 early() {
     catn("Done.");
+    sim.demandPhenotype(NULL);
     sim.treeSeqOutput(TREES_FILE, simplify=F);
     sim.simulationFinished();
 }

tests/test_recipes/restart_and_run_nonWF.slim

@@ -29,6 +29,7 @@ initialize()
 
 20 early() {
     catn("Done.");
+    sim.demandPhenotype(NULL);
     sim.treeSeqOutput(TREES_FILE, simplify=F);
     sim.simulationFinished();
 }

tests/test_recipes/restart_multispecies_WF.slim

@@ -5,6 +5,7 @@ species all initialize()
 {
     setSeed(23);
     initializeSLiMModelType("WF");
+    if (!exists("PHENOTYPES")) defineGlobal("PHENOTYPES", T);
 }
 
 species fox initialize()
@@ -38,6 +39,7 @@ ticks all 1 early() {
   if (STAGE == "early") {
     for (sp in community.allSpecies) {
 	    sp.readFromPopulationFile(TREES_IN.getValue(sp.name), subpopMap=SUBPOP_MAP.getValue(sp.name));
+       if (PHENOTYPES) sp.demandPhenotype(NULL);
    	 sp.treeSeqOutput(executeLambda("TREES_FILE_" + sp.name + ";"), simplify=F);
     }
   }
@@ -46,6 +48,7 @@ ticks all 1 late() {
   if (STAGE == "late") {
     for (sp in community.allSpecies) {
 	    sp.readFromPopulationFile(TREES_IN.getValue(sp.name), subpopMap=SUBPOP_MAP.getValue(sp.name));
+       if (PHENOTYPES) sp.demandPhenotype(NULL);
    	 sp.treeSeqOutput(executeLambda("TREES_FILE_" + sp.name + ";"), simplify=F);
     }
   }