From 54c202ae7b744bfc35c3d3e71182581d653147ce Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Wed, 10 Jun 2026 16:48:33 +0000 Subject: [PATCH 01/62] debug_util: Excess -v's should still show output Without -vvv means no output, which isn't what you were hoping for. --- oz_tree_build/utilities/debug_util.py | 2 +- tests/test_debug_util.py | 33 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/oz_tree_build/utilities/debug_util.py b/oz_tree_build/utilities/debug_util.py index cea334d4..86b7735c 100644 --- a/oz_tree_build/utilities/debug_util.py +++ b/oz_tree_build/utilities/debug_util.py @@ -54,7 +54,7 @@ def parse_args_and_add_logging_switch(parser): logging.basicConfig(stream=sys.stderr, level=logging.WARNING) elif args.verbosity == 1: logging.basicConfig(stream=sys.stderr, level=logging.INFO) - elif args.verbosity == 2: + else: logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) if _error_handler is None: diff --git a/tests/test_debug_util.py b/tests/test_debug_util.py index f1bf131e..c2082b8b 100644 --- a/tests/test_debug_util.py +++ b/tests/test_debug_util.py @@ -72,7 +72,40 @@ def test_default_verbosity_suppresses_info(): assert "hello info" not in result.stderr +def test_verbose_flag_suppresses_debug(): + result = _run("logging.debug('hello debug')", "-v") + assert result.returncode == 0 + assert "hello debug" not in result.stderr + + +def test_v_still_emits_errors(): + result = _run("logging.error('boom')", "-v") + assert result.returncode == 1 + assert "boom" in result.stderr + assert "Exiting with status 1: 1 error(s) were logged" in result.stderr + + def test_vv_enables_debug_output(): result = _run("logging.debug('hello debug')", "-vv") assert result.returncode == 0 assert "hello debug" in result.stderr + + +def test_vv_still_emits_errors(): + result = _run("logging.error('boom')", "-vv") + assert result.returncode == 1 + assert "boom" in result.stderr + assert "Exiting with status 1: 1 error(s) were logged" in result.stderr + + +def test_vvv_enables_debug_output(): + result = _run("logging.debug('hello debug')", "-vvv") + assert result.returncode == 0 + assert "hello debug" in result.stderr + + +def test_vvv_still_emits_errors(): + result = _run("logging.error('boom')", "-vvv") + assert result.returncode == 1 + assert "boom" in result.stderr + assert "Exiting with status 1: 1 error(s) were logged" in result.stderr From c664b8da047538c11c756e9ca1710b2e6b22330b Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Wed, 10 Jun 2026 17:14:33 +0000 Subject: [PATCH 02/62] OTT_popularity_mapping: Add __repr__ to WikidataItem --- .../OTT_popularity_mapping.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py b/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py index cc72d936..89f1ce49 100755 --- a/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py +++ b/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py @@ -187,6 +187,16 @@ class WikidataItem: exclude_langs = frozenset(("species", "commons")) + def __repr__(self): + return ( + "WikidataItem[" + + " ".join( + (f"{a}={getattr(self, a)} " if hasattr(self, a) else "") + for a in ("Q", "ipni", "EoL", "iucn", "wd_ott", "raw_popularity", "l") + ) + + "]" + ) + def __init__(self, json_item): """ Create a basic item with an (integer) 'Q' attribute and an 'l' for sitelinks. From 1a184090674d0bfe955ec270598e2a5390626a20 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 26 May 2026 14:49:56 +0000 Subject: [PATCH 03/62] node_ages: download_node_ages pipeline stage Generate node_ages.json using chronosynth. --- data/.gitignore | 1 + dvc.lock | 11 +++- dvc.yaml | 7 +++ .../download_node_ages/chronosynth_config.ini | 42 +++++++++++++ .../download_node_ages/download_node_ages.py | 60 +++++++++++++++++++ .../download_node_ages/peyotl_config.ini | 15 +++++ pyproject.toml | 5 ++ 7 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 oz_tree_build/download_node_ages/chronosynth_config.ini create mode 100644 oz_tree_build/download_node_ages/download_node_ages.py create mode 100644 oz_tree_build/download_node_ages/peyotl_config.ini diff --git a/data/.gitignore b/data/.gitignore index 0992b36e..2e8386a7 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1,2 +1,3 @@ /js_output /output_files +/node_ages.json diff --git a/dvc.lock b/dvc.lock index 81c07fe6..3f087215 100644 --- a/dvc.lock +++ b/dvc.lock @@ -174,8 +174,8 @@ stages: size: 221682224 build_oz_tree: cmd: - - cd data/OZTreeBuild/AllLife && build_oz_tree - BespokeTree/include_OT_v16.1/Base.PHY OpenTreeParts/OpenTree_all/ + - cd data/OZTreeBuild/AllLife && build_oz_tree --nodeages + ../../node_ages.json BespokeTree/include_OT_v16.1/Base.PHY AllLife_full_tree.phy deps: - path: data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1/ @@ -279,3 +279,10 @@ stages: hash: md5 md5: 1018a5c664d01747fdd7e218190cb4ac size: 72 + download_node_ages: + cmd: download_node_ages data/node_ages.json + outs: + - path: data/node_ages.json + hash: md5 + md5: 2440f9bb2139301bf352797a9802c2d9 + size: 14044769 diff --git a/dvc.yaml b/dvc.yaml index c3b40492..101f6b37 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -9,6 +9,11 @@ stages: outs: - data/OpenTree/${ot_version}/ + download_node_ages: + cmd: .venv/bin/download_node_ages data/node_ages.json + outs: + - data/node_ages.json + # ~20 secs add_ott_numbers_to_trees: cmd: @@ -51,10 +56,12 @@ stages: - >- cd data/OZTreeBuild/${oz_tree} && build_oz_tree + --nodeages ../../node_ages.json BespokeTree/include_OT_${ot_version}/Base.PHY OpenTreeParts/OpenTree_all/ ${oz_tree}_full_tree.phy deps: + - data/node_ages.json - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OpenTree_all/ params: diff --git a/oz_tree_build/download_node_ages/chronosynth_config.ini b/oz_tree_build/download_node_ages/chronosynth_config.ini new file mode 100644 index 00000000..e368e342 --- /dev/null +++ b/oz_tree_build/download_node_ages/chronosynth_config.ini @@ -0,0 +1,42 @@ +# Copied from https://github.com/OpenTreeOfLife/chronosynth/blob/main/default.config + +[paths] +cache_file_dir = /tmp/ + +[params] +ultrametricity_precision=0.01 + + +### +# logging configuration +# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +### + +[loggers] +keys = root + +[handlers] +keys = console, chronosynth + +[formatters] +keys = generic + +[logger_root] +level = DEBUG +handlers = chronosynth, console + +[handler_console] +class = StreamHandler +level = NOTSET +formatter = generic +args = (sys.stderr,) + +[handler_chronosynth] +class = FileHandler +args = ('chronosynth.log', 'a') +level = DEBUG +formatter = generic + + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s diff --git a/oz_tree_build/download_node_ages/download_node_ages.py b/oz_tree_build/download_node_ages/download_node_ages.py new file mode 100644 index 00000000..746ead90 --- /dev/null +++ b/oz_tree_build/download_node_ages/download_node_ages.py @@ -0,0 +1,60 @@ +""" +Populate node_ages.json by calling out to the OpenTree API via. chronosynth + +Usage: download_node_ages node_ages.json + +NB: The output is not based on the tree downloaded in other steps, +internally chronosynth will call out to the OpenTree API & +github.com/OpenTreeOfLife/phylesystem-1. + +""" + +import argparse +import json +import logging +import os +import os.path +import sys +import time + +os.environ["CHRONOSYNTH_CONFIG_FILE"] = os.path.join(os.path.dirname(__file__), "chronosynth_config.ini") +os.environ["PEYOTL_CONFIG_FILE"] = os.path.join(os.path.dirname(__file__), "peyotl_config.ini") + +import chronosynth.chronogram # noqa: E402 - we need to set env first + + +def download_node_ages(): + node_ages = chronosynth.chronogram.build_synth_node_source_ages(fresh=True) + return node_ages + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--verbosity", + "-v", + action="count", + default=0, + help="verbosity level: output extra non-essential info", + ) + parser.add_argument("output_path", help="Path to where output data should be saved") + args = parser.parse_args() + + if args.verbosity == 0: + logging.basicConfig(stream=sys.stderr, level=logging.WARNING) + elif args.verbosity == 1: + logging.basicConfig(stream=sys.stderr, level=logging.INFO) + elif args.verbosity == 2: + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + + start = time.time() + + with open(args.output_path, "w") as f: + json.dump(download_node_ages(), f) + + end = time.time() + logging.debug(f"Time taken: {end - start} seconds") + + +if __name__ == "__main__": + main() diff --git a/oz_tree_build/download_node_ages/peyotl_config.ini b/oz_tree_build/download_node_ages/peyotl_config.ini new file mode 100644 index 00000000..cff1a75a --- /dev/null +++ b/oz_tree_build/download_node_ages/peyotl_config.ini @@ -0,0 +1,15 @@ +# From: https://github.com/OpenTreeOfLife/peyotl/blob/master/peyotl/default.conf + +[logging] +level = info +filepath = /tmp/peyotl-log +formatter = simple + +[apis] +phylesystem_api = https://devapi.opentreeoflife.org +collections_api = https://devapi.opentreeoflife.org +amendments_api = https://devapi.opentreeoflife.org +oti = https://devapi.opentreeoflife.org +taxomachine = https://api.opentreeoflife.org +treemachine = https://api.opentreeoflife.org + diff --git a/pyproject.toml b/pyproject.toml index bcfbac61..f2697af4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,10 @@ dependencies = [ "mwparserfromhell>=0.6.6", "requests-cache>=1.2.1", "dvc[s3]>=3.0", + "chronosynth @ git+https://github.com/OpenTreeOfLife/chronosynth@7fc31d2bb3bfbf0786d31530579c916499f3616a", + # Undeclared chronosynth dependencies + "peyotl @ git+https://github.com/lentinj/peyotl.git", + "opentree @ git+https://github.com/OpenTreeOfLife/python-opentree@9b9afce7d0a526a3328af0f9c328ef1194aec1b9#egg=opentree", ] [project.optional-dependencies] @@ -60,6 +64,7 @@ find_in_file = "oz_tree_build.utilities.find_in_file:main" wiki_clade_extractor = "oz_tree_build.wiki_extraction.wiki_clade_extractor:main" newick_combiner = "oz_tree_build.wiki_extraction.newick_combiner:main" add_dates_and_species_to_tree = "oz_tree_build.wiki_extraction.add_dates_and_species_to_tree:main" +download_node_ages = "oz_tree_build.download_node_ages.download_node_ages:main" [tool.setuptools] packages = ["oz_tree_build"] From 8a3e65f69cc61fe0bce04e2f444a3cbce361c135 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 8 Jun 2026 15:55:28 +0000 Subject: [PATCH 04/62] requirements: Add requirements.txt to freeze git versions We need loose package requirements so we don't clash with dated-complete-tree, but should still be installing known versions. Use requirements.txt to achieve this. requirements: peyotl changes merged --- .github/workflows/dvc.yml | 1 + .github/workflows/tests.yml | 2 +- README.markdown | 2 +- pyproject.toml | 7 ++++--- requirements.txt | 7 +++++++ 5 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 requirements.txt diff --git a/.github/workflows/dvc.yml b/.github/workflows/dvc.yml index 9c6b8bde..37d98b78 100644 --- a/.github/workflows/dvc.yml +++ b/.github/workflows/dvc.yml @@ -26,6 +26,7 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install '.[dev]' + python3 -m pip install -r requirements.txt - name: Checkout merge ref if: github.event_name == 'pull_request_target' uses: actions/checkout@v7 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index df9233ac..d203e83a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,7 +40,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - python3 -m pip install '.[dev]' + python3 -m pip install -r requirements.txt - name: Test with pytest run: | python3 -m pytest tests --conf-file tests/appconfig.ini diff --git a/README.markdown b/README.markdown index 08109ef1..016532f9 100644 --- a/README.markdown +++ b/README.markdown @@ -13,7 +13,7 @@ The first step to using this repo is to create a Python virtual environment and source .venv/bin/activate # Install it - pip install -e '.[dev]' + pip install -r requirements.txt # Set up git hooks including linting and DVC pre-commit install --hook-type pre-push --hook-type post-checkout --hook-type pre-commit diff --git a/pyproject.toml b/pyproject.toml index f2697af4..abfe6d28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,10 +22,11 @@ dependencies = [ "mwparserfromhell>=0.6.6", "requests-cache>=1.2.1", "dvc[s3]>=3.0", - "chronosynth @ git+https://github.com/OpenTreeOfLife/chronosynth@7fc31d2bb3bfbf0786d31530579c916499f3616a", + "dated_complete_tree", + "chronosynth", # Undeclared chronosynth dependencies - "peyotl @ git+https://github.com/lentinj/peyotl.git", - "opentree @ git+https://github.com/OpenTreeOfLife/python-opentree@9b9afce7d0a526a3328af0f9c328ef1194aec1b9#egg=opentree", + "peyotl", + "opentree", ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c0369395 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +dated_complete_tree @ git+https://github.com/OneZoom/dated-complete-tree.git +chronosynth @ git+https://github.com/OpenTreeOfLife/chronosynth@7fc31d2bb3bfbf0786d31530579c916499f3616a +# Undeclared chronosynth dependencies +peyotl @ git+https://github.com/OpenTreeOfLife/peyotl.git@72ccb5369bef07b76f57eb6852f5b90fe677b09b +opentree @ git+https://github.com/OpenTreeOfLife/python-opentree@9b9afce7d0a526a3328af0f9c328ef1194aec1b9#egg=opentree + +-e .[dev] From fe6b473b2ce6496fdab905d832d2b77296ca2ae9 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 8 Jun 2026 15:42:52 +0000 Subject: [PATCH 05/62] date_tree: Copy main.py from dated-complete-tree --- data/.gitignore | 1 + dvc.yaml | 14 ++ oz_tree_build/date_tree/date_tree.py | 295 +++++++++++++++++++++++++++ pyproject.toml | 1 + ruff.toml | 2 + 5 files changed, 313 insertions(+) create mode 100644 oz_tree_build/date_tree/date_tree.py diff --git a/data/.gitignore b/data/.gitignore index 2e8386a7..89500714 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1,3 +1,4 @@ /js_output /output_files /node_ages.json +/data/dated_tree/ diff --git a/dvc.yaml b/dvc.yaml index 101f6b37..ecd67c61 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -14,6 +14,20 @@ stages: outs: - data/node_ages.json + date_tree: + cmd: + - rm -rf data/dated_tree + - mkdir -p data/dated_tree + - >- + .venv/bin/date_tree + --output_folder=data/dated_tree + --date_cache data/node_ages.json + --annotations data/OpenTree/${ot_version}/annotations.json + --taxonomy data/OpenTree/${ot_version}/taxonomy.tsv + --supertree data/OpenTree/${ot_version}/labelled_supertree_ottnames.tre + outs: + - data/dated_tree/dated_tree_topo_sample_1.tre + # ~20 secs add_ott_numbers_to_trees: cmd: diff --git a/oz_tree_build/date_tree/date_tree.py b/oz_tree_build/date_tree/date_tree.py new file mode 100644 index 00000000..907c8434 --- /dev/null +++ b/oz_tree_build/date_tree/date_tree.py @@ -0,0 +1,295 @@ +# BSD 3-Clause License + +# Copyright (c) 2025, Jonathan David Duke + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import sys +import gc +import numpy as np +import datetime + +from dated_complete_tree import tree_loading +from dated_complete_tree import tree_labelling +from dated_complete_tree import tree_fixing +from dated_complete_tree import tree_dating +from dated_complete_tree import tree_metrics + +import argparse +import logging +logger = logging.getLogger(__name__) +logging.basicConfig(filename="main.log", filemode="w", force=True, level=logging.ERROR) + + +def generate_trees(args): + if not os.path.exists(args.output_folder): + os.makedirs(args.output_folder) + + sys.setrecursionlimit(10000) + + rng = np.random.default_rng(seed=1) + date_interpolation_rng = np.random.default_rng(seed=1) + date_source_rng = np.random.default_rng(seed=1) + + ##################################################################################################################### + # Load and prune tree + + # Load metadata for tree from Open Tree and Chronosynth + dates, phylogeny_nodes, taxa = tree_loading.load_metadata(date_cache=args.date_cache, + annotations=args.annotations, + taxonomy=args.taxonomy) + + # Create ETE3 tree structure for entire Open Tree of Life, with my annotations + whole_tre_unmodified = tree_loading.build_and_annotate_tree(phylogeny_nodes, taxa, tree_filename=args.supertree) + + tree_fixing.strip_birds(whole_tre_unmodified) + tree_fixing.strip_turtles(whole_tre_unmodified) + + rng = np.random.default_rng(seed=1) + + tree_fixing.remove_subspecies(whole_tre_unmodified, rng) + tree_fixing.impute_species_into_empty_taxa(whole_tre_unmodified) + + tree_fixing.fix_taxonomy_ordering(whole_tre_unmodified) + + tree_labelling.add_anc_ranks(whole_tre_unmodified) + tree_labelling.add_desc_ranks(whole_tre_unmodified) + + tree_fixing.forced_taxa_moves(whole_tre_unmodified) + + if args.pd_clades: + topo_pd_clades = [cld.strip() for cld in list(open(args.pd_clades))] + topo_pd_dict = {} + topo_dates_dict = {} + topo_spp_dict = {} + for clade in topo_pd_clades: + topo_pd_dict[clade] = [] + topo_dates_dict[clade] = [] + topo_spp_dict[clade] = [] + + if args.num_date_samples > 0: + both_pd_clades = [cld.strip() for cld in list(open(args.pd_clades))] + both_pd_dict = {} + both_dates_dict = {} + both_spp_dict = {} + for clade in both_pd_clades: + both_pd_dict[clade] = [] + both_dates_dict[clade] = [] + both_spp_dict[clade] = [] + + if args.compute_ed: + topo_ed_scores = {} + if args.num_date_samples > 0: + both_ed_scores = {} + + itr_start = datetime.datetime.now() + + for n in range(args.num_trees): + print("Tree number", n+1, "/ projected end time:", itr_start + args.num_trees*(datetime.datetime.now() - itr_start)/n if n > 0 else "first iteration, no estimate yet") + + # Copy tree - we will change the copy, and keep the original unchanged so we can restore it next iteration without + # reloading everything + whole_tre = whole_tre_unmodified.copy() + + ##################################################################################################################### + # Fix topology + + # First, do labelling for steps 1-3: + # - 1-2 are independent of each other; step 3 collects up nodes not labelled in 1-2. + # - tree is only labelled at this stage; modifications are made in tree_fixing functions. + genus_dict = {} # step 1, nodes below genus nodes + nmp_genus_dict = {} # step 2, non-monophyletic genera + tree_labelling.populate_genus_dict(whole_tre, genus_dict, nmp_genus_dict, None) + + tofix_dict = {} # step 3, all other nodes from taxonomy (not phylogenies) to + # be moved to a suitable place in the tree, such that we + # generated a plausible hypothetical tree + tree_labelling.populate_tofix_dict(whole_tre, tofix_dict, nmp_genus_dict) + + # Second, fix the topology based on the labels. + # Fix steps 1 and 2. + tree_fixing.fix_polyphyly(genus_dict, rng) + tree_fixing.fix_polyphyly(nmp_genus_dict, rng) + + tree_fixing.remove_nonspecies_leaves(whole_tre) + + # Find and label backbone for step 3, after steps 1 an 2 already fixed. + tree_labelling.populate_tofix_bkb(whole_tre, tofix_dict, []) + fix_dict = tree_labelling.process_tofix_bkb(tofix_dict) + + # Finally, fix step 3. + tree_fixing.fix_polyphyly(fix_dict, rng, expand_parent_backbones=True) + + tree_fixing.remove_nonspecies_leaves(whole_tre) + + # Last of all, polytomy resolution. + tree_fixing.fix_all_polytomies(whole_tre, rng) + + # Remove one-child nodes. Gives a fully bifurcating tree. + whole_tre = tree_fixing.delete_one_child_nodes(whole_tre) + + ##################################################################################################################### + # Assign and interpolate median dates + + # Assign dates + tree_dating.assign_dates(whole_tre, dates) + + # Date cleaning to ensure time consistency down the tree + tree_dating.label_older_descendants(whole_tre) + tree_dating.dq_date_removal(whole_tre) + + # Date imputation + tree_dating.date_labelling(whole_tre) + if args.use_birth_model: + tree_dating.impute_missing_dates(whole_tre, use_birth_model=True, rng=date_interpolation_rng) + else: + tree_dating.impute_missing_dates(whole_tre, l=0.25) + + # All nodes now dated - set dists in ete and write out tree. + tree_dating.compute_branch_lengths(whole_tre) + tree_dating.write_tree_with_branch_lengths(whole_tre, filename="%s/%s_topo_sample_%d.tre" % (args.output_folder, args.output_tree_filename, n+1)) + + if args.compute_ed: + tree_metrics.compute_ed_scores(whole_tre, topo_ed_scores) + + if args.pd_clades: + tree_metrics.compute_pd(whole_tre) + tree_metrics.save_pd_for_clades(whole_tre, topo_pd_clades, topo_pd_dict, topo_dates_dict, topo_spp_dict) + + # Now do date sampling, if desired + for s in range(args.num_date_samples): + print(" Tree number", n+1, "; Date sample", s+1) + for node in whole_tre.traverse(strategy="preorder"): + # reset all dates + node.props["date"] = None + node.props["imputed_date"] = False + node.props["imputation_type"] = 0 + + # Assign dates + tree_dating.assign_dates(whole_tre, dates, sample_dates=True, rng=date_source_rng) + + # Date cleaning to ensure time consistency down the tree + tree_dating.label_older_descendants(whole_tre) + tree_dating.dq_date_removal(whole_tre) + + # Date imputation + tree_dating.date_labelling(whole_tre) + if args.use_birth_model: + tree_dating.impute_missing_dates(whole_tre, use_birth_model=True, rng=date_interpolation_rng) + else: + tree_dating.impute_missing_dates(whole_tre, l=0.25) + + # All nodes now dated - set dists in ete and write out tree. + tree_dating.compute_branch_lengths(whole_tre) + tree_dating.write_tree_with_branch_lengths(whole_tre, filename="%s/%s_both_sample_%d.tre" % (args.output_folder, args.output_tree_filename, n*args.num_date_samples+s+1)) + + if args.compute_ed: + tree_metrics.compute_ed_scores(whole_tre, both_ed_scores) + + if args.pd_clades: + tree_metrics.compute_pd(whole_tre) + tree_metrics.save_pd_for_clades(whole_tre, both_pd_clades, both_pd_dict, both_dates_dict, both_spp_dict) + + del whole_tre + gc.collect() + + + if args.compute_ed: + print("Writing out ED score distributions for all species (takes ~5 minutes)") + tree_metrics.write_ed_scores("%s/%s_topo_ed_scores.txt" % (args.output_folder, args.output_tree_filename), topo_ed_scores) + + if args.num_date_samples > 0: + tree_metrics.write_ed_scores("%s/%s_both_ed_scores.txt" % (args.output_folder, args.output_tree_filename), both_ed_scores) + + if args.pd_clades: + tree_metrics.write_pd_dists("%s/%s_topo" % (args.output_folder, args.output_tree_filename), topo_pd_dict, topo_dates_dict, topo_spp_dict) + + if args.num_date_samples > 0: + tree_metrics.write_pd_dists("%s/%s_both" % (args.output_folder, args.output_tree_filename), both_pd_dict, both_dates_dict, both_spp_dict) + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Generate a set of dated trees of all life, based on the Open Tree of Life and Chronosynth. " + "Optionally, also generate evolutionary distinctiveness scores for the trees" + ) + ) + + parser.add_argument("--num_trees", + help="How many trees to generate. Default: 1", + type=int, + default=1) + + parser.add_argument("--num_date_samples", + help="How many times to sample a set of dates for each tree (in addition to the tree using median dates). Default: 0", + type=int, + default=0) + + parser.add_argument("--output_folder", + help="Path of folder where output trees will be written in Newick format, Default: output", + default="output") + + parser.add_argument("--output_tree_filename", + help="Filename for output trees, e.g. the default 'dated_tree' would result in trees named 'dated_tree_topo_sample_1', '..._2' etc.", + default="dated_tree") + + parser.add_argument("--supertree", + help="Path of the labelled_supertree_ottnames.tre file from the Open Tree of Life. Default: opentree16.1_tree/labelled_supertree/labelled_supertree_ottnames.tre", + default="opentree16.1_tree/labelled_supertree/labelled_supertree_ottnames.tre") + + parser.add_argument("--date_cache", + help="Path of the date cache generated by Chronosynth. Default: chronosynth_date_info/node_ages.json", + default="chronosynth_date_info/node_ages.json") + + parser.add_argument("--annotations", + help="Path of the annotations.json file from the Open Tree of Life. Default: opentree16.1_tree/annotations.json", + default="opentree16.1_tree/annotations.json") + + parser.add_argument("--taxonomy", + help="Path of the taxonomy.tsv file from the Open Tree Taxonomy. Default: ott3.7.3/taxonomy.tsv", + default="ott3.7.3/taxonomy.tsv") + + parser.add_argument("--use_birth_model", + help="Flag: whether use the date interpolation method based on a birth model, rather than the EQS-LS method. Default: False", + action="store_true") + + parser.add_argument("--pd_clades", + help="Path of a text file containing a list of node names (one on each line) for which to output PD estimates. Default: None", + default=None) + + parser.add_argument("--compute_ed", + help="Flag: whether to compute a distribution of ED scores. A csv file summarising the scores will be placed in the output folder. Default: False", + action="store_true") + + args = parser.parse_args() + generate_trees(args) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index abfe6d28..121c69fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ wiki_clade_extractor = "oz_tree_build.wiki_extraction.wiki_clade_extractor:main" newick_combiner = "oz_tree_build.wiki_extraction.newick_combiner:main" add_dates_and_species_to_tree = "oz_tree_build.wiki_extraction.add_dates_and_species_to_tree:main" download_node_ages = "oz_tree_build.download_node_ages.download_node_ages:main" +date_tree = "oz_tree_build.date_tree.date_tree:main" [tool.setuptools] packages = ["oz_tree_build"] diff --git a/ruff.toml b/ruff.toml index 36d9229a..64ce5172 100644 --- a/ruff.toml +++ b/ruff.toml @@ -28,6 +28,8 @@ exclude = [ "*.PHY", "*.md", "*.markdown", + # date_tree is actually from https://github.com/jdduke24/dated-complete-tree/blob/main/main.py, preserve its formatting + "oz_tree_build/date_tree/date_tree.py", ] line-length = 120 From 80726829acafc7c2666fc942aeeefc04288ed0df Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 8 Jun 2026 15:47:04 +0000 Subject: [PATCH 06/62] date_tree: Use date_cache directly, don't chronosynth it --- oz_tree_build/date_tree/date_tree.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/oz_tree_build/date_tree/date_tree.py b/oz_tree_build/date_tree/date_tree.py index 907c8434..ddee9fa2 100644 --- a/oz_tree_build/date_tree/date_tree.py +++ b/oz_tree_build/date_tree/date_tree.py @@ -28,6 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import json import os import sys import gc @@ -60,9 +61,13 @@ def generate_trees(args): # Load and prune tree # Load metadata for tree from Open Tree and Chronosynth - dates, phylogeny_nodes, taxa = tree_loading.load_metadata(date_cache=args.date_cache, - annotations=args.annotations, - taxonomy=args.taxonomy) + with open(args.date_cache) as f: + dates = json.load(f) + phylogeny_nodes, taxa = tree_loading.load_metadata( + date_cache=None, + annotations=args.annotations, + taxonomy=args.taxonomy, + ) # Create ETE3 tree structure for entire Open Tree of Life, with my annotations whole_tre_unmodified = tree_loading.build_and_annotate_tree(phylogeny_nodes, taxa, tree_filename=args.supertree) From e1b61df5d63ae596b8a3425a66e7071b84d35f1b Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 8 Jun 2026 15:59:11 +0000 Subject: [PATCH 07/62] download_node_ages: Move filtering from dated_complete_tree/tree_loading.py Since we don't let dated-complete-tree do it's own filtering, copy it here. --- .../download_node_ages/download_node_ages.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/oz_tree_build/download_node_ages/download_node_ages.py b/oz_tree_build/download_node_ages/download_node_ages.py index 746ead90..3bf14e06 100644 --- a/oz_tree_build/download_node_ages/download_node_ages.py +++ b/oz_tree_build/download_node_ages/download_node_ages.py @@ -24,8 +24,25 @@ def download_node_ages(): - node_ages = chronosynth.chronogram.build_synth_node_source_ages(fresh=True) - return node_ages + dates = chronosynth.chronogram.build_synth_node_source_ages(fresh=True) + + # Remove sources, from dated_complete_tree/tree_loading.py + sources_to_delete = set(["ot_1250@tree2"]) + deletions = [] + for ott_name in dates["node_ages"]: + for i, source in enumerate(dates["node_ages"][ott_name]): + if source["source_id"] in sources_to_delete: + deletions.append((ott_name, i)) + + deletions.sort(reverse=True) + + for ott_name, i in deletions: + del dates["node_ages"][ott_name][i] + if len(dates["node_ages"][ott_name]) == 0: + del dates["node_ages"][ott_name] + #### + + return dates def main(): From 09d59b884888b828b4023852cefba918c569f8a4 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 8 Jun 2026 16:00:06 +0000 Subject: [PATCH 08/62] download_opentree: Fetch files required by dated-complete-tree instead The opentree is going to go via. dated-complete-tree and the pipeline will assume it's dated. To achieve this, fetch the files it needs rather than what the old pipeline needed. --- dvc.yaml | 5 ++- oz_tree_build/utilities/download_opentree.py | 36 ++++++-------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/dvc.yaml b/dvc.yaml index ecd67c61..56e34c85 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -3,7 +3,10 @@ vars: stages: download_opentree: - cmd: download_opentree --version ${ot_version} --output-dir data/OpenTree + cmd: + - rm -rf data/OpenTree/${ot_version}/ + - mkdir -p data/OpenTree/${ot_version}/ + - .venv/bin/download_opentree --version ${ot_version} --output-dir data/OpenTree params: - ot_version outs: diff --git a/oz_tree_build/utilities/download_opentree.py b/oz_tree_build/utilities/download_opentree.py index 4105a908..e592546c 100644 --- a/oz_tree_build/utilities/download_opentree.py +++ b/oz_tree_build/utilities/download_opentree.py @@ -14,7 +14,7 @@ import argparse import os -import re +import os.path import shutil import tarfile import tempfile @@ -42,36 +42,19 @@ def find_synthesis_entry(synthesis_json, version): raise SystemExit(f"Version '{version}' not found in synthesis.json. " f"Available versions: {', '.join(available)}") -def strip_mrca_prefixes(content: str) -> str: - # Clean up synthetically named mrca (most recent common ancestor) node labels, no use to us - content = re.sub(r"\)mrcaott\d+ott\d+", ")", content) - # Also clean up unwanted spaces - content = re.sub(r"[ _]+", "_", content) - return content - - -def download_tree(version, output_dir): +def download_file(version, output_dir, download_file="/labelled_supertree/labelled_supertree_ottnames.tre"): """Download the labelled supertree and produce the processed draftversion.""" assert version.startswith("v") version_without_v = version[1:] - tree_url = ( - f"https://files.opentreeoflife.org/synthesis/opentree{version_without_v}" - f"/output/labelled_supertree/labelled_supertree_simplified_ottnames.tre" - ) - print(f"Downloading tree from {tree_url} ...") - response = requests.get(tree_url, verify=OT_SSL_VERIFY) + url = f"https://files.opentreeoflife.org/synthesis/opentree{version_without_v}/output/{download_file}" + out_path = os.path.join(output_dir, os.path.basename(url)) + + print(f"Downloading {url} -> {out_path} ...") + response = requests.get(url, verify=OT_SSL_VERIFY) response.raise_for_status() - raw_path = os.path.join(output_dir, "labelled_supertree_simplified_ottnames.tre") - with open(raw_path, "w") as f: + with open(out_path, "w") as f: f.write(response.text) - print(f" Saved raw tree to {raw_path}") - - draft_path = os.path.join(output_dir, "draftversion.tre") - print(" Stripping mrca prefixes ...") - with open(draft_path, "w") as f: - f.write(strip_mrca_prefixes(response.text)) - print(f" Saved processed tree to {draft_path}") def download_taxonomy(ott_version_raw, output_dir): @@ -130,7 +113,8 @@ def main(): output_dir = os.path.join(args.output_dir, version) os.makedirs(output_dir, exist_ok=True) - download_tree(version, output_dir) + download_file(version, output_dir, "/labelled_supertree/labelled_supertree_ottnames.tre") + download_file(version, output_dir, "/annotated_supertree/annotations.json") download_taxonomy(entry["OTT_version"], output_dir) print(f"Done. All files written to {output_dir}/") From 3683803af2781a8316fb32c4515a8c323915d98d Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Wed, 10 Jun 2026 07:50:49 +0000 Subject: [PATCH 09/62] date_tree: Exit early, write out tree We nneed the tree before we've dated it, and can't remove nonspecies / unary nodes at this point. Leave the rest of the code as-is to borrow it later. --- data/.gitignore | 2 +- dvc.yaml | 4 ++- oz_tree_build/date_tree/date_tree.py | 37 ++++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/data/.gitignore b/data/.gitignore index 89500714..4bf6b42b 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1,4 +1,4 @@ /js_output /output_files /node_ages.json -/data/dated_tree/ +/dated_tree/ diff --git a/dvc.yaml b/dvc.yaml index 56e34c85..3e195dd3 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -28,8 +28,10 @@ stages: --annotations data/OpenTree/${ot_version}/annotations.json --taxonomy data/OpenTree/${ot_version}/taxonomy.tsv --supertree data/OpenTree/${ot_version}/labelled_supertree_ottnames.tre + deps: + - data/node_ages.json outs: - - data/dated_tree/dated_tree_topo_sample_1.tre + - data/dated_tree/dated_tree_pre.tre # ~20 secs add_ott_numbers_to_trees: diff --git a/oz_tree_build/date_tree/date_tree.py b/oz_tree_build/date_tree/date_tree.py index ddee9fa2..3a8d2a67 100644 --- a/oz_tree_build/date_tree/date_tree.py +++ b/oz_tree_build/date_tree/date_tree.py @@ -35,6 +35,8 @@ import numpy as np import datetime +import ete4 + from dated_complete_tree import tree_loading from dated_complete_tree import tree_labelling from dated_complete_tree import tree_fixing @@ -47,6 +49,32 @@ logging.basicConfig(filename="main.log", filemode="w", force=True, level=logging.ERROR) +def nwk_write(tree, outfile): + """ + Tidy properties and write out newick tree + """ + for n in tree.traverse(): + # Tidy up attributes, only output useful values + if "date" in n.props and n.props["date"] is None: + n.del_prop("date") + tree.write( + outfile=outfile, + props=["date"], + parser=1, + ) + + +def nwk_read(infile): + """ + Re-reads trees written by nwk_write, used by downstream processes + """ + tree = ete4.Tree(infile, parser=1) + for n in tree.traverse(): + if "date" in n.props: + n.props["date"] = float(n.props["date"]) + return tree + + def generate_trees(args): if not os.path.exists(args.output_folder): os.makedirs(args.output_folder) @@ -150,13 +178,13 @@ def generate_trees(args): # Finally, fix step 3. tree_fixing.fix_polyphyly(fix_dict, rng, expand_parent_backbones=True) - tree_fixing.remove_nonspecies_leaves(whole_tre) + # tree_fixing.remove_nonspecies_leaves(whole_tre) # Last of all, polytomy resolution. tree_fixing.fix_all_polytomies(whole_tre, rng) # Remove one-child nodes. Gives a fully bifurcating tree. - whole_tre = tree_fixing.delete_one_child_nodes(whole_tre) + # whole_tre = tree_fixing.delete_one_child_nodes(whole_tre) ##################################################################################################################### # Assign and interpolate median dates @@ -164,6 +192,11 @@ def generate_trees(args): # Assign dates tree_dating.assign_dates(whole_tre, dates) + ############# Write out early, before imputing dates + nwk_write(whole_tre, "%s/%s_pre.tre" % (args.output_folder, args.output_tree_filename)) + sys.exit(0) + ######################## + # Date cleaning to ensure time consistency down the tree tree_dating.label_older_descendants(whole_tre) tree_dating.dq_date_removal(whole_tre) From 7083bdaba031778210e3837290dedc9401620454 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Wed, 10 Jun 2026 08:09:39 +0000 Subject: [PATCH 10/62] tree_build: Replace get_open_trees_from_one_zoom / build_oz_tree Start reimplementing new pipeline, fully ete4-based. Re-implement pruning/grafting in memory using ete4, so bespoke trees / OpenTree can have date properties. utilities/ete: Generic OTT-extractor function Keep returning output as str. It's a bit confusing, but is generally what we're dealing with one way or another so saves a bunch of conversion. --- dvc.yaml | 38 +---- oz_tree_build/tree_build/step_graft.py | 190 ++++++++++++++++++++++ oz_tree_build/tree_build/step_parse.py | 89 +++++++++++ oz_tree_build/tree_build/tree_build.py | 86 ++++++++++ oz_tree_build/utilities/ete.py | 15 ++ pyproject.toml | 1 + tests/test_tree_build_step_graft.py | 161 +++++++++++++++++++ tests/test_tree_build_step_parse.py | 208 +++++++++++++++++++++++++ 8 files changed, 756 insertions(+), 32 deletions(-) create mode 100644 oz_tree_build/tree_build/step_graft.py create mode 100644 oz_tree_build/tree_build/step_parse.py create mode 100644 oz_tree_build/tree_build/tree_build.py create mode 100644 oz_tree_build/utilities/ete.py create mode 100644 tests/test_tree_build_step_graft.py create mode 100644 tests/test_tree_build_step_parse.py diff --git a/dvc.yaml b/dvc.yaml index 3e195dd3..aa3bee68 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -51,43 +51,17 @@ stages: outs: - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ - # ~a few secs - get_open_trees_from_one_zoom: + tree_build: cmd: - >- - cd data/OZTreeBuild/${oz_tree} && - get_open_trees_from_one_zoom - ../../OpenTree/${ot_version}/draftversion.tre - OpenTreeParts/OpenTree_all/ - BespokeTree/include_OT_${ot_version}/*.PHY + .venv/bin/tree_build + --bespoke_dir data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ + --orphan_dir data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ + --opentree data/dated_tree/dated_tree_pre.tre deps: - - data/OpenTree/${ot_version}/draftversion.tre - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ - params: - - oz_tree - - ot_version - outs: - - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OpenTree_all/ - - build_oz_tree: - cmd: - - >- - cd data/OZTreeBuild/${oz_tree} && - build_oz_tree - --nodeages ../../node_ages.json - BespokeTree/include_OT_${ot_version}/Base.PHY - OpenTreeParts/OpenTree_all/ - ${oz_tree}_full_tree.phy - deps: - - data/node_ages.json - - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ - - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OpenTree_all/ - params: - - oz_tree - - ot_version - outs: - - data/OZTreeBuild/${oz_tree}/${oz_tree}_full_tree.phy + - data/dated_tree/dated_tree_pre.tre download_eol: # EOL doesn't version the provider ids file, so capture the last-modified header instead diff --git a/oz_tree_build/tree_build/step_graft.py b/oz_tree_build/tree_build/step_graft.py new file mode 100644 index 00000000..5bb5358e --- /dev/null +++ b/oz_tree_build/tree_build/step_graft.py @@ -0,0 +1,190 @@ +import re + +from ..utilities.ete import node_get_ott + +OT_INCLUSION_SYNTAX_RE = re.compile(r"(\w+)[_ ]ott(\d*)~?([-\d]*)@$") + + +def graft_tree(t, additional_trees, prefer_subtree_name=False, disable_recursion=False): + """ + Resolve OneZoom inclusion syntax in ``t`` by grafting subtrees in place, + replacing from ``additional_trees`` (a dict inclusion string -> tree). + + Walks ``t`` and, for every node whose name matches the inclusion syntax (a + label ending in ``@``, see ``decypher_inclusion_syntax``). + ``additional_trees`` is a dict of inclusion labels to ete4 trees. + + Naming of the grafted node: + - default: use the name derived from the inclusion token (e.g. ``"Sub + ott1"``), falling back to the subtree's root name if the token has no + derived name + - ``prefer_subtree_name=True``: use the subtree's root name, falling + back to the token-derived name if the subtree root is unnamed + + By default, the function recurses into each grafted subtree so nested + inclusions are also resolved. Pass ``disable_recursion=True`` to graft + only one level, i.e. for OpenTree subtrees which won't contain inclusion syntax. + + Returns a list of inclusion labels that had no match in ``additional_trees`` + (including those found while recursing). Missing inclusions leave the + placeholder node unchanged in ``t``. + """ + missing_inclusions = [] + + def is_leaf_fn(n): + r = decypher_inclusion_syntax(n.name) + if r is None: + # No inclusion syntax, recurse + return n.is_leaf + + if n.name not in additional_trees: + # Not present, ignore for now + missing_inclusions.append(n.name) + return True + + # Graft sub_t at this point + sub_t = additional_trees[n.name] + if not disable_recursion: + missing_inclusions.extend(graft_tree(sub_t, additional_trees, prefer_subtree_name)) + if prefer_subtree_name: + n.name = sub_t.root.name or r["node_name"] + else: + n.name = r["node_name"] or sub_t.root.name + if sub_t.root.dist is not None: + n.dist = sub_t.root.dist + for key, val in sub_t.root.props.items(): + if val is not None: + n.props[key] = val + n.children = sub_t.root.children + + # Replaced children, no point recursing through the old ones + return True + + for _ in t.traverse(strategy="levelorder", is_leaf_fn=is_leaf_fn): + # NB: We do all the work in the is_leaf_fn, so we can influence whether to recurse + pass + + return missing_inclusions + + +def graft_extract_ot_subtrees(opentree_t, inclusions): + """ + Extract the subtrees needed to satisfy a list of OneZoom inclusion labels + from the full OpenTree tree. + + ``inclusions`` is a list of inclusion-syntax labels (e.g. ``"Sub_ott1@"``, + ``"Renamed_ott~5@"``); each is parsed for its base OTT + (see ``decypher_inclusion_syntax``). ``opentree_t`` is walked and any node + whose OTT matches one of those base OTTs is detached and returned as a + standalone subtree. ``opentree_t`` is mutated in place — every extracted + subtree is removed from it. + + The extraction recurses into each detached subtree, so a requested OTT + that lies inside another requested subtree is still extracted (and its + outer subtree no longer contains it). The returned dict is keyed by the + *original* inclusion label, preserving any rebase / exclusion syntax so + callers can pass the result straight to ``graft_tree``. + + Inclusions whose base OTT does not appear in ``opentree_t`` are silently + absent from the result. + """ + # Organise inclusions by base_ott (as string) + start_otts = {} + for i in inclusions: + r = decypher_inclusion_syntax(i) + start_otts[str(r["base_ott"])] = r + + def prune_ot_subtrees(ot_t): + out_trees = {} + + def is_leaf_fn(n): + # No point checking leaves + if n.is_leaf: + return True + + # Does this node have a required OTT? If not, ignore it + node_ott = node_get_ott(n) + if node_ott is None or node_ott not in start_otts: + return n.is_leaf + r = start_otts[node_ott] + del start_otts[node_ott] + + # Prune this tree, extract any required subtrees from this subtree + sub_t = n.detach() + out_trees.update(prune_ot_subtrees(sub_t)) # NB: node_ott now removed from start_otts, so won't loop + out_trees[r["orig_name"]] = sub_t + return True + + for _ in ot_t.traverse(strategy="preorder", is_leaf_fn=is_leaf_fn): + # NB: We do all the work in the is_leaf_fn, so we can influence whether to recurse + pass + return out_trees + + return prune_ot_subtrees(opentree_t) + + +def present_in_tree(t, inclusion): + """ + Is a node matching (inclusion) present anywhere in (t)? + """ + r = decypher_inclusion_syntax(inclusion) + to_find = "ott" + str(r["base_ott"]) + for n in t.traverse(): + if n.name.endswith(to_find): + return n + return None + + +def decypher_inclusion_syntax(node_name): + """ + Parse inclusion syntax from node label + Parse a single OneZoom token from label name + """ + if not node_name or not node_name.endswith("@"): + return None + + result = dict( + orig_name=node_name, + ) + + match = OT_INCLUSION_SYNTAX_RE.match(node_name) + if not match: + # Has an @, but not ott syntax. Assume bespoke + result["node_name"] = node_name[:-1] + return result + + # split by minus signs + result["excluded_otts"] = (match.group(3) or "").split("-") + + # If present, the first number after '=' is the tree to extract. + first_number_after_equal = result["excluded_otts"].pop(0) + result["base_ott"] = first_number_after_equal or match.group(2) + + # Note that we don't append the ott in the name if it came after the '=' + result["node_name"] = match.group(1) + if not first_number_after_equal: + result["node_name"] += f" ott{result['base_ott']}" + return result + + +def remove_exclusions(t, exclusion_otts): + """ + Given (t), prune any (exclusion_otts) from tree + """ + orphan_ns = [] + + def is_leaf_fn(n): + node_ott = node_get_ott(n) + if node_ott is None or node_ott not in exclusion_otts: + return n.is_leaf + + orphan_ns.append(n.detach()) + return True + + if len(exclusion_otts) == 0: + return orphan_ns + exclusion_otts = set(str(x) for x in exclusion_otts) + for _ in t.traverse(strategy="preorder", is_leaf_fn=is_leaf_fn): + # NB: We do all the work in the is_leaf_fn, so we can influence whether to recurse + pass + return orphan_ns diff --git a/oz_tree_build/tree_build/step_parse.py b/oz_tree_build/tree_build/step_parse.py new file mode 100644 index 00000000..fd72e7a0 --- /dev/null +++ b/oz_tree_build/tree_build/step_parse.py @@ -0,0 +1,89 @@ +# https://github.com/etetoolkit/ete/blob/ete4/ete4/core/tree.pyx +# https://github.com/etetoolkit/ete/blob/ete4/ete4/parser/newick.pyx +import glob +import logging +import os.path + +import ete4 + +from .step_graft import decypher_inclusion_syntax, remove_exclusions +from .token_to_oz_tree_file_mapping import token_to_file_map + +logger = logging.getLogger(__name__) + +NWK_READ_PARSER = 1 + + +def parse_ot_orphans(orphan_dir, inclusions): + """ + Load orphan OpenTree subtrees from ``orphan_dir`` that match the given + inclusion labels. + + ``orphan_dir`` is expected to contain ``.nwk`` files, one per OpenTree + subtree that lives outside the main OpenTree synthesis (the "orphan" + pieces). Each ``inclusions`` label is parsed for its base OTT + (see ``decypher_inclusion_syntax``); if a file named + ``.nwk`` exists, it is loaded and returned, keyed by the + *original* inclusion label so callers can pass the result straight to + ``graft_tree``. + + Any ``excluded_otts`` carried by the inclusion syntax are pruned from the + loaded subtree via ``remove_exclusions`` before it is returned. Orphan + files that don't correspond to any requested inclusion are skipped, and + inclusions with no matching orphan file are silently absent from the + result. + """ + # Organise inclusions by base_ott (as string) + start_otts = {} + for i in inclusions: + r = decypher_inclusion_syntax(i) + start_otts[str(r["base_ott"])] = r + + # Find orphan trees that match inclusion points + out_trees = {} + for orphan_path in glob.glob(os.path.join(orphan_dir, "*.nwk")): + node_ott = os.path.splitext(os.path.basename(orphan_path))[0] + if node_ott not in start_otts: + continue + r = start_otts[node_ott] + del start_otts[node_ott] + + sub_t = ete4.Tree(orphan_path, parser=NWK_READ_PARSER) + remove_exclusions(sub_t, r["excluded_otts"]) + out_trees[r["orig_name"]] = sub_t + return out_trees + + +def parse_bespoke_trees(bespoke_dir, base_name="Base.PHY"): + """ + Load the base tree and every hand-curated ("bespoke") subtree referenced + by ``token_to_file_map`` from ``bespoke_dir``. + + Returns a ``(base_t, bespoke_t)`` tuple: + - ``base_t`` is the tree parsed from ``/`` and + forms the trunk that everything else hangs off. + - ``bespoke_t`` is a dict keyed by ``"@"`` (matching the + inclusion syntax used inside ``base_t``) mapping to the parsed + subtree. For each token, the entry in ``token_to_file_map`` may + override the subtree's root name (``taxon``) and the length of the + edge connecting it to its parent (``edge_length``). + + The result is suitable for passing straight to ``graft_tree`` as the + ``additional_trees`` argument. Tokens whose file is missing from + ``bespoke_dir`` are logged as errors and omitted from ``bespoke_t`` + rather than raising. + """ + base_t = ete4.Tree(os.path.join(bespoke_dir, base_name), parser=NWK_READ_PARSER) + bespoke_t = {} + for key, x in token_to_file_map.items(): + key = key + "@" + sub_path = os.path.join(bespoke_dir, x["file"]) + if not os.path.exists(sub_path): + logger.error(f"Sub-tree {x['file']} referenced in token_to_oz_tree_file_mapping missing") + continue + bespoke_t[key] = ete4.Tree(sub_path, parser=NWK_READ_PARSER) + if x.get("taxon") is not None: + bespoke_t[key].root.name = x["taxon"] + if x.get("edge_length") is not None: + bespoke_t[key].root.dist = x["edge_length"] + return base_t, bespoke_t diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py new file mode 100644 index 00000000..daa31b82 --- /dev/null +++ b/oz_tree_build/tree_build/tree_build.py @@ -0,0 +1,86 @@ +""" +Splice together a set of trees using OZ inclusion syntax +""" + +import argparse +import logging + +from ..date_tree import date_tree +from ..utilities.debug_util import parse_args_and_add_logging_switch +from .step_graft import graft_extract_ot_subtrees, graft_tree +from .step_parse import parse_bespoke_trees, parse_ot_orphans + +logger = logging.getLogger(__name__) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--bespoke_dir", + help=("Directory containing bespoke trees, including Base.PHY, the root"), + ) + parser.add_argument( + "--orphan_dir", + help=("Directory containing orphan OpenTree subtrees"), + ) + parser.add_argument( + "--opentree", + help=("Newick tree with OpenTree"), + ) + parser.add_argument( + "--out_dir", + default="data/out", + help=("Directory to write output files to"), + ) + args = parse_args_and_add_logging_switch(parser) + + logger.info("Parse & graft bespoke tree together") + base_t, bespoke_ts = parse_bespoke_trees(args.bespoke_dir) + missing_inclusions = graft_tree(base_t, additional_trees=bespoke_ts, prefer_subtree_name=True) + + logger.info("Random resoution of polytomies for bespoke trees") + # tidy_resolve_polytomy_random(base_t) + + logger.info("Resolve branch lengths to dates bottom-up. Remove (or not care about) branch lengths") + # tidy_resolve_bl_to_dates(base_t) + + logger.info("Top-down conflict resolution in bespoke tree, delete entries that conflict with higher ages") + # tidy_resolve_date_conflicts(base_t) + + logger.info("Graft OT subtrees onto our trees. Already polytomy-resolved & date pins from chronosynth applied") + opentree_ts = graft_extract_ot_subtrees(date_tree.nwk_read(args.opentree), missing_inclusions) + opentree_ts.update(parse_ot_orphans(args.orphan_dir, missing_inclusions)) + missing_inclusions = graft_tree( + base_t, additional_trees=opentree_ts, prefer_subtree_name=False, disable_recursion=True + ) + for i in missing_inclusions: + logger.error(f"No subtree found for {i}") + + logger.info( + "Popularity calculations for entire tree (including any remaining subspecies from bespoke tree, " + "Jonathan's will have them already removed) (stop caring about polytomy vs. popularity calculations, " + "and just apply them post-resolution). Apply popularity based on OTT -> popularity map, percolate " + "using existing rules (which preserves popularity from removed subspecies)" + ) + # prop_add_popularity(base_t) + + logger.info("Remove subspecies (now popularity has percolated)") + # tidy_remove_subspecies(base_t) + + logger.info( + "Top-down conflict resolution. If there's conflict with higher ages, remove ages until conflict goes away" + ) + # resolve_date_conflicts(base_t) + + logger.info("Remove unary nodes (they are likely uninteresting, and make a mess of the tree rendering)") + # tidy_remove_unary(base_t) + + logger.info("Re-interpoltate missing dates") + # interpolate_dates(base_t) + + logger.info("Add properies to tree") + pass + + +if __name__ == "__main__": + main() diff --git a/oz_tree_build/utilities/ete.py b/oz_tree_build/utilities/ete.py new file mode 100644 index 00000000..506a3d63 --- /dev/null +++ b/oz_tree_build/utilities/ete.py @@ -0,0 +1,15 @@ +import re + +NODE_OTT_RE = re.compile(r"[_ ]ott(\d+)$") + + +def node_get_ott(n): + """ + Extract OTT from node if present, None otherwise + + NB: OTT is returned as string, not int + """ + if not n.name: + return None + m = NODE_OTT_RE.search(n.name) + return m.group(1) if m else None diff --git a/pyproject.toml b/pyproject.toml index 121c69fa..0e23d882 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ newick_combiner = "oz_tree_build.wiki_extraction.newick_combiner:main" add_dates_and_species_to_tree = "oz_tree_build.wiki_extraction.add_dates_and_species_to_tree:main" download_node_ages = "oz_tree_build.download_node_ages.download_node_ages:main" date_tree = "oz_tree_build.date_tree.date_tree:main" +tree_build = "oz_tree_build.tree_build.tree_build:main" [tool.setuptools] packages = ["oz_tree_build"] diff --git a/tests/test_tree_build_step_graft.py b/tests/test_tree_build_step_graft.py new file mode 100644 index 00000000..bd5aa009 --- /dev/null +++ b/tests/test_tree_build_step_graft.py @@ -0,0 +1,161 @@ +import ete4 + +from oz_tree_build.tree_build.step_graft import graft_extract_ot_subtrees, graft_tree, present_in_tree + + +class TestGraftTree: + def test_no_inclusions_is_noop(self): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + missing = graft_tree(t, {}) + assert missing == [] + assert t.write() == "(A_ott1,B_ott2);" + + def test_simple_graft(self): + # The grafted node inherits the subtree's children and its dist. + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Y_ott12)SubRoot_ott1;", parser=1) + sub.root.dist = 3.5 + + missing = graft_tree(t, {"Sub_ott1@": sub}) + + assert missing == [] + assert t.write() == "(A_ott99,(X_ott11,Y_ott12):3.5);" + + def test_missing_inclusion_reported(self): + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + missing = graft_tree(t, {}) + assert missing == ["Sub_ott1@"] + # Original tree untouched at the inclusion point. + assert t.write() == "(A_ott99,Sub_ott1@);" + + def test_multiple_missing_inclusions(self): + t = ete4.Tree("(A_ott1@,(B_ott2@,C_ott3@)Sub)Root;", parser=1) + missing = graft_tree(t, {}) + assert sorted(missing) == ["A_ott1@", "B_ott2@", "C_ott3@"] + + def test_prefer_subtree_name_uses_subroot_name(self): + # With prefer_subtree_name=True, the subtree's root name wins. + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Y_ott12)SubRoot_ott1;", parser=1) + + graft_tree(t, {"Sub_ott1@": sub}, prefer_subtree_name=True) + + grafted_names = [n.name for n in t.traverse() if not n.is_leaf and n.name] + assert "SubRoot_ott1" in grafted_names + + def test_prefer_subtree_name_falls_back_when_subroot_unnamed(self): + # If the subtree's root is unnamed, fall back to the inclusion's + # derived node_name ("Sub ott1"). + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Y_ott12);", parser=1) + + graft_tree(t, {"Sub_ott1@": sub}, prefer_subtree_name=True) + + grafted_names = [n.name for n in t.traverse() if not n.is_leaf and n.name] + assert "Sub ott1" in grafted_names + + def test_recursion_resolves_nested_inclusions(self): + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Inner_ott2@)SubRoot;", parser=1) + inner = ete4.Tree("(I1_ott21,I2_ott22)InnerRoot;", parser=1) + + missing = graft_tree(t, {"Sub_ott1@": sub, "Inner_ott2@": inner}) + + assert missing == [] + assert t.write() == "(A_ott99,(X_ott11,(I1_ott21,I2_ott22)));" + + def test_disable_recursion_leaves_nested_inclusions(self): + # disable_recursion=True skips the inner graft_tree call, so the + # nested inclusion token is left in place and not reported missing + # (since the traversal treats the grafted node as a leaf). + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Inner_ott2@)SubRoot;", parser=1) + inner = ete4.Tree("(I1_ott21,I2_ott22)InnerRoot;", parser=1) + + missing = graft_tree( + t, + {"Sub_ott1@": sub, "Inner_ott2@": inner}, + disable_recursion=True, + ) + + assert missing == [] + assert t.write() == "(A_ott99,(X_ott11,Inner_ott2@));" + + def test_recursion_reports_missing_from_nested(self): + # A nested inclusion that has no provided subtree is reported. + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Missing_ott2@)SubRoot;", parser=1) + + missing = graft_tree(t, {"Sub_ott1@": sub}) + + assert missing == ["Missing_ott2@"] + + def test_subtree_props_copied(self): + # Non-None props from the subtree root are copied onto the grafted node. + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Y_ott12)SubRoot_ott1;", parser=1) + sub.root.props["custom_prop"] = "hello" + + graft_tree(t, {"Sub_ott1@": sub}) + + grafted = next(n for n in t.traverse() if n.props.get("custom_prop") == "hello") + assert grafted is not None + + +class TestGraftExtractOtSubtrees: + def test_extracts_subtrees_by_base_ott(self, tmp_path): + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((X1_ott11,X2_ott12)Sub1_ott1,(Y1_ott21,Y2_ott22)Sub2_ott2)Inner_ott3,Z_ott4)Root_ott99;") + result = graft_extract_ot_subtrees(ete4.Tree(str(ot_file), parser=1), ["Sub1_ott1@", "Sub2_ott2@"]) + assert set(result.keys()) == {"Sub1_ott1@", "Sub2_ott2@"} + assert result["Sub1_ott1@"].write() == "(X1_ott11,X2_ott12);" + assert result["Sub2_ott2@"].write() == "(Y1_ott21,Y2_ott22);" + + def test_missing_otts_not_in_result(self, tmp_path): + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("((A_ott1,B_ott2)Sub_ott3)Root_ott4;") + result = graft_extract_ot_subtrees(ete4.Tree(str(ot_file), parser=1), ["NotThere_ott99@"]) + assert result == {} + + def test_renaming_inclusion_uses_orig_name_as_key(self, tmp_path): + # The key is the original inclusion string, including any rebase syntax. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("((A_ott1,B_ott2)Sub_ott5,C_ott3)Root_ott99;") + result = graft_extract_ot_subtrees(ete4.Tree(str(ot_file), parser=1), ["Renamed_ott~5@"]) + assert list(result.keys()) == ["Renamed_ott~5@"] + assert result["Renamed_ott~5@"].write() == "(A_ott1,B_ott2);" + + def test_recurses_into_extracted_subtrees(self, tmp_path): + # If an extracted subtree itself contains another requested OTT, that + # nested subtree is also extracted. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((((I1_ott11,I2_ott12)Inner_ott1)Filler_ott99)Sub_ott2)Outer_ott3)Root_ott99;") + result = graft_extract_ot_subtrees(ete4.Tree(str(ot_file), parser=1), ["Outer_ott2@", "Nested_ott1@"]) + assert set(result.keys()) == {"Outer_ott2@", "Nested_ott1@"} + assert result["Nested_ott1@"].write() == "(I1_ott11,I2_ott12);" + # NB: Outer tree no longer contains inner tree + assert result["Outer_ott2@"].write() == "(Filler_ott99);" + + +class TestPresentInTree: + def test_finds_matching_node(self): + t = ete4.Tree("(A_ott99,(Sub_ott1,C_ott4)B_ott7)Root_ott42;", parser=1) + n = present_in_tree(t, "Anything_ott1@") + assert n is not None + assert n.name == "Sub_ott1" + + def test_matches_internal_node(self): + t = ete4.Tree("(A_ott99,(Sub_ott1,C_ott4)B_ott7)Root_ott42;", parser=1) + n = present_in_tree(t, "Anything_ott7@") + assert n is not None + assert n.name == "B_ott7" + + def test_returns_none_when_absent(self): + t = ete4.Tree("(A_ott99,(Sub_ott1,C_ott4)B_ott7)Root_ott42;", parser=1) + assert present_in_tree(t, "Anything_ott999@") is None + + def test_matches_root(self): + t = ete4.Tree("(A_ott99,B_ott7)Root_ott42;", parser=1) + n = present_in_tree(t, "Anything_ott42@") + assert n is not None + assert n.name == "Root_ott42" diff --git a/tests/test_tree_build_step_parse.py b/tests/test_tree_build_step_parse.py new file mode 100644 index 00000000..7175c0de --- /dev/null +++ b/tests/test_tree_build_step_parse.py @@ -0,0 +1,208 @@ +import ete4 +import pytest + +from oz_tree_build.tree_build import step_parse +from oz_tree_build.tree_build.step_parse import ( + decypher_inclusion_syntax, + parse_bespoke_trees, + parse_ot_orphans, + remove_exclusions, +) + + +class TestDecypherInclusionSyntax: + def test_returns_none_for_falsy_input(self): + assert decypher_inclusion_syntax(None) is None + assert decypher_inclusion_syntax("") is None + + def test_returns_none_without_trailing_at(self): + assert decypher_inclusion_syntax("Foo_ott1") is None + assert decypher_inclusion_syntax("BASE") is None + + def test_bespoke_token_no_ott(self): + # Bespoke tokens lack the _ottN suffix; they fall through to the + # "@-but-no-ott-syntax" branch. + assert decypher_inclusion_syntax("BASE@") == { + "orig_name": "BASE@", + "node_name": "BASE", + } + assert decypher_inclusion_syntax("AMORPHEA@") == { + "orig_name": "AMORPHEA@", + "node_name": "AMORPHEA", + } + + def test_simple_ott(self): + # node_name preserves the "ottN" suffix (space-separated) when no + # rebase via ~ occurred. + assert decypher_inclusion_syntax("Foo_ott1@") == { + "orig_name": "Foo_ott1@", + "excluded_otts": [], + "base_ott": "1", + "node_name": "Foo ott1", + } + + def test_ott_with_rebase(self): + # `~N` rebases the subtree extraction onto ott N; the parent's ott + # number is dropped from node_name. + assert decypher_inclusion_syntax("Foo_ott1~2@") == { + "orig_name": "Foo_ott1~2@", + "excluded_otts": [], + "base_ott": "2", + "node_name": "Foo", + } + + def test_ott_with_exclusions(self): + assert decypher_inclusion_syntax("Foo_ott1~-2-3@") == { + "orig_name": "Foo_ott1~-2-3@", + "excluded_otts": ["2", "3"], + "base_ott": "1", + "node_name": "Foo ott1", + } + + def test_ott_with_rebase_and_exclusions(self): + assert decypher_inclusion_syntax("Foo_ott~4-2-3@") == { + "orig_name": "Foo_ott~4-2-3@", + "excluded_otts": ["2", "3"], + "base_ott": "4", + "node_name": "Foo", + } + + +class TestRemoveExclusions: + def test_empty_exclusion_list_is_noop(self): + t = ete4.Tree("(A_ott1,B_ott2)R_ott3;", parser=1) + orphans = remove_exclusions(t, []) + assert orphans == [] + assert t.write() == "(A_ott1,B_ott2);" + + def test_removes_matching_otts(self): + t = ete4.Tree( + "((A_ott1,B_ott2)Sub_ott3,(C_ott4,D_ott5)Other_ott6)Root_ott7;", + parser=1, + ) + orphans = remove_exclusions(t, ["2", "4"]) + assert t.write() == "((A_ott1),(D_ott5));" + orphan_names = sorted(o.root.name for o in orphans) + assert orphan_names == ["B_ott2", "C_ott4"] + + def test_accepts_int_exclusions(self): + # Values are stringified before matching, so int input is fine. + t = ete4.Tree("(A_ott1,B_ott2)R_ott3;", parser=1) + orphans = remove_exclusions(t, [2]) + assert t.write() == "(A_ott1);" + assert [o.root.name for o in orphans] == ["B_ott2"] + + def test_non_matching_otts_left_alone(self): + t = ete4.Tree("(A_ott1,B_ott2)R_ott3;", parser=1) + orphans = remove_exclusions(t, ["99"]) + assert orphans == [] + assert t.write() == "(A_ott1,B_ott2);" + + +class TestParseOtOrphans: + def test_picks_up_matching_files(self, tmp_path): + for ott in ["1", "2", "3"]: + (tmp_path / f"{ott}.nwk").write_text(f"(A_ott{ott}0,B_ott{ott}1);") + result = parse_ot_orphans(str(tmp_path), ["Sub1_ott1@", "Sub3_ott3@"]) + assert set(result.keys()) == {"Sub1_ott1@", "Sub3_ott3@"} + assert result["Sub1_ott1@"].write() == "(A_ott10,B_ott11);" + assert result["Sub3_ott3@"].write() == "(A_ott30,B_ott31);" + + def test_no_matches_returns_empty(self, tmp_path): + (tmp_path / "5.nwk").write_text("(A,B);") + result = parse_ot_orphans(str(tmp_path), ["Sub_ott99@"]) + assert result == {} + + def test_empty_directory(self, tmp_path): + assert parse_ot_orphans(str(tmp_path), ["Sub_ott1@"]) == {} + + def test_applies_exclusions(self, tmp_path): + # Exclusion otts named in the inclusion syntax are pruned from the + # loaded orphan tree. + (tmp_path / "1.nwk").write_text("((A_ott11,B_ott12)Inner_ott13,(C_ott14,D_ott15)Other_ott16)R_ott1;") + result = parse_ot_orphans(str(tmp_path), ["Sub_ott1~-12-14@"]) + assert set(result.keys()) == {"Sub_ott1~-12-14@"} + assert result["Sub_ott1~-12-14@"].write() == "((A_ott11),(D_ott15));" + + def test_non_nwk_files_ignored(self, tmp_path): + (tmp_path / "1.txt").write_text("(A,B);") + assert parse_ot_orphans(str(tmp_path), ["Sub_ott1@"]) == {} + + +class TestParseBespokeTrees: + def test_reads_base_and_listed_files(self, tmp_path, monkeypatch): + # Restrict the token map so we only have to provide a few files. + monkeypatch.setattr( + step_parse, + "token_to_file_map", + { + "AMORPHEA": {"file": "Amorphea.PHY", "edge_length": 50, "taxon": None}, + "AMBULACRARIA": { + "file": "Ambulacraria.PHY", + "edge_length": 20, + "taxon": "AmbulacrariaOverride", + }, + }, + ) + (tmp_path / "Base.PHY").write_text("(AMORPHEA@,AMBULACRARIA@)Root;") + (tmp_path / "Amorphea.PHY").write_text("(A_ott1,B_ott2)Amorphea;") + (tmp_path / "Ambulacraria.PHY").write_text("(C_ott3,D_ott4)OriginalName;") + + base_t, bespoke_t = parse_bespoke_trees(str(tmp_path)) + + assert base_t.write() == "(AMORPHEA@,AMBULACRARIA@);" + assert set(bespoke_t.keys()) == {"AMORPHEA@", "AMBULACRARIA@"} + + # edge_length from token_to_file_map is applied to the subtree root. + assert bespoke_t["AMORPHEA@"].root.dist == 50 + # AMORPHEA has no taxon override, so its root name is the file's own. + assert bespoke_t["AMORPHEA@"].root.name == "Amorphea" + + # AMBULACRARIA has a taxon override, which replaces the root name. + assert bespoke_t["AMBULACRARIA@"].root.dist == 20 + assert bespoke_t["AMBULACRARIA@"].root.name == "AmbulacrariaOverride" + + def test_missing_file_logged_and_skipped(self, tmp_path, monkeypatch, caplog): + monkeypatch.setattr( + step_parse, + "token_to_file_map", + { + "AMORPHEA": {"file": "Amorphea.PHY", "edge_length": 50, "taxon": None}, + "MISSING": {"file": "DoesNotExist.PHY", "edge_length": 1, "taxon": None}, + }, + ) + (tmp_path / "Base.PHY").write_text("(AMORPHEA@,MISSING@)Root;") + (tmp_path / "Amorphea.PHY").write_text("(A_ott1,B_ott2)Amorphea;") + + with caplog.at_level("ERROR", logger=step_parse.__name__): + _, bespoke_t = parse_bespoke_trees(str(tmp_path)) + + assert "AMORPHEA@" in bespoke_t + assert "MISSING@" not in bespoke_t + assert any("DoesNotExist.PHY" in r.message for r in caplog.records) + + def test_no_edge_length_leaves_dist_none(self, tmp_path, monkeypatch): + monkeypatch.setattr( + step_parse, + "token_to_file_map", + { + "CRUMS": {"file": "CRuMs.PHY", "edge_length": None, "taxon": None}, + }, + ) + (tmp_path / "Base.PHY").write_text("(CRUMS@)Root;") + (tmp_path / "CRuMs.PHY").write_text("(A_ott1,B_ott2)CRuMs;") + + _, bespoke_t = parse_bespoke_trees(str(tmp_path)) + assert bespoke_t["CRUMS@"].root.dist is None + + def test_alternate_base_name(self, tmp_path, monkeypatch): + monkeypatch.setattr(step_parse, "token_to_file_map", {}) + (tmp_path / "OtherBase.PHY").write_text("(A_ott1,B_ott2)R;") + base_t, bespoke_t = parse_bespoke_trees(str(tmp_path), base_name="OtherBase.PHY") + assert base_t.write() == "(A_ott1,B_ott2);" + assert bespoke_t == {} + + def test_missing_base_file_raises(self, tmp_path, monkeypatch): + monkeypatch.setattr(step_parse, "token_to_file_map", {}) + with pytest.raises(FileNotFoundError): + parse_bespoke_trees(str(tmp_path)) From 27030ab76b135d0d62f1da149965875caca7f88c Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Wed, 10 Jun 2026 17:40:33 +0000 Subject: [PATCH 11/62] taxon_map: Pull mapping from CSV_base_table_creator Lift and shift code from CSV_base_table_creator into a separate file that generates taxon mapping & raw popularity based purely on OT taxonomy.tsv, rather than reading the tree. This may process more nodes than required, but I'm sure we'll manage. We also no longer try and process mrca nodes, but given we explicitly filter them anyway, this has been broken for a long time. References: https://github.com/OneZoom/tree-build/issues/131 --- data/.gitignore | 1 + dvc.yaml | 19 + .../taxon_mapping_and_popularity/taxon_map.py | 496 ++++++++++++++++++ pyproject.toml | 1 + 4 files changed, 517 insertions(+) create mode 100644 oz_tree_build/taxon_mapping_and_popularity/taxon_map.py diff --git a/data/.gitignore b/data/.gitignore index 4bf6b42b..be7a0ec8 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -2,3 +2,4 @@ /output_files /node_ages.json /dated_tree/ +/taxon_map.csv diff --git a/dvc.yaml b/dvc.yaml index aa3bee68..c5cdeb31 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -158,6 +158,25 @@ stages: - data/filtered/pageviews/: persist: true + # ~4 mins + taxon_map: + cmd: >- + .venv/bin/taxon_map + --OpenTreeTaxonomy data/OpenTree/${ot_version}/taxonomy.tsv + --wikidataDumpFile data/filtered/OneZoom_latest-all.json + --wikipediaSQLDumpFile data/filtered/OneZoom_enwiki-latest-page.sql + --wikipedia_totals_bz2_pageviews data/filtered/pageviews/ + --EOLidentifiers data/filtered/OneZoom_provider_ids.csv + -o data/taxon_map.csv + deps: + - data/OpenTree/${ot_version}/taxonomy.tsv + - data/filtered/OneZoom_latest-all.json + - data/filtered/OneZoom_enwiki-latest-page.sql + - data/filtered/pageviews/ + - data/filtered/OneZoom_provider_ids.csv + outs: + - data/taxon_map.csv + # ~10 mins CSV_base_table_creator: cmd: diff --git a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py new file mode 100644 index 00000000..bdec42e9 --- /dev/null +++ b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py @@ -0,0 +1,496 @@ +""" +Read in OpenTree taxonomy.csv, join it with wikidata / wikipedia data to +generate a CSV mapping OTT to various IDs, and raw popularity. +""" + +import argparse +import collections +import csv +import glob +import logging +import os.path + +from ..taxon_mapping_and_popularity import OTT_popularity_mapping +from ..utilities.debug_util import parse_args_and_add_logging_switch +from ..utilities.file_utils import open_file_based_on_extension + +logger = logging.getLogger(__name__) + + +def map_wiki_info( + source_ptrs, + source_order, + OTT_ptrs, + WD_filename, + lang, + WP_SQL_filename, + WP_pageviews_filenames, +): + """ + - source_ptrs: + - OTT_ptrs: {ott: { ott: X, sources: {} }} (generated by get_OTT_list) + + 1) use the wikidata JSON dump to map identifiers from the source_ptrs structure to + wikidata Qids, and then on to wikipedia pages + 2) if sql and pagevisits filenames are given + + Return True if popularity is mapped + """ + logger.debug(f"Processing wikidata json dump in parallel for {lang}") + popularity_steps = 0 + WDitems, WPnames, swap_Qs = set_wikidata(WD_filename, source_ptrs, lang) + + if WP_SQL_filename is not None: + logger.info(f" > Adding wikipedia page sizes from {WP_SQL_filename}") + # Can't easily parallelize this as it is gzip compressed (not a block format) + OTT_popularity_mapping.add_pagesize_for_titles(WPnames, WP_SQL_filename) + popularity_steps += 1 + + if len(WP_pageviews_filenames) > 0: + logger.info(f" > Adding wikipedia visit counts from {len(WP_pageviews_filenames)} files") + set_wikipedia_pageviews(WP_pageviews_filenames, WPnames, lang) + popularity_steps += 1 + + if popularity_steps == 2: + logger.info(" > Calculating raw popularity measures") + tot = 0 + for WDinstance in WDitems.values(): + if WDinstance.set_raw_popularity(): + tot += 1 + logger.info(f" ✔ Raw popularity measures set on {tot} wikidata items") + else: + logger.info(" x Skipping popularity calculations") + + # Here we might want to multiply up some taxa, e.g. plants, + # see https://github.com/OneZoom/OZtree/issues/130 + logger.info(" > Finding best wiki matches") + OTT_popularity_mapping.identify_best_wikidata(OTT_ptrs, lang, source_order) + + logger.info(" > Swapping vernacular wikidata items into taxon items") + OTT_popularity_mapping.overwrite_wd(WDitems, swap_Qs, only_if_more_popular=(popularity_steps == 2), check_lang=lang) + + logger.info(" > Supplementing ids (EOL/IPNI) with ones from wikidata") + supplement_from_wikidata(OTT_ptrs) + + logger.info("✔ Wikidata/wikipedia data mapped") + + display_WD_ott_stats(OTT_ptrs) + + return popularity_steps == 2 + + +def display_WD_ott_stats(OTT_ptrs): + """ + Display some stats about OTTs coming from Wikidata + """ + matching_otts = 0 + mismatching_otts = 0 + no_wd_otts = 0 + for ott in OTT_ptrs: + try: + if OTT_ptrs[ott]["rank"] == "species": + if OTT_ptrs[ott]["wd"].get("wd_ott") is not None: + if ott == OTT_ptrs[ott]["wd"].wd_ott: + matching_otts += 1 + else: + logger.debug(f"Q{OTT_ptrs[ott]['wd'].Q}: OTT {ott} does not match {OTT_ptrs[ott]['wd'].wd_ott}") + mismatching_otts += 1 + else: + no_wd_otts += 1 + except (KeyError, AttributeError): + pass + + logger.info("✔ Stats on Wikidata OTT matching:") + logger.info(f" Leaves where the WD ott matches the ott: {matching_otts}") + logger.info(f" Leaves where the WD ott does not match the wd_ott: {mismatching_otts}") + logger.info(f" Leaves where WD does not have an ott: {no_wd_otts}") + + +def set_wikidata(bz2_filename, source_ptrs, lang): + """ + Will alter the source_ptrs. + Returns WDitems (Q->WD), WPnames (name-WD), common_name_Qs (Q->Q) + """ + WDitems = {} + WPnames = {} + common_name_Qs = {} + sum_info = collections.defaultdict(int) + + ( + Q_to_WD, + WPname_to_WD, + src_to_WD, + replace_Q, + info, + ) = OTT_popularity_mapping.wikidata_info(bz2_filename, source_ptrs, lang) + + WDitems.update(Q_to_WD) + WPnames.update(WPname_to_WD) + common_name_Qs.update(replace_Q) + # Add 'wd' item to source_ptrs + for src, ids in src_to_WD.items(): + for src_id, WD in ids.items(): + source_ptrs[src][src_id]["wd"] = WD + for k, v in info.items(): + sum_info[k] += v + + logger.info( + f"✔ {len(WDitems)} wikidata matches, of which " + f"{sum_info['n_eol']} have EOL ids, {sum_info['n_iucn']} have IUCN ids, " + f"{sum_info['n_ipni']} have IPNI, and {len(WPnames)} " + f"({(len(WPnames)/len(WDitems)*100):.2f}%) have titles that exist on " + f"{lang}.wikipedia. Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + return WDitems, WPnames, common_name_Qs + + +def set_wikipedia_pageviews(filenames, WPnames, lang): + names_found = 0 + for fn in filenames: + WPnames_views = OTT_popularity_mapping.pageviews_for_titles(fn, set(WPnames.keys()), lang) + for name, n_views in WPnames_views.items(): + if not hasattr(WPnames[name], "pageviews"): + names_found += 1 + WPnames[name].pageviews = [] + WPnames[name].pageviews.append(n_views) + logger.info( + f" ✔ Of {len(WPnames)} WikiData taxon entries, {names_found} " + f"({(names_found/len(WPnames) * 100):.2f}%) have pageview data for '{lang}' in " + f"{len(filenames)} files. Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + + +def supplement_from_wikidata(OTT_ptrs): + """ + If no OTT_ptrs[OTTid]['eol'] exists, but there is an + OTT_ptrs[OTTid]['wd']['initial_wiki_item']['EoL'] then put this into + OTT_ptrs[OTTid]['eol'] + Similarly for IPNI (although this is currently unpopulated) + """ + EOLalready = n_eol = n_ipni = n = 0 + for data in OTT_ptrs.values(): + n += 1 + if data.get("eol") is None: + try: + data["eol"] = int(data["wd"].EoL) + n_eol += 1 + except (AttributeError, KeyError, TypeError, ValueError): + pass + else: + EOLalready += 1 + if data.get("ipni") is None: + try: + data["ipni"] = int(data["wd"].ipni) + n_ipni += 1 + except (AttributeError, KeyError, TypeError, ValueError): + pass + logger.info( + f"✔ Out of {n} OTT taxa, {EOLalready} ({(EOLalready/n * 100):.2f}%) already " + f"have EOL ids from the EOL file. Supplementing these with {n_eol} EOL ids from " + f"wikidata gives a coverage of {((EOLalready + n_eol)/n * 100):.1f} %." + + (f" An addition {n_ipni} IPNI identifiers added via wikidata" if n_ipni else "") + ) + + +def add_eol_IDs_from_EOL_table_dump(source_ptrs, identifiers_filename, source_mapping): + used = 0 + EOL2OTT = {v: k for k, v in source_mapping.items()} + with open_file_based_on_extension(identifiers_filename, "rt") as identifiers_file: + reader = csv.DictReader(identifiers_file) + for EOLrow in reader: + if reader.line_num % 1000000 == 0: + logger.info( + f"... {reader.line_num} rows read, {used} used, " + f"mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + provider = int(EOLrow["resource_id"]) + if provider in EOL2OTT: + src = source_ptrs[EOL2OTT[provider]] + if EOL2OTT[provider] == "gbif" and not EOLrow["resource_pk"].isdigit(): + # The EoL file has duplicate (non numeric) IDs for GBIF: ignore these + continue + providerid = EOLrow["resource_pk"] + EOLid = int(EOLrow["page_id"]) + try: + if int(providerid) in src: + used += 1 + src[int(providerid)]["EoL"] = EOLid + except ValueError: + if providerid in src: + used += 1 + src[providerid]["EoL"] = EOLid + logger.info( + f"✔ Matched {used} EoL entries in the EoL identifiers file. " + f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + + +def identify_best_EoLdata(OTT_ptrs, sources): + """ + Each OTT number may point to several EoL entries, one for the NCBI number, + another for the WORMS number, etc etc. Hopefully these will be the same entry, + but they may not be. If they are different we need to choose the best one + to use. We take the one with the most sources supporting this entry: + if there is a tie, we take the lowest, as recommended by JRice from EoL + """ + validOTTs = OTTs_with_EOLmatch = dups = 0 + for OTTid, data in OTT_ptrs.items(): + validOTTs += 1 + choose = {} + for src in sources: + if src in data["sources"] and data["sources"][src] is not None: + if "EoL" in data["sources"][src]: + EOLid = int(data["sources"][src]["EoL"]) + if EOLid not in choose: + choose[EOLid] = [] + choose[EOLid] += [src] + if len(choose) == 0: + data["eol"] = None + else: + OTTs_with_EOLmatch += 1 + errstr = None + if len(choose) > 1: + # weed out those EOLids with the least support. + errstr = f"More than one EoL ID {choose} for taxon OTT: {OTTid}" + dups += 1 + max_refs = max([len(choose[i]) for i in choose]) + choose = [EOLid for EOLid in choose if len(choose[EOLid]) == max_refs] + best = min(choose) + data["eol"] = best + if errstr: + logger.debug(f" {errstr}, chosen {best}") + logger.info( + f" ✔ Of {validOTTs} OpenTree taxa, {OTTs_with_EOLmatch} " + f"({OTTs_with_EOLmatch / validOTTs * 100:.2f}%) have EoL entries in the EoL " + f"identifiers file, and {dups} have multiple possible EOL ids. " + f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + + +def populate_iucn(OTT_ptrs, identifiers_filename, verbosity=0, iucn_num=5): + """ + Port the IUCN number from both EoL and Wikidata, and keep both if there is a conflict + """ + used = 0 + + eol_mapping = {} # to store eol=>iucn + for OTTid, data in OTT_ptrs.items(): + if "eol" in data: + if data["eol"] in eol_mapping: + eol_mapping[data["eol"]].append(OTTid) + else: + eol_mapping[data["eol"]] = [OTTid] + + with open_file_based_on_extension(identifiers_filename, "rt") as identifiers_file: + reader = csv.DictReader(identifiers_file) + for EOLrow in reader: + if reader.line_num % 1000000 == 0: + logger.info( + f" - {reader.line_num} rows read, {used} used. " f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + if int(EOLrow["resource_id"]) == iucn_num and EOLrow["resource_pk"].isdigit(): + # there are lots of non-species IUCN rows with pk == str (e.g. Animalia) + try: + for ott in eol_mapping[int(EOLrow["page_id"])]: + OTT_ptrs[ott]["iucn"] = EOLrow["resource_pk"] + used += 1 + except LookupError: + pass # no equivalent eol id in eol_mapping + logger.info( + f" > matched {used} IUCN entries in the EoL identifiers file. " + f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + + # now go through and double-check against IUCN stored on wikidata + for OTTid, data in OTT_ptrs.items(): + try: + wd_iucn = str(int(data["wd"].iucn)) + if "iucn" not in data: + data["iucn"] = wd_iucn + used += 1 + else: + if wd_iucn not in data["iucn"].split("|"): + data["iucn"] += "|" + wd_iucn + logger.debug( + f' conflicting IUCN IDs for OTT {OTTid}: EoL = {data["iucn"]} ' + f'(via http://eol.org/pages/{data["eol"]}), wikidata = ' + f'{wd_iucn} (via http://http://wikidata.org/wiki/Q{data["wd"].Q}).' + ) + except ValueError: + logger.warning(f" Cannot convert wikidata IUCN ID {data['wd'].iucn} to integer.") + except (KeyError, AttributeError): + pass # can't find a wd instance or an iucn within the wd instance. Oh well. + + logger.info(f" > Increased IUCN coverage to {used} taxa using wikidata") + + +def read_ot_taxonomy(path="./data/OpenTree/v16.1/taxonomy.tsv"): + """Yield each row of an OpenTree taxonomy file as a dict keyed by header. + + Fields are separated by ``\\t|\\t`` and each line ends with a trailing + ``\\t|\\t``, so this is not parseable as a plain TSV. + """ + with open(path, encoding="utf-8") as f: + header = next(f).rstrip("\n").split("\t|\t")[:-1] + for line in f: + fields = line.rstrip("\n").split("\t|\t")[:-1] + out = dict(zip(header, fields)) + + out["uid"] = int(out["uid"]) + out["parent_uid"] = None if out["parent_uid"] == "" else int(out["parent_uid"]) + + # Deparse sourceinfo to a dict of IDs + sourceinfo = {} + for i in out["sourceinfo"].split(","): + k, v = i.split(":", 1) + sourceinfo[k] = int(v) if v.isdigit() else v + out["sourceinfo"] = sourceinfo + + yield out + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--OpenTreeTaxonomy", + default="data/OpenTree/v16.1/taxonomy.tsv", + help="The OpenTree taxonomy.tsv file, from http://files.opentreeoflife.org/ott/", + ) + parser.add_argument( + "--wikidataDumpFile", + default="data/filtered/OneZoom_latest-all.json", + help=( + "The very large wikidata JSON dump, " + "from https://dumps.wikimedia.org/wikidatawiki/entities/ (latest-all.json.bz2)." + "A filtered version can be used for faster processing." + ), + ) + parser.add_argument( + "--wikilang", + default="en", + help=( + 'The language wikipedia to check for popularity, e.g. "en". ' + "Where there are multiple Wikidata items for a taxon " + "(e.g. one under the common name, one under the scientific name), " + "then we also default to using the WD item with the sitelink in this language." + ), + ) + parser.add_argument( + "--wikipediaSQLDumpFile", + default="data/filtered/OneZoom_enwiki-latest-page.sql", + help=( + "The gzipped >1GB wikipedia -latest-page.sql.gz dump, " + "from https://dumps.wikimedia.org/enwiki/latest/ (enwiki-page.sql.gz) " + ), + ) + parser.add_argument( + "--wikipedia_totals_bz2_pageviews", + default="data/filtered/pageviews/", + help=( + 'Directory of b2zipped "totals" pageview count files, ' + "from https://dumps.wikimedia.org/other/pagecounts-ez/merged/ " + "(e.g. pagecounts-2016-01-views-ge-5-totals.bz2, or pagecounts*totals.bz2)" + ), + ) + parser.add_argument( + "--EOLidentifiers", + default="data/filtered/OneZoom_provider_ids.csv", + help=("EOL identifiers file, from " "https://opendata.eol.org/dataset/identifiers-csv-gz"), + ) + parser.add_argument( + "-o", + type=argparse.FileType("w"), + default="-", + help=("File to output CSV OTT map to"), + ) + parser.add_argument( + "--skip_popularity", + action="store_true", + help="Skip popularity calculations (no wikipedia SQL dump or pageviews required)", + ) + args = parse_args_and_add_logging_switch(parser) + + # Generate both OTT_ptrs / source_ptrs from taxonomy + # Replaces get_OTT_list & OTT_popularity_mapping.create_from_taxonomy respectively + logger.info("Generating OTT_ptrs / source_ptrs from taxonomy") + OTT_ptrs = {} + source_ptrs = {} + for r in read_ot_taxonomy(args.OpenTreeTaxonomy): + OTTid = r["uid"] + OTT_ptrs[OTTid] = {"ott": OTTid, "sources": {}} + + has_ncbi = False + for src in reversed( + r["sourceinfo"].keys() + ): # NB: look at sources in reverse order, overwriting, so 1st ones take priority + src_id = r["sourceinfo"][src] + if src == "ncbi": + has_ncbi = True + elif not has_ncbi and src == "ncbi_silva": + # only use the ncbi_via_silva id if no 'normal' ncbi already set + src = "ncbi" + if src not in source_ptrs: + source_ptrs[src] = {} + source_ptrs[src][src_id] = {"id": src_id} + OTT_ptrs[OTTid]["sources"][src] = source_ptrs[src][src_id] + OTT_ptrs[OTTid]["rank"] = r["rank"] + + eol_sources = { + "ncbi": 676, + "worms": 459, + "gbif": 767, + } # update when EoL has harvested index fungorum & IRMNG + add_eol_IDs_from_EOL_table_dump(source_ptrs, args.EOLidentifiers, eol_sources) + identify_best_EoLdata(OTT_ptrs, eol_sources) + + map_wiki_info( + source_ptrs=source_ptrs, + source_order=["ncbi", "if", "worms", "irmng", "gbif"], + OTT_ptrs=OTT_ptrs, + WD_filename=args.wikidataDumpFile, + lang=args.wikilang, + WP_SQL_filename=(None if args.skip_popularity else args.wikipediaSQLDumpFile), + WP_pageviews_filenames=( + None if args.skip_popularity else glob.glob(os.path.join(args.wikipedia_totals_bz2_pageviews, "*")) + ), + ) + populate_iucn(OTT_ptrs, args.EOLidentifiers) + + # Write collated data out into CSV format + writer = csv.writer(args.o, dialect="excel") + writer.writerow( + ( + "ott", + "wikidata", + "wikipedia_lang_flag", + "iucn", + "eol", + "rank", + "raw_popularity", + "ncbi", + "ifung", + "worms", + "irmng", + "gbif", + "ipni", + ) + ) + for o in OTT_ptrs.values(): + writer.writerow( + ( + o["ott"], + o.get("wd", {}).get("Q"), + o.get("wd", {}).get("wikipedia_lang_flag"), + o.get("iucn"), + o.get("eol"), + o.get("rank"), + o.get("wd", {}).get("raw_popularity"), + o["sources"].get("ncbi", {}).get("id"), + o["sources"].get("ifung", {}).get("id"), + o["sources"].get("worms", {}).get("id"), + o["sources"].get("irmng", {}).get("id"), + o["sources"].get("gbif", {}).get("id"), + o.get("ipni"), + ) + ) + args.o.close() diff --git a/pyproject.toml b/pyproject.toml index 0e23d882..38d4a3bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ add_dates_and_species_to_tree = "oz_tree_build.wiki_extraction.add_dates_and_spe download_node_ages = "oz_tree_build.download_node_ages.download_node_ages:main" date_tree = "oz_tree_build.date_tree.date_tree:main" tree_build = "oz_tree_build.tree_build.tree_build:main" +taxon_map = "oz_tree_build.taxon_mapping_and_popularity.taxon_map:main" [tool.setuptools] packages = ["oz_tree_build"] From d94bf3ce4eb24b20ede4b6a8dc8fdc96503b7d7d Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 11 Jun 2026 10:30:29 +0000 Subject: [PATCH 12/62] step_taxon: Apply taxon_map to built tree Import csv generated by taxon_map, and distribute references by OTT about the tree. --- dvc.yaml | 2 + .../taxon_mapping_and_popularity/taxon_map.py | 30 +++++++ oz_tree_build/tree_build/step_taxon.py | 16 ++++ oz_tree_build/tree_build/tree_build.py | 10 +++ tests/test_tree_build_step_taxon.py | 82 +++++++++++++++++++ 5 files changed, 140 insertions(+) create mode 100644 oz_tree_build/tree_build/step_taxon.py create mode 100644 tests/test_tree_build_step_taxon.py diff --git a/dvc.yaml b/dvc.yaml index c5cdeb31..0469e9c3 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -58,10 +58,12 @@ stages: --bespoke_dir data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ --orphan_dir data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ --opentree data/dated_tree/dated_tree_pre.tre + --taxon_map data/taxon_map.csv deps: - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ - data/dated_tree/dated_tree_pre.tre + - data/taxon_map.csv download_eol: # EOL doesn't version the provider ids file, so capture the last-modified header instead diff --git a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py index bdec42e9..c6c6dedf 100644 --- a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py +++ b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py @@ -494,3 +494,33 @@ def main(): ) ) args.o.close() + + +def read_taxon_map(path): + """Read a taxon map CSV (as written by :func:`main`) into a dict keyed by ott. + + Empty fields become ``None``. Numeric fields are converted to ``int`` or + ``float``; ``iucn`` is left as a string because multiple values may be + joined with ``|``. Source IDs (``ncbi``, ``ifung``, ``worms``, ``irmng``, + ``gbif``) are converted to ``int`` where possible and otherwise left as + strings. + """ + int_fields = ("ott", "wikidata", "wikipedia_lang_flag", "eol", "ipni") + float_fields = ("raw_popularity",) + source_fields = ("ncbi", "ifung", "worms", "irmng", "gbif") + out = {} + with open(path, encoding="utf-8", newline="") as f: + reader = csv.DictReader(f, dialect="excel") + for row in reader: + r = {k: (v if v != "" else None) for k, v in row.items()} + for k in int_fields: + if r.get(k) is not None: + r[k] = int(r[k]) + for k in float_fields: + if r.get(k) is not None: + r[k] = float(r[k]) + for k in source_fields: + if r.get(k) is not None and r[k].isdigit(): + r[k] = int(r[k]) + out[r["ott"]] = r + return out diff --git a/oz_tree_build/tree_build/step_taxon.py b/oz_tree_build/tree_build/step_taxon.py new file mode 100644 index 00000000..afd58718 --- /dev/null +++ b/oz_tree_build/tree_build/step_taxon.py @@ -0,0 +1,16 @@ +from ..utilities.ete import node_get_ott + + +def taxon_add_prop( + tree, + taxon_map, +): + """ + Add references to relevant lines in taxon_map to tree nodes + + We should also check that there are not multiple uses of the same Qid + (https://github.com/OneZoom/OZtree/issues/132) + """ + for n in tree.traverse(): + node_ott = node_get_ott(n) + n.props["taxon"] = taxon_map.get(int(node_ott), {}) if node_ott else {} diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index daa31b82..97063dcd 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -6,9 +6,11 @@ import logging from ..date_tree import date_tree +from ..taxon_mapping_and_popularity.taxon_map import read_taxon_map from ..utilities.debug_util import parse_args_and_add_logging_switch from .step_graft import graft_extract_ot_subtrees, graft_tree from .step_parse import parse_bespoke_trees, parse_ot_orphans +from .step_taxon import taxon_add_prop logger = logging.getLogger(__name__) @@ -27,6 +29,11 @@ def main(): "--opentree", help=("Newick tree with OpenTree"), ) + parser.add_argument( + "--taxon_map", + default="data/taxon_map.csv", + help=("Taxon map CSV as generated by taxon_mapping_and_popularity.taxon_map"), + ) parser.add_argument( "--out_dir", default="data/out", @@ -56,6 +63,9 @@ def main(): for i in missing_inclusions: logger.error(f"No subtree found for {i}") + logger.info("Attach taxon information to nodes") + taxon_add_prop(base_t, read_taxon_map(args.taxon_map)) + logger.info( "Popularity calculations for entire tree (including any remaining subspecies from bespoke tree, " "Jonathan's will have them already removed) (stop caring about polytomy vs. popularity calculations, " diff --git a/tests/test_tree_build_step_taxon.py b/tests/test_tree_build_step_taxon.py new file mode 100644 index 00000000..4c4de994 --- /dev/null +++ b/tests/test_tree_build_step_taxon.py @@ -0,0 +1,82 @@ +import ete4 + +from oz_tree_build.tree_build.step_taxon import taxon_add_prop + + +class TestTaxonAddProp: + def test_empty_taxon_map_assigns_empty_dict(self): + # Every node — leaf, internal, root — gets {} when the map is empty. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + taxon_add_prop(t, {}) + for n in t.traverse(): + assert n.props["taxon"] == {} + + def test_leaf_match_assigns_taxon_entry(self): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + a_entry = {"ott": 1, "wikidata": 42, "raw_popularity": 1.5} + taxon_add_prop(t, {1: a_entry}) + + a = next(n for n in t.traverse() if n.name == "A_ott1") + b = next(n for n in t.traverse() if n.name == "B_ott2") + assert a.props["taxon"] == a_entry + assert b.props["taxon"] == {} + + def test_internal_node_match_assigns_taxon_entry(self): + # Internal nodes are also looked up via their ottN suffix. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + root_entry = {"ott": 3, "rank": "family"} + taxon_add_prop(t, {3: root_entry}) + + root = next(n for n in t.traverse() if n.name == "Root_ott3") + assert root.props["taxon"] == root_entry + + def test_node_without_ott_gets_empty_dict(self): + # "Foo" has no ottN suffix, so node_get_ott returns None and the + # entry must be {} regardless of what is in the taxon map. + t = ete4.Tree("(Foo,B_ott2)Root_ott3;", parser=1) + taxon_add_prop(t, {1: {"ott": 1}, 2: {"ott": 2}, 3: {"ott": 3}}) + + foo = next(n for n in t.traverse() if n.name == "Foo") + assert foo.props["taxon"] == {} + + def test_unnamed_node_gets_empty_dict(self): + # An unnamed internal node (e.g. result of resolve_polytomy) has no + # OTT and must end up with {}. + t = ete4.Tree("(A_ott1,B_ott2,C_ott4)Root_ott3;", parser=1) + t.resolve_polytomy() + + taxon_add_prop(t, {1: {"ott": 1}}) + + for n in t.traverse(): + if not n.name: + assert n.props["taxon"] == {} + + def test_unmatched_ott_gets_empty_dict(self): + # Node has an ottN but it's absent from the map. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + taxon_add_prop(t, {99: {"ott": 99}}) + + for n in t.traverse(): + assert n.props["taxon"] == {} + + def test_space_separated_ott_in_name_is_matched(self): + # node_get_ott accepts both "_ottN" and " ottN" suffixes. + t = ete4.Tree("(A_ott1,B_ott2)Root;", parser=1) + # Rename a node to use the space-separated form. + a = next(n for n in t.traverse() if n.name == "A_ott1") + a.name = "Some name ott1" + entry = {"ott": 1} + taxon_add_prop(t, {1: entry}) + + assert a.props["taxon"] == entry + + def test_taxon_entry_is_assigned_by_reference(self): + # The function stores the same dict object on the node, so callers + # who mutate the taxon map afterwards see the change reflected on + # the tree (and vice versa). + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + a_entry = {"ott": 1} + taxon_add_prop(t, {1: a_entry}) + + a = next(n for n in t.traverse() if n.name == "A_ott1") + assert a.props["taxon"] is a_entry From fbfca90bc951c4f79c6090ca4d83645cfdd0b0d2 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 11 Jun 2026 10:32:46 +0000 Subject: [PATCH 13/62] step_popularity: Port popularity from CSV_base_table_creator Given raw_popularity from the taxon_map, lift-and-shift popularity percolation code out of CSV_base_table_creator into step_popularity. --- oz_tree_build/tree_build/step_popularity.py | 204 +++++++++++++++ oz_tree_build/tree_build/tree_build.py | 3 +- tests/test_tree_build_step_popularity.py | 264 ++++++++++++++++++++ 3 files changed, 470 insertions(+), 1 deletion(-) create mode 100644 oz_tree_build/tree_build/step_popularity.py create mode 100644 tests/test_tree_build_step_popularity.py diff --git a/oz_tree_build/tree_build/step_popularity.py b/oz_tree_build/tree_build/step_popularity.py new file mode 100644 index 00000000..ccb19016 --- /dev/null +++ b/oz_tree_build/tree_build/step_popularity.py @@ -0,0 +1,204 @@ +import logging +from math import log + +from ..utilities.ete import node_get_ott + +logger = logging.getLogger(__name__) + + +def popularity_add_prop( + tree, + exclude_taxa=None, +): + """ + Compute a phylogenetic popularity score for every node and store it on + ``node.props["popularity"]`` (rounded to 2 dp). + + The raw per-node popularity is taken from ``node.props["taxon"]["raw_popularity"]`` + (see ``sum_popularity_over_tree``). Each node's score combines the raw popularity + of its ancestors and descendants — so a node inherits some weight from its + relatives, not just from its own ``raw_popularity``. See ``popularity_function`` + for the exact combination. + + ``exclude_taxa`` is forwarded to ``sum_popularity_over_tree`` and lets the + caller zero out the raw popularity of named nodes before summation (e.g. + excluding Dinosauria so its popularity is not credited to birds). + + A warning is logged for any Wikidata Qid that appears on more than one node, + since that causes the same popularity to be counted twice. + + Must be run before monotomies / unary nodes are removed: those nodes often + carry useful popularity that needs to percolate to their relatives first. + Nodes synthesised by polytomy resolution are handled the same way as any + other node. + """ + sum_popularity_over_tree(tree, exclude_taxa=exclude_taxa) + + # now apply the popularity function + Qids = set() + for node in tree.traverse(strategy="preorder"): + Q = node.props["taxon"].get("wikidata") + if Q is not None: + if Q in Qids: + logger.warning( + f"duplicate wikidata Qids used (Q{Q}) - this will cause " + f"popularity double-counting for OTT {node_get_ott(node)}" + ) + else: + Qids.add(Q) + pop = popularity_function( + node.props["ancestors_popsum"], + node.props["descendants_popsum"], + node.props["n_ancestors"], + node.props["n_descendants"], + ) + + # Round to 2 decimal places + node.props["popularity"] = round(pop, 2) + + +def popularity_function( + sum_of_all_ancestor_popularities, + sum_of_all_descendant_popularities, + number_of_ancestors, + number_of_descendants, +): + """ + a) Dividing by number_of_ancestors+number_of_descendants would mean averaging + popularity over all nodes, which would bias against taxa which have many + unvisited/unpopular children + b) Alternatively, dividing by a constant is equivalent to summing popularity over + all nodes, which biases towards taxa with many fine taxonomic divisions + We do something between the two by dividing by the log of the number of nodes. + """ + if ( + (sum_of_all_ancestor_popularities is None) + or (sum_of_all_descendant_popularities is None) + or (number_of_ancestors is None) + or (number_of_descendants is None) + ): + return None + elif number_of_ancestors + number_of_descendants == 1: + # Avoid a divide by zero error if this adds up to 1 + # Though the need for this makes me think that the log calculation + # may not be mathematically sound + return sum_of_all_ancestor_popularities + sum_of_all_descendant_popularities + else: + return (sum_of_all_ancestor_popularities + sum_of_all_descendant_popularities) / log( + number_of_ancestors + number_of_descendants + ) + + +def popularity_add_info( + tree, + focal_labels, +): + """ + Print debug info for ete4 nodes whose names appear in ``focal_labels``: + each node's own popularity, its descendant popularity sum, a sample of + its leaves, and the chain of ancestors with non-zero popularity. + """ + remaining = set(focal_labels) + for node in tree.traverse(): + if not remaining: + break + if node.name not in remaining: + continue + remaining.discard(node.name) + print( + "{}: own pop = {} (Q{}) descendant pop sum = {}".format( + node.name, + node.props["pop"], + node.props["taxon"].get("wikidata", " absent"), + node.props["descendants_popsum"], + ) + ) + for t, tip in enumerate(node.leaves()): + print( + "Tip {} = {}: own_pop = {}, Qid = Q{}".format( + t, + tip.name, + tip.props.get("pop"), + tip.props["taxon"].get("wikidata", " absent"), + ) + ) + if t > 100: + print("More tips exist, but have been omitted") + break + ancestor = node.up + while ancestor: + if ancestor.props.get("pop"): + print(f"Ancestors: {ancestor.name} = {ancestor.props['pop']:.2f}") + ancestor = ancestor.up + for missing in remaining: + logger.warning(f"Problem reporting on focal taxon '{missing}': not found") + + +def sum_popularity_over_tree(tree, exclude_taxa=None): + """ + Sum raw popularity values up and down an ete4 phylogenetic tree. + + Each node's raw popularity is taken from ``node.props["taxon"]["raw_popularity"]`` + It is copied onto ``node.props["pop"]`` and then summed across ancestors and + descendants. + + We might want to exclude some names from the popularity metric (e.g. exclude + archosaurs, to ensure birds don't gather popularity intended for dinosaurs). + This is done by passing an array such as + ``['Dinosauria_ott90215', 'Archosauria_ott335588']`` as the ``exclude_taxa`` argument + -- the names are matched against ``node.name``. + + After running, the following props are set on every node: + pop raw popularity for this node + has_pop whether raw popularity was available + descendants_popsum popularity summed over all descendants + n_descendants number of descendants + ancestors_popsum popularity summed over all ancestors + n_ancestors number of ancestors + n_pop_ancestors number of ancestors with a popularity measure + """ + exclude_taxa = set(exclude_taxa or []) + + logger.info("Tree read for phylogenetic popularity calc") + + # put popularity into the "pop" attribute + for node in tree.traverse(strategy="preorder"): + if node.name in exclude_taxa or node.props["taxon"].get("raw_popularity") is None: + node.props["pop"] = 0 + node.props["has_pop"] = False + else: + node.props["pop"] = node.props["taxon"]["raw_popularity"] + node.props["has_pop"] = True + + # go up the tree from the tips, summing up the popularity indices beneath and + # adding the number of descendants + for node in tree.traverse(strategy="postorder"): + if node.is_leaf: + node.props["descendants_popsum"] = 0 + node.props["n_descendants"] = 0 + parent = node.up + if parent is None: + continue + parent.props["n_descendants"] = parent.props.get("n_descendants", 0) + 1 + node.props["n_descendants"] + parent.props["descendants_popsum"] = ( + parent.props.get("descendants_popsum", 0) + node.props["pop"] + node.props["descendants_popsum"] + ) + + # go down the tree from the root, summing up the popularity indices above, + # and summing up numbers of nodes + for node in tree.traverse(strategy="preorder"): + parent = node.up + if parent is None: + # this is the root. + node.props["n_ancestors"] = 0 + node.props["n_pop_ancestors"] = 0 + node.props["ancestors_popsum"] = 0.0 + else: + node.props["n_ancestors"] = parent.props["n_ancestors"] + 1 + node.props["ancestors_popsum"] = parent.props["ancestors_popsum"] + node.props["pop"] + if node.props.get("has_pop"): + node.props["n_pop_ancestors"] = parent.props["n_pop_ancestors"] + 1 + else: + node.props["n_pop_ancestors"] = parent.props["n_pop_ancestors"] + + return tree diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 97063dcd..b9ec4600 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -10,6 +10,7 @@ from ..utilities.debug_util import parse_args_and_add_logging_switch from .step_graft import graft_extract_ot_subtrees, graft_tree from .step_parse import parse_bespoke_trees, parse_ot_orphans +from .step_popularity import popularity_add_prop from .step_taxon import taxon_add_prop logger = logging.getLogger(__name__) @@ -72,7 +73,7 @@ def main(): "and just apply them post-resolution). Apply popularity based on OTT -> popularity map, percolate " "using existing rules (which preserves popularity from removed subspecies)" ) - # prop_add_popularity(base_t) + popularity_add_prop(base_t) logger.info("Remove subspecies (now popularity has percolated)") # tidy_remove_subspecies(base_t) diff --git a/tests/test_tree_build_step_popularity.py b/tests/test_tree_build_step_popularity.py new file mode 100644 index 00000000..2e6bd399 --- /dev/null +++ b/tests/test_tree_build_step_popularity.py @@ -0,0 +1,264 @@ +import csv +from math import log + +import ete4 +import pytest + +from oz_tree_build.taxon_mapping_and_popularity.taxon_map import read_taxon_map +from oz_tree_build.tree_build.step_popularity import ( + popularity_add_prop, + popularity_function, + sum_popularity_over_tree, +) +from oz_tree_build.tree_build.step_taxon import taxon_add_prop + +TAXON_CSV_FIELDS = [ + "ott", + "wikidata", + "wikipedia_lang_flag", + "iucn", + "eol", + "rank", + "raw_popularity", + "ncbi", + "ifung", + "worms", + "irmng", + "gbif", + "ipni", +] + + +def _attach_taxa(tmp_path, tree, rows): + """Write ``rows`` to a CSV in the same shape as taxon_map.main produces, + read it back with read_taxon_map, and attach to ``tree`` via taxon_add_prop. + """ + path = tmp_path / "taxon.csv" + with open(path, "w", newline="", encoding="utf-8") as f: + w = csv.writer(f, dialect="excel") + w.writerow(TAXON_CSV_FIELDS) + for row in rows: + w.writerow([row.get(k, "") for k in TAXON_CSV_FIELDS]) + taxon_add_prop(tree, read_taxon_map(path)) + + +class TestPopularityFunction: + def test_returns_none_when_any_input_is_none(self): + assert popularity_function(None, 1.0, 1, 1) is None + assert popularity_function(1.0, None, 1, 1) is None + assert popularity_function(1.0, 1.0, None, 1) is None + assert popularity_function(1.0, 1.0, 1, None) is None + + def test_single_node_special_case_returns_sum(self): + # n_ancestors + n_descendants == 1 dodges the log(1)=0 divide-by-zero + # and just returns the sum of ancestor + descendant popsums. + assert popularity_function(3.0, 7.0, 1, 0) == 10.0 + assert popularity_function(3.0, 7.0, 0, 1) == 10.0 + + def test_general_case_divides_by_log_of_node_count(self): + # (anc + desc) / log(n_anc + n_desc) + assert popularity_function(10.0, 20.0, 1, 3) == pytest.approx(30.0 / log(4)) + + +class TestSumPopularityOverTree: + def test_pop_copied_from_taxon_raw_popularity(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "raw_popularity": 20.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + + by_name = {n.name: n for n in t.traverse()} + assert by_name["A_ott1"].props["pop"] == 10.0 + assert by_name["A_ott1"].props["has_pop"] is True + assert by_name["B_ott2"].props["pop"] == 20.0 + assert by_name["Root_ott3"].props["pop"] == 30.0 + + def test_no_taxon_means_pop_zero(self, tmp_path): + # A node with no matching taxon entry gets pop=0, has_pop=False. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + ], + ) + + sum_popularity_over_tree(t) + + b = next(n for n in t.traverse() if n.name == "B_ott2") + assert b.props["pop"] == 0 + assert b.props["has_pop"] is False + + def test_taxon_present_but_raw_popularity_missing(self, tmp_path): + # A taxon row exists for the node but its raw_popularity field is empty, + # so read_taxon_map gives raw_popularity=None. The node should still + # be treated as un-populated (pop=0, has_pop=False) rather than + # propagating None as a popularity value. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "wikidata": 42}, # row present, raw_popularity empty + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + + b = next(n for n in t.traverse() if n.name == "B_ott2") + # Taxon dict is populated (wikidata is set) but raw_popularity is None. + assert b.props["taxon"].get("wikidata") == 42 + assert b.props["taxon"].get("raw_popularity") is None + assert b.props["pop"] == 0 + assert b.props["has_pop"] is False + + def test_descendant_sums_aggregate_upwards(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "raw_popularity": 20.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + # Leaves: nothing beneath them. + assert by_name["A_ott1"].props["descendants_popsum"] == 0 + assert by_name["A_ott1"].props["n_descendants"] == 0 + # Root: own pop excluded from descendants_popsum. + assert by_name["Root_ott3"].props["descendants_popsum"] == 30.0 + assert by_name["Root_ott3"].props["n_descendants"] == 2 + + def test_ancestor_sums_accumulate_downwards(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "raw_popularity": 20.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["Root_ott3"].props["n_ancestors"] == 0 + assert by_name["Root_ott3"].props["ancestors_popsum"] == 0.0 + # ancestors_popsum at a child = parent.ancestors_popsum + own pop. + assert by_name["A_ott1"].props["n_ancestors"] == 1 + assert by_name["A_ott1"].props["ancestors_popsum"] == 10.0 + assert by_name["B_ott2"].props["ancestors_popsum"] == 20.0 + + def test_n_pop_ancestors_only_counts_nodes_with_pop(self, tmp_path): + # Root has popularity, intermediate node does not, leaf does. + t = ete4.Tree("((A_ott1)Mid_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 5.0}, + # ott 2 deliberately missing -> has_pop=False at Mid + {"ott": 3, "raw_popularity": 7.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + # Root counts as a "pop ancestor" only when its own has_pop counts up + # via descendants — n_pop_ancestors counts populated nodes on the way down. + assert by_name["Mid_ott2"].props["n_pop_ancestors"] == 0 # mid itself missing + assert by_name["A_ott1"].props["n_pop_ancestors"] == 1 # only A itself + + def test_exclude_taxa_zeroes_pop_for_named_node(self, tmp_path): + # Excluded node's own pop becomes 0 / has_pop=False, but descendants + # still contribute to its descendants_popsum (used by children below). + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "raw_popularity": 20.0}, + {"ott": 3, "raw_popularity": 999.0}, + ], + ) + + sum_popularity_over_tree(t, exclude_taxa=["Root_ott3"]) + root = next(n for n in t.traverse() if n.name == "Root_ott3") + + assert root.props["pop"] == 0 + assert root.props["has_pop"] is False + # Children's pop still aggregates upwards. + assert root.props["descendants_popsum"] == 30.0 + + +class TestPopularityAddProp: + def test_popularity_set_on_every_node_and_rounded(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "raw_popularity": 20.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + popularity_add_prop(t) + + by_name = {n.name: n for n in t.traverse()} + # Leaves: n_anc + n_desc == 1 -> sum of ancestor + descendant popsums. + # A: anc=10, desc=0 -> 10. B: anc=20, desc=0 -> 20. + assert by_name["A_ott1"].props["popularity"] == 10.0 + assert by_name["B_ott2"].props["popularity"] == 20.0 + # Root: (0 + 30) / log(2) — rounded to 2 dp. + assert by_name["Root_ott3"].props["popularity"] == round(30.0 / log(2), 2) + + def test_nodes_without_pop_get_zero_popularity(self, tmp_path): + # All zeros in -> popularity is 0 (special case applies at leaves). + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa(tmp_path, t, []) + + popularity_add_prop(t) + for n in t.traverse(): + assert n.props["popularity"] == 0 + + def test_exclude_taxa_passes_through(self, tmp_path): + # popularity_add_prop forwards exclude_taxa to sum_popularity_over_tree; + # the excluded node's own pop is zeroed so its rendered popularity + # reflects only descendants. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "raw_popularity": 20.0}, + {"ott": 3, "raw_popularity": 999.0}, + ], + ) + + popularity_add_prop(t, exclude_taxa=["Root_ott3"]) + root = next(n for n in t.traverse() if n.name == "Root_ott3") + # Root pop=0, descendants_popsum=30, n_desc=2 -> 30/log(2). + assert root.props["popularity"] == round(30.0 / log(2), 2) From 9b000e6231e2fd8c1e681506bdf2046e35d1c841 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 11 Jun 2026 10:36:41 +0000 Subject: [PATCH 14/62] tree_build: Use ete4-default resolve_polytomy Note that unlike previously, we don't shuffle the children before ladderizing. However we did so with a fixed seed so I'm not sure the difference is worth the re-implementation. --- oz_tree_build/tree_build/tree_build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index b9ec4600..ea09d304 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -47,7 +47,9 @@ def main(): missing_inclusions = graft_tree(base_t, additional_trees=bespoke_ts, prefer_subtree_name=True) logger.info("Random resoution of polytomies for bespoke trees") - # tidy_resolve_polytomy_random(base_t) + # https://etetoolkit.org/docs/latest/reference/reference_tree.html#ete3.TreeNode.resolve_polytomy + # NB: Doesn't shuffle children like the DendroPy equivalent, but given a fixed seed do we care? + base_t.resolve_polytomy() logger.info("Resolve branch lengths to dates bottom-up. Remove (or not care about) branch lengths") # tidy_resolve_bl_to_dates(base_t) From 314b540640dd8ec4d957e38b766d9c15c66e0408 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 11 Jun 2026 14:43:13 +0000 Subject: [PATCH 15/62] step_tidy: Add tree tidying steps Use dated-complete-tree where we can, otherwise implement our own in step_tidy --- oz_tree_build/tree_build/step_tidy.py | 27 +++ oz_tree_build/tree_build/tree_build.py | 11 +- tests/test_tree_build_step_tidy.py | 235 +++++++++++++++++++++++++ 3 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 oz_tree_build/tree_build/step_tidy.py create mode 100644 tests/test_tree_build_step_tidy.py diff --git a/oz_tree_build/tree_build/step_tidy.py b/oz_tree_build/tree_build/step_tidy.py new file mode 100644 index 00000000..33bdad08 --- /dev/null +++ b/oz_tree_build/tree_build/step_tidy.py @@ -0,0 +1,27 @@ +def tidy_infill_dates_bottomup(tree): + """ + Working bottom-upwards, fill in missing date properties based on branch lengths. + """ + for node in tree.traverse(strategy="postorder"): + if node.is_leaf: + node.props["date"] = 0 + else: + for c in node.children: + if c.props.get("date") is not None and c.dist is not None: + c_date = c.props["date"] + c.dist + if node.props.get("date") is None or node.props["date"] < c_date: + node.props["date"] = c_date + + +def tidy_clear_conflicting_dates_topdown(parent, mrad=None): + """ + Work through tree, removing dates older than their most recent ancestor. + """ + if parent.props.get("date") is not None: + if mrad is not None and (parent.props["date"] - mrad) > 1e-5: + # date is greater than mrad, this shouldn't happen + del parent.props["date"] + else: + mrad = parent.props["date"] + for c in parent.children: + tidy_clear_conflicting_dates_topdown(c, mrad) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index ea09d304..387db475 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -5,6 +5,8 @@ import argparse import logging +from dated_complete_tree import tree_fixing + from ..date_tree import date_tree from ..taxon_mapping_and_popularity.taxon_map import read_taxon_map from ..utilities.debug_util import parse_args_and_add_logging_switch @@ -12,6 +14,7 @@ from .step_parse import parse_bespoke_trees, parse_ot_orphans from .step_popularity import popularity_add_prop from .step_taxon import taxon_add_prop +from .step_tidy import tidy_clear_conflicting_dates_topdown, tidy_infill_dates_bottomup logger = logging.getLogger(__name__) @@ -52,10 +55,10 @@ def main(): base_t.resolve_polytomy() logger.info("Resolve branch lengths to dates bottom-up. Remove (or not care about) branch lengths") - # tidy_resolve_bl_to_dates(base_t) + tidy_infill_dates_bottomup(base_t) logger.info("Top-down conflict resolution in bespoke tree, delete entries that conflict with higher ages") - # tidy_resolve_date_conflicts(base_t) + tidy_clear_conflicting_dates_topdown(base_t) logger.info("Graft OT subtrees onto our trees. Already polytomy-resolved & date pins from chronosynth applied") opentree_ts = graft_extract_ot_subtrees(date_tree.nwk_read(args.opentree), missing_inclusions) @@ -83,10 +86,10 @@ def main(): logger.info( "Top-down conflict resolution. If there's conflict with higher ages, remove ages until conflict goes away" ) - # resolve_date_conflicts(base_t) + tidy_clear_conflicting_dates_topdown(base_t) logger.info("Remove unary nodes (they are likely uninteresting, and make a mess of the tree rendering)") - # tidy_remove_unary(base_t) + tree_fixing.delete_one_child_nodes(base_t) logger.info("Re-interpoltate missing dates") # interpolate_dates(base_t) diff --git a/tests/test_tree_build_step_tidy.py b/tests/test_tree_build_step_tidy.py new file mode 100644 index 00000000..d41bcfaa --- /dev/null +++ b/tests/test_tree_build_step_tidy.py @@ -0,0 +1,235 @@ +import ete4 + +from oz_tree_build.tree_build.step_tidy import ( + tidy_clear_conflicting_dates_topdown, + tidy_infill_dates_bottomup, +) + + +def _by_name(tree): + return {n.name: n for n in tree.traverse()} + + +class TestTidyInfillDatesBottomup: + def test_single_leaf_gets_date_zero(self): + t = ete4.Tree("A;", parser=1) + tidy_infill_dates_bottomup(t) + assert t.props["date"] == 0 + + def test_all_leaves_get_date_zero(self): + # Every leaf is reset to 0 regardless of any earlier value. + t = ete4.Tree("(A:1,B:2)Root;", parser=1) + nodes = _by_name(t) + nodes["A"].props["date"] = 99 # will be overwritten + tidy_infill_dates_bottomup(t) + assert nodes["A"].props["date"] == 0 + assert nodes["B"].props["date"] == 0 + + def test_parent_date_is_max_child_branch_length(self): + # Root date = max(child.date + child.dist) over leaves at date 0. + t = ete4.Tree("(A:1,B:2)Root;", parser=1) + tidy_infill_dates_bottomup(t) + assert t.props["date"] == 2 + + def test_multilevel_tree_accumulates_branch_lengths(self): + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + tidy_infill_dates_bottomup(t) + # Internal (parent of A,B) = max(0+1, 0+2) = 2 + # Root = max(2+3, 0+4) = 5 + internal = next(c for c in t.children if not c.is_leaf) + assert internal.props["date"] == 2 + assert t.props["date"] == 5 + + def test_existing_internal_date_preserved_when_larger(self): + # Pre-existing date on an internal node is kept if no child's + # accumulated date exceeds it. + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + internal = next(c for c in t.children if not c.is_leaf) + internal.props["date"] = 100 + tidy_infill_dates_bottomup(t) + assert internal.props["date"] == 100 + # Root sees the preserved internal date: max(100+3, 0+4) = 103 + assert t.props["date"] == 103 + + def test_existing_internal_date_overwritten_when_smaller(self): + # A child-derived date larger than the pre-existing one wins. + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + internal = next(c for c in t.children if not c.is_leaf) + internal.props["date"] = 0.5 + tidy_infill_dates_bottomup(t) + assert internal.props["date"] == 2 + assert t.props["date"] == 5 + + def test_child_without_dist_does_not_contribute(self): + # A child whose dist is None must not bump its parent's date. + t = ete4.Tree("(A,B:5)Root;", parser=1) + tidy_infill_dates_bottomup(t) + # Only B contributes: Root = 0 + 5 = 5 + assert t.props["date"] == 5 + + def test_internal_without_datable_descendants_has_no_date(self): + # If every child of an internal lacks dist, the internal stays + # without a date and cannot in turn propagate upward. + t = ete4.Tree("((A,B):4,C:1)Root;", parser=1) + tidy_infill_dates_bottomup(t) + internal = next(c for c in t.children if not c.is_leaf) + # Internal has no datable children (A and B both lack dist). + assert internal.props.get("date") is None + # Root sees only C contributing (internal has no date). + assert t.props["date"] == 1 + + def test_picks_oldest_subtree_when_branches_differ(self): + # Two subtrees with different total ages — root takes the older. + t = ete4.Tree("((A:1,B:1):10,(C:1,D:1):2)Root;", parser=1) + tidy_infill_dates_bottomup(t) + # left subtree internal = 1, +10 = 11; right subtree internal = 1, +2 = 3 + assert t.props["date"] == 11 + + +class TestTidyClearConflictingDatesTopdown: + def test_consistent_dates_are_all_kept(self): + # Descending dates: root oldest, leaves youngest. Nothing removed. + t = ete4.Tree("((A,B)I,C)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["I"].props["date"] = 5 + nodes["A"].props["date"] = 0 + nodes["B"].props["date"] = 2 + nodes["C"].props["date"] = 3 + tidy_clear_conflicting_dates_topdown(t) + assert nodes["Root"].props["date"] == 10 + assert nodes["I"].props["date"] == 5 + assert nodes["A"].props["date"] == 0 + assert nodes["B"].props["date"] == 2 + assert nodes["C"].props["date"] == 3 + + def test_child_older_than_parent_is_cleared(self): + # A child date that is older than its ancestor is a conflict + # and must be removed. + t = ete4.Tree("(A,B)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["A"].props["date"] = 20 # older than Root — conflict + nodes["B"].props["date"] = 3 + tidy_clear_conflicting_dates_topdown(t) + assert "date" not in nodes["A"].props + assert nodes["B"].props["date"] == 3 + assert nodes["Root"].props["date"] == 10 + + def test_conflict_clears_only_that_node(self): + # Removing an intermediate conflicting date should not stop the + # descent — its descendants are still compared against the + # original ancestor's date. + t = ete4.Tree("((A,B)I,C)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["I"].props["date"] = 20 # conflict — older than Root + nodes["A"].props["date"] = 5 # consistent with Root (10) + nodes["B"].props["date"] = 50 # conflict — older than Root + tidy_clear_conflicting_dates_topdown(t) + assert nodes["Root"].props["date"] == 10 + assert "date" not in nodes["I"].props + assert nodes["A"].props["date"] == 5 + assert "date" not in nodes["B"].props + + def test_missing_ancestor_date_does_not_constrain_descendants(self): + # With no ancestor date set, the first node we meet on each path + # establishes the ceiling for everything below it. + t = ete4.Tree("(A,(B,C)I)Root;", parser=1) + nodes = _by_name(t) + # Root has no date. + nodes["A"].props["date"] = 100 # nothing above — kept + nodes["I"].props["date"] = 50 + nodes["B"].props["date"] = 1 + nodes["C"].props["date"] = 200 # conflict — older than I (50) + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] == 100 + assert nodes["I"].props["date"] == 50 + assert nodes["B"].props["date"] == 1 + assert "date" not in nodes["C"].props + + def test_nodes_without_date_are_skipped_and_pass_mrad_through(self): + # An intermediate node without a date must not reset the ceiling; + # its descendants are still compared against the nearest ancestor + # that does have a date. + t = ete4.Tree("((A)I)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + # I has no date. + nodes["A"].props["date"] = 50 # still in conflict with Root via I + tidy_clear_conflicting_dates_topdown(t) + assert nodes["Root"].props["date"] == 10 + assert "date" not in nodes["I"].props + assert "date" not in nodes["A"].props + + def test_small_overshoot_within_tolerance_is_kept(self): + # The function tolerates floating-point noise up to 1e-5. + t = ete4.Tree("(A)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["A"].props["date"] = 10 + 1e-6 # just within tolerance + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] == 10 + 1e-6 + + def test_overshoot_beyond_tolerance_is_cleared(self): + t = ete4.Tree("(A)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["A"].props["date"] = 10 + 1e-3 # outside tolerance + tidy_clear_conflicting_dates_topdown(t) + assert "date" not in nodes["A"].props + + def test_equal_dates_are_kept(self): + # parent.date == ancestor.date is not a conflict. + t = ete4.Tree("(A)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["A"].props["date"] = 10 + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] == 10 + + def test_mrad_tightens_as_we_descend(self): + # After we descend past a node with a smaller date, that smaller + # date becomes the new ceiling for everything below. + t = ete4.Tree("((A,B)I)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 100 + nodes["I"].props["date"] = 5 # tighter than Root + nodes["A"].props["date"] = 3 # below I — fine + nodes["B"].props["date"] = 50 # would be fine vs Root but conflicts with I + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] == 3 + assert "date" not in nodes["B"].props + + def test_runs_on_tree_with_no_dates_at_all(self): + # Nothing to do; should not raise. + t = ete4.Tree("((A,B),C)Root;", parser=1) + tidy_clear_conflicting_dates_topdown(t) + for n in t.traverse(): + assert n.props.get("date") is None + + +class TestTidyPipeline: + def test_bottomup_then_topdown_on_clean_tree_is_stable(self): + # End-to-end: branch-length-derived dates should already be + # self-consistent, so the topdown pass changes nothing. + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + tidy_infill_dates_bottomup(t) + before = {id(n): n.props.get("date") for n in t.traverse()} + tidy_clear_conflicting_dates_topdown(t) + after = {id(n): n.props.get("date") for n in t.traverse()} + assert before == after + + def test_bottomup_then_topdown_clears_oversized_pin(self): + # If a pre-pinned internal date is older than the root's + # inferred date, the topdown pass should strip it. + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + internal = next(c for c in t.children if not c.is_leaf) + internal.props["date"] = 100 # pin older than root could ever be + tidy_infill_dates_bottomup(t) + # Root is forced up to 103 by the pin; that's fine. + # Now imagine an externally fixed Root date that's smaller: + t.props["date"] = 50 + tidy_clear_conflicting_dates_topdown(t) + assert t.props["date"] == 50 + assert "date" not in internal.props From cfbaaca409f572171e40f4bb5f21f3ad60ca6f49 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 11 Jun 2026 17:24:39 +0000 Subject: [PATCH 16/62] step_popularity: Add popularity_add_rank Copy over popularity_add_rank, convert to ete4, add to the pipeline. --- oz_tree_build/tree_build/step_popularity.py | 32 ++++++ oz_tree_build/tree_build/tree_build.py | 5 +- tests/test_tree_build_step_popularity.py | 103 ++++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) diff --git a/oz_tree_build/tree_build/step_popularity.py b/oz_tree_build/tree_build/step_popularity.py index ccb19016..bfb43c38 100644 --- a/oz_tree_build/tree_build/step_popularity.py +++ b/oz_tree_build/tree_build/step_popularity.py @@ -1,3 +1,4 @@ +import collections import logging from math import log @@ -57,6 +58,37 @@ def popularity_add_prop( node.props["popularity"] = round(pop, 2) +def popularity_add_rank(tree): + """ + Rank every leaf by its ``node.props["popularity"]`` and write the position + to ``node.props["popularity_rank"]``. Rank 1 is the most popular leaf. + + Ties use standard competition ranking ("1224"): tied leaves share the lower + rank and the next distinct value skips ahead by the size of the tie. For + example, popularities [100, 50, 50, 50, 1] produce ranks [1, 2, 2, 2, 5]. + + Internal nodes are neither ranked nor used as tie-breakers, and their + ``popularity`` prop (if any) is ignored. + + Should be run after invalid tips and unary nodes have been removed, so the + ranking reflects the final set of leaves. + """ + leaf_popularities = collections.defaultdict(int) + for node in tree.traverse(): + if node.is_leaf: + leaf_popularities[node.props.get("popularity")] += 1 + cumsum = 1 + if None in leaf_popularities: + return + for k in sorted(leaf_popularities.keys(), reverse=True): + add_next = leaf_popularities[k] + leaf_popularities[k] = cumsum + cumsum += add_next + for node in tree.traverse(): + if node.is_leaf: + node.props["popularity_rank"] = leaf_popularities[node.props.get("popularity")] + + def popularity_function( sum_of_all_ancestor_popularities, sum_of_all_descendant_popularities, diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 387db475..175266f1 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -12,7 +12,7 @@ from ..utilities.debug_util import parse_args_and_add_logging_switch from .step_graft import graft_extract_ot_subtrees, graft_tree from .step_parse import parse_bespoke_trees, parse_ot_orphans -from .step_popularity import popularity_add_prop +from .step_popularity import popularity_add_prop, popularity_add_rank from .step_taxon import taxon_add_prop from .step_tidy import tidy_clear_conflicting_dates_topdown, tidy_infill_dates_bottomup @@ -94,6 +94,9 @@ def main(): logger.info("Re-interpoltate missing dates") # interpolate_dates(base_t) + logger.info("Rank popularities, post-node removal") + popularity_add_rank(base_t) + logger.info("Add properies to tree") pass diff --git a/tests/test_tree_build_step_popularity.py b/tests/test_tree_build_step_popularity.py index 2e6bd399..e263571f 100644 --- a/tests/test_tree_build_step_popularity.py +++ b/tests/test_tree_build_step_popularity.py @@ -7,6 +7,7 @@ from oz_tree_build.taxon_mapping_and_popularity.taxon_map import read_taxon_map from oz_tree_build.tree_build.step_popularity import ( popularity_add_prop, + popularity_add_rank, popularity_function, sum_popularity_over_tree, ) @@ -262,3 +263,105 @@ def test_exclude_taxa_passes_through(self, tmp_path): root = next(n for n in t.traverse() if n.name == "Root_ott3") # Root pop=0, descendants_popsum=30, n_desc=2 -> 30/log(2). assert root.props["popularity"] == round(30.0 / log(2), 2) + + +class TestPopularityAddRank: + @staticmethod + def _set_leaf_pops(tree, pops): + # Manually attach popularity values to leaves (bypassing the full + # pipeline). popularity_add_rank only reads node.props["popularity"]. + for n in tree.traverse(): + if n.is_leaf and n.name in pops: + n.props["popularity"] = pops[n.name] + + def test_distinct_popularities_get_sequential_ranks(self): + # Higher popularity -> lower (better) rank, starting at 1. + t = ete4.Tree("(A,B,C)R;", parser=1) + self._set_leaf_pops(t, {"A": 30, "B": 20, "C": 10}) + + popularity_add_rank(t) + + by_name = {n.name: n for n in t.traverse()} + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 2 + assert by_name["C"].props["popularity_rank"] == 3 + + def test_ties_use_standard_competition_ranking(self): + # Two leaves tied at the top share rank 1; the next leaf gets + # rank 3 (not 2). Same for ties further down. + t = ete4.Tree("(A,B,C,D,E)R;", parser=1) + self._set_leaf_pops(t, {"A": 10, "B": 10, "C": 5, "D": 3, "E": 3}) + + popularity_add_rank(t) + + by_name = {n.name: n for n in t.traverse()} + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 1 + assert by_name["C"].props["popularity_rank"] == 3 + assert by_name["D"].props["popularity_rank"] == 4 + assert by_name["E"].props["popularity_rank"] == 4 + + def test_all_leaves_tied_share_rank_one(self): + t = ete4.Tree("(A,B,C)R;", parser=1) + self._set_leaf_pops(t, {"A": 7, "B": 7, "C": 7}) + + popularity_add_rank(t) + + for name in ("A", "B", "C"): + leaf = next(n for n in t.traverse() if n.name == name) + assert leaf.props["popularity_rank"] == 1 + + def test_single_leaf_gets_rank_one(self): + # The function ranks even the root if it is a leaf. + t = ete4.Tree("A;", parser=1) + t.props["popularity"] = 42 + popularity_add_rank(t) + assert t.props["popularity_rank"] == 1 + + def test_internal_nodes_do_not_get_a_rank(self): + # Only leaves are ranked; the internal node's popularity, even + # if set, is ignored both as a tie-breaker and as an output. + t = ete4.Tree("((A,B)I,C)R;", parser=1) + self._set_leaf_pops(t, {"A": 10, "B": 5, "C": 1}) + by_name = {n.name: n for n in t.traverse()} + by_name["I"].props["popularity"] = 999 # should not affect anything + by_name["R"].props["popularity"] = 999 + + popularity_add_rank(t) + + # Leaves ranked normally. + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 2 + assert by_name["C"].props["popularity_rank"] == 3 + # Internal nodes untouched by ranking. + assert "popularity_rank" not in by_name["I"].props + assert "popularity_rank" not in by_name["R"].props + + def test_internal_node_without_popularity_does_not_trigger_skip(self): + # The None-guard only inspects leaves. An internal node that + # has no popularity prop must not cause the early return. + t = ete4.Tree("((A,B)I,C)R;", parser=1) + self._set_leaf_pops(t, {"A": 10, "B": 5, "C": 1}) + # I and R deliberately have no popularity prop set. + + popularity_add_rank(t) + + by_name = {n.name: n for n in t.traverse()} + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 2 + assert by_name["C"].props["popularity_rank"] == 3 + + def test_rank_cumsum_with_mixed_group_sizes(self): + # 1 leaf at top, 3 tied below, 1 at the bottom. + # Expected ranks: top=1, tied group=2, bottom=5. + t = ete4.Tree("(A,B,C,D,E)R;", parser=1) + self._set_leaf_pops(t, {"A": 100, "B": 50, "C": 50, "D": 50, "E": 1}) + + popularity_add_rank(t) + + by_name = {n.name: n for n in t.traverse()} + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 2 + assert by_name["C"].props["popularity_rank"] == 2 + assert by_name["D"].props["popularity_rank"] == 2 + assert by_name["E"].props["popularity_rank"] == 5 From dee1fe16c8b16886ff16b33299fc340869ffe840 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 11 Jun 2026 17:25:30 +0000 Subject: [PATCH 17/62] tree_build: Add ladderizing step As with CSV_base_table_creator, add a ladderizing step. --- oz_tree_build/tree_build/tree_build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 175266f1..7aee3df3 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -97,6 +97,10 @@ def main(): logger.info("Rank popularities, post-node removal") popularity_add_rank(base_t) + logging.info("ladderizing tree (groups with fewer leaves first)") + # warning: ladderize ascending is needed for the short OZ newick-like form + base_t.ladderize(topological=False, reverse=False) + logger.info("Add properies to tree") pass From 1b8e8a73abe7fd08a1e7caa31b66c055cc53c956 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 11 Jun 2026 20:25:10 +0000 Subject: [PATCH 18/62] step_output: Add tree_build CSV-writing Copy code to write CSVs from dendropy_extras. Don't bother with the non-MySQL CSVs, go straight for the files we use. --- oz_tree_build/tree_build/step_output.py | 233 ++++++++++++ oz_tree_build/tree_build/tree_build.py | 5 + oz_tree_build/utilities/ete.py | 9 + tests/test_tree_build_step_output.py | 484 ++++++++++++++++++++++++ 4 files changed, 731 insertions(+) create mode 100644 oz_tree_build/tree_build/step_output.py create mode 100644 tests/test_tree_build_step_output.py diff --git a/oz_tree_build/tree_build/step_output.py b/oz_tree_build/tree_build/step_output.py new file mode 100644 index 00000000..b37af266 --- /dev/null +++ b/oz_tree_build/tree_build/step_output.py @@ -0,0 +1,233 @@ +import csv +import os.path + +from ..utilities.ete import node_name_without_ott + + +def output_add_prop_ids(tree): + """ + Annotate every internal node with the integer ids the OneZoom MySQL + schema uses for nested-set descendant queries. + + Sets four props on each internal node: + - ``id`` : 1-based preorder index among internal nodes. + - ``leaf_lft`` : preorder position of the leftmost leaf in the subtree. + - ``leaf_rgt`` : preorder position of the rightmost leaf in the subtree. + - ``node_rgt`` : id of the rightmost internal-node descendant + (equals ``id`` when every child is a leaf). + + Leaves are not annotated — they are implicitly numbered by their + preorder position. Ids and leaf positions both start at 1 to line up + with MySQL row numbering. + + Assumes the tree is ladderized *ascending* (smallest subtree first): + ``node_rgt`` is derived by walking postorder and trusting that the + last-visited child sits at the right of its parent, which only holds + when the rightmost child carries the largest subtree. A descending + tree where a leaf sits to the right of an internal sibling will get + the wrong ``node_rgt`` (the parent will look terminal). + """ + # allocate node numbers + internal_node_number = 0 + leaf_count = 1 + for node in tree.traverse("preorder"): + if node.is_leaf: + leaf_count += 1 + else: + # NB: increment first, since we use a 1-base numbering system, for mySQL row numbering + internal_node_number += 1 + node.props["id"] = internal_node_number + node.props["leaf_lft"] = leaf_count + + # postorder traversal to allocate rgt side of ranges + internal_leaf_count = 0 + prev_node = None + for node in tree.traverse("postorder"): + # find rightmost leaf by postorder iteration. + # For rightmost node, if previously visited node is a leaf, then (because we ladderize + # ascending) the rightmost node must be self (i.e. this is a terminal internal node). + # Otherwise it is the previously visted node + if node.is_leaf: + internal_leaf_count += 1 + else: + node.props["leaf_rgt"] = internal_leaf_count # should have counted all the internal leaves by now + if prev_node.is_leaf: + node.props["node_rgt"] = node.props["id"] # node_rgt == self + else: + # the node_rgt should be the same as the node_rgt of the previous node + node.props["node_rgt"] = prev_node.props["node_rgt"] + prev_node = node + + +def output_mysqlexport(tree, out_dir): + """ + Write the three files needed to load the tree into the OneZoom MySQL + database: + + - ``ordered_leaves.csv`` : one row per leaf, in preorder. + - ``ordered_nodes.csv`` : one row per internal node, in preorder. + - ``import.sql`` : TRUNCATE + ``LOAD DATA LOCAL INFILE`` + script that loads both CSVs. + + Preconditions + ------------- + Every node must carry ``props["taxon"]``, a dict of taxon-derived + columns (``ott``, ``wikidata``, ``ncbi``, ...); missing keys are + written as ``\\N``. Every internal node must additionally carry + ``id`` / ``node_rgt`` / ``leaf_lft`` / ``leaf_rgt`` as produced by + `output_add_prop_ids`. + + Encoding conventions + -------------------- + - ``\\N`` is the marker for missing values (MySQL ``LOAD DATA`` + treats it as NULL). + - ``real_parent`` walks past randomly-resolved polytomies: any + ancestor with ``dist == 0`` is skipped so the column points at the + nearest biologically meaningful parent. The raw ``parent`` column + still references the immediate parent. + - A node that is itself a polytomy resolution (``dist == 0``) records + its ``real_parent`` as the *negative* of the resolved parent's id, + flagging the relationship as artificial. + - The leaf ``name`` column has any trailing ``_ottNNN`` suffix + stripped (the OTT is carried separately in its own column). + - An internal node's ``date`` prop is exposed via the ``age`` column. + - The root's ``parent`` is ``\\N`` but its ``real_parent`` is the + sentinel ``0``. + """ + + with ( + open(os.path.join(out_dir, "ordered_leaves.csv"), "w+", encoding="utf-8") as leaf_file, + open(os.path.join(out_dir, "ordered_nodes.csv"), "w+", encoding="utf-8") as node_file, + ): + leaf_csv = csv.writer(leaf_file, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") + node_csv = csv.writer(node_file, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") + leaf_csv.writerow( + [ + "parent", + "real_parent", + "name", + "extinction_date", + "ott", + "wikidata", + "wikipedia_lang_flag", + "iucn", + "eol", + "raw_popularity", + "popularity", + "popularity_rank", + "price", + "ncbi", + "ifung", + "worms", + "irmng", + "gbif", + "ipni", + ] + ) + node_csv.writerow( + [ + "parent", + "real_parent", + "node_rgt", + "leaf_lft", + "leaf_rgt", + "name", + "age", + "ott", + "wikidata", + "wikipedia_lang_flag", + "eol", + "rnk", # We avoid using 'rank' as it is a reserved word in mysql + "raw_popularity", + "popularity", + "ncbi", + "ifung", + "worms", + "irmng", + "gbif", + "ipni", + "vern_synth", + ] + + [rit + str(i + 1) for rit in ("rep", "rtr", "rpd") for i in range(8)] + + ["iucn" + t for t in ("NE", "DD", "LC", "NT", "VU", "EN", "CR", "EW", "EX")] + ) + + for node in tree.traverse("preorder"): + # Find our real parent, ignoring randomly resolved polytomies + real_parent = node.parent + while real_parent and real_parent.dist == 0: # TODO: Is this still how we identify polytomies? + real_parent = real_parent.parent + + if not real_parent: + real_parent_id = 0 + elif node.dist == 0: + # real_parent is negative iff we're a polytomy + real_parent_id = -real_parent.props["id"] + else: + real_parent_id = real_parent.props["id"] + + if node.is_leaf: + leaf_csv.writerow( + [ + node.parent.props["id"] if node.parent else "\\N", # "parent" + # TODO: negative real_parent ids if this is a polytomy + real_parent_id, + node_name_without_ott(node), + node.props.get("extinction_date", "\\N"), + node.props["taxon"].get("ott", "\\N"), + node.props["taxon"].get("wikidata", "\\N"), + node.props["taxon"].get("wikipedia_lang_flag", "\\N"), + node.props["taxon"].get("iucn", "\\N"), + node.props["taxon"].get("eol", "\\N"), + node.props["taxon"].get("raw_popularity", "\\N"), + node.props.get("popularity", "\\N"), + node.props.get("popularity_rank", "\\N"), + None, # "price" + node.props["taxon"].get("ncbi", "\\N"), + node.props["taxon"].get("ifung", "\\N"), + node.props["taxon"].get("worms", "\\N"), + node.props["taxon"].get("irmng", "\\N"), + node.props["taxon"].get("gbif", "\\N"), + node.props["taxon"].get("ipni", "\\N"), + ] + ) + else: + node_csv.writerow( + [ + node.parent.props["id"] if node.parent else "\\N", # "parent" + real_parent_id, + node.props["node_rgt"], + node.props["leaf_lft"], + node.props["leaf_rgt"], + node_name_without_ott(node), + node.props.get("date", "\\N"), # TODO: But only if it's not imputed + node.props["taxon"].get("ott", "\\N"), + node.props["taxon"].get("wikidata", "\\N"), + node.props["taxon"].get("wikipedia_lang_flag", "\\N"), + node.props["taxon"].get("eol", "\\N"), + node.props["taxon"].get("rnk", "\\N"), + node.props["taxon"].get("raw_popularity", "\\N"), + node.props.get("popularity", "\\N"), + node.props["taxon"].get("ncbi", "\\N"), + node.props["taxon"].get("ifung", "\\N"), + node.props["taxon"].get("worms", "\\N"), + node.props["taxon"].get("irmng", "\\N"), + node.props["taxon"].get("gbif", "\\N"), + node.props["taxon"].get("ipni", "\\N"), + None, # "vern_synth" + ] + + ["\\N" for _ in ("rep", "rtr", "rpd") for _ in range(8)] + + ["\\N" for _ in ("NE", "DD", "LC", "NT", "VU", "EN", "CR", "EW", "EX")] + ) + + with open(os.path.join(out_dir, "import.sql"), "w", encoding="utf-8") as sql_f: + for csvfile in ("ordered_leaves.csv", "ordered_nodes.csv"): + table = os.path.splitext(csvfile)[0] + sql_f.writelines( + [ + f"TRUNCATE TABLE {table};\n" + f"LOAD DATA LOCAL INFILE '{csvfile}' REPLACE INTO TABLE `{table}` \n" + f" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' \n" + f" IGNORE 1 LINES ({open(os.path.join(out_dir,csvfile)).readline().rstrip()}) SET id = NULL;\n" + ] + ) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 7aee3df3..9a8e2fdc 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -11,6 +11,7 @@ from ..taxon_mapping_and_popularity.taxon_map import read_taxon_map from ..utilities.debug_util import parse_args_and_add_logging_switch from .step_graft import graft_extract_ot_subtrees, graft_tree +from .step_output import output_add_prop_ids, output_mysqlexport from .step_parse import parse_bespoke_trees, parse_ot_orphans from .step_popularity import popularity_add_prop, popularity_add_rank from .step_taxon import taxon_add_prop @@ -104,6 +105,10 @@ def main(): logger.info("Add properies to tree") pass + logger.info("Output MySQL CSV files") + output_add_prop_ids(base_t) + output_mysqlexport(base_t, args.out_dir) + if __name__ == "__main__": main() diff --git a/oz_tree_build/utilities/ete.py b/oz_tree_build/utilities/ete.py index 506a3d63..e7650099 100644 --- a/oz_tree_build/utilities/ete.py +++ b/oz_tree_build/utilities/ete.py @@ -13,3 +13,12 @@ def node_get_ott(n): return None m = NODE_OTT_RE.search(n.name) return m.group(1) if m else None + + +def node_name_without_ott(n): + """ + Remove any OTT at the end of the label, return node name + """ + if not n.name: + return None + return NODE_OTT_RE.sub("", n.name) diff --git a/tests/test_tree_build_step_output.py b/tests/test_tree_build_step_output.py new file mode 100644 index 00000000..4c026f56 --- /dev/null +++ b/tests/test_tree_build_step_output.py @@ -0,0 +1,484 @@ +import csv +import os + +import ete4 + +from oz_tree_build.tree_build.step_output import ( + output_add_prop_ids, + output_mysqlexport, +) + + +def _by_name(tree): + return {n.name: n for n in tree.traverse()} + + +def _prep(tree, taxon_overrides=None): + """ + Give every node the minimum props output_mysqlexport requires: + a (possibly empty) taxon dict and the id props from output_add_prop_ids. + `taxon_overrides` is `{node_name: {key: value, ...}}`. + """ + taxon_overrides = taxon_overrides or {} + for n in tree.traverse(): + n.props["taxon"] = dict(taxon_overrides.get(n.name, {})) + output_add_prop_ids(tree) + + +def _read_csv(out_dir, name): + with open(os.path.join(out_dir, name), encoding="utf-8") as f: + return list(csv.reader(f)) + + +class TestOutputAddPropIds: + def test_minimal_two_leaf_tree(self): + # Single internal node with two leaves. The root is the only node + # that receives id/leaf_lft/leaf_rgt/node_rgt. + t = ete4.Tree("(A,B)R;", parser=1) + output_add_prop_ids(t) + assert t.props["id"] == 1 + assert t.props["leaf_lft"] == 1 + assert t.props["leaf_rgt"] == 2 + # No internal node sits below the root, so its rightmost-internal + # descendant is itself. + assert t.props["node_rgt"] == 1 + + def test_leaves_get_no_id_props(self): + # Leaves are implicitly numbered by their preorder position; the + # function must not write any of the id props onto leaf nodes. + t = ete4.Tree("((A,B)I,(C,D)J)R;", parser=1) + output_add_prop_ids(t) + for leaf in t.leaves(): + assert "id" not in leaf.props + assert "leaf_lft" not in leaf.props + assert "leaf_rgt" not in leaf.props + assert "node_rgt" not in leaf.props + + def test_internal_ids_are_one_based_preorder(self): + # Internal nodes receive ids 1..N in preorder. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + ids = [n.props["id"] for n in t.traverse("preorder") if not n.is_leaf] + assert ids == [1, 2, 3, 4] + nodes = _by_name(t) + assert nodes["R"].props["id"] == 1 + assert nodes["I"].props["id"] == 2 + assert nodes["K"].props["id"] == 3 + assert nodes["J"].props["id"] == 4 + + def test_leaf_lft_is_position_of_leftmost_descendant_leaf(self): + # leaf_lft is the 1-based preorder position of the subtree's + # leftmost leaf. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + # Preorder leaf order is A, B, C, D, E → positions 1..5. + assert nodes["R"].props["leaf_lft"] == 1 # leftmost descendant is A + assert nodes["I"].props["leaf_lft"] == 1 # leftmost descendant is A + assert nodes["K"].props["leaf_lft"] == 3 # leftmost descendant is C + assert nodes["J"].props["leaf_lft"] == 4 # leftmost descendant is D + + def test_leaf_rgt_is_position_of_rightmost_descendant_leaf(self): + # leaf_rgt is the 1-based preorder position of the subtree's + # rightmost leaf. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + assert nodes["R"].props["leaf_rgt"] == 5 # rightmost descendant is E + assert nodes["I"].props["leaf_rgt"] == 2 # rightmost descendant is B + assert nodes["K"].props["leaf_rgt"] == 5 # rightmost descendant is E + assert nodes["J"].props["leaf_rgt"] == 5 # rightmost descendant is E + + def test_node_rgt_for_terminal_internal_is_self(self): + # If every child of an internal node is a leaf, its rightmost + # internal descendant is itself. + t = ete4.Tree("((A,B)I,(C,D)J)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + assert nodes["I"].props["node_rgt"] == nodes["I"].props["id"] + assert nodes["J"].props["node_rgt"] == nodes["J"].props["id"] + + def test_node_rgt_walks_rightmost_internal_descendant(self): + # For a non-terminal internal node, node_rgt is the id of the + # rightmost internal descendant (assuming ascending ladderization + # so the rightmost child holds the biggest subtree). + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + # R's rightmost internal descendant is J (id=4) via K. + assert nodes["R"].props["node_rgt"] == 4 + # K's rightmost internal descendant is J (id=4). + assert nodes["K"].props["node_rgt"] == 4 + + def test_unnamed_internal_nodes_still_get_props(self): + # The function keys off is_leaf, not the node name, so unnamed + # internal nodes still receive the id props. + t = ete4.Tree("(C,(A,B));", parser=1) + output_add_prop_ids(t) + internals = [n for n in t.traverse("preorder") if not n.is_leaf] + assert [n.props["id"] for n in internals] == [1, 2] + # Root (id=1) spans all three leaves; its rightmost internal + # descendant is the (A,B) subtree (id=2). + assert internals[0].props["leaf_lft"] == 1 + assert internals[0].props["leaf_rgt"] == 3 + assert internals[0].props["node_rgt"] == 2 + # (A,B) (id=2) spans leaves 2..3; terminal, so node_rgt is itself. + assert internals[1].props["leaf_lft"] == 2 + assert internals[1].props["leaf_rgt"] == 3 + assert internals[1].props["node_rgt"] == 2 + + def test_relies_on_ascending_ladderization(self): + # node_rgt is computed by walking postorder and trusting that the + # last-visited child sits at the right of its parent — which holds + # only when the tree is ladderized ascending (small subtree first, + # so the rightmost child carries the largest subtree). If a leaf + # sits to the right of an internal sibling, the function wrongly + # treats the parent as terminal. This test pins that assumption. + t = ete4.Tree("((A,B)I,C)R;", parser=1) # descending: leaf C on the right + output_add_prop_ids(t) + nodes = _by_name(t) + # R's true rightmost-internal descendant is I (id=2), but because + # C is the rightmost child the function records R as terminal. + assert nodes["R"].props["id"] == 1 + assert nodes["R"].props["node_rgt"] == 1 + + def test_deeply_nested_ladder(self): + # A right-leaning ladder: each level's rightmost child is the + # bigger subtree, so node_rgt should chain down to the deepest + # internal node. + t = ete4.Tree("(A,(B,(C,(D,E)J)K)L)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + # Preorder of internals: R, L, K, J → ids 1, 2, 3, 4. + assert nodes["R"].props["id"] == 1 + assert nodes["L"].props["id"] == 2 + assert nodes["K"].props["id"] == 3 + assert nodes["J"].props["id"] == 4 + # Every ancestor's rightmost internal descendant is J. + assert nodes["R"].props["node_rgt"] == 4 + assert nodes["L"].props["node_rgt"] == 4 + assert nodes["K"].props["node_rgt"] == 4 + assert nodes["J"].props["node_rgt"] == 4 # terminal → self + # leaf_lft / leaf_rgt span the leaves below each node. + assert nodes["R"].props["leaf_lft"] == 1 + assert nodes["R"].props["leaf_rgt"] == 5 + assert nodes["L"].props["leaf_lft"] == 2 + assert nodes["L"].props["leaf_rgt"] == 5 + assert nodes["K"].props["leaf_lft"] == 3 + assert nodes["K"].props["leaf_rgt"] == 5 + assert nodes["J"].props["leaf_lft"] == 4 + assert nodes["J"].props["leaf_rgt"] == 5 + + def test_leaf_lft_leq_leaf_rgt_for_every_internal(self): + # Sanity invariant — leftmost-leaf position never exceeds the + # rightmost-leaf position within the same subtree. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + for n in t.traverse(): + if n.is_leaf: + continue + assert n.props["leaf_lft"] <= n.props["leaf_rgt"] + + def test_node_rgt_never_exceeds_max_internal_id(self): + # node_rgt always points at an existing internal node id, so it + # cannot exceed the count of internal nodes. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + internals = [n for n in t.traverse() if not n.is_leaf] + max_id = max(n.props["id"] for n in internals) + for n in internals: + assert 1 <= n.props["node_rgt"] <= max_id + + +# Header columns in the order written by output_mysqlexport. +LEAF_HEADER = [ + "parent", + "real_parent", + "name", + "extinction_date", + "ott", + "wikidata", + "wikipedia_lang_flag", + "iucn", + "eol", + "raw_popularity", + "popularity", + "popularity_rank", + "price", + "ncbi", + "ifung", + "worms", + "irmng", + "gbif", + "ipni", +] + +NODE_HEADER = ( + [ + "parent", + "real_parent", + "node_rgt", + "leaf_lft", + "leaf_rgt", + "name", + "age", + "ott", + "wikidata", + "wikipedia_lang_flag", + "eol", + "rnk", + "raw_popularity", + "popularity", + "ncbi", + "ifung", + "worms", + "irmng", + "gbif", + "ipni", + "vern_synth", + ] + + [rit + str(i + 1) for rit in ("rep", "rtr", "rpd") for i in range(8)] + + ["iucn" + t for t in ("NE", "DD", "LC", "NT", "VU", "EN", "CR", "EW", "EX")] +) + + +class TestOutputMysqlExport: + def test_creates_three_output_files(self, tmp_path): + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + assert (tmp_path / "ordered_leaves.csv").exists() + assert (tmp_path / "ordered_nodes.csv").exists() + assert (tmp_path / "import.sql").exists() + + def test_leaf_header_matches_expected_columns(self, tmp_path): + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + rows = _read_csv(tmp_path, "ordered_leaves.csv") + assert rows[0] == LEAF_HEADER + + def test_node_header_matches_expected_columns(self, tmp_path): + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + rows = _read_csv(tmp_path, "ordered_nodes.csv") + assert rows[0] == NODE_HEADER + + def test_leaf_and_node_row_widths_match_their_headers(self, tmp_path): + # Each emitted row must have exactly as many fields as its header, + # otherwise MySQL's LOAD DATA INFILE will reject it. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + for row in leaves[1:]: + assert len(row) == len(LEAF_HEADER) + for row in nodes[1:]: + assert len(row) == len(NODE_HEADER) + + def test_leaves_go_to_leaf_csv_internals_to_node_csv(self, tmp_path): + # Five leaves under four internal nodes → 5 data rows in leaves, + # 4 data rows in nodes. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + leaf_names = [r[LEAF_HEADER.index("name")] for r in leaves[1:]] + node_names = [r[NODE_HEADER.index("name")] for r in nodes[1:]] + assert sorted(leaf_names) == ["A", "B", "C", "D", "E"] + assert sorted(node_names) == ["I", "J", "K", "R"] + + def test_leaf_rows_are_in_preorder(self, tmp_path): + # The function traverses preorder; leaf rows should reflect that. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + leaf_names = [r[LEAF_HEADER.index("name")] for r in leaves[1:]] + assert leaf_names == ["A", "B", "C", "D", "E"] + + def test_leaf_name_strips_ott_suffix(self, tmp_path): + # The "_ottNNN" suffix carries the OTT id and is removed from the + # name written to the CSV. + t = ete4.Tree("(A_ott1234,B_ott5678)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + names = [r[LEAF_HEADER.index("name")] for r in leaves[1:]] + assert names == ["A", "B"] + + def test_root_parent_field_is_backslash_N(self, tmp_path): + # Root has no parent → "parent" column is \N (MySQL NULL marker). + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root_row = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + assert root_row[NODE_HEADER.index("parent")] == "\\N" + + def test_root_real_parent_is_zero(self, tmp_path): + # Root has no parent, so real_parent is the sentinel 0. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root_row = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + assert root_row[NODE_HEADER.index("real_parent")] == "0" + + def test_non_root_parent_is_parent_id(self, tmp_path): + # A leaf's "parent" field is the id of its (internal) parent. + t = ete4.Tree("((A,B)I,(C,D)J)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + nodes = _by_name(t) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + i_id = str(nodes["I"].props["id"]) + j_id = str(nodes["J"].props["id"]) + rows = {r[LEAF_HEADER.index("name")]: r for r in leaves[1:]} + assert rows["A"][LEAF_HEADER.index("parent")] == i_id + assert rows["B"][LEAF_HEADER.index("parent")] == i_id + assert rows["C"][LEAF_HEADER.index("parent")] == j_id + assert rows["D"][LEAF_HEADER.index("parent")] == j_id + + def test_node_row_writes_id_range_columns(self, tmp_path): + # Internal nodes carry node_rgt / leaf_lft / leaf_rgt across to + # their CSV row. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + nodes_by_name = _by_name(t) + rows = _read_csv(tmp_path, "ordered_nodes.csv") + by_name = {r[NODE_HEADER.index("name")]: r for r in rows[1:]} + for nm in ("R", "I", "K", "J"): + row = by_name[nm] + n = nodes_by_name[nm] + assert row[NODE_HEADER.index("node_rgt")] == str(n.props["node_rgt"]) + assert row[NODE_HEADER.index("leaf_lft")] == str(n.props["leaf_lft"]) + assert row[NODE_HEADER.index("leaf_rgt")] == str(n.props["leaf_rgt"]) + + def test_taxon_props_are_written_to_leaf_row(self, tmp_path): + # Values supplied via node.props["taxon"] are projected onto the + # matching CSV columns; absent keys become \N. + t = ete4.Tree("(A,B)R;", parser=1) + _prep( + t, + taxon_overrides={ + "A": { + "ott": "111", + "wikidata": "Q1", + "iucn": "LC", + "eol": "42", + "ncbi": "999", + }, + }, + ) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + rows = {r[LEAF_HEADER.index("name")]: r for r in leaves[1:]} + a = rows["A"] + assert a[LEAF_HEADER.index("ott")] == "111" + assert a[LEAF_HEADER.index("wikidata")] == "Q1" + assert a[LEAF_HEADER.index("iucn")] == "LC" + assert a[LEAF_HEADER.index("eol")] == "42" + assert a[LEAF_HEADER.index("ncbi")] == "999" + # B had no overrides → \N everywhere taxon-derived. + b = rows["B"] + assert b[LEAF_HEADER.index("ott")] == "\\N" + assert b[LEAF_HEADER.index("ncbi")] == "\\N" + + def test_taxon_props_are_written_to_node_row(self, tmp_path): + # Internal nodes get the same taxon projection — but with `rnk` + # in place of the leaf-only `iucn`/`extinction_date` columns. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t, taxon_overrides={"R": {"ott": "777", "rnk": "family"}}) + output_mysqlexport(t, str(tmp_path)) + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + assert root[NODE_HEADER.index("ott")] == "777" + assert root[NODE_HEADER.index("rnk")] == "family" + + def test_missing_extinction_date_and_popularity_are_backslash_N(self, tmp_path): + # Leaf-only props (extinction_date, popularity, popularity_rank) + # default to \N when not set. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + for row in leaves[1:]: + assert row[LEAF_HEADER.index("extinction_date")] == "\\N" + assert row[LEAF_HEADER.index("popularity")] == "\\N" + assert row[LEAF_HEADER.index("popularity_rank")] == "\\N" + + def test_leaf_extinction_date_and_popularity_are_emitted(self, tmp_path): + # Values on the leaf itself flow through unchanged. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + nodes = _by_name(t) + nodes["A"].props["extinction_date"] = "2020" + nodes["A"].props["popularity"] = 1.5 + nodes["A"].props["popularity_rank"] = 3 + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + a = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "A") + assert a[LEAF_HEADER.index("extinction_date")] == "2020" + assert a[LEAF_HEADER.index("popularity")] == "1.5" + assert a[LEAF_HEADER.index("popularity_rank")] == "3" + + def test_internal_node_date_written_as_age(self, tmp_path): + # An internal node's "date" property is exposed via the "age" column. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + t.props["date"] = 12.5 + output_mysqlexport(t, str(tmp_path)) + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + assert root[NODE_HEADER.index("age")] == "12.5" + + def test_polytomy_parent_is_skipped_for_real_parent(self, tmp_path): + # A non-polytomy node whose immediate parent has dist==0 (a + # randomly-resolved polytomy node) should attribute its + # real_parent to the next ancestor with dist!=0. + # Tree shape: G -> P (dist=0 polytomy) -> X (dist=1 leaf). + # X's real_parent must be G, not P. + t = ete4.Tree("((X:1,Y:1)P:0,Z:1)G:1;", parser=1) + nodes = _by_name(t) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + x = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "X") + # parent (raw) is still P; real_parent skips it up to G. + assert x[LEAF_HEADER.index("parent")] == str(nodes["P"].props["id"]) + assert x[LEAF_HEADER.index("real_parent")] == str(nodes["G"].props["id"]) + + def test_polytomy_self_emits_negative_real_parent(self, tmp_path): + # A node that is itself a polytomy resolution (dist=0) writes a + # negative real_parent_id, flagging the relationship as artificial. + t = ete4.Tree("((X:1,Y:1)P:0,Z:1)G:1;", parser=1) + nodes = _by_name(t) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + node_rows = _read_csv(tmp_path, "ordered_nodes.csv") + p = next(r for r in node_rows[1:] if r[NODE_HEADER.index("name")] == "P") + # P's dist is 0; G is its (non-polytomy) parent. + assert p[NODE_HEADER.index("real_parent")] == str(-nodes["G"].props["id"]) + + def test_import_sql_contains_load_data_for_both_tables(self, tmp_path): + # The SQL script should truncate-and-load both CSV files. The + # column list inside `LOAD DATA INFILE` is read from the first + # line of the CSV, so it must match the CSV header exactly. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + sql = (tmp_path / "import.sql").read_text(encoding="utf-8") + assert "TRUNCATE TABLE ordered_leaves;" in sql + assert "TRUNCATE TABLE ordered_nodes;" in sql + assert "LOAD DATA LOCAL INFILE 'ordered_leaves.csv'" in sql + assert "LOAD DATA LOCAL INFILE 'ordered_nodes.csv'" in sql + # Header echoed inside the LOAD DATA column list. + assert "(" + ",".join(LEAF_HEADER) + ")" in sql + assert "(" + ",".join(NODE_HEADER) + ")" in sql + # `id` is auto-assigned by MySQL, not loaded from CSV. + assert "SET id = NULL;" in sql From 775e1442368d66444320e4d23947c61e8319ccb7 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 29 May 2026 16:42:54 +0000 Subject: [PATCH 19/62] tree_build/step_treeprop: date-derived tree properties #1007 Functions to populate required algorithms into tree properties. --- oz_tree_build/tree_build/step_treeprop.py | 186 +++++++++++++++ tests/test_tree_build_step_treeprop.py | 274 ++++++++++++++++++++++ 2 files changed, 460 insertions(+) create mode 100644 oz_tree_build/tree_build/step_treeprop.py create mode 100644 tests/test_tree_build_step_treeprop.py diff --git a/oz_tree_build/tree_build/step_treeprop.py b/oz_tree_build/tree_build/step_treeprop.py new file mode 100644 index 00000000..e54f6287 --- /dev/null +++ b/oz_tree_build/tree_build/step_treeprop.py @@ -0,0 +1,186 @@ +import logging +import math + +logger = logging.getLogger(__name__) + + +# Sourced from https://stratigraphy.org/supplementary#data +# fmt: off +GEOLOGICAL_PERIODS = [ + {"eon": "Unknown","era": "Unknown","period": "Unknown","epoch": "Unknown","short_text": "Unknown","long_text": "Unknown","color": "#1A1A1A","mya_start": -1e9,"number": 1}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Anthropocene","short_text": "Anthropocene","long_text": "Anthropocene mass extinction event","color": "#1A1A1A","mya_start": 0.000246,"number": 1}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Holocene","short_text": "Holocene Epoch","long_text": "Holocene Epoch, Quaternary Period","color": "#7A7A72","mya_start": 0.0117,"number": 2}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Pleistocene","short_text": "Pleistocene Epoch","long_text": "Pleistocene Epoch, Quaternary Period","color": "#7A7A72","mya_start": 2.58,"number": 3}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Neogene","epoch": "Pliocene","short_text": "Neogene Period","long_text": "Pliocene Epoch, Neogene Period","color": "#A08050","mya_start": 5.333,"number": 4}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Neogene","epoch": "Miocene","short_text": "Neogene Period","long_text": "Miocene Epoch, Neogene Period","color": "#A08050","mya_start": 23.04,"number": 5}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Oligocene","short_text": "Paleogene Period","long_text": "Oligocene Epoch, Paleogene Period","color": "#8A6A3A","mya_start": 33.9,"number": 6}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Eocene","short_text": "Paleogene Period","long_text": "Eocene Epoch, Paleogene Period","color": "#8A6A3A","mya_start": 56,"number": 7}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Paleocene","short_text": "Paleogene Period","long_text": "Paleocene Epoch, Paleogene Period","color": "#8A6A3A","mya_start": 66,"number": 8}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Upper","short_text": "Cretaceous–Paleogene extinction","long_text": "Cretaceous–Paleogene extinction","color": "#1A1A1A","mya_start": 65.9999,"number": 9}, # noqa: E501 RUF001 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Upper","short_text": "Cretaceous Period","long_text": "(Upper) Cretaceous Period","color": "#6C7A4D","mya_start": 100.5,"number": 10}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Lower","short_text": "Cretaceous Period","long_text": "(Lower) Cretaceous Period","color": "#6C7A4D","mya_start": 143.1,"number": 11}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Upper","short_text": "Jurassic Period","long_text": "(Upper) Jurassic Period","color": "#3E5B3A","mya_start": 161.5,"number": 12}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Middle","short_text": "Jurassic Period","long_text": "(Middle) Jurassic Period","color": "#3E5B3A","mya_start": 174.7,"number": 13}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Lower","short_text": "Jurassic Period","long_text": "(Lower) Jurassic Period","color": "#3E5B3A","mya_start": 201.4,"number": 14}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Upper","short_text": "Triassic–Jurassic extinction event","long_text": "Triassic–Jurassic extinction event","color": "#1A1A1A","mya_start": 201.3,"number": 15}, # noqa: E501 RUF001 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Upper","short_text": "Triassic Period","long_text": "(Upper) Triassic Period","color": "#7A4A2B","mya_start": 237,"number": 16}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Middle","short_text": "Triassic Period","long_text": "(Middle) Triassic Period","color": "#7A4A2B","mya_start": 246.7,"number": 17}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Lower","short_text": "Triassic Period","long_text": "(Lower) Triassic Period","color": "#7A4A2B","mya_start": 251.902,"number": 18}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Lopingian","short_text": "Permian–Triassic extinction event","long_text": "Permian–Triassic extinction event ""Great dying""","color": "#1A1A1A","mya_start": 252,"number": 19}, # noqa: E501 RUF001 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Lopingian","short_text": "Permian Period","long_text": "Lopingian Epoch, Permian Period","color": "#6A5D35","mya_start": 259.51,"number": 20}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Guadalupian","short_text": "Permian Period","long_text": "Guadalupian Epoch, Permian Period","color": "#6A5D35","mya_start": 274.4,"number": 21}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Cisuralian","short_text": "Permian Period","long_text": "Cisuralian Epoch, Permian Period","color": "#6A5D35","mya_start": 298.9,"number": 22}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Carboniferous","epoch": "Pennsylvanian","short_text": "Carboniferous Period","long_text": "Pennsylvanian Epoch, Carboniferous Period","color": "#2F4F2F","mya_start": 323.4,"number": 23}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Carboniferous","epoch": "Mississippian","short_text": "Carboniferous Period","long_text": "Mississippian Epoch, Carboniferous Period","color": "#2F4F2F","mya_start": 358.86,"number": 24}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Upper","short_text": "Late Devonian mass extinction","long_text": "Late Devonian mass extinction","color": "#1A1A1A","mya_start": 372,"number": 25}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Upper","short_text": "Devonian Period","long_text": "(Upper Epoch, Devonian Period","color": "#6B6B2F","mya_start": 382.31,"number": 26}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Middle","short_text": "Devonian Period","long_text": "(Middle) Devonian Period","color": "#6B6B2F","mya_start": 393.47,"number": 27}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Lower","short_text": "Devonian Period","long_text": "(Lower) Devonian Period","color": "#6B6B2F","mya_start": 419.62,"number": 28}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Pridoli","short_text": "Silurian Period","long_text": "Pridoli Epoch, Silurian Period","color": "#5A6A3A","mya_start": 422.7,"number": 29}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Ludlow","short_text": "Silurian Period","long_text": "Ludlow Epoch, Silurian Period","color": "#5A6A3A","mya_start": 426.7,"number": 30}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Wenlock","short_text": "Silurian Period","long_text": "Wenlock Epoch, Silurian Period","color": "#5A6A3A","mya_start": 432.9,"number": 31}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Llandovery","short_text": "Silurian Period","long_text": "Llandovery Epoch, Silurian Period","color": "#5A6A3A","mya_start": 443.1,"number": 32}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Upper","short_text": "Late Ordovician mass extinction","long_text": "Late Ordovician mass extinction","color": "#1A1A1A","mya_start": 445,"number": 33}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Upper","short_text": "Ordovician Period","long_text": "(Upper) Ordovician Period","color": "#486B4A","mya_start": 458.2,"number": 34}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Middle","short_text": "Ordovician Period","long_text": "(Middle) Ordovician Period","color": "#486B4A","mya_start": 471.3,"number": 35}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Lower","short_text": "Ordovician Period","long_text": "(Lower) Ordovician Period","color": "#486B4A","mya_start": 486.85,"number": 36}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Furongian","short_text": "Cambrian Period","long_text": "Furongian Epoch, Cambrian Period","color": "#2E5D50","mya_start": 497,"number": 37}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Miaolingian","short_text": "Cambrian Period","long_text": "Miaolingian Epoch, Cambrian Period","color": "#2E5D50","mya_start": 506.5,"number": 38}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Series 2","short_text": "Cambrian Period","long_text": "Series 2 Epoch, Cambrian Period","color": "#2E5D50","mya_start": 521,"number": 39}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Terreneuvian","short_text": "Cambrian Period","long_text": "Terreneuvian Epoch, Cambrian Period","color": "#2E5D50","mya_start": 538.8,"number": 40}, # noqa: E501 + {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Ediacaran","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Ediacaran Period, Neo-proterozoic Era","color": "#3B4A52","mya_start": 635,"number": 41}, # noqa: E501 + {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Cryogenian","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Cryogenian Period, Neo-proterozoic Era","color": "#3B4A52","mya_start": 720,"number": 42}, # noqa: E501 + {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Tonian","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Tonian Period, Neo-proterozoic Era","color": "#3B4A52","mya_start": 1000,"number": 43}, # noqa: E501 + {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Stenian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Stenian Period, Meso-proterozoic Era","color": "#3B4A52","mya_start": 1200,"number": 44}, # noqa: E501 + {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Ectasian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Ectasian Period, Meso-proterozoic Era","color": "#3B4A52","mya_start": 1400,"number": 45}, # noqa: E501 + {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Calymmian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Calymmian Period, Meso-proterozoic Era","color": "#3B4A52","mya_start": 1600,"number": 46}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Statherian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Statherian Period, Paleo-proterozoic Era","color": "#3B4A52","mya_start": 1800,"number": 47}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Orosirian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Orosirian Period, Paleo-proterozoic Era","color": "#3B4A52","mya_start": 2050,"number": 48}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Rhyacian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Rhyacian Period, Paleo-proterozoic Era","color": "#3B4A52","mya_start": 2300,"number": 49}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Siderian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Siderian Period, Paleo-proterozoic Era","color": "#3B4A52","mya_start": 2500,"number": 50}, # noqa: E501 + {"eon": "Archean","era": "Neo-Archean","period": "-","epoch": "-","short_text": "Neo-Archean Era","long_text": "Neo-Archean Era","color": "#2C2A28","mya_start": 2800,"number": 51}, # noqa: E501 + {"eon": "Archean","era": "Meso-Archean","period": "-","epoch": "-","short_text": "Meso-Archean Era","long_text": "Meso-Archean Era","color": "#2C2A28","mya_start": 3200,"number": 52}, # noqa: E501 + {"eon": "Archean","era": "Paleo-Archean","period": "-","epoch": "-","short_text": "Paleo-Archean Era","long_text": "Paleo-Archean Era","color": "#2C2A28","mya_start": 3600,"number": 53}, # noqa: E501 + {"eon": "Archean","era": "Eo-Archean","period": "-","epoch": "-","short_text": "Eo-Archean Era","long_text": "Eo-Archean Era","color": "#2C2A28","mya_start": 4031,"number": 54}, # noqa: E501 + {"eon": "Hadean","era": "-","period": "-","epoch": "-","short_text": "Hadean Eon","long_text": "Hadean Eon","color": "#1A1A1A","mya_start": 4567,"number": 55}, # noqa: E501 +] +# fmt: on + + +def treeprop_geological(tree): + """ + Given an ete4 tree object, add a "geological" prop to each node, + representing a 1-based period index. + + Assumes the tree already has an "age" prop representing an absolute age in Mya. + + Return name of prop just added. + """ + # Turn array into (mya, idx) pairs + lookup = [(p["mya_start"], idx) for idx, p in enumerate(GEOLOGICAL_PERIODS)] + + for node in tree.traverse("preorder"): + n_age = node.props.get("age") + if n_age is None: + logger.warning(f"Node {node.name} has no age property") + node.props["geological"] = 0 + else: + for mya_start, idx in lookup: # noqa: B007 # idx is used outside the lookup, not inside + if n_age <= mya_start: + break + else: + # Fell off end + idx = None + node.props["geological"] = idx + + prop_format = tree.root.props.setdefault("prop_format", {}) + prop_format["geological"] = "u8" + + return "geological" + + +def treeprop_sliding_window(tree, local_mean_width=5): + """ + Given an ete4 tree object, add a "sliding_window" prop to each node, + representing the log-ratio between the node's edge length and the mean + edge length of nearby ancestors (up to local_mean_width upwards) and + descendants (up to local_mean_width deep). + + Return name of prop just added. + """ + for node in tree.traverse("preorder"): + edge_length = node.dist + if edge_length == 0: + node.props["sliding_window"] = 0 + continue + if edge_length is None or edge_length < 0: + logger.warning(f"Node {node.name} has no / negative branch length {edge_length}") + node.props["sliding_window"] = 0.0 + continue + window_count = 0 + window_sum = 0 + + # Work upwards, adding parents to window + nn = node + for _ in range(local_mean_width): + if nn.up is None: + break + nn_length = nn.dist + if nn_length is None or nn_length < 0: + logger.warning(f"Node {nn.name} has no / negative branch length {nn_length}") + break + window_sum += nn_length + window_count += 1 + nn = nn.up + + # Work downwards, adding descendents to window + stack = [(c, 1) for c in node.children] + while stack: + dn, depth = stack.pop() + dn_length = dn.dist + if dn_length is None or dn_length < 0: + logger.warning(f"Node {dn.name} has no / negative branch length {dn_length}") + continue + window_sum += dn_length + window_count += 1 + if depth < local_mean_width: + stack.extend((c, depth + 1) for c in dn.children) + + if window_count == 0: + node.props["sliding_window"] = 0.0 + else: + node.props["sliding_window"] = math.log(edge_length / (window_sum / window_count)) + + prop_format = tree.root.props.setdefault("prop_format", {}) + prop_format["sliding_window"] = "f16" + + return "sliding_window" + + +def treeprop_weighted_mean(tree, weighting=0.8): + """ + Given an ete4 tree object, add a "weighted_mean_ratio" prop. + + Return name of prop just added. + """ + # Calculate weighted mean vs. ancestors + for node in tree.traverse("preorder"): + edge_length = node.dist + parent = node.up + if parent is None: + node.props["weighted_mean"] = 0.0 + node.props["weighted_mean_ratio"] = 0.0 + elif edge_length is None or parent.props.get("weighted_mean") is None: + if edge_length is None: + logger.warning(f"Node {node.name} has no branch length") + node.props["weighted_mean"] = 0.0 + node.props["weighted_mean_ratio"] = 0.0 + else: + node.props["weighted_mean"] = (edge_length + (parent.props["weighted_mean"] * weighting)) / (1 + weighting) + node.props["weighted_mean_ratio"] = edge_length / node.props["weighted_mean"] + + prop_format = tree.root.props.setdefault("prop_format", {}) + prop_format["weighted_mean"] = "f16" + prop_format["weighted_mean_ratio"] = "f16" + + return "weighted_mean_ratio" diff --git a/tests/test_tree_build_step_treeprop.py b/tests/test_tree_build_step_treeprop.py new file mode 100644 index 00000000..b492083a --- /dev/null +++ b/tests/test_tree_build_step_treeprop.py @@ -0,0 +1,274 @@ +import logging +import math +import random + +import ete4 + +from oz_tree_build.tree_build.step_treeprop import ( + GEOLOGICAL_PERIODS, + treeprop_geological, + treeprop_sliding_window, + treeprop_weighted_mean, +) + +######################################## +# treeprop_geological +######################################## + + +def set_ages_from_dist(tree): + """ + Postorder pass: leaves get age 0, interior nodes get max(child.age + child.dist). + If any child has an unknown age or dist, the parent's age becomes None. + """ + for node in tree.traverse("postorder"): + if node.is_leaf: + node.props["age"] = 0 + continue + parent_age = 0 + for c in node.children: + if c.props.get("age") is None or c.dist is None: + parent_age = None + break + new_age = c.props["age"] + c.dist + if new_age > parent_age: + parent_age = new_age + node.props["age"] = parent_age + + +def do_treeprop_geological(nwk, date_tree=True): + t = ete4.Tree(nwk, parser=1) + # Our tree needs to have the age prop set for this to work + if date_tree: + set_ages_from_dist(t) + assert treeprop_geological(t) == "geological" + + # Traverse tree, returning all periods + return [(n.name, n.props.get("age"), n.props["geological"]) for n in t.traverse("preorder")] + + +class TestTreepropGeological: + def test_undated_tree(self): + """Undated trees get 0 set""" + assert do_treeprop_geological("(A:10)B;", date_tree=False) == [ + ("B", None, 0), + ("A", None, 0), + ] + + def test_incomplete_date_tree(self): + """If not all dates set, we do what we can""" + assert do_treeprop_geological("((C:5,D:4)B)A:15;") == [ + ("A", None, 0), + ("B", 5.0, 4), + ("C", 0, 1), + ("D", 0, 1), + ] + + def test_complete_date_tree(self): + """If all dates set""" + assert do_treeprop_geological("((C:5,D:4)B:10)A:15;") == [ + ("A", 15.0, 5), + ("B", 5.0, 4), + ("C", 0, 1), + ("D", 0, 1), + ] + + def test_period_inclusive(self): + """Mya ranges are incclusive""" + + def get_period(x): + p = GEOLOGICAL_PERIODS[do_treeprop_geological(f"(B:{x})A;")[0][2]] + return (p["period"], p["epoch"], p["mya_start"]) + + assert get_period(520.99) == ("Cambrian", "Series 2", 521) + assert get_period(521) == ("Cambrian", "Series 2", 521) + assert get_period(521.01) == ("Cambrian", "Terreneuvian", 538.8) + + +######################################## +# treeprop_weighted_mean +######################################## + + +def do_treeprop_weighted_mean(nwk, weighting=0.8): + t = ete4.Tree(nwk, parser=1) + assert treeprop_weighted_mean(t, weighting=weighting) == "weighted_mean_ratio" + + # Traverse tree, returning all periods + return [(n.name, n.props.get("date"), n.props["weighted_mean_ratio"]) for n in t.traverse("preorder")] + + +def generate_tree(dists): + tree_str = "" + for i, d in enumerate(dists): + if tree_str != "": + tree_str = f"({tree_str})" + tree_str += f"n{i}" + if d is not None: + tree_str += f":{d}" + tree_str += ";" + return tree_str + + +def expected_results_wm(dists, weighting): + """ + Compute expected weighted_mean_ratio values for a linear caterpillar tree + built from dists, where dists[k] is the branch length of node nk, nk's + parent is n(k+1), and n(len-1) is the root. + + Traversal is preorder, so results start at the root (n_last) and walk + down to the leaf (n0). The root and any node with a missing branch + length get weighted_mean and weighted_mean_ratio of 0.0; that 0.0 then + feeds back into the recurrence for descendants like any other value, + so a single missing dist no longer poisons the whole subtree below it. + """ + n = len(dists) + weighted_mean = [None] * n + + # Walk from root (index n-1) down to leaf (index 0) + for i in range(n - 1, -1, -1): + if i == n - 1 or dists[i] is None: + weighted_mean[i] = 0.0 + else: + weighted_mean[i] = (dists[i] + weighted_mean[i + 1] * weighting) / (1 + weighting) + + results = [] + for i in range(n - 1, -1, -1): + if i == n - 1 or dists[i] is None: + ratio = 0.0 + else: + ratio = dists[i] / weighted_mean[i] + results.append((f"n{i}", None, ratio)) + return results + + +class TestTreepropWeightedMean: + def test_weighting(self): + """weighting param honoured""" + dists = [random.randrange(10, 100) for _ in range(20)] + tree_str = generate_tree(dists) + + assert do_treeprop_weighted_mean(tree_str) == expected_results_wm(dists, weighting=0.8) + assert do_treeprop_weighted_mean(tree_str, weighting=3) == expected_results_wm(dists, weighting=3) + assert expected_results_wm(dists, 0.8) != expected_results_wm(dists, 3) + + def test_missing_branch_length(self, caplog): + """Nodes with missing branch length get a 0.0 weighted_mean and emit a warning. + + The missing node's 0.0 feeds back into the recurrence for its descendants + like any other value, so only the missing node itself (and the root, which + is always 0.0) shows a zero ratio. + """ + dists = [random.randrange(10, 100) for _ in range(20)] + dists[10] = None + tree_str = generate_tree(dists) + + with caplog.at_level( + logging.WARNING, + logger="oz_tree_build.taxon_mapping_and_popularity.tree_props.weighted_mean", + ): + result = do_treeprop_weighted_mean(tree_str, weighting=3) + + assert result == expected_results_wm(dists, weighting=3) + # Preorder visits n19..n0. Only the root (index 0 → n19) and the missing + # node (index 9 → n10) have ratio 0.0; descendants of n10 compute normally. + assert [i for i, x in enumerate(result) if x[2] == 0.0] == [0, 9] + assert any("n10" in r.message and r.levelno == logging.WARNING for r in caplog.records) + + +######################################## +# treeprop_sliding_window +######################################## + + +def do_treeprop_sliding_window(nwk, local_mean_width=5): + t = ete4.Tree(nwk, parser=1) + assert treeprop_sliding_window(t, local_mean_width=local_mean_width) == "sliding_window" + + # Traverse tree, returning all periods + return [(n.name, n.props.get("date"), n.props["sliding_window"]) for n in t.traverse("preorder")] + + +def expected_results_sw(dists, local_mean_width): + """ + Compute expected sliding_window values from a linear caterpillar tree built + from dists, where dists[k] is the branch length of node nk, nk's parent is + n(k+1), and n(len-1) is the root. + + Traversal order is preorder: root (n_last) down to leaf (n0). The window + walks up to local_mean_width ancestors (stopping at the root, whose own + edge length is never counted, or at any None / negative edge length) and + up to local_mean_width descendants downwards (in this linear tree a single + chain; again stopping at a None / negative edge length). A node whose own + edge length is exactly 0 short-circuits to 0; a None / negative own edge + length short-circuits to 0.0; a node that contributes nothing to the + window returns 0.0. + """ + n = len(dists) + + def node_sw(k): + if dists[k] == 0: + return 0 + if dists[k] is None or dists[k] < 0: + return 0.0 + window = [] + kk = k + for _ in range(local_mean_width): + if kk >= n - 1: + break + if dists[kk] is None or dists[kk] < 0: + break + window.append(dists[kk]) + kk += 1 + kk = k - 1 + for _ in range(local_mean_width): + if kk < 0: + break + if dists[kk] is None or dists[kk] < 0: + break + window.append(dists[kk]) + kk -= 1 + if not window: + return 0.0 + return math.log(dists[k] / (sum(window) / len(window))) + + return [(f"n{i}", None, node_sw(i)) for i in range(n - 1, -1, -1)] + + +class TestSlidingWindow: + def test_local_mean_width(self): + """local_mean_width param honoured""" + dists = [random.randrange(10, 100) for _ in range(20)] + tree_str = generate_tree(dists) + + assert do_treeprop_sliding_window(tree_str) == expected_results_sw(dists, local_mean_width=5) + assert do_treeprop_sliding_window(tree_str, local_mean_width=3) == expected_results_sw( + dists, local_mean_width=3 + ) + assert expected_results_sw(dists, 5) != expected_results_sw(dists, 3) + + def test_missing_branch_length(self, caplog): + """Nodes with missing branch length get sliding_window 0.0 and emit a warning""" + dists = [random.randrange(10, 100) for _ in range(20)] + dists[10] = None + tree_str = generate_tree(dists) + + with caplog.at_level( + logging.WARNING, logger="oz_tree_build.taxon_mapping_and_popularity.tree_props.sliding_window" + ): + result = do_treeprop_sliding_window(tree_str, local_mean_width=3) + + assert result == expected_results_sw(dists, local_mean_width=3) + assert any("n10" in r.message and r.levelno == logging.WARNING for r in caplog.records) + + def test_zero_branch_length(self): + """Nodes with edge length == 0 short-circuit to sliding_window 0 without affecting siblings""" + dists = [random.randrange(10, 100) for _ in range(20)] + dists[10] = 0 + tree_str = generate_tree(dists) + + result = do_treeprop_sliding_window(tree_str, local_mean_width=3) + + assert result == expected_results_sw(dists, local_mean_width=3) + # n10 is at preorder index 9 and was given dist 0, so it should be exactly 0. + assert result[9] == ("n10", None, 0) From 7e9eb1d76a9c746caefb985be58785aafa46a3ed Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 2 Jun 2026 09:13:36 +0000 Subject: [PATCH 20/62] tree_build/step_output: Assemble an ete4 tree property into 2 arrays Write an ete4 property out to a data file, using the same sorting as ordered leaves/nodes. --- oz_tree_build/tree_build/step_output.py | 41 ++++++++++++ tests/test_tree_build_step_output.py | 89 +++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/oz_tree_build/tree_build/step_output.py b/oz_tree_build/tree_build/step_output.py index b37af266..5fd8d2f8 100644 --- a/oz_tree_build/tree_build/step_output.py +++ b/oz_tree_build/tree_build/step_output.py @@ -1,5 +1,6 @@ import csv import os.path +import struct from ..utilities.ete import node_name_without_ott @@ -231,3 +232,43 @@ def output_mysqlexport(tree, out_dir): f" IGNORE 1 LINES ({open(os.path.join(out_dir,csvfile)).readline().rstrip()}) SET id = NULL;\n" ] ) + + +def output_proparray(tree, out_dir, prop_name): + """ + Given an ete4 tree and prop_name, write out 2 packed arrays to out_dir: + + (prop_name)_leaves_(pack format).dat + (prop_name)_nodes_(pack format).dat + + The ordering will match ordered_leaves/ordered_nodes + """ + PROP_FORMAT_TO_PACK = dict( + c8="c", # 8-bit chars + i8="b", # Signed 8-bit ints + u8="B", # Unsigned 8-bit ints + f16=" leaves [a, b, c], internals [root, x] + assert read_packed(leaf_path, "B") == [1, 2, 3] + assert read_packed(node_path, "B") == [5, 12] + + def test_float_packing(self, tmp_path): + """Float properties pack as 2-byte half-floats.""" + t = build_tree( + "((a,b)x,c)root;", + "myprop", + {"root": 5.0, "x": 12.0, "a": 1.0, "b": 2.0, "c": 3.0}, + prop_format="f32", + ) + leaf_path, node_path = output_proparray(t, str(tmp_path), "myprop") + + assert leaf_path == str(tmp_path / "myprop_leaves_f32.dat") + assert node_path == str(tmp_path / "myprop_nodes_f32.dat") + + assert struct.calcsize(" Date: Fri, 12 Jun 2026 11:00:49 +0000 Subject: [PATCH 21/62] step_output: Output dicts as JS source Given a dict, output a JS source file that defines each entry as a variable. Will be used for the cutmap & brief newick. --- oz_tree_build/tree_build/step_output.py | 12 ++++++ tests/test_tree_build_step_output.py | 54 +++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/oz_tree_build/tree_build/step_output.py b/oz_tree_build/tree_build/step_output.py index 5fd8d2f8..9316ec19 100644 --- a/oz_tree_build/tree_build/step_output.py +++ b/oz_tree_build/tree_build/step_output.py @@ -1,4 +1,5 @@ import csv +import json import os.path import struct @@ -234,6 +235,17 @@ def output_mysqlexport(tree, out_dir): ) +def output_jssource(tree, out_dir, file_name, data): + """ + Turn ``data`` dict into a js source file that defines it's keys as variables, with JSON encoded values + """ + with open(os.path.join(out_dir, file_name), "w") as f: + for k, v in data.items(): + f.write(f"var {k} = ") + json.dump(v, f) + f.write(";\n") + + def output_proparray(tree, out_dir, prop_name): """ Given an ete4 tree and prop_name, write out 2 packed arrays to out_dir: diff --git a/tests/test_tree_build_step_output.py b/tests/test_tree_build_step_output.py index 1bd2d9b7..87a3410d 100644 --- a/tests/test_tree_build_step_output.py +++ b/tests/test_tree_build_step_output.py @@ -1,4 +1,5 @@ import csv +import json import os import struct @@ -7,6 +8,7 @@ from oz_tree_build.tree_build.step_output import ( output_add_prop_ids, + output_jssource, output_mysqlexport, output_proparray, ) @@ -487,6 +489,58 @@ def test_import_sql_contains_load_data_for_both_tables(self, tmp_path): assert "SET id = NULL;" in sql +######################################## +# output_jssource +######################################## + + +class TestOutputJsSource: + def test_writes_file_at_given_path(self, tmp_path): + # The file is created at out_dir/file_name. + output_jssource(None, str(tmp_path), "out.js", {"x": 1}) + assert (tmp_path / "out.js").exists() + + def test_single_key_emits_var_declaration(self, tmp_path): + # Each dict key becomes a `var = ;` line. + output_jssource(None, str(tmp_path), "out.js", {"rawData": "abc"}) + assert (tmp_path / "out.js").read_text() == 'var rawData = "abc";\n' + + def test_multiple_keys_emit_separate_lines(self, tmp_path): + # Every dict entry produces its own line, in insertion order. + data = {"a": 1, "b": "two", "c": [3, 4]} + output_jssource(None, str(tmp_path), "out.js", data) + assert (tmp_path / "out.js").read_text() == ("var a = 1;\n" 'var b = "two";\n' "var c = [3, 4];\n") + + def test_values_are_json_encoded(self, tmp_path): + # Non-trivial Python values are serialised through json.dump, so + # nested dicts/lists and unicode survive a JSON round-trip. + value = {"nested": [1, 2, {"k": "v"}], "u": "café"} + output_jssource(None, str(tmp_path), "out.js", {"obj": value}) + text = (tmp_path / "out.js").read_text() + # Strip the `var obj = ` prefix and trailing `;\n` to recover the JSON. + assert text.startswith("var obj = ") + assert text.endswith(";\n") + json_payload = text[len("var obj = ") : -len(";\n")] + assert json.loads(json_payload) == value + + def test_string_values_are_quoted(self, tmp_path): + # JSON encoding wraps strings in double quotes — without it the + # generated JS would reference an undefined identifier. + output_jssource(None, str(tmp_path), "out.js", {"s": "hello"}) + assert (tmp_path / "out.js").read_text() == 'var s = "hello";\n' + + def test_empty_dict_writes_empty_file(self, tmp_path): + # An empty data dict still creates the file, just with no content. + output_jssource(None, str(tmp_path), "out.js", {}) + assert (tmp_path / "out.js").read_text() == "" + + def test_special_characters_in_strings_are_escaped(self, tmp_path): + # Quotes/backslashes/newlines inside values must be JSON-escaped so + # the emitted JS parses. + output_jssource(None, str(tmp_path), "out.js", {"s": 'a"b\\c\nd'}) + assert (tmp_path / "out.js").read_text() == 'var s = "a\\"b\\\\c\\nd";\n' + + ###### From f4f0b739637c4f407201d7c847d4fba1e4b3ff23 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 12 Jun 2026 11:04:27 +0000 Subject: [PATCH 22/62] step_jsnewick: Convert tree JS output to ete4 ete4 versions of make_js_treefiles' JS output conversions. Output native data structures so we can add the conversions later (and potentially remove them). --- oz_tree_build/tree_build/step_jsnewick.py | 175 ++++++++++++++++ tests/test_tree_build_step_jsnewick.py | 231 ++++++++++++++++++++++ 2 files changed, 406 insertions(+) create mode 100644 oz_tree_build/tree_build/step_jsnewick.py create mode 100644 tests/test_tree_build_step_jsnewick.py diff --git a/oz_tree_build/tree_build/step_jsnewick.py b/oz_tree_build/tree_build/step_jsnewick.py new file mode 100644 index 00000000..585dc5af --- /dev/null +++ b/oz_tree_build/tree_build/step_jsnewick.py @@ -0,0 +1,175 @@ +""" +Tree-to-string converters used by the JS frontend. + +OneZoom's frontend slurps a "bracket-only" newick that has had every +leaf name, branch length, comma, and semicolon stripped — only ``(`` +and ``)`` characters remain. Internal nodes are identified by their +character position in that string, and a cut-position map tells the +frontend where to split each internal node's two children. + +Historically these strings were built by reading a named-leaf newick +back off disk and string-munging it (see +``oz_tree_build.utilities.make_js_treefiles.tidy_newick`` and +``generate_binary_cut_position_map``). This module produces the same +output directly from an ete4 tree, with no on-disk round-trip. +""" + + +def jsnewick_brief_newick(tree, polytomy_braces="()"): + """ + Return the bracket-only string for ``tree``. + + Equivalent to writing the tree as newick, dropping leaf names and + branch lengths, then stripping commas, semicolons, and newlines: + each internal node contributes a matching ``(`` and ``)``, leaves + contribute nothing. The ete4 tree built from ``((A,B),C);`` yields + ``"(())"``. + + ``polytomy_braces`` is a two-character string overriding the + brackets used for any *non-root* internal whose ``dist == 0`` — + the marker ``resolve_polytomy`` leaves on an artificial split. + Pass e.g. ``"{}"`` to flag those nodes for the frontend (matches + ``dendropy_extras.write_brief_newick``). + """ + parts = [] + for node, action in _walk_internal(tree): + braces = polytomy_braces if (node.up is not None and node.dist == 0) else "()" + parts.append(braces[0] if action == "open" else braces[1]) + return "".join(parts) + + +def jsnewick_cutpositionmap_binary(tree, threshold=10000): + """ + Cut-position map for an ete4 ``tree``. + + Keys are the tidy-string position of an internal node's ``)``; + the value is the position of the last character of that node's + first child — i.e. where the frontend should split the subtree + string into its two children. An internal whose first child is a + leaf records the parent's ``(`` position as the cut (the leaf + occupies an empty range immediately after). + + Only the root and any internal node whose own subtree contributes + more than ``threshold`` characters to the tidied string get an + entry; the recursion mirrors ``make_js_treefiles`` exactly. An + internal with two leaf children produces no entry (there is no + further split for the frontend to make). + + The tree is assumed to be binary; nodes with fewer than two + children are skipped and any third-or-later child of a polytomy + is ignored — callers should resolve polytomies first. + + Equivalent to ``make_js_treefiles.generate_binary_cut_position_map`` + """ + start_pos = {} + end_pos = {} + for pos, (node, action) in enumerate(_walk_internal(tree)): + if action == "open": + start_pos[id(node)] = pos + else: + end_pos[id(node)] = pos + + cut_map = {} + worklist = [tree] + while worklist: + node = worklist.pop(0) + children = list(node.children) + if len(children) < 2: + continue + c1, c2 = children[0], children[1] + + if c1.is_leaf and c2.is_leaf: + continue + + cut = start_pos[id(node)] if c1.is_leaf else end_pos[id(c1)] + cut_map[end_pos[id(node)]] = cut + + for child in (c1, c2): + if child.is_leaf: + continue + if end_pos[id(child)] - start_pos[id(child)] + 1 > threshold: + worklist.append(child) + + return cut_map + + +def jsnewick_cutpositionmap_polytomy(tree, threshold=10000): + """ + Cut-position map for an ete4 ``tree``. + + Keys are the tidy-string position of an internal node's ``)``; + the value is a flat ``[start1, end1, start2, end2]`` list describing + each of the node's two children: + - An internal child contributes its bracket range ``(start, end)``. + - A leaf child contributes an inverted range marking the empty + position the leaf occupies between siblings (``start > end``). + - An internal whose children are *both* leaves falls back to the + degenerate ``[start, start, end, end]`` form — using the + parent's own bracket positions — matching the original + algorithm's cut-point-not-found case. + + Only the root and any internal node whose own subtree contributes + more than ``threshold + 1`` characters to the tidied string get an + entry; the (off-by-one) threshold check mirrors + ``make_js_treefiles`` exactly. + + The tree is assumed to be binary; nodes with fewer than two + children are skipped and any third-or-later child of a polytomy + is ignored — callers should resolve polytomies first. + + Equivalent to ``make_js_treefiles.generate_polytomy_cut_position_map`` + """ + start_pos = {} + end_pos = {} + for pos, (node, action) in enumerate(_walk_internal(tree)): + if action == "open": + start_pos[id(node)] = pos + else: + end_pos[id(node)] = pos + + cut_map = {} + worklist = [tree] + while worklist: + node = worklist.pop(0) + children = list(node.children) + if len(children) < 2: + continue + c1, c2 = children[0], children[1] + s_n = start_pos[id(node)] + e_n = end_pos[id(node)] + + if c1.is_leaf and c2.is_leaf: + cut_map[e_n] = [s_n, s_n, e_n, e_n] + else: + pair1 = [s_n + 1, s_n] if c1.is_leaf else [start_pos[id(c1)], end_pos[id(c1)]] + pair2 = [e_n, e_n - 1] if c2.is_leaf else [start_pos[id(c2)], end_pos[id(c2)]] + cut_map[e_n] = pair1 + pair2 + + for child in (c1, c2): + if child.is_leaf: + continue + if end_pos[id(child)] - start_pos[id(child)] > threshold: + worklist.append(child) + + return cut_map + + +def _walk_internal(tree): + """ + Iterative DFS yielding ``(node, action)`` for every internal node + in the order tidy_newick would emit them. ``action`` is ``"open"`` + on first visit and ``"close"`` on return. Leaves contribute no + character to the tidied string and are skipped. + """ + stack = [(tree, False)] + while stack: + node, visited = stack.pop() + if node.is_leaf: + continue + if visited: + yield node, "close" + else: + yield node, "open" + stack.append((node, True)) + for child in reversed(list(node.children)): + stack.append((child, False)) diff --git a/tests/test_tree_build_step_jsnewick.py b/tests/test_tree_build_step_jsnewick.py new file mode 100644 index 00000000..3e25848a --- /dev/null +++ b/tests/test_tree_build_step_jsnewick.py @@ -0,0 +1,231 @@ +import json +import random + +import ete4 + +from oz_tree_build.tree_build.step_jsnewick import ( + jsnewick_brief_newick, + jsnewick_cutpositionmap_binary, + jsnewick_cutpositionmap_polytomy, +) +from oz_tree_build.utilities import make_js_treefiles + +######################################## +# jsnewick_brief_newick +######################################## + + +def test_brief_newick_single_internal(): + """A single internal with two leaf children collapses to a bare ``()``.""" + t = ete4.Tree("(A,B);", parser=1) + assert jsnewick_brief_newick(t) == "()" + + +def test_brief_newick_nested_left(): + """``((A,B),C)`` and ``(C,(A,B))`` both yield ``(())`` — leaves are invisible.""" + t = ete4.Tree("((A,B),C);", parser=1) + assert jsnewick_brief_newick(t) == "(())" + t = ete4.Tree("(C,(A,B));", parser=1) + assert jsnewick_brief_newick(t) == "(())" + + +def test_brief_newick_two_subtrees(): + t = ete4.Tree("((A,B),(C,D));", parser=1) + assert jsnewick_brief_newick(t) == "(()())" + + +def test_brief_newick_deep_caterpillar(): + t = ete4.Tree("(A,(B,(C,(D,E))));", parser=1) + assert jsnewick_brief_newick(t) == "(((())))" + + +def test_brief_newick_polytomy_braces_default(): + """With the default polytomy_braces the dist=0 marker is invisible.""" + t = ete4.Tree("((A:1,B:1):0,C:2);", parser=1) + assert jsnewick_brief_newick(t) == "(())" + + +def test_brief_newick_polytomy_braces_overridden(): + """An internal with dist=0 gets the override braces; non-zero dist does not.""" + t = ete4.Tree("((A:1,B:1):0,C:2);", parser=1) + assert jsnewick_brief_newick(t, polytomy_braces="{}") == "({})" + + t_nonzero = ete4.Tree("((A:1,B:1):3,C:2);", parser=1) + assert jsnewick_brief_newick(t_nonzero, polytomy_braces="{}") == "(())" + + +def test_brief_newick_polytomy_root_excluded(): + """The root's own dist is ignored even when set to 0.""" + t = ete4.Tree("(A:1,B:1):0;", parser=1) + assert jsnewick_brief_newick(t, polytomy_braces="{}") == "()" + + +def test_brief_newick_polytomy_braces_nested(): + """Multiple dist=0 ancestors each get the polytomy braces.""" + t = ete4.Tree("(((A:1,B:1):0,C:2):0,D:1);", parser=1) + assert jsnewick_brief_newick(t, polytomy_braces="{}") == "({{}})" + + +######################################## +# jsnewick_cutpositionmap_binary +######################################## + + +def test_cutmap_binary_two_leaves_empty(): + """A single internal with two leaf children produces no entry — nothing to split.""" + t = ete4.Tree("(A,B);", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {} + + +def test_cutmap_binary_internal_then_leaf(): + """First child internal → cut is the position of that child's ``)``.""" + # brief = '(())': root open=0, inner open=1, inner close=2, root close=3. + t = ete4.Tree("((A,B),C);", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {3: 2} + + +def test_cutmap_binary_leaf_then_internal(): + """First child leaf → cut is the parent's ``(`` position.""" + t = ete4.Tree("(C,(A,B));", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {3: 0} + + +def test_cutmap_binary_two_internals(): + """Both children internal → cut is the first child's ``)`` position.""" + # brief = '(()())': positions root=0/5, (A,B)=1/2, (C,D)=3/4. + t = ete4.Tree("((A,B),(C,D));", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {5: 2} + + +def test_cutmap_binary_caterpillar(): + """Recursive descent records each non-trivial internal along the spine.""" + # (A,(B,(C,(D,E)))) → brief = '(((())))', root close at pos 7. + t = ete4.Tree("(A,(B,(C,(D,E))));", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {7: 0, 6: 1, 5: 2} + + +def test_cutmap_binary_threshold_skips_small_subtrees(): + """Only subtrees whose bracket span exceeds the threshold get recursed into.""" + t = ete4.Tree("(A,(B,(C,(D,E))));", parser=1) + # Subtree spans: root=8, (B,(C,(D,E)))=6, (C,(D,E))=4, (D,E)=2. + # threshold=5 admits the first two; threshold=6 admits only the root. + assert jsnewick_cutpositionmap_binary(t, threshold=5) == {7: 0, 6: 1} + assert jsnewick_cutpositionmap_binary(t, threshold=6) == {7: 0} + + +def test_cutmap_binary_matches_legacy_on_ladderized_tree(): + """On a tree ladderized smallest-subtree-first, the new map matches the legacy + string-based generator byte-for-byte. The legacy algorithm assumes leaf-first + child ordering — ladderize(ascending=True) is what the OZ pipeline runs to + enforce that.""" + random.seed(1234) + nwk = _random_binary_newick(25) + t = ete4.Tree(nwk, parser=1) + t.ladderize() # ete4 ladderize is ascending by default + + brief = jsnewick_brief_newick(t) + new_map = jsnewick_cutpositionmap_binary(t, threshold=0) + assert new_map == _legacy_binary_map(brief, 0) + + +######################################## +# jsnewick_cutpositionmap_polytomy +######################################## + + +def test_cutmap_polytomy_two_leaves_degenerate(): + """An internal with two leaf children falls back to the parent's own positions.""" + t = ete4.Tree("(A,B);", parser=1) + # threshold=0 still enqueues the root (the worklist always seeds with it). + assert jsnewick_cutpositionmap_polytomy(t, threshold=0) == {1: [0, 0, 1, 1]} + + +def test_cutmap_polytomy_internal_then_leaf(): + """Internal child gets its (start, end); trailing leaf gets the inverted + [parent_close, parent_close-1] empty range.""" + t = ete4.Tree("((A,B),C);", parser=1) + # Top-level root cut: c1=(A,B) spans [1,2], c2=C trails at [3,2]. + # Inner two-leaf node also gets degenerate entry at threshold=0. + assert jsnewick_cutpositionmap_polytomy(t, threshold=0) == { + 3: [1, 2, 3, 2], + 2: [1, 1, 2, 2], + } + + +def test_cutmap_polytomy_leaf_then_internal(): + """Leading leaf gets the inverted [parent_open+1, parent_open] empty range.""" + t = ete4.Tree("(C,(A,B));", parser=1) + assert jsnewick_cutpositionmap_polytomy(t, threshold=0) == { + 3: [1, 0, 1, 2], + 2: [1, 1, 2, 2], + } + + +def test_cutmap_polytomy_two_internals(): + """Both children internal → flat list of both children's spans.""" + t = ete4.Tree("((A,B),(C,D));", parser=1) + assert jsnewick_cutpositionmap_polytomy(t, threshold=0) == { + 5: [1, 2, 3, 4], + 2: [1, 1, 2, 2], + 4: [3, 3, 4, 4], + } + + +def test_cutmap_polytomy_threshold_off_by_one_vs_binary(): + """The polytomy recursion uses ``span > threshold`` (raw span), while binary + uses ``span + 1 > threshold``; at the same threshold the polytomy map admits + one fewer level than the binary map.""" + t = ete4.Tree("(A,(B,(C,(D,E))));", parser=1) + # The (B,(C,(D,E))) child of root has bracket span 6-1 = 5. + # Binary admits it (5+1 > 5); polytomy does not (5 > 5 is false), so only + # the root entry survives in polytomy at threshold=5. + assert jsnewick_cutpositionmap_binary(t, threshold=5) == {7: 0, 6: 1} + assert jsnewick_cutpositionmap_polytomy(t, threshold=5) == {7: [1, 0, 1, 6]} + + # Dropping the threshold below the span lets polytomy recurse one more level. + assert jsnewick_cutpositionmap_polytomy(t, threshold=3) == { + 7: [1, 0, 1, 6], + 6: [2, 1, 2, 5], + } + + +def test_cutmap_polytomy_matches_legacy_on_ladderized_tree(): + """Matches the legacy polytomy generator on a leaf-first-ordered tree.""" + random.seed(5678) + nwk = _random_binary_newick(25) + t = ete4.Tree(nwk, parser=1) + t.ladderize() + + brief = jsnewick_brief_newick(t) + new_map = jsnewick_cutpositionmap_polytomy(t, threshold=0) + assert new_map == _legacy_polytomy_map(brief, 0) + + +######################################## +# helpers +######################################## + + +def _random_binary_newick(n_leaves): + """Generate a random binary newick string with ``n_leaves`` leaves.""" + + def rec(i, j): + if j - i == 1: + return f"L{i}" + m = random.randint(i + 1, j - 1) + return f"({rec(i, m)},{rec(m, j)})" + + return rec(0, n_leaves) + ";" + + +def _legacy_binary_map(brief, threshold): + """Run the legacy generator and parse the embedded JSON back into a dict.""" + js = make_js_treefiles.generate_binary_cut_position_map(brief, threshold) + payload = js.split("'", 2)[1] + return {int(k): v for k, v in json.loads(payload).items()} + + +def _legacy_polytomy_map(brief, threshold): + js = make_js_treefiles.generate_polytomy_cut_position_map(brief, threshold) + payload = js.split("'", 2)[1] + return {int(k): v for k, v in json.loads(payload).items()} From 123f4e4acdbe41c9a197814a85b443bfcb0c3be8 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 12 Jun 2026 16:41:21 +0000 Subject: [PATCH 23/62] step_tidy: Preseve date properties when we're removing dated-complete-tree expects everything to have a date property, even if it's None. Stop removing them here. --- oz_tree_build/tree_build/step_tidy.py | 2 +- tests/test_tree_build_step_tidy.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/oz_tree_build/tree_build/step_tidy.py b/oz_tree_build/tree_build/step_tidy.py index 33bdad08..696e566a 100644 --- a/oz_tree_build/tree_build/step_tidy.py +++ b/oz_tree_build/tree_build/step_tidy.py @@ -20,7 +20,7 @@ def tidy_clear_conflicting_dates_topdown(parent, mrad=None): if parent.props.get("date") is not None: if mrad is not None and (parent.props["date"] - mrad) > 1e-5: # date is greater than mrad, this shouldn't happen - del parent.props["date"] + parent.props["date"] = None else: mrad = parent.props["date"] for c in parent.children: diff --git a/tests/test_tree_build_step_tidy.py b/tests/test_tree_build_step_tidy.py index d41bcfaa..f0d06ddf 100644 --- a/tests/test_tree_build_step_tidy.py +++ b/tests/test_tree_build_step_tidy.py @@ -112,7 +112,7 @@ def test_child_older_than_parent_is_cleared(self): nodes["A"].props["date"] = 20 # older than Root — conflict nodes["B"].props["date"] = 3 tidy_clear_conflicting_dates_topdown(t) - assert "date" not in nodes["A"].props + assert nodes["A"].props["date"] is None assert nodes["B"].props["date"] == 3 assert nodes["Root"].props["date"] == 10 @@ -128,9 +128,9 @@ def test_conflict_clears_only_that_node(self): nodes["B"].props["date"] = 50 # conflict — older than Root tidy_clear_conflicting_dates_topdown(t) assert nodes["Root"].props["date"] == 10 - assert "date" not in nodes["I"].props + assert nodes["I"].props["date"] is None assert nodes["A"].props["date"] == 5 - assert "date" not in nodes["B"].props + assert nodes["B"].props["date"] is None def test_missing_ancestor_date_does_not_constrain_descendants(self): # With no ancestor date set, the first node we meet on each path @@ -146,7 +146,7 @@ def test_missing_ancestor_date_does_not_constrain_descendants(self): assert nodes["A"].props["date"] == 100 assert nodes["I"].props["date"] == 50 assert nodes["B"].props["date"] == 1 - assert "date" not in nodes["C"].props + assert nodes["C"].props["date"] is None def test_nodes_without_date_are_skipped_and_pass_mrad_through(self): # An intermediate node without a date must not reset the ceiling; @@ -160,7 +160,7 @@ def test_nodes_without_date_are_skipped_and_pass_mrad_through(self): tidy_clear_conflicting_dates_topdown(t) assert nodes["Root"].props["date"] == 10 assert "date" not in nodes["I"].props - assert "date" not in nodes["A"].props + assert nodes["A"].props["date"] is None def test_small_overshoot_within_tolerance_is_kept(self): # The function tolerates floating-point noise up to 1e-5. @@ -177,7 +177,7 @@ def test_overshoot_beyond_tolerance_is_cleared(self): nodes["Root"].props["date"] = 10 nodes["A"].props["date"] = 10 + 1e-3 # outside tolerance tidy_clear_conflicting_dates_topdown(t) - assert "date" not in nodes["A"].props + assert nodes["A"].props["date"] is None def test_equal_dates_are_kept(self): # parent.date == ancestor.date is not a conflict. @@ -199,7 +199,7 @@ def test_mrad_tightens_as_we_descend(self): nodes["B"].props["date"] = 50 # would be fine vs Root but conflicts with I tidy_clear_conflicting_dates_topdown(t) assert nodes["A"].props["date"] == 3 - assert "date" not in nodes["B"].props + assert nodes["B"].props["date"] is None def test_runs_on_tree_with_no_dates_at_all(self): # Nothing to do; should not raise. @@ -232,4 +232,4 @@ def test_bottomup_then_topdown_clears_oversized_pin(self): t.props["date"] = 50 tidy_clear_conflicting_dates_topdown(t) assert t.props["date"] == 50 - assert "date" not in internal.props + assert internal.props["date"] is None From 826916f44e8d18e3faf4dd9fd4c29b33e44524c0 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 12 Jun 2026 16:42:14 +0000 Subject: [PATCH 24/62] date_tree: Reconstitute date property dated-complete-tree expects everything to have a date property, even if it's None. Stop removing them here. --- oz_tree_build/date_tree/date_tree.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oz_tree_build/date_tree/date_tree.py b/oz_tree_build/date_tree/date_tree.py index 3a8d2a67..8ebe4a69 100644 --- a/oz_tree_build/date_tree/date_tree.py +++ b/oz_tree_build/date_tree/date_tree.py @@ -70,8 +70,7 @@ def nwk_read(infile): """ tree = ete4.Tree(infile, parser=1) for n in tree.traverse(): - if "date" in n.props: - n.props["date"] = float(n.props["date"]) + n.props["date"] = float(n.props["date"]) if "date" in n.props else None return tree From 93327ce572a612cef099b5d5be681dba8fc740d8 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 12 Jun 2026 16:45:13 +0000 Subject: [PATCH 25/62] tree_build: Output JS files Hook up step_jsnewick into main pipeline --- oz_tree_build/tree_build/tree_build.py | 33 +++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 9a8e2fdc..1894b338 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -3,6 +3,7 @@ """ import argparse +import json import logging from dated_complete_tree import tree_fixing @@ -11,7 +12,13 @@ from ..taxon_mapping_and_popularity.taxon_map import read_taxon_map from ..utilities.debug_util import parse_args_and_add_logging_switch from .step_graft import graft_extract_ot_subtrees, graft_tree -from .step_output import output_add_prop_ids, output_mysqlexport +from .step_jsnewick import jsnewick_brief_newick, jsnewick_cutpositionmap_binary, jsnewick_cutpositionmap_polytomy +from .step_output import ( + output_add_prop_ids, + output_jssource, + output_mysqlexport, + output_proparray, +) from .step_parse import parse_bespoke_trees, parse_ot_orphans from .step_popularity import popularity_add_prop, popularity_add_rank from .step_taxon import taxon_add_prop @@ -109,6 +116,30 @@ def main(): output_add_prop_ids(base_t) output_mysqlexport(base_t, args.out_dir) + logger.info("Output JS newick / cut position map") + output_jssource( + base_t, + args.out_dir, + "completetree.js", + dict( + rawData=jsnewick_brief_newick(base_t, polytomy_braces="{}"), + ), + ) + cutmap_threshold = 10000 + output_jssource( + base_t, + args.out_dir, + "cut_position_map.js", + dict( + # NB: For legacy reasons the variable contains a JSON string, not JSON + cut_position_map_json_str=json.dumps(jsnewick_cutpositionmap_binary(base_t, threshold=cutmap_threshold)), + polytomy_cut_position_map_json_str=json.dumps( + jsnewick_cutpositionmap_polytomy(base_t, threshold=cutmap_threshold) + ), + threshold=cutmap_threshold, + ), + ) + if __name__ == "__main__": main() From 5a93185311493e9c91d73ad3b36da001d3600893 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 12 Jun 2026 16:46:01 +0000 Subject: [PATCH 26/62] step_treeprop: Avoid div/0 in weighted mean There's probably a deeper bug, but gets us going for now. --- oz_tree_build/tree_build/step_treeprop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oz_tree_build/tree_build/step_treeprop.py b/oz_tree_build/tree_build/step_treeprop.py index e54f6287..5ef5d764 100644 --- a/oz_tree_build/tree_build/step_treeprop.py +++ b/oz_tree_build/tree_build/step_treeprop.py @@ -177,7 +177,7 @@ def treeprop_weighted_mean(tree, weighting=0.8): node.props["weighted_mean_ratio"] = 0.0 else: node.props["weighted_mean"] = (edge_length + (parent.props["weighted_mean"] * weighting)) / (1 + weighting) - node.props["weighted_mean_ratio"] = edge_length / node.props["weighted_mean"] + node.props["weighted_mean_ratio"] = edge_length / max(node.props["weighted_mean"], 1e-6) prop_format = tree.root.props.setdefault("prop_format", {}) prop_format["weighted_mean"] = "f16" From 12808c6a95605415f8e87a45c126511184480bac Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 12 Jun 2026 16:46:38 +0000 Subject: [PATCH 27/62] tree_build: Wire up prop arrays Generate prop arrays in main pipeline --- oz_tree_build/tree_build/tree_build.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 1894b338..32e61737 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -23,6 +23,11 @@ from .step_popularity import popularity_add_prop, popularity_add_rank from .step_taxon import taxon_add_prop from .step_tidy import tidy_clear_conflicting_dates_topdown, tidy_infill_dates_bottomup +from .step_treeprop import ( + treeprop_geological, + treeprop_sliding_window, + treeprop_weighted_mean, +) logger = logging.getLogger(__name__) @@ -109,9 +114,6 @@ def main(): # warning: ladderize ascending is needed for the short OZ newick-like form base_t.ladderize(topological=False, reverse=False) - logger.info("Add properies to tree") - pass - logger.info("Output MySQL CSV files") output_add_prop_ids(base_t) output_mysqlexport(base_t, args.out_dir) @@ -140,6 +142,11 @@ def main(): ), ) + logger.info("Generate tree properties and output arrays") + output_proparray(base_t, args.out_dir, treeprop_geological(base_t)) + output_proparray(base_t, args.out_dir, treeprop_sliding_window(base_t)) + output_proparray(base_t, args.out_dir, treeprop_weighted_mean(base_t)) + if __name__ == "__main__": main() From fc4c4b0cf5fbb8abe4d240408ad96f4e36b4c578 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 12 Jun 2026 16:58:54 +0000 Subject: [PATCH 28/62] tree_build: Bodge tree_dating into life Work through tree fixing cases it gets confused by, and get it running at least. --- oz_tree_build/tree_build/tree_build.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 32e61737..d291fd0c 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -6,7 +6,7 @@ import json import logging -from dated_complete_tree import tree_fixing +from dated_complete_tree import tree_dating, tree_fixing from ..date_tree import date_tree from ..taxon_mapping_and_popularity.taxon_map import read_taxon_map @@ -105,7 +105,24 @@ def main(): tree_fixing.delete_one_child_nodes(base_t) logger.info("Re-interpoltate missing dates") - # interpolate_dates(base_t) + for n in base_t.traverse(): # First do some tidying to force tree_dating to work + if n.is_leaf and n.name == "mrcaimp": + # Bin imputed mrca nodes made by fix_polyphyly left dangling by grafting process + n.detach() + continue + + if not n.name: + # dated-complete-tree will assume all nodes have a name + n.name = "" + + if n.props.get("date") is None: + # Ensure we have a date property on every leaf + n.props["date"] = 0 if n.is_leaf else None + else: + # If we do have a date, we also have to set imputed date + n.props["imputed_date"] = n.props.get("imputed_date") or False + tree_dating.date_labelling(base_t) + tree_dating.impute_missing_dates(base_t, l=0.25) logger.info("Rank popularities, post-node removal") popularity_add_rank(base_t) From 33e19cc6e185e8f348ea369c5bf2af53f717959d Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 12 Jun 2026 19:47:21 +0000 Subject: [PATCH 29/62] dvc.yaml: Manage tree_build output --- data/.gitignore | 1 + dvc.yaml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/data/.gitignore b/data/.gitignore index be7a0ec8..2a258bcf 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -3,3 +3,4 @@ /node_ages.json /dated_tree/ /taxon_map.csv +/out diff --git a/dvc.yaml b/dvc.yaml index 0469e9c3..9980547f 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -53,17 +53,22 @@ stages: tree_build: cmd: + - rm -rf data/out + - mkdir -p data/out - >- .venv/bin/tree_build --bespoke_dir data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ --orphan_dir data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ --opentree data/dated_tree/dated_tree_pre.tre --taxon_map data/taxon_map.csv + --out_dir data/out deps: - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ - data/dated_tree/dated_tree_pre.tre - data/taxon_map.csv + outs: + - data/out download_eol: # EOL doesn't version the provider ids file, so capture the last-modified header instead From ec960d74b4bd03f005029b089917d77c6d121919 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 12 Jun 2026 19:48:11 +0000 Subject: [PATCH 30/62] versioned_outputs: Final copy-with-version-number/gzip stage Replace make_js_files with a step that just does versioning/gzipping. Instead of using the current time of building the tree, we use the time that the files were created, which should amount to the same thing. As the data CSVs are now unversioned, instead add the version as a final step in the import.sql script. We have to modify the file anyway to add the versioned file names. As with make_js_treefiles, mark it as always_changed so we always re-run this step. The outputs will always be different. --- data/.gitignore | 1 + dvc.yaml | 24 +-- .../versioned_outputs/versioned_outputs.py | 99 ++++++++++ pyproject.toml | 1 + tests/test_versioned_outputs.py | 169 ++++++++++++++++++ 5 files changed, 283 insertions(+), 11 deletions(-) create mode 100644 oz_tree_build/versioned_outputs/versioned_outputs.py create mode 100644 tests/test_versioned_outputs.py diff --git a/data/.gitignore b/data/.gitignore index 2a258bcf..700db5ee 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -4,3 +4,4 @@ /dated_tree/ /taxon_map.csv /out +/out_versioned diff --git a/dvc.yaml b/dvc.yaml index 9980547f..0d0f0689 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -70,6 +70,19 @@ stages: outs: - data/out + versioned_outputs: + cmd: + # NB: Don't delete it, we can keep old versions at this point + - mkdir -p data/out_versioned + - >- + .venv/bin/versioned_outputs + --out_dir data/out_versioned + data/out/* + -vv + deps: + - data/out/ + always_changed: true + download_eol: # EOL doesn't version the provider ids file, so capture the last-modified header instead cmd: @@ -216,14 +229,3 @@ stages: - exclude_from_popularity outs: - data/output_files/ - - make_js_treefiles: - cmd: - - rm -r data/js_output ; mkdir -p data/js_output - - >- - make_js_treefiles - --outdir data/js_output - data/output_files/ordered_tree_*.poly - deps: - - data/output_files/ - always_changed: true diff --git a/oz_tree_build/versioned_outputs/versioned_outputs.py b/oz_tree_build/versioned_outputs/versioned_outputs.py new file mode 100644 index 00000000..2f532ea3 --- /dev/null +++ b/oz_tree_build/versioned_outputs/versioned_outputs.py @@ -0,0 +1,99 @@ +""" +Add version number / gzip output files as a final stage of the pipeline + +In addition to versioning data files, the SQL import script is also modified. +CSV filenames have their version added, and as a final step the version number +is inserted into the DB as the parent of the root. +""" + +import argparse +import logging +import os +import re +import shutil +import subprocess + +from ..utilities.debug_util import parse_args_and_add_logging_switch + +logger = logging.getLogger(__name__) + + +def process(in_files, out_dir, version_number): + """ + Copy list of ``in_files`` to ``out_dir``, with ``version_number`` appended to name + """ + + def add_version(f_name): + return re.sub( + # Extract any existing version number / extension from filename + r"(_\d+)?(\.[a-zA-Z]+)$", + # Replace with verison number / extension + "_" + str(version_number) + r"\2", + f_name, + ) + + for input_path in in_files: + input_name = os.path.basename(input_path) + output_path = os.path.join(out_dir, add_version(input_name)) + + logger.info(f"{input_path} -> {output_path}") + if input_name == "import.sql": + with open(input_path) as in_f, open(output_path, "w") as out_f: + for l in in_f: + # Replace any instance of an input filename with it's versioned equivalent + for repl_path in in_files: + repl_name = os.path.basename(repl_path) + l = l.replace("'" + repl_name + "'", "'" + add_version(repl_name) + "'") + out_f.write(l) + # Extra command to bodge version number into root's parent + out_f.writelines(f"UPDATE ordered_nodes SET parent = -{version_number} WHERE id = 1;\n") + else: + shutil.copyfile(input_path, output_path) + subprocess.call(["gzip", "-9fk", output_path]) + logger.info("Done") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--outdir", + "-o", + default=os.path.join( + os.path.dirname(__file__), + "..", + "..", + "..", + "OZtree", + "static", + "FinalOutputs", + "data", + ), + help="output filepath of cut_position_map", + ) + parser.add_argument( + "in_files", + nargs="+", + metavar="FILE", + help="Files to move to outdir, with versions appended if not present", + ) + parser.add_argument( + "--version", + type=int, + help=("Version number / serial to append to file names, if not provided use mtime of first in_file"), + ) + parser.add_argument( + "--out_dir", + default="data/out", + help=("Directory to write output files to"), + ) + args = parse_args_and_add_logging_switch(parser) + + process( + args.in_files, + args.out_dir, + int(os.path.getmtime(args.in_files[0])) if args.version is None else args.version, + ) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 38d4a3bd..1c586c15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ download_node_ages = "oz_tree_build.download_node_ages.download_node_ages:main" date_tree = "oz_tree_build.date_tree.date_tree:main" tree_build = "oz_tree_build.tree_build.tree_build:main" taxon_map = "oz_tree_build.taxon_mapping_and_popularity.taxon_map:main" +versioned_outputs = "oz_tree_build.versioned_outputs.versioned_outputs:main" [tool.setuptools] packages = ["oz_tree_build"] diff --git a/tests/test_versioned_outputs.py b/tests/test_versioned_outputs.py new file mode 100644 index 00000000..bd34ebde --- /dev/null +++ b/tests/test_versioned_outputs.py @@ -0,0 +1,169 @@ +""" +Unit tests for versioned_outputs.process +""" + +import gzip + +from oz_tree_build.versioned_outputs.versioned_outputs import process + + +def _write(path, content): + with open(path, "w") as f: + f.write(content) + + +def test_copies_with_version_appended(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + src = in_dir / "data.csv" + _write(src, "hello,world\n") + + process([str(src)], str(out_dir), 42) + + out_file = out_dir / "data_42.csv" + assert out_file.read_text() == "hello,world\n" + + +def test_replaces_existing_version_in_filename(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + src = in_dir / "data_99.csv" + _write(src, "row\n") + + process([str(src)], str(out_dir), 7) + + assert (out_dir / "data_7.csv").exists() + assert not (out_dir / "data_99_7.csv").exists() + + +def test_produces_gzip_alongside_plain_file(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + src = in_dir / "data.csv" + _write(src, "the quick brown fox\n") + + process([str(src)], str(out_dir), 3) + + gz_path = out_dir / "data_3.csv.gz" + assert gz_path.exists() + with gzip.open(gz_path, "rt") as f: + assert f.read() == "the quick brown fox\n" + + +def test_multiple_files_each_versioned(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + a = in_dir / "a.csv" + b = in_dir / "b.json" + _write(a, "A\n") + _write(b, "B\n") + + process([str(a), str(b)], str(out_dir), 11) + + assert (out_dir / "a_11.csv").read_text() == "A\n" + assert (out_dir / "b_11.json").read_text() == "B\n" + + +def test_import_sql_rewrites_filename_references(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + nodes = in_dir / "ordered_nodes.csv" + leaves = in_dir / "ordered_leaves.csv" + sql = in_dir / "import.sql" + _write(nodes, "n\n") + _write(leaves, "l\n") + _write( + sql, + "LOAD DATA INFILE 'ordered_nodes.csv' INTO TABLE ordered_nodes;\n" + "LOAD DATA INFILE 'ordered_leaves.csv' INTO TABLE ordered_leaves;\n", + ) + + process([str(nodes), str(leaves), str(sql)], str(out_dir), 55) + + out_sql = (out_dir / "import_55.sql").read_text() + assert "'ordered_nodes_55.csv'" in out_sql + assert "'ordered_leaves_55.csv'" in out_sql + assert "'ordered_nodes.csv'" not in out_sql + assert "'ordered_leaves.csv'" not in out_sql + + +def test_import_sql_appends_root_parent_update(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + sql = in_dir / "import.sql" + _write(sql, "-- nothing to rewrite\n") + + process([str(sql)], str(out_dir), 123) + + out_sql = (out_dir / "import_123.sql").read_text() + assert out_sql.endswith("UPDATE ordered_nodes SET parent = -123 WHERE id = 1;\n") + assert "-- nothing to rewrite\n" in out_sql + + +def test_import_sql_only_replaces_quoted_names(tmp_path): + """Filename substring inside other identifiers (no quotes) should not be touched.""" + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + nodes = in_dir / "ordered_nodes.csv" + sql = in_dir / "import.sql" + _write(nodes, "n\n") + # Mention the bare filename (no quotes) in a comment; it must NOT be rewritten. + _write( + sql, + "-- see ordered_nodes.csv for schema\n" "LOAD DATA INFILE 'ordered_nodes.csv' INTO TABLE ordered_nodes;\n", + ) + + process([str(nodes), str(sql)], str(out_dir), 9) + + out_sql = (out_dir / "import_9.sql").read_text() + assert "-- see ordered_nodes.csv for schema\n" in out_sql + assert "'ordered_nodes_9.csv'" in out_sql + + +def test_input_file_outside_in_dir_uses_basename(tmp_path): + """Output is named from basename, regardless of input path.""" + nested = tmp_path / "deep" / "nested" / "dir" + nested.mkdir(parents=True) + out_dir = tmp_path / "out" + out_dir.mkdir() + src = nested / "thing.txt" + _write(src, "x\n") + + process([str(src)], str(out_dir), 1) + + assert (out_dir / "thing_1.txt").exists() + assert (out_dir / "thing_1.txt.gz").exists() + + +def test_overwrites_existing_output(tmp_path): + """gzip -f and shutil.copyfile both clobber prior outputs without error.""" + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + src = in_dir / "data.csv" + _write(src, "fresh\n") + + # Pre-existing stale outputs from an earlier run. + _write(out_dir / "data_5.csv", "stale\n") + _write(out_dir / "data_5.csv.gz", "not a real gzip") + + process([str(src)], str(out_dir), 5) + + assert (out_dir / "data_5.csv").read_text() == "fresh\n" + with gzip.open(out_dir / "data_5.csv.gz", "rt") as f: + assert f.read() == "fresh\n" From c09d720e240a1b5965b30f6f70a2d5935d5368f2 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 18 Aug 2026 10:35:29 +0000 Subject: [PATCH 31/62] dvc.yaml: Add back missing params --- dvc.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dvc.yaml b/dvc.yaml index 0d0f0689..33d08ef2 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -30,6 +30,8 @@ stages: --supertree data/OpenTree/${ot_version}/labelled_supertree_ottnames.tre deps: - data/node_ages.json + params: + - ot_version outs: - data/dated_tree/dated_tree_pre.tre @@ -67,6 +69,9 @@ stages: - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ - data/dated_tree/dated_tree_pre.tre - data/taxon_map.csv + params: + - oz_tree + - ot_version outs: - data/out @@ -194,6 +199,8 @@ stages: - data/filtered/OneZoom_enwiki-latest-page.sql - data/filtered/pageviews/ - data/filtered/OneZoom_provider_ids.csv + params: + - ot_version outs: - data/taxon_map.csv From 9eed26def4c3ec673d132a35bad74462c7608330 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 24 Aug 2026 15:05:20 +0000 Subject: [PATCH 32/62] dvc.yaml: Refer to .venv paths for python executables Use .venv paths directly, to avoid venv activation mishap --- dvc.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dvc.yaml b/dvc.yaml index 33d08ef2..79578477 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -41,7 +41,7 @@ stages: - rm -rf data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version} - mkdir -p data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version} - >- - add_ott_numbers_to_trees + .venv/bin/add_ott_numbers_to_trees --savein data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version} --output_info data/add_ott_numbers_to_trees.log data/OZTreeBuild/${oz_tree}/BespokeTree/include_noAutoOTT/*.[pP][hH][yY] @@ -104,7 +104,7 @@ stages: filter_eol: cmd: >- - filter_eol + .venv/bin/filter_eol data/EOL/provider_ids.csv.gz data/OpenTree/${ot_version}/taxonomy.tsv -o data/filtered/OneZoom_provider_ids.csv @@ -118,14 +118,14 @@ stages: discover_latest_wikidata_dump_url: cmd: >- - discover_latest_wikidata_dump_url > data/Wiki/wd_JSON/latest-all-json-bz2-url.txt + .venv/bin/discover_latest_wikidata_dump_url > data/Wiki/wd_JSON/latest-all-json-bz2-url.txt outs: - data/Wiki/wd_JSON/latest-all-json-bz2-url.txt # >6 hours (streams ~90 GB dump from wikidata server) download_and_filter_wikidata: cmd: >- - download_and_filter_wikidata + .venv/bin/download_and_filter_wikidata --url "$(cat data/Wiki/wd_JSON/latest-all-json-bz2-url.txt)" -o data/filtered/OneZoom_latest-all.json deps: @@ -135,7 +135,7 @@ stages: extract_wikidata_titles: cmd: >- - extract_wikidata_titles + .venv/bin/extract_wikidata_titles data/filtered/OneZoom_latest-all.json -o data/filtered/wikidata_titles.txt deps: @@ -145,7 +145,7 @@ stages: discover_latest_enwiki_sql_url: cmd: >- - discover_latest_enwiki_sql_url > data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt + .venv/bin/discover_latest_enwiki_sql_url > data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt outs: - data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt @@ -160,7 +160,7 @@ stages: filter_wikipedia_sql: cmd: >- - filter_wikipedia_sql + .venv/bin/filter_wikipedia_sql data/Wiki/wp_SQL/enwiki-page.sql.gz data/filtered/wikidata_titles.txt -o data/filtered/OneZoom_enwiki-latest-page.sql @@ -173,7 +173,7 @@ stages: # ~several hours (streams 12 ~5GB monthly dumps) download_and_filter_pageviews: cmd: >- - download_and_filter_pageviews + .venv/bin/download_and_filter_pageviews --titles-file data/filtered/wikidata_titles.txt --months 12 -o data/filtered/pageviews @@ -209,7 +209,7 @@ stages: cmd: - mkdir -p data/output_files - >- - CSV_base_table_creator + .venv/bin/CSV_base_table_creator data/OZTreeBuild/${oz_tree}/${oz_tree}_full_tree.phy data/OpenTree/${ot_version}/taxonomy.tsv data/filtered/OneZoom_provider_ids.csv From dfeb6b6aa40436dda25a30830794df03aed739a9 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 25 Aug 2026 16:04:59 +0000 Subject: [PATCH 33/62] BespokeTree: Replace sharks with self-contained tree #130 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the collection of bespoke files with a single sharks tree, with a note on how it was sourced. In the output tree, 3 species were duplicated, these have been removed manually from the median_PD_shark_tree.txt file produced. ┌────────────────────┬──────────────────────────────────┬────────────────┐ │ Dropped tip │ Duplicate of │ Distance apart │ ├────────────────────┼──────────────────────────────────┼────────────────┤ │ Raja atriventralis │ Okamejei kenojei (ott167107) │ 68.2 Ma │ ├────────────────────┼──────────────────────────────────┼────────────────┤ │ Torpedo zugmayeri │ Torpedo sinuspersici (ott254411) │ 12.0 Ma │ ├────────────────────┼──────────────────────────────────┼────────────────┤ │ Rhinoptera sewelli │ Rhinoptera jayakari (ott801133) │ 53.0 Ma │ └────────────────────┴──────────────────────────────────┴────────────────┘ Claude has this to say about the resulting change in species counts: Resolving the 94 renamed names through TNRS (84 match as synonyms) gives the real accounting: 397 OTTs dropped, 126 gained, 1043 shared. Of the 397 dropped: - 199 are OpenTree barcode/voucher junk — Squalus_cf._megalops_FOAL613-10, Himantura_uarnak_3_GJPN-2012, Squalus_mitsukurii_complex_sp._A_TDE-2018, Callorhinchus_environmental_sample. These came in via the OpenTree grafts at Rajidae_ott978560@, Arhynchobatidae_ott406376@ etc. Losing them is a cleanup, not a loss. - 3 are bare genus-level placeholder tips — Chaenogaleus, Hemigaleus, Paragaleus. The new tree resolves those same genera into 7 real species, so this is a gain dressed as a loss. - 195 are clean binomials genuinely gone. A dozen are fossils (Crassodontidanidae 6, Paracestracion 6 — Jurassic hornsharks). The rest are real extant species the 2018 tree never sampled, concentrated in Dasyatidae (35), Rajidae (25), Rhinobatidae (16), Potamotrygonidae (12), Squalidae (11) — including post-2018 descriptions like Squalus_clarkae, S. margaretsmithae, S. albicaudus. Pulling the other way, the families the Naylor2012 hand-trees covered worst gain substantially: Scyliorhinidae +34, Etmopteridae +23, Triakidae +11, Squatinidae +10. --- .../Batoids_Aschliman2012.PHY | 14 ---- .../Chondrichthyes_Renz2013.phy | 7 -- .../Chondrichthyes_Stein2018.PHY | 49 ++++++++++++++ .../Holocephali_Inoue2010.PHY | 4 -- .../Naylor2012Carcharhinicae_minus.PHY | 9 --- .../Naylor2012Dalatiidae.PHY | 4 -- .../Naylor2012Etmopteridae.phy | 5 -- .../Naylor2012Pristiophoridae.phy | 6 -- .../Naylor2012Scyliorhinidae2.PHY | 5 -- .../Naylor2012Scyliorhinidae3.PHY | 5 -- .../Naylor2012Selachimorpha.PHY | 37 ----------- .../Naylor2012Somniosidae_Oxynotidae.PHY | 6 -- .../Naylor2012Squatinidae.phy | 5 -- .../token_to_oz_tree_file_mapping.py | 64 ++----------------- 14 files changed, 55 insertions(+), 165 deletions(-) delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Batoids_Aschliman2012.PHY delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Renz2013.phy create mode 100644 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Stein2018.PHY delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Holocephali_Inoue2010.PHY delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Carcharhinicae_minus.PHY delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Dalatiidae.PHY delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Etmopteridae.phy delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Pristiophoridae.phy delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae2.PHY delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae3.PHY delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Selachimorpha.PHY delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Somniosidae_Oxynotidae.PHY delete mode 100755 data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Squatinidae.phy diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Batoids_Aschliman2012.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Batoids_Aschliman2012.PHY deleted file mode 100755 index e48d543b..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Batoids_Aschliman2012.PHY +++ /dev/null @@ -1,14 +0,0 @@ -[Batoids & dates from: -Body plan convergence in the evolution of skates and rays (Chondrichthyes: Batoidea) -N.C. Aschliman et al. / Molecular Phylogenetics and Evolution 63 (2012) 28–42 https://doi.org/10.1016/j.ympev.2011.12.012 with basal divergence set to 300Ma, and ray/other divergence at 200Ma, i.e. - -(Squalus:300.0,((Rajidae_ott978560@:78.3688,(Anacanthobatidae_ott802681@:69.5035,Arhynchobatidae_ott406376@:69.5035):8.8652)Rajiformes:121.6312,(((Platyrhinodis_ott1032962@:63.8298,Platyrhina_ott456578@:63.8298)Platyrhinidae:100.3546,((Torpedinidae_ott553102,Hypnidae_ott356637@):72.695,(Narcinidae_ott818997@:63.8298,Narkidae_ott932203@:63.8298):8.8652)Torpediniformes:91.4894):13.4752,( - -From Aschliman fig 1 I split Rhiniformes into 2 groups, and move Zanobatidae (which in OpenTree v5 is in this Pristiformes/Rhiniformes group) outside both -(Zapteryx_ott356651@:79.078,Trygonorrhina_ott1041304@:79.078)Rhiniformes1_:79.7872,(Rhiniformes2__ott356644~-456585-356651-1041304@:152.4823,(Zanobatidae_ott456585@:142.1986, - - -(Hexatrygonidae_ott456584@:92.9078,Myliobatiformes_minus_Hexatrygon_ott~706576-456584@:92.9078)Myliobatiformes:49.2908):10.2837):6.383):18.7943):22.3404):100.0); - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((Rajidae_ott978560@:78.3688,(Anacanthobatidae_ott802681@:69.5035,Arhynchobatidae_ott406376@:69.5035):8.8652)Rajiformes:121.6312,(((Platyrhinodis_ott1032962@:63.8298,Platyrhina_ott456578@:63.8298)Platyrhinidae:100.3546,((Torpedinidae_ott553102@,Hypnidae_ott356637@):72.695,(Narcinidae_ott818997@:63.8298,Narkidae_ott932203@:63.8298):8.8652)Torpediniformes:91.4894):13.4752,((Zapteryx_ott356651@:79.078,Trygonorrhina_ott1041304@:79.078)Rhiniformes1_:79.7872,(Rhiniformes2__ott356644~-456585-356651-1041304@:152.4823,(Zanobatidae_ott456585@:142.1986,(Hexatrygonidae_ott456584@:92.9078,Myliobatiformes_minus_Hexatrygon__ott~706576-456584@:92.9078)Myliobatiformes:49.2908):10.2837):6.383):18.7943):22.3404)Batoidea; diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Renz2013.phy b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Renz2013.phy deleted file mode 100755 index b3acaa90..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Renz2013.phy +++ /dev/null @@ -1,7 +0,0 @@ -[Chondrichthyes tree: - -Basal split from Revealing Less Derived Nature of Cartilaginous Fish Genomes with Their Evolutionary Time Scale Inferred with Nuclear Genes ( https://doi.org/10.1371/journal.pone.0066400 ), dates need setting so that -(HOLOCEPHALI@:420,(BATOIDEA@:300,SELACHIMORPHA@:300):120); - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(HOLOCEPHALI@,(BATOIDEA@,SELACHII@:300):120)CHONDRICHTHYES; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Stein2018.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Stein2018.PHY new file mode 100644 index 00000000..e515ec3a --- /dev/null +++ b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Stein2018.PHY @@ -0,0 +1,49 @@ +[Chondrichthyes (sharks, rays and chimaeras), 1189 spp., crown at 375.51Mya. + +Sourced from https://vertlife.org/data/sharks/ , i.e. Stein et al. (2018) "Global priorities for +conserving the evolutionary history of sharks, rays and chimaeras", +https://www.nature.com/articles/s41559-017-0448-4 + +My method (for records) - read in the distribution of 10,000 trees, calculate phylogenetic +diversity of each, find out which tree yields the median value, interpret this as a median tree +to move forward with, output as newick. NB: R's double-square-bracket list indexing is written +with curly braces below, since a closing square bracket would terminate this newick comment: + +rm(list=ls()) + +# install useful packages +require('ape') +require('caper') + +# Read all trees +trees <- ape::read.tree(file='Chond.10Cal.10kTreeSet.tre') + +# Calculate PD for each tree +pd <- sapply(trees, function(tr) sum(tr$edge.length)) + +# Median PD value +median_pd <- median(pd) + +# Tree closest to the median +median_idx <- which.min(abs(pd - median_pd)) +median_tree <- trees{{median_idx}} + +# Newick string +newick <- write.tree(median_tree) +writeLines(newick, "median_PD_shark_tree.txt") +Three tips of the published 1192 have been dropped, as WoRMS and GBIF agree each is a junior +synonym of a species that is *also* already a tip in this tree, i.e. the source tree scores one +taxon twice. In each case the two tips are not sisters, so this is not a resolution question: + Raja atriventralis = Okamejei kenojei (ott167107, other tip 68.2Ma away) + Torpedo zugmayeri = Torpedo sinuspersici (ott254411, other tip 12.0Ma away) + Rhinoptera sewelli = Rhinoptera jayakari (ott801133, other tip 53.0Ma away) +Each was removed by splicing its sibling onto its grandparent with the two branch lengths summed, +so every surviving tip keeps its exact original depth. + +Five further tips have no OTT in OpenTree v16.1, so will carry no metadata. Narcine nigra +(gbif:9209986) and Glaucostegus spinosus (worms:1577337, gbif:11555527) are valid species merely +absent from OpenTree; Rajella alia, Narcine bicolor and Rhinoptera hainanensis have no +species-level record in either WoRMS or the GBIF backbone. + +#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] +(((Callorhinchus_callorynchus:8.358674601,(Callorhinchus_milii:6.062028924,Callorhinchus_capensis:6.062028924):2.296645677):183.545478,(((Neoharriotta_pinnata:23.47814296,(Neoharriotta_pumila:5.362641919,Neoharriotta_carri:5.362641919):18.11550104):98.09750975,((Harriotta_raleighana:2.167434783,Harriotta_haeckeli:2.167434783):45.49684829,(Rhinochimaera_pacifica:28.17523151,(Rhinochimaera_africana:10.67730884,Rhinochimaera_atlantica:10.67730884):17.49792266):19.48905057):73.91136963):29.19241684,((((Hydrolagus_mitsukurii:7.770233588,Hydrolagus_africanus:7.770233588):63.10423394,((Hydrolagus_matallanasi:8.086410705,Hydrolagus_pallidus:8.086410705):41.79791888,(((Hydrolagus_alberti:37.29381117,Hydrolagus_alphus:37.29381117):1.001440697,Chimaera_bahamaensis:38.29525187):4.716001781,(Chimaera_jordani:31.28687314,Chimaera_cubana:31.28687314):11.7243805):6.873075936):20.99013895):22.57191234,((Chimaera_phantasma:56.11744559,((Hydrolagus_lemures:0.7134329691,Hydrolagus_ogilbyi:0.7134329691):3.531579029,Chimaera_owstoni:4.245011998):51.87243359):36.52182889,((Hydrolagus_melanophasma:24.11616128,((Hydrolagus_macrophthalmus:0.6714718955,Hydrolagus_purpurescens:0.6714718955):13.77257212,Hydrolagus_marmoratus:14.44404401):9.672118271):33.60542615,((((Hydrolagus_trolli:4.335658096,(Hydrolagus_affinis:2.263558323,Hydrolagus_mirabilis:2.263558323):2.072099773):25.16991186,Hydrolagus_novaezealandiae:29.50556995):10.64297717,(Chimaera_macrospina:0.2239522907,Chimaera_monstrosa:0.2239512907):39.92459584):10.37280832,((Chimaera_fulva:7.366992129,Hydrolagus_lusitanicus:7.366992129):35.56949083,((Chimaera_opalescens:21.28026693,(Chimaera_lignaria:0.3028240304,Chimaera_notafricana:0.3028240304):20.9774429):13.49399672,(Hydrolagus_mccoskeri:28.45812427,(Chimaera_panthera:24.23156051,Hydrolagus_bemisi:24.23156051):4.226563753):6.316139386):8.162219305):7.584873489):7.200231985):34.91768605):0.8071063885):22.08228939,((Chimaera_obscura:33.90186107,(Hydrolagus_deani:31.1475967,(Hydrolagus_barbouri:1.228210918,Chimaera_argiloba:1.228210918):29.91938578):2.754264375):20.18437018,(Hydrolagus_colliei:10.37477941,(Hydrolagus_homonycteris:5.829567273,Hydrolagus_eidolon:5.829567273):4.545212136):43.71145184):61.442439):35.23939929):41.13608306):183.6071783,((((((Sinobatis_borneensis:39.17614414,(Sinobatis_filicauda:5.646536957,Sinobatis_bulbicauda:5.646536957):33.52960718):26.84262147,(Sinobatis_melanosoma:53.38554254,Sinobatis_caerulea:53.38554254):12.63322307):65.92837501,(((((Irolita_waitii:7.040727833,Irolita_westraliensis:7.040727833):51.81997166,(((((Brochiraja_vittacauda:10.22961535,Brochiraja_heuresa:10.22961535):3.055060773,Brochiraja_aenigma:13.28467612):14.02245351,Brochiraja_microspinifera:27.30712963):0.6018088931,((Brochiraja_asperula:4.976380405,Brochiraja_spinifera:4.976380405):6.968529938,(Brochiraja_leviveneta:0.6397389676,Brochiraja_albilabiata:0.6397389676):11.30517037):15.96402918):11.49240089,((((Pavoraja_alleni:3.473419762,Pavoraja_umbrosa:3.473419762):10.75805432,(Pavoraja_mosaica:5.08545657,Pavoraja_pseudonitida:5.08545657):9.14601751):2.857309725,(Pavoraja_nitida:7.871088436,Pavoraja_arenaria:7.871088436):9.217695368):20.66914121,(((((Notoraja_sticta:9.925666549,(Notoraja_longiventralis:4.738130075,Notoraja_inusitata:4.738130075):5.187536474):1.648937547,(Notoraja_tobitukai:1.266431615,Notoraja_lira:1.266431615):10.30817248):0.8839082218,Notoraja_alisae:12.45851232):1.011039725,((Notoraja_azurea:8.755770931,(Notoraja_ochroderma:4.410926525,(Notoraja_sapphira:3.841511327,Notoraja_hirticauda:3.841511327):0.5694151984):4.344844406):1.542934181,Notoraja_fijiensis:10.29870511):3.170846932):0.7775969335,(Insentiraja_subtilispinosa:10.96454473,Insentiraja_laxipella:10.96454473):3.282604249):23.51077503):1.643415396):19.45935909):15.8676792,(((Rioraja_agassizii:33.3705862,(Atlantoraja_castelnaui:28.19579882,(Atlantoraja_platana:18.13784829,Atlantoraja_cyclophora:18.13784829):10.05795052):5.174787384):10.87431488,((Bathyraja_andriashevi:19.11773684,((Bathyraja_simoterus:6.560069018,Bathyraja_tzinovskii:6.560069018):6.17497096,Bathyraja_notoroensis:12.73503998):6.382696862):17.45591983,((((Bathyraja_scaphiops:10.47632346,((Bathyraja_brachyurops:0.6969068085,Bathyraja_diplotaenia:0.6969048085):4.88199018,(Bathyraja_panthera:1.92466328,Rhinoraja_magellanica:1.92466528):3.654231709):4.897428468):11.14091704,(Bathyraja_griseocauda:18.89341279,(((Bathyraja_kincaidii:0.1790584979,Bathyraja_violacea:0.1790584979):1.148156186,(Bathyraja_mariposa:0.6875087035,Bathyraja_aguja:0.6875067035):0.6397069803):4.581032185,(Bathyraja_matsubarai:1.845381476,Rhinoraja_taranetzi:1.845382476):4.062864392):12.98516592):2.723828705):6.643969351,(((((Bathyraja_tunae:3.32733383,Bathyraja_peruana:3.32733383):1.577556814,Bathyraja_minispinosa:4.904891643):5.841272448,(Bathyraja_longicauda:5.967620675,(Bathyraja_fedorovi:2.416912331,Bathyraja_cousseauae:2.416913331):3.550708344):4.778542416):5.43735687,((Bathyraja_pallida:10.5806181,((Rhinoraja_murrayi:5.435463142,(Bathyraja_irrasa:2.87594671,Bathyraja_papilionifera:2.87594571):2.559516432):3.756099099,Bathyraja_meridionalis:9.19156224):1.389056861):3.237185484,((Bathyraja_parmifera:3.825072969,(Rhinoraja_odai:1.881583418,Bathyraja_smirnovi:1.881585418):1.943487551):7.712375304,Bathyraja_shuntovi:11.53744727):2.280357312):2.365716377):1.035349203,(Bathyraja_schroederi:8.146900523,(Rhinoraja_albomaculata:2.330425709,Rhinoraja_macloviana:2.330425709):5.816476814):9.071968642):11.04234068):1.07702797,(((Bathyraja_ishiharai:14.84805665,Bathyraja_spinosissima:14.84805665):11.19580746,(((Rhinoraja_longicauda:0.7550978383,Rhinoraja_multispinis:0.7550988383):14.28914483,(Bathyraja_maculata:1.327631923,Bathyraja_lindbergi:1.327631923):13.71661174):5.722606066,((Bathyraja_abyssicola:12.01095504,Bathyraja_trachura:12.01095504):8.381503389,((((Bathyraja_isotrachys:12.72829711,Bathyraja_aleutica:12.72829811):1.213623875,((Bathyraja_eatonii:4.392058529,Bathyraja_spinicauda:4.392059529):5.595300283,Bathyraja_microtrachys:9.987358811):3.954562172):2.950771985,((Bathyraja_maccaini:0.9494173912,Rhinoraja_kujiensis:0.9494173912):15.24487839,Bathyraja_bergi:16.19429578):0.6983971896):3.474385439,Bathyraja_hesperafricana:20.36707841):0.02537902003):0.3743913068):5.277015375):2.078710296,(Bathyraja_richardsoni:25.49240586,(Bathyraja_smithii:10.0701836,(Bathyraja_leucomelanos:2.716500375,Bathyraja_trachouros:2.716500375):7.353683224):15.42222126):2.630169545):1.215663409):7.235418858):7.671244406):4.483878586,((((Sympterygia_acuta:8.263860235,Sympterygia_lima:8.263860235):17.50978683,(Sympterygia_bonapartii:20.53289355,Sympterygia_brevicaudata:20.53289355):5.240753521):13.80557435,((((Psammobatis_rutrum:8.333778171,Psammobatis_parvacauda:8.333778171):10.02832979,Psammobatis_extenta:18.36210796):3.192798292,(Psammobatis_scobina:0.1046593915,Psammobatis_bergi:0.1046593915):21.45024687):14.19881628,(Psammobatis_rudis:14.6054161,(Psammobatis_normani:6.748291324,Psammobatis_lentiginosa:6.748291324):7.857124776):21.14830643):3.825498887):4.097396449,(Arhynchobatis_asperrimus:7.393882799,Pseudoraja_fischeri:7.393882799):36.28273507):5.052161795):25.99959803):21.46362374,(Cruriraja_parcomaculata:32.96965624,(Cruriraja_poeyi:16.56034743,(Cruriraja_rugosa:14.07928554,(Cruriraja_atlantis:7.735276647,Cruriraja_durbanensis:7.735276647):6.344008892):2.481061888):16.40930982):63.22234519):4.976529961,(Cruriraja_hulleyi:77.48091226,((Anacanthobatis_longirostris:32.40952188,(Anacanthobatis_folirostris:18.83687194,(((Anacanthobatis_nanhaiensis:5.3918596,Anacanthobatis_marmoratus:5.3918596):8.05536903,Anacanthobatis_donghaiensis:13.44722863):5.102418714,((Anacanthobatis_ori:9.385791171,Anacanthobatis_americanus:9.385791171):6.188292923,Anacanthobatis_stenosoma:15.57408409):2.97556325):0.2872245957):13.57264994):7.526547653,(Cruriraja_cadenati:7.519803767,Cruriraja_andamanica:7.519803767):32.41626677):37.54484173):23.68762013):30.77860923):0.4428895613,(((((Raja_texana:3.166585749,Raja_cervigoni:3.166585749):7.316143459,((Raja_rouxi:6.971347569,((Raja_herwigi:0.5569348506,Raja_undulata:0.5569348506):1.77734269,Raja_equatorialis:2.33427754):4.637070028):1.786027024,Raja_miraletus:8.757374593):1.725354615):34.71744076,((Raja_asterias:25.7065491,((Raja_africana:11.191193391,(((Raja_straeleni:0.512673198,Raja_cortezensis:0.512673198):6.896253357,Raja_polystigma:7.408926555):3.745552218,(Raja_maderensis:8.453013722,(Raja_radula:4.772818913,Raja_clavata:4.772818913):3.680194808):2.701465051):0.03671461803):2.023333395,Raja_bahamensis:13.21452679):12.49202232):6.320659222,(Raja_montagui:22.4338096,(((Raja_brachyura:3.855638977,Raja_chinensis:3.855638977):7.809305279,(Raja_rondeleti:5.867856493,(Raja_ackleyi:4.325066784,Raja_macrorynchus:4.325066784):1.542789709):5.797087763):3.853175257,Raja_microocellata:15.51811951):6.915690085):9.593398727):13.17296164):22.96236808,((((Okamejei_leptoura:16.85192558,(Okamejei_meerdervoortii:3.16553417,Okamejei_mengae:3.16553417):13.68639141):20.70399803,Okamejei_powelli:37.55592361):11.93197211,(((Okamejei_acutispina:23.65243425,(Okamejei_porosa:23.39886067,Okamejei_heemstrai:23.39886067):0.2535725731):11.08388534,(Okamejei_pita:25.26613029,((Okamejei_philipi:6.045728956,Okamejei_kenojei:6.045729956):4.230855597,((Okamejei_boesemani:6.896892747,Okamejei_cairae:6.896892747):0.2855586432,Okamejei_schmidti:7.18245039):3.094134162):14.98954574):9.470188301):5.877338636,(Okamejei_arafurensis:8.123219002,Okamejei_hollandi:8.123220002):32.49043822):8.874238497):9.906376556,((Hongeo_koreana:49.4451325,((Beringraja_binoculata:17.25206725,Beringraja_pulchra:17.25206725):25.80350595,((Raja_stellulata:5.678675253,Raja_rhina:5.678675253):25.44142192,Raja_inornata:31.12009618):11.93547703):6.389559302):2.649090585,((Dipturus_oculus:39.20229066,(((Dipturus_melanospilus:28.65631037,((Dipturus_pullopunctata:5.511556864,(Dipturus_olseni:0.736989903,Dipturus_flavirostris:0.736989903):4.774566961):4.488935698,Dipturus_oregoni:10.00049256):18.65581781):9.534985746,((Dipturus_bullisi:7.3559195,Dipturus_garricki:7.3559195):28.76926653,(((Dipturus_springeri:5.391234476,Dipturus_queenslandicus:5.391234476):28.13086063,((((Dipturus_gigas:10.02264915,(Dipturus_grahami:4.066597938,Dipturus_stenorhynchus:4.066597938):5.956051212):6.277449585,Dipturus_nidarosiensis:16.30009874):3.91787814,((Spiniraja_whitleyi:0.6994862958,Dipturus_polyommata:0.6994862958):5.301824778,Dipturus_endeavouri:6.001311074):14.2166658):8.651483567,(((Zearaja_chilensis:3.574525543,Zearaja_nasuta:3.574525543):11.37020723,(Zearaja_argentinensis:13.01696807,Zearaja_maugeana:13.01696807):1.927764699):9.22426834,(Dipturus_trachydermus:0.7414619029,Dipturus_leptocaudus:0.7414619029):23.42753921):4.70046033):4.652634664):0.1296576574,(((Dipturus_gudgeri:12.3875517,Dipturus_innominatus:12.3875517):8.73735124,((Dipturus_confusus:5.265413949,(Dipturus_australis:2.606517517,Dipturus_cerva:2.606517517):2.658896432):5.00625359,((Dipturus_canutus:4.687016366,(Dentiraja_lemprieri:3.023365297,Dentiraja_flindersi:3.023365297):1.663653068):3.385802388,Dipturus_johannisdavesi:8.072818753):2.198846786):10.8532354):5.575985752,(((Dipturus_lanceorostratus:12.16586385,Dipturus_mennii:12.16586385):6.014323239,(Dipturus_crosnieri:10.0690654,(Dipturus_healdi:8.148318605,(Dipturus_teevani:6.013058991,(Dipturus_wengi:3.800047011,(Dipturus_falloargus:2.122104733,Dipturus_wuhanlingi:2.122104733):1.677942278):2.21301198):2.135258615):1.920747793):8.111121686):0.01180091955,(Dipturus_macrocauda:0.2515679286,Dipturus_tengu:0.2515679286):17.94042108):8.508898692):6.950866067):2.47343327):2.066110081):0.2104667985,((((Dipturus_apricus:2.487523728,Dipturus_batis:2.487522728):3.750537835,Dipturus_oxyrinchus:6.238060563):10.5374782,(Dipturus_acrobelus:11.20098378,(Dipturus_ecuadoriensis:9.118548994,Dipturus_intermedia:9.118548994):2.082434789):5.57455598):7.425229406,((Dipturus_laevis:2.63344629,Dipturus_campbelli:2.63344629):11.85892953,Dipturus_doutrei:14.49237582):9.708393344):14.20099374):0.8005277506):7.808813209,Dipturus_kwangtungensis:47.01110387):5.083120216):7.300049191):8.768264774):23.95119521,((((((Gurgesiella_dorsalifera:9.193446746,Gurgesiella_atlantica:9.193446746):15.31929839,Gurgesiella_furvescens:24.51274514):35.09986193,(Okamejei_jensenae:58.97528969,((Raja_velezi:0.6990403687,Raja_eglanteria:0.6990403687):34.74950308,Rostroraja_alba:35.44854344):23.52674625):0.637316378):21.68274158,(((((Leucoraja_ocellata:10.44357804,((Leucoraja_yucatanensis:0.5084598457,Leucoraja_leucosticta:0.5084598457):7.193407559,Leucoraja_virginica:7.701867405):2.74171164):8.128054054,Leucoraja_erinacea:18.5716321):7.085630183,((Leucoraja_garmani:19.55854395,Leucoraja_lentiginosa:19.55854495):5.887003073,Leucoraja_compagnoi:25.44554802):0.2117152589):27.72626899,(((Rajella_bathyphila:31.76419226,(((Rajella_leopardus:2.962789352,Rajella_purpuriventralis:2.962789352):6.211535839,Rajella_dissimilis:9.174325191):20.95394229,((Rajella_ravidula:23.24397328,(Rajella_lintea:19.47028017,Rajella_nigerrima:19.47028017):3.773693108):0.4004479477,(((Rajella_kukujevi:8.897621508,Rajella_fuliginea:8.897622508):2.175991339,((Rajella_alia:0.2268500174,Rajella_sadowskii:0.2268500174):7.787764885,Rajella_annandalei:8.014614903):3.058998944):11.05590689,Rajella_barnardi:22.12951974):1.514900492):6.483846251):1.63592578):3.670537043,((Rajella_challengeri:15.7985789,Rajella_bigelowi:15.7985779):5.589226473,((Rajella_eisenhardti:14.10028167,Rajella_caudaspinosa:14.10028067):3.527269429,Rajella_fyllae:17.6275501):3.760254282):14.04692493):4.547874484,((Leucoraja_caribbaea:20.43016075,Leucoraja_pristispina:20.43016075):7.768825148,((Amblyraja_robertsi:12.8370791,((Amblyraja_taaf:1.957320159,Amblyraja_georgiana:1.957320159):6.173320086,((Amblyraja_frerichsi:3.537299873,Amblyraja_doellojuradoi:3.537299873):2.131030062,((Amblyraja_reversa:1.499503147,Amblyraja_hyperborea:1.499503147):1.039350746,(Amblyraja_jenseni:0.04960359504,Amblyraja_badia:0.04960359504):2.489250298):3.129476042):2.462310311):4.706438859):2.665534523,Amblyraja_radiata:15.50261363):12.69637227):11.78361889):13.40092749):7.438985845,(Leucoraja_circularis:25.02143978,(Leucoraja_naevus:10.90476428,(Leucoraja_wallacei:8.875025509,(Leucoraja_fullonica:4.48038987,Leucoraja_melitensis:4.48039087):4.39463464):2.029738766):14.1166755):35.80107934):20.47282953):0.6501199846,((((Malacoraja_kreffti:4.589430327,Malacoraja_spinacidermis:4.589430327):21.07207038,(Malacoraja_obscura:6.873534683,Malacoraja_senta:6.873534683):18.78796602):16.41535361,(Dactylobatus_armatus:38.55805552,Dactylobatus_clarkii:38.55805552):3.518798788):11.5473781,(Neoraja_stehmanni:19.47148225,(Neoraja_carolinensis:14.78936932,((Neoraja_africana:0.6125252306,Neoraja_iberica:0.6125252306):10.20121094,Neoraja_caerulea:10.81373617):3.975633148):4.682112933):34.15275016):28.32123623):6.695045356,(((((Fenestraja_cubensis:7.542189809,Fenestraja_sinusmexicanus:7.542189809):4.40795778,Fenestraja_maceachrani:11.95014759):8.841224208,Fenestraja_mamillidens:20.7913718):7.088910034,(Fenestraja_ishiyamai:26.94523382,((Fenestraja_plutonia:3.718224306,Fenestraja_atripinna:3.718224306):1.512678504,Fenestraja_sibogae:5.230902811):21.71433101):0.9350480135):33.88379945,(((Breviraja_colesi:6.999599827,Breviraja_claramaculata:6.999599827):1.497909702,Breviraja_spinosa:8.497509529):6.387544282,Breviraja_nigriventralis:14.88505381):46.87902747):26.87643271):3.473220277):40.27629691):98.13618487,(((Platyrhinoidis_triseriata:119.9002818,((Platyrhina_sinensis:30.06173956,Platyrhina_tangi:30.06173956):7.895263815,Platyrhina_hyugaensis:37.95700338):81.9432784):96.36647121,(Hypnos_monopterygius:141.3830394,((((Narcine_nelsoni:25.18935277,(((Narcine_insolita:8.234021176,Narcine_brunnea:8.234021176):0.398777927,Narcine_nigra:8.632799103):7.461296303,(Narcine_lingula:7.798639659,(Narcine_timlei:5.192366724,Narcine_bancroftii:5.192366724):2.606272935):8.295455747):9.095257362):20.32197661,((Narcine_ornata:36.57948628,((Narcine_firma:3.643040183,Narcine_maculata:3.643040183):19.50524876,(Narcine_rierai:3.966175777,Narcine_brevilabiata:3.966175777):19.18211317):13.43119734):5.613996735,((Narcine_oculifera:14.4769021,Narcine_atzi:14.4769021):7.183710371,(Narcine_brasiliensis:3.021940785,Narcine_vermiculatus:3.021940785):18.63867169):20.53287055):3.317846364):44.10934401,((Narcine_prodorsalis:19.68586978,(Narcine_bicolor:13.57147845,Narcine_entemedor:13.57147845):6.114391324):13.09420591,Narcine_leoparda:32.78007568):56.84059771):29.92629281,(((((((Diplobatis_ommata:10.42239302,Diplobatis_pictus:10.42239302):5.575263305,Diplobatis_colombiensis:15.99765632):28.75059464,Diplobatis_guamachensis:44.74825096):34.72188584,(Discopyge_castelloi:1.721389097,Discopyge_tschudii:1.721389097):77.7487477):8.938328581,(((Torpedo_mackayana:38.05239618,Torpedo_tokionis:38.05239618):1.706995777,((Torpedo_torpedo:9.019999346,Torpedo_fuscomaculata:9.019999345999999):3.016684923,(Torpedo_marmorata:11.2143906,(Torpedo_andersoni:8.748488211,(Torpedo_sinuspersici:2.781079598,Torpedo_tremens:2.781079598):5.967408613):2.46590239):0.8222926684):27.72270869):24.49537913,(((((((Torpedo_fairchildi:8.620533445,Torpedo_alexandrinsis:8.620533445):7.639430498,Torpedo_microdiscus:16.25996394):12.13761001,(Torpedo_californica:3.314106777,Torpedo_puelcha:3.314106777):25.08346718):4.683021765,((Torpedo_formosa:1.447130747,Torpedo_adenensis:1.447130747):12.32644516,(Torpedo_macneilli:5.634587003,(Torpedo_nobiliana:1.561207266,Torpedo_polleni:1.561206266):4.073379737):8.138989904):19.30701982):4.415574925,Torpedo_semipelagica:37.49617065):12.64928991,(Torpedo_peruana:7.671210488,Torpedo_panthera:7.671210488):42.47425007):13.70452567,(Torpedo_suessii:19.01594773,Torpedo_bauchotae:19.01594773):44.8340385):0.404784859):24.15369429):5.550481244,((((Heteronarce_bentuviai:2.643170898,Heteronarce_mollis:2.643170898):5.014099166,Heteronarce_prabhui:7.657270064):7.247006764,Heteronarce_garmani:14.90427683):71.65860573,((Crassinarke_dormitor:13.99829617,((Electrolux_addisoni:6.689196302,((Narke_dipterygia:0.3921102731,Narke_capensis:0.3921102731):1.918801914,Narke_japonica:2.310912187):4.378284115):4.772910039,Temera_hardwickii:11.46210634):2.536189826):7.325293331,(Typhlonarke_aysoni:4.830802912,Typhlonarke_tarakea:4.830802912):16.49278658):65.23929306):7.396064065):9.601013473,((Benthobatis_marcida:35.84723151,(Benthobatis_moresbyi:14.01411778,(Benthobatis_kreffti:2.854311606,Benthobatis_yangi:2.854311606):11.15980617):21.83311373):40.7505562,(Narcine_westraliensis:47.94212957,(Narcine_lasti:20.82209261,Narcine_tasmaniensis:20.82209261):27.12003696):28.65565713):26.96217239):15.9870061):21.83607325):74.88371354):9.155085965,((((Trygonorrhina_dumerilii:25.68937262,Trygonorrhina_fasciata:25.68937262):37.73137024,Trygonorrhina_melaleuca:63.42074286):87.0534649,((Zapteryx_brevirostris:78.54367083,(Zapteryx_xyster:4.180896346,Zapteryx_exasperata:4.180896346):74.36277448):42.11287348,((Aptychotrema_timorensis:5.250606906,Aptychotrema_rostrata:5.250606906):28.32078516,Aptychotrema_vincentiana:33.57139207):87.08515124):29.81766445):59.55858144,((((Rhina_ancylostoma:98.28393074,((Rhynchobatus_palpebratus:9.034476135,(Rhynchobatus_laevis:1.20464812,Rhynchobatus_springeri:1.20464812):7.829828015):14.08607483,((Rhynchobatus_djiddensis:5.309800286,Rhynchobatus_australiae:5.309800286):7.526694087,Rhynchobatus_luebberti:12.83649437):10.2840566):75.16337978):42.91154007,((((Glaucostegus_cemiculus:3.596731869,Glaucostegus_granulatus:3.596731869):17.50155713,Glaucostegus_halavi:21.098289):33.52481676,(((Glaucostegus_microphthalmus:2.838521318,Glaucostegus_thouin:2.838521318):3.841786822,(Glaucostegus_obtusus:2.266205897,Glaucostegus_spinosus:2.266205897):4.414102243):3.476136604,Glaucostegus_typus:10.15644474):44.46666102):73.96671497,(Anoxypristis_cuspidata:121.4912742,(Pristis_pristis:86.46471243,(Pristis_clavata:67.40996125,(Pristis_zijsron:55.50754067,Pristis_pectinata:55.50754067):11.90242159):19.05475118):35.02656073):7.098546561):12.60565009):16.14049037,((((Rhinobatos_irvinei:8.515294888,Rhinobatos_lentiginosus:8.515294888):23.84950284,Acroteriobatus_leucospilus:32.36479773):62.32153268,((((Rhinobatos_horkelii:14.57186817,Acroteriobatus_zanzibarensis:14.57186817):2.041070533,Acroteriobatus_ocellatus:16.6129387):45.22996714,(((Acroteriobatus_salalah:27.90728152,Rhinobatos_glaucostigma:27.90728152):3.523539446,Rhinobatos_albomaculatus:31.43082097):5.441804393,(Rhinobatos_productus:23.75569542,Rhinobatos_leucorhynchus:23.75569542):13.11692994):24.97028048):11.19236706,(Rhinobatos_planiceps:18.56568436,(Rhinobatos_sainsburyi:7.834219269,Rhinobatos_percellens:7.834219269):10.73146509):54.46958854):21.65105752):55.061003,(((Acroteriobatus_annulatus:29.09716844,(Rhinobatos_nudidorsalis:4.942349945,(Rhinobatos_annandalei:0.4155657118,Acroteriobatus_variegatus:0.4155657118):4.526785233):24.1548175):30.05673669,((((Rhinobatos_schlegelii:3.59019968,Rhinobatos_formosensis:3.59019968):3.711698758,Rhinobatos_hynnicephalus:7.301898438):11.25426297,Rhinobatos_punctifer:18.55616141):23.00296064,(((Rhinobatos_rhinobatos:6.97377461,Rhinobatos_prahli:6.97377361):13.47524679,Acroteriobatus_blochii:20.4490204):16.19413788,Rhinobatos_lionotus:36.64315828):4.915963766):17.59478208):54.96684368,(Rhinobatos_penggali:63.18266917,(Rhinobatos_jimbaranensis:5.319046449,Rhinobatos_holcorhynchus:5.319046449):57.86362273):50.93807864):35.6265856):7.588627765):48.31474389,(Zanobatus_schoenleinii:192.2080066,(Hexatrygon_bickelli:138.3849967,((((Aetobatus_flagellum:46.6446165,Aetobatus_narinari:46.6446165):69.95446405,(((Aetomylaeus_vespertilio:78.73843232,(Aetomylaeus_maculatus:47.54500358,(Aetomylaeus_nichofii:31.43173836,(Pteromylaeus_bovinus:10.89042125,Pteromylaeus_asperrimus:10.89042125):20.54131711):16.11326523):31.19342873):17.54098204,((Myliobatis_californicus:40.34839692,((Myliobatis_freminvillei:18.00077731,Myliobatis_longirostris:18.00077731):11.77966248,(Myliobatis_peruvianus:10.96873197,Myliobatis_tenuicaudatus:10.96873197):18.81170882):10.56795613):27.75548727,((Myliobatis_chilensis:31.05809892,(((Myliobatis_rhombus:1.978671904,Myliobatis_hamlyni:1.978671904):11.58887439,(Myliobatis_ridens:5.681763206,Myliobatis_tobijei:5.681763206):7.885783084):14.81855788,Myliobatis_aquila:28.38610417):2.671994751):17.53018513,(Myliobatis_australis:0.9710848798,Myliobatis_goodei:0.9710848798):47.61719916):19.51560015):28.17553017):14.59774079,((((Rhinoptera_jayakari:26.97373417,Rhinoptera_steindachneri:26.97373417):2.250287622,Rhinoptera_bonasus:29.22402279):23.76489512,(Rhinoptera_hainanensis:22.08866197,(Rhinoptera_marginata:19.325558787000002,(((Rhinoptera_javanica:3.63613518,Rhinoptera_adspersa:3.63613418):2.332327264,Rhinoptera_neglecta:5.968462444):7.747452472,(Rhinoptera_brasiliensis:12.18654202,Rhinoptera_peli:12.18654202):1.529371895):5.609644867):2.763103183):30.90025494):12.02837268,(((Manta_birostris:11.91938335,Manta_alfredi:11.91938335):35.11961696,((Mobula_tarapacana:5.255516264,Mobula_japanica:5.255516264):35.8487036,((Mobula_thurstoni:10.32470245,Mobula_kuhlii:10.32470245):24.3401836,Mobula_rochebrunei:34.66488605):6.439333805):5.93478044):13.85046125,((Mobula_mobular:16.00237161,Mobula_diabolus:16.00237161):28.52895421,(Mobula_eregoodootenkee:31.48935902,(Mobula_munkiana:10.11769162,Mobula_hypostoma:10.11769162):21.3716674):13.0419668):16.35813573):4.127828043):45.85986456):5.721925401):10.98139957,(Plesiobatis_daviesi:118.3929008,(((Trygonoptera_ovalis:43.25067798,Trygonoptera_galba:43.25067798):14.2669157,((Trygonoptera_testacea:33.41323343,(Trygonoptera_mucosa:10.47827509,Trygonoptera_imitata:10.47827509):22.93495834):12.13157807,Trygonoptera_personata:45.5448115):11.97278118):52.33068576,(((((((Urolophus_javanicus:4.676652958,(Urolophus_piperatus:2.692627086,Urolophus_deforgesi:2.692627086):1.984025873):7.058608284,Urolophus_papilio:11.73526124):2.241394263,(Urolophus_armatus:13.44642159,Urolophus_westraliensis:13.44642259):0.5302339137):12.64550216,Urolophus_aurantiacus:26.62215866):6.792264437,((((Urolophus_sufflavus:1.161252086,(Urolophus_cruciatus:0.9809657173,Urolophus_paucimaculatus:0.9809657173):0.1802863682):13.57730048,Urolophus_orarius:14.73855256):15.48198989,(Urolophus_kapalensis:0.1629809544,Urolophus_lobatus:0.1629799544):30.0575625):0.3003893461,(Urolophus_kaianus:23.9749954,((Urolophus_viridis:6.404147975,Urolophus_neocaledoniensis:6.404146975):1.259339021,Urolophus_expansus:7.663486996):16.31150941):6.5459364):2.893490297):20.26640125,Urolophus_gigas:53.68082335):24.31071648,(Urolophus_mitosis:70.05391057,((Urolophus_flavomosaicus:2.793087349,Urolophus_bucculentus:2.793087349):47.09514786,Urolophus_circularis:49.88823521):20.16567536):7.937629262):31.85673961):8.54462132):9.187578364):3.0838548,((((((Gymnura_altavela:1.003578991,Gymnura_tentaculata:1.003578991):12.64017567,Gymnura_natalensis:13.64375466):17.55706166,(Gymnura_marmorata:9.113746944,Gymnura_crebripunctata:9.113746944):22.08706938):53.78547763,(Gymnura_afuerae:24.3219565,Gymnura_micrura:24.3219565):60.66433746):14.48528783,(((Gymnura_poecilura:26.70871372,Gymnura_hirundo:26.70871372):37.65412001,((Gymnura_zonura:14.9934195,Gymnura_bimaculata:14.9934195):13.24167512,Gymnura_crooki:28.23509463):36.1277391):17.5127712,(Gymnura_japonica:8.642964755,Gymnura_australis:8.642964755):73.23264017):17.59597686):15.49008132,((((Urobatis_halleri:23.01344967,(Urobatis_maculatus:3.784106032,Urobatis_concentricus:3.784106032):19.22934364):49.78050761,((Urobatis_marmoratus:37.66096818,(Urobatis_tumbesensis:16.19197771,Urobatis_jamaicensis:16.19197771):21.46899047):23.33160834,((((Urotrygon_serrula:4.569639857,Urotrygon_microphthalmum:4.569639857):19.43510139,Urotrygon_reticulata:24.00474125):4.496135171,(Urotrygon_rogersi:27.42416404,(((Urotrygon_munda:5.185434779,(Urotrygon_simulatrix:1.434351486,Urotrygon_caudispinosus:1.434351486):3.751083293):3.321145402,Urotrygon_venezuelae:8.50658018):9.717290073,(Urotrygon_cimar:2.031257341,Urotrygon_nana:2.031257341):16.19261291):9.200293791):1.076712372):11.350753,(Urotrygon_aspidura:16.25541205,(Urotrygon_chilensis:5.490024193,Urotrygon_peruanus:5.490024193):10.76538786):23.59621737):21.14094711):11.80138175):26.97451415,((Himantura_schmardae:21.00392628,Himantura_pacifica:21.00392628):48.67470127,(((Potamotrygon_constellata:34.15146208,((((Potamotrygon_marinae:9.791352925,Potamotrygon_falkneri:9.791352925):15.85315491,Potamotrygon_schuemacheri:25.64450784):1.24617883,(Potamotrygon_brachyura:15.56881797,((Potamotrygon_henlei:12.93304235,Potamotrygon_humerosa:12.93304235):0.2347456696,((Potamotrygon_schroederi:5.778233393,Potamotrygon_tigrina:5.778233393):2.88440591,((Potamotrygon_orbignyi:5.290202191,Potamotrygon_castexi:5.290202191):0.8720133288,(Potamotrygon_dumerilii:1.949356052,(Potamotrygon_scobina:0.8323102727,Potamotrygon_motoro:0.8323102727):1.117045779):4.212859467):2.500423783):4.505148713):2.401029957):11.32186869):6.199393309,(Potamotrygon_tatianae:26.42922045,(((Plesiotrygon_iwamae:8.140454789,Plesiotrygon_nana:8.140454789):11.89682724,((Potamotrygon_boesemani:7.566258501,(Potamotrygon_ocellata:4.4909062,Potamotrygon_yepezi:4.4909062):3.075352302):8.26559798,(Potamotrygon_signata:1.602703281,Potamotrygon_leopoldi:1.602703281):14.2291532):4.205425543):5.083948972,(Potamotrygon_magdalenae:23.20066416,Potamotrygon_hystrix:23.20066416):1.920566837):1.30798945):6.660859528):1.061382102):5.626480181,Paratrygon_aiereba:39.77794226):4.294282415,(Heliotrygon_gomesi:6.194699588,Heliotrygon_rosai:6.194699588):37.87752508):25.60640288):30.08984487):11.17388531,(((Taeniura_lymma:62.52887828,((Neotrygon_kuhlii:34.15370367,(Neotrygon_leylandi:8.287011947,(Neotrygon_picta:5.585087182,Neotrygon_annotata:5.585088182):2.701924765):25.86669073):14.60678344,Neotrygon_ningalooensis:48.76048711):13.76839116):33.81315457,((((Dasyatis_sabina:22.08146439,Dasyatis_sinensis:22.08146439):42.77626045,(Dasyatis_gigantea:11.83878879,(Dasyatis_americana:10.94243584,(Dasyatis_marianae:6.211223363,Dasyatis_longa:6.211222363):4.731213481):0.8963519514):53.01893604):6.74349893,((Dasyatis_dipterura:25.64525288,(Dasyatis_say:12.57079926,Dasyatis_acutirostra:12.57079926):13.07445362):31.01472413,(((Dasyatis_guttata:0.4518722463,Dasyatis_hypostigma:0.4518722463):11.05350182,Dasyatis_geijskesi:11.50537407):13.40840928,Dasyatis_colarensis:24.91378335):31.74619366):14.94124676):15.80078832,((((((Dasyatis_garouaensis:2.206612624,(Dasyatis_fluviorum:0.6754233275,Dasyatis_tortonesei:0.6754243275):1.531188297):19.76099128,((Dasyatis_akajei:2.657677184,Dasyatis_hastata:2.657678184):4.251614106,Dasyatis_laosensis:6.909291291):15.05831162):6.208149952,(Dasyatis_parvonigra:14.70960625,(Dasyatis_bennetti:0.4322282121,Dasyatis_navarrae:0.4322282121):14.27737804):13.46614661):21.64236123,(Dasyatis_izuensis:9.2521103,Dasyatis_laevigata:9.2521103):40.56600279):11.51894307,Dasyatis_zugei:61.33705716):10.94806241,((((Dasyatis_ushiei:10.19611082,(Dasyatis_thetidis:3.112784547,Dasyatis_centroura:3.112784547):7.083326269):38.69947863,(Pteroplatytrygon_violacea:40.80396813,((Dasyatis_matsubarai:5.65689369,Dasyatis_brevicaudata:5.65689369):5.614220823,Dasyatis_multispinosa:11.27111551):29.53285361):8.09162032):1.516733331,Dasyatis_marmorata:50.41232278):6.526621418,((Taeniurops_grabata:41.10593149,Taeniurops_meyeni:41.10593149):10.23939925,((Dasyatis_rudis:23.25591861,Dasyatis_pastinaca:23.25591761):2.642200187,Dasyatis_chrysonota:25.8981178):25.44721394):5.59361145):15.34617638):15.11689151):8.940020764):12.08153846,((Dasyatis_microps:85.80139935,((Dasyatis_margaritella:16.1479049,Dasyatis_margarita:16.1479049):49.93428254,((((((Himantura_kittipongi:1.445502181,Himantura_fai:1.445502181):20.98134335,Himantura_hortlei:22.42684553):10.44385934,((Himantura_uarnak:13.74454599,(Himantura_marginata:10.98247985,Himantura_undulata:10.98248085):2.762065139):10.21940705,Himantura_leoparda:23.96395204):8.906752824):5.451015932,(((Himantura_toshi:0.9928705457,Himantura_astra:0.9928705457):2.750259474,Himantura_pastinacoides:3.74312902):33.0828758,((Himantura_signifer:24.89887026,Himantura_oxyrhyncha:24.89887026):9.683152292,(((Himantura_imbricata:0.2560154644,Dasyatis_lata:0.2560154644):12.59589248,(Himantura_walga:3.646202386,Himantura_dalyensis:3.646202386):9.20570556):12.39337829,Himantura_gerrardi:25.24528624):9.336735319):2.243983262):1.495715982):7.17636378,(Himantura_uarnacoides:23.97461919,(Himantura_jenkinsii:12.64499635,Himantura_randalli:12.64499535):11.32962284):21.52346639):11.48986326,(((Urogymnus_asperrimus:21.52653296,Urogymnus_ukpam:21.52653296):20.21474893,(Himantura_polylepis:1.66908045,Himantura_granulata:1.66908045):40.07220144):6.412166046,Himantura_lobistoma:48.15344794):8.834499902):9.094238607):19.7192129):9.643849013,(Makararaja_chindwinensis:45.06544892,((Pastinachus_stellurostris:22.17693541,Pastinachus_solocirostris:22.17693541):10.53528892,((Pastinachus_sephen:9.801828233,Pastinachus_gracilicaudus:9.801828233):1.899023909,Pastinachus_atrus:11.70085114):21.01137319):12.35322359):50.37979944):12.97832195):2.518787433):4.019305366):15.70267182):7.720662796):53.82300883):13.44269852):4.382084126):15.38904876):5.104377093):46.83220985,((((Chlamydoselachus_anguineus:0.007482449505,Chlamydoselachus_africana:0.007482449505):145.2157711,(Notorynchus_cepedianus:81.34156367,((Hexanchus_vitulus:31.74940647,(Hexanchus_griseus:2.130500002,Hexanchus_nakamurai:2.130500002):29.61890647):13.14305906,Heptranchias_perlo:44.89246553):36.44909814):63.88168992):96.54902134,(((((Deania_quadrispinosa:12.68890734,Deania_profundorum:12.68890734):26.17505355,(Deania_hystricosa:4.370110851,Deania_calcea:4.370110851):34.49385005):76.48501793,(((Centrophorus_squamosus:45.63838927,(Centrophorus_granulosus:8.094764847,Centrophorus_tessellatus:8.094763847):37.54362542):3.806351082,((((Centrophorus_harrissoni:10.5961307,Centrophorus_seychellorum:10.5961297):2.607291109,Centrophorus_isodon:13.20342181):11.9716952,(Centrophorus_lusitanicus:3.527989816,Centrophorus_moluccensis:3.527989816):21.64712619):7.546269764,Centrophorus_westraliensis:32.72138577):16.72335458):47.02953903,(Centrophorus_atromarginatus:38.29852257,Centrophorus_zeehaani:38.29852257):58.1757568):18.87469945):52.50371713,((((Aculeola_nigra:29.83898388,(((Centroscyllium_granulatum:3.977778692,Centroscyllium_excelsum:3.977778692):1.420277553,(Centroscyllium_kamoharai:0.407860159,Centroscyllium_nigrum:0.407860159):4.990196086):9.239732255,((Centroscyllium_fabricii:8.583871699,Centroscyllium_ritteri:8.583871699):2.561024524,Centroscyllium_ornatum:11.14489622):3.492892278):15.20119538):71.25157141,(Trigonognathus_kabeyai:100.7445743,(((((Etmopterus_dianthus:16.4730376,((Etmopterus_unicolor:10.67136892,Etmopterus_litvinovi:10.67136892):0.590435874,(((Etmopterus_bullisi:0.3514589177,Etmopterus_villosus:0.3514589177):0.08591578253,Etmopterus_spinax:0.4373747002):4.698804343,(Etmopterus_princeps:1.118831664,Etmopterus_granulosus:1.118832664):4.017347379):6.125625754):5.211231806):5.865991676,Etmopterus_compagnoi:22.33902828):28.02093677,((Etmopterus_gracilispinis:35.15946206,(Etmopterus_schultzi:32.76960336,(Etmopterus_polli:20.51165759,Etmopterus_virens:20.51165759):12.25794577):2.389858699):12.34156966,Etmopterus_schmidti:47.50103072):2.85893433):6.030278684,(((Etmopterus_robinsi:41.97012785,((Etmopterus_caudistigmus:7.571816072,Etmopterus_decacuspidatus:7.571816072):21.72816884,(Etmopterus_sculptus:3.182478294,Etmopterus_sentosus:3.182478294):26.11750662):12.67014294):5.413346116,(Etmopterus_pusillus:28.60879292,Etmopterus_bigelowi:28.60879292):18.77468105):1.878429206,(((Etmopterus_splendidus:18.33271446,(Etmopterus_burgessi:15.19831161,Etmopterus_fusus:15.19831161):3.134402847):22.96376579,((Etmopterus_pycnolepis:2.019557697,Etmopterus_carteri:2.019557697):10.71401361,(Etmopterus_pseudosqualiolus:1.69604974,Etmopterus_perryi:1.69604974):11.03752157):28.56290894):7.919852134,(Etmopterus_evansi:16.84862558,Etmopterus_hillianus:16.84862558):32.3677068):0.04557078673):7.12834056):27.6488323,((((Etmopterus_dislineatus:13.18228509,Etmopterus_joungi:13.18228409):4.614555323,Etmopterus_molleri:17.79684041):10.65117952,Etmopterus_lucifer:28.44801993):11.51949607,((Etmopterus_brachyurus:10.01544602,Etmopterus_viator:10.01544502):5.944409694,Etmopterus_sheikoi:15.95985571):24.00766029):44.07156103):16.70549827):0.3459809901):45.8250891,(((Squaliolus_laticaudus:62.82489696,(Mollisquama_parini:61.30760484,((Heteroscymnoides_marleyi:12.24488571,Squaliolus_aliae:12.24488571):17.20399701,(Euprotomicroides_zantedeschia:3.433907158,Euprotomicrus_bispinatus:3.433907158):26.01497557):31.85872211):1.517292123):64.20837255,(((Isistius_labialis:4.453458847,Isistius_brasiliensis:4.453458847):0.2585786812,Isistius_plutodus:4.712037528):80.74217458,Dalatias_licha:85.45421211):41.5790574):16.87340225,((Cirrhigaleus_asper:56.62274364,(Cirrhigaleus_barbifer:16.26028245,Cirrhigaleus_australis:16.26028245):40.36246119):28.02037768,(((((Squalus_notocaudatus:11.54517606,Squalus_suckleyi:11.54517606):3.348574088,Squalus_acanthias:14.89375014):0.982797746,Squalus_altipinnis:15.87654789):3.620252502,Squalus_griffini:19.49680039):60.46530723,((Squalus_megalops:31.0995569,((Squalus_melanurus:5.549231904,Squalus_albifrons:5.549231904):6.711899161,Squalus_brevirostris:12.26113206):18.83842484):20.44138766,((Squalus_grahami:15.16814855,((Squalus_mitsukurii:10.89410676,(Squalus_rancureli:9.256679907,(Squalus_montalbani:0.2121823748,Squalus_chloroculus:0.2121823748):9.044497533):1.637426852):2.670714492,Squalus_cubensis:13.56482125):1.603327303):19.74048663,((((Squalus_japonicus:7.840485518,(Squalus_blainville:5.925567146,(Squalus_nasutus:2.123959344,Squalus_raoulensis:2.123959344):3.801607802):1.914918372):5.16888256,(Squalus_edmundsi:9.427592629,Squalus_hemipinnis:9.427592629):3.581775449):16.76605042,((Squalus_lalannei:3.523880279,Squalus_formosus:3.523880279):10.83367267,Squalus_crassispinus:14.35755295):15.41786555):4.889925312,Squalus_bucephalus:34.66534381):0.2432913743):16.63230837):28.42116407):4.681012695):59.26355144):3.008971639):11.13974098,(((Somniosus_rostratus:18.77438309,Somniosus_longus:18.77438309):5.47100368,((Somniosus_antarcticus:2.957963686,Somniosus_pacificus:2.957963686):4.865521765,Somniosus_microcephalus:7.823484452):16.42190232):133.6791905,(((Centroscymnus_coelolepis:46.44653144,((Scymnodalatias_garricki:36.91845704,Scymnodalatias_oligodon:36.91845704):6.036486787,(Scymnodalatias_sherwoodi:13.57875359,Scymnodalatias_albicauda:13.57875359):29.37619023):3.491587614):7.876093264,(((Proscymnodon_plunketi:2.219131166,Proscymnodon_macracanthus:2.219131166):16.8221658,Centroscymnus_owstonii:19.04129697):26.26245382,((Oxynotus_caribbaeus:37.8724312,((Oxynotus_japonicus:0.6878710176,Oxynotus_centrina:0.6878710176):29.69117686,(Oxynotus_bruniensis:22.53812185,Oxynotus_paradoxus:22.53812185):7.840926034):7.493383322):0.07144831636,Scymnodon_ringens:37.94387952):7.35987227):9.018872916):18.23334479,((Zameus_squamulosus:17.00885569,Zameus_ichiharai:17.00885569):37.83023217,Centroselachus_crepidater:54.83908785):17.71688164):85.36860775):0.1308081315):9.797311583):39.66946926,(((Echinorhinus_brucus:27.72473684,Echinorhinus_cookei:27.72473684):77.95912971,(Pliotrema_warreni:92.13477035,(Pristiophorus_japonicus:65.30120692,(((Pristiophorus_delicatus:6.547860922,Pristiophorus_nancyae:6.547860922):12.963068,Pristiophorus_cirratus:19.51092892):29.43889705,(Pristiophorus_nudipinnis:3.312010713,Pristiophorus_schroederi:3.312010713):45.63781526):16.35138094):26.83356243):13.5490972):8.323688048,(((((Squatina_legnota:9.604536318,Squatina_tergocellatoides:9.604536318):17.49546032,(Squatina_formosa:9.314628034,Squatina_nebulosa:9.314628034):17.7853686):22.45677473,Squatina_japonica:49.55677137):4.913014869,((((Squatina_californica:6.859721993,Squatina_dumeril:6.859721993):17.31768277,(Squatina_occulta:13.49984386,Squatina_guggenheim:13.49984386):10.6775609):11.51363075,Squatina_armata:35.69103452):1.745392124,Squatina_africana:37.43642664):17.0333596):24.60483021,(((Squatina_albipunctata:5.899470433,(Squatina_tergocellata:4.249635109,Squatina_pseudocellata:4.249635109):1.649836324):24.4988268,(Squatina_oculata:11.54457034,Squatina_australis:11.54457034):18.85372689):16.13258772,((Squatina_argentina:26.24570098,Squatina_aculeata:26.24570098):13.89943507,(Squatina_squatina:38.99982089,Squatina_caillieti:38.99982089):1.145315161):6.385748895):32.54373049):34.93293915):93.51461062):34.25010871):12.72360209,((Heterodontus_francisci:105.6242033,(((((Heterodontus_zebra:1.753230771,Heterodontus_quoyi:1.753229771):40.13547186,(Heterodontus_portusjacksoni:15.55596067,Heterodontus_galeatus:15.55596067):26.33274196):14.2828199,Heterodontus_ramalheira:56.17152153):2.99347869,(Heterodontus_japonicus:2.763072658,Heterodontus_omanensis:2.763072658):56.40192756):26.50798964,Heterodontus_mexicanus:85.67299086):19.95121344):132.9841361,((((((Parascyllium_variolatum:8.749520313,(Parascyllium_ferrugineum:4.631975505,Parascyllium_elongatum:4.631975505):4.117544808):3.397765747,Parascyllium_collare:12.14728606):13.17740164,Parascyllium_sparsimaculatum:25.3246877):10.23383615,((Cirrhoscyllium_expolitum:7.100178173,Cirrhoscyllium_formosanum:7.100178173):8.724796254,Cirrhoscyllium_japonicum:15.82497443):19.73354942):177.53247,(((Brachaelurus_colcloughi:58.96101233,Brachaelurus_waddi:58.96101233):89.43427313,(Eucrossorhinus_dasypogon:87.6432052,(((Orectolobus_ornatus:48.7988366,((Orectolobus_floridus:13.20647288,Orectolobus_reticulatus:13.20647288):14.24983141,Sutorectus_tentaculatus:27.45630428):21.34253232):7.869474173,((Orectolobus_leptolineatus:2.79886054,Orectolobus_wardi:2.79886054):2.491146835,Orectolobus_halei:5.290007376):51.3783034):7.058662484,((Orectolobus_japonicus:3.465035802,Orectolobus_maculatus:3.465035802):39.59176907,(Orectolobus_hutchinsi:34.28860258,Orectolobus_parvimaculatus:34.28860258):8.768202295):20.67016938):23.91623194):60.75208026):22.1035266,((Pseudoginglymostoma_brevicaudatum:89.85151626,((Rhincodon_typus:1.715041029,Nebrius_ferrugineus:1.715042029):43.30663305,(Ginglymostoma_cirratum:0.9474585788,Stegostoma_fasciatum:0.9474575788):44.0742165):44.82984218):46.3422819,((Hemiscyllium_ocellatum:63.53364463,((Hemiscyllium_trispeculare:6.991417458,Hemiscyllium_henryi:6.991417458):19.93514048,(Hemiscyllium_galei:17.96942478,((Hemiscyllium_freycineti:3.950928032,Hemiscyllium_hallstromi:3.950928032):0.6781280339,(Hemiscyllium_strahani:2.208443563,Hemiscyllium_michaeli:2.208443563):2.420612503):13.34036871):8.957133162):36.60708669):1.310389773,(((Chiloscyllium_arabicum:7.932681842,(Chiloscyllium_hasseltii:0.524748455,Chiloscyllium_griseum:0.524748455):7.407933387):4.875320699,Chiloscyllium_burmensis:12.80800254):47.28612329,((Chiloscyllium_indicum:34.10675713,Chiloscyllium_plagiosum:34.10675713):13.7129619,Chiloscyllium_punctatum:47.81971904):12.2744078):4.749908564):71.34976376):34.3050139):42.59218175):10.60480141,((Mitsukurina_owstoni:134.2111241,((Alopias_superciliosus:99.96974986,((Alopias_vulpinus:72.96925002,Alopias_pelagicus:72.96925002):17.07828172,((Megachasma_pelagios:32.12557447,(Odontaspis_noronhai:14.41254216,Odontaspis_ferox:14.41254216):17.7130323):11.22494932,Pseudocarcharias_kamoharai:43.35052379):46.69700795):9.922218118):18.32518103,(Carcharias_taurus:102.5805044,(Cetorhinus_maximus:91.2720053,((Lamna_ditropis:42.00458329,Lamna_nasus:42.00458329):19.00560201,(Carcharodon_carcharias:53.86403447,(Isurus_oxyrinchus:40.40124976,Isurus_paucus:40.40124976):13.46278471):7.14615083):30.26182):11.30849913):15.71442646):15.91619222):61.34502001,(((Cephalurus_cephalus:9.955041462,(Poroderma_pantherinum:3.364614017,Poroderma_africanum:3.364614017):6.590427444):75.65488141,(((((Scyliorhinus_torrei:3.314098104,Scyliorhinus_haeckelii:3.314098104):19.39441154,(Scyliorhinus_canicula:16.16749142,Scyliorhinus_stellaris:16.16749142):6.541017221):5.79368327,(Scyliorhinus_cervigoni:2.083816541,Scyliorhinus_retifer:2.083816541):26.41837637):6.802751424,Scyliorhinus_capensis:35.30494434):36.59685206,((((Cephaloscyllium_fasciatum:7.189015599,Cephaloscyllium_pictum:7.189015599):22.36660692,(Cephaloscyllium_ventriosum:4.504281183,(Cephaloscyllium_signourum:0.8699138181,Cephaloscyllium_stevensi:0.8699138181):3.634367365):25.05134134):18.08144583,(Cephaloscyllium_maculatum:41.63865528,(((Cephaloscyllium_silasi:28.7542332,(Cephaloscyllium_pardelotum:20.97856321,Cephaloscyllium_speccum:20.97856321):7.775669992):2.408632527,((Cephaloscyllium_variegatum:14.94241244,(Cephaloscyllium_laticeps:13.81944885,((Cephaloscyllium_isabellum:11.17429585,(Cephaloscyllium_albipinnum:10.49348232,((Cephaloscyllium_zebrum:8.841682101,Cephaloscyllium_hiscosellum:8.841681101):1.118112451,Cephaloscyllium_umbratile:9.959794552):0.5336887676):0.680812532):0.3117720506,Cephaloscyllium_sarawakensis:11.4860679):2.333381953):1.122963588):8.946962885,Cephaloscyllium_cooki:23.88937633):7.273489403):5.506260095,Cephaloscyllium_sufflans:36.66912583):4.969529456):5.998413067):4.723532436,(((Scyliorhinus_garmani:12.32715371,Scyliorhinus_tokubee:12.32715371):7.726367517,(Scyliorhinus_comoroensis:18.01809592,(Scyliorhinus_meadi:7.433352724,Scyliorhinus_boa:7.433352724):10.58474319):2.035425309):20.11875643,(Scyliorhinus_hesperius:36.2008802,(Scyliorhinus_besnardi:13.17757833,Scyliorhinus_torazame:13.17757833):23.02330187):3.971397461):12.18832313):19.54119561):13.70812548):92.43974859,((((Schroederichthys_bivius:8.1387007,Schroederichthys_tenuis:8.1387007):101.5515934,(Schroederichthys_maculatus:6.766576398,(Schroederichthys_saurisqualus:5.890957235,Schroederichthys_chilensis:5.890957235):0.8756191631):102.9237177):42.81891313,((Aulohalaelurus_labiosus:2.891202693,Aulohalaelurus_kanakorum:2.891202693):64.37069769,((Atelomycterus_baliensis:18.22823845,Atelomycterus_marmoratus:18.22823845):39.00973161,((Atelomycterus_fasciatus:5.591448261,Atelomycterus_macleayi:5.591448261):1.697598664,Atelomycterus_marnkalha:7.289046925):49.94892313):10.02393033):85.24730688):15.00625565,(((((Eridacnis_barbouri:8.710690946,(Eridacnis_sinuans:2.845802643,Eridacnis_radcliffei:2.845802643):5.864888303):7.914124045,Ctenacis_fehlmanni:16.62481499):64.86875846,((Proscyllium_magnificum:4.064034456,Proscyllium_venustum:4.064034456):14.76182168,Proscyllium_habereri:18.82585614):62.66771731):36.50912235,(Pseudotriakis_microdon:66.95646575,(Planonasus_parini:58.23666935,(Gollum_attenuatus:15.26942161,Gollum_suluensis:15.26942161):42.96724774):8.719796402):51.04623005):31.27796832,(((Leptocharias_smithii:107.0922393,((Hemipristis_elongata:94.46893312,(((Paragaleus_tengi:17.6427788,((Paragaleus_pectoralis:3.909892049,Paragaleus_randalli:3.909892049):1.508668647,Paragaleus_leucolomatus:5.418560696):12.2242181):27.98934651,Chaenogaleus_macrostoma:45.63212531):11.63883623,(Hemigaleus_microstoma:27.07340816,Hemigaleus_australiensis:27.07340816):30.19755338):37.19797258):0.5613666624,(Galeocerdo_cuvier:86.9459618,((Eusphyra_blochii:53.1393355,(Sphyrna_mokarran:47.42667078,(Sphyrna_zygaena:43.41342972,(Sphyrna_lewini:34.24620219,(Sphyrna_corona:22.19814277,(Sphyrna_tiburo:16.53687161,(Sphyrna_tudes:12.59117963,Sphyrna_media:12.59117963):3.94569198):5.661270164):12.04805941):9.167227532):4.013242061):5.712664722):26.62345531,(((Loxodon_macrorhinus:48.90488323,(Scoliodon_macrorhynchos:8.820852934,Scoliodon_laticaudus:8.820852934):40.0840303):16.57297315,((Rhizoprionodon_oligolinx:18.88932096,(Rhizoprionodon_acutus:16.18622104,Rhizoprionodon_taylori:16.18622104):2.703099917):28.935825,((Rhizoprionodon_longurio:14.32369121,Rhizoprionodon_lalandii:14.32369121):6.111505624,(Rhizoprionodon_terraenovae:5.234978623,Rhizoprionodon_porosus:5.234978623):15.20021821):27.38994912):17.65270943):7.022891076,(((((((Triaenodon_obesus:2.54764861,Carcharhinus_sorrah:2.54764961):19.78250327,Carcharhinus_amboinensis:22.33015288):14.81174703,(((Carcharhinus_borneensis:24.30103648,Carcharhinus_macloti:24.30103648):1.015984226,(Carcharhinus_sealei:16.7032399,Carcharhinus_dussumieri:16.7032399):8.613779806):6.798182979,Carcharhinus_porosus:32.11520369):5.026696227):0.9944017699,((Carcharhinus_perezii:29.38734982,((Carcharhinus_tjutjot:10.65879678,Carcharhinus_cerdale:10.65879678):8.05936036,((Isogomphodon_oxyrhynchus:1.600992739,Carcharhinus_longimanus:1.600992739):15.24528018,(Carcharhinus_coatesi:11.45412096,(Carcharhinus_obscurus:1.812165757,Carcharhinus_galapagensis:1.812165757):9.641956198):5.392151969):1.87188422):10.66919268):6.524379538,(Carcharhinus_brachyurus:18.48480739,Carcharhinus_brevipinna:18.48480739):17.42692097):2.224572324):6.76243056,(((((Prionace_glauca:10.05156668,Carcharhinus_albimarginatus:10.05156668):14.00699862,Carcharhinus_hemiodon:24.0585663):6.323652981,(Carcharhinus_falciformis:6.55314985,Carcharhinus_wheeleri:6.55315085):23.82906843):7.400774404,Carcharhinus_amblyrhynchos:37.78299268):3.009925421,((Carcharhinus_leucas:8.410255553,(Carcharhinus_altimus:6.337232296,Carcharhinus_plumbeus:6.337232296):2.073024257):13.73914499,(Carcharhinus_isodon:19.94134125,(Nasolamia_velox:8.570558778,Carcharhinus_acronotus:8.570558778):11.37078247):2.208059295):18.64351756):4.105813144):11.7376862,(Carcharhinus_signatus:40.09984274,((Carcharhinus_fitzroyensis:33.31310476,(Carcharhinus_melanopterus:9.226559455,Carcharhinus_cautus:9.226559455):24.08654531):5.711069596,((Carcharhinus_limbatus:6.667418612,(Carcharhinus_tilstoni:5.79104573,Carcharhinus_amblyrhynchoides:5.79104573):0.876371882):2.130438024,Carcharhinus_leiodon:8.797855636):30.22631872):1.075669385):16.53657471):7.707605265,((Negaprion_acutidens:24.74747595,Negaprion_brevirostris:24.74747595):25.05448423,((Lamiopsis_temminckii:25.12177474,Lamiopsis_tephrodes:25.12177474):10.17607834,((Glyphis_glyphis:17.66547909,(Glyphis_garricki:8.820389447,Glyphis_siamensis:8.820389447):8.845089648):10.2966767,((Glyphis_fowlerae:1.984076128,Glyphis_gangeticus:1.984076128):23.14678293,Glyphis_sp.1:25.13085906):2.83129674):7.33569729):14.5041061):14.54206353):8.156723744):7.262044354):7.183169985):8.084337986):12.06193951):2.229628852,(((Triakis_semifasciata:41.32357614,((Triakis_maculata:1.483507267,Triakis_acutipinna:1.483507267):1.959037845,Triakis_scyllium:3.442545112):37.88103103):45.14173476,(Furgaleus_macki:54.91111657,(Hemitriakis_indroyonoi:41.69888773,((Hemitriakis_japanica:3.14389151,(Hemitriakis_abdita:2.849363983,Hemitriakis_leucoperiptera:2.849363983):0.2945275274):23.37255922,(Hemitriakis_falcata:4.388073915,Hemitriakis_complicofasciata:4.388073915):22.12837682):15.182437):13.21222884):31.55419434):22.01328173,(((Gogolia_filewoodi:45.87822513,(Iago_garricki:45.18324133,Iago_omanensis:45.18324133):0.694983798):19.24910388,(Galeorhinus_galeus:63.06271669,Hypogaleus_hyugaensis:63.06271669):2.064612314):39.85048517,((((((Mustelus_mento:1.007166999,Mustelus_albipinnis:1.007166999):10.47476897,((Mustelus_palumbes:3.80104806,Mustelus_asterias:3.80104806):7.457029063,(Mustelus_stevensi:6.330832452,(Mustelus_antarcticus:5.339255298,(Mustelus_dorsalis:2.337101569,Mustelus_lenticulatus:2.337101569):3.002153728):0.9915781542):4.927243671):0.223858845):1.892883402,Mustelus_manazo:13.37481837):6.81737378,Mustelus_schmitti:20.19219315):35.78000078,((Mustelus_griseus:36.5700188,((Scylliogaleus_quecketti:34.12247321,Triakis_megalopterus:34.12247321):0.9692223434,Mustelus_mangalorensis:35.09169555):1.478323253):3.709686355,Mustelus_whitneyi:40.27970516):15.69248877):13.30129623,((Mustelus_sinusmexicanus:41.1552099,((Mustelus_norrisi:23.98723683,((Mustelus_canis:18.8959876,(Mustelus_higmani:10.19952419,Mustelus_henlei:10.19952419):8.696463411):0.3851234186,Mustelus_walkeri:19.28111102):4.70612581):4.643084131,(((Mustelus_lunulatus:18.79953762,(Mustelus_mosis:10.90941679,(Mustelus_ravidus:2.616055768,Mustelus_widodoi:2.616055768):8.293361022):7.890120833):2.292606408,(Mustelus_mustelus:2.088403143,Mustelus_fasciatus:2.088403143):19.00374189):0.9500885653,Mustelus_californicus:22.0422346):6.588087366):12.52488894):15.07172521,(Mustelus_minicanis:5.372763639,Mustelus_punctulatus:5.372763639):50.85417147):13.04655505):35.70432302):3.500779463):0.8432745072):18.67871258,((((((Bythaelurus_hispidus:9.483733485,(Bythaelurus_immaculatus:2.314472081,Bythaelurus_clevai:2.314472081):7.169261404):28.69656724,Bythaelurus_canescens:38.18030072):28.10877385,((Bythaelurus_alcockii:5.691939533,Bythaelurus_giddingsi:5.691939533):25.68566054,(Bythaelurus_lutarius:6.89256777,(Bythaelurus_incanus:1.061122649,Bythaelurus_dawsoni:1.061122649):5.831445122):24.48503231):34.91147449):17.95220284,((Figaro_boardmani:30.75405074,Figaro_striatus:30.75405074):17.55586784,(Asymbolus_funebris:35.54050268,(Asymbolus_pallidus:29.40412131,(Asymbolus_galacticus:26.88628887,((Asymbolus_occiduus:2.775123752,(Asymbolus_vincenti:2.339680136,Asymbolus_parvus:2.339680136):0.435443616):15.51525631,((Asymbolus_submaculatus:1.173350135,Asymbolus_rubiginosus:1.173350135):2.022175151,Asymbolus_analis:3.195525285):15.09485478):8.595908808):2.517832444):6.136381367):12.76941589):35.93135783):24.35045098,(((Galeus_antillensis:7.343732315,Galeus_murinus:7.343732315):60.42023634,((((Galeus_piperatus:1.780975267,Galeus_arae:1.780975267):5.102239248,(Galeus_gracilis:6.829413204,Galeus_springeri:6.829413204):0.05380131062):21.48643347,Galeus_longirostris:28.36964798):3.164008547,((Galeus_mincaronei:21.53901611,(Galeus_schultzi:5.219641029,Galeus_nipponensis:5.219641029):16.31937509):5.282193644,(Galeus_polli:19.12528105,((Galeus_atlanticus:11.53266903,(Galeus_cadenati:7.510371514,Galeus_priapus:7.510371514):4.022297513):0.8642701572,Galeus_melastomus:12.39693918):6.728341865):7.695928709):4.712446773):36.23031213):31.89060977,((((Apristurus_canutus:70.89205627,(((((Apristurus_sibogae:15.72528345,Apristurus_brunneus:15.72528345):8.072755835,(Apristurus_laurussonii:22.44892634,(Apristurus_melanoasper:7.591394095,(Apristurus_platyrhynchus:7.397969596,Apristurus_sinensis:7.397969596):0.1934244992):14.85753325):1.349111944):5.252417669,Apristurus_japonicus:29.05045696):1.850256656,Apristurus_internatus:30.90071361):33.0284104,((((Apristurus_herklotsi:12.5794805,Apristurus_micropterygeus:12.5794805):3.125912202,Apristurus_spongiceps:15.7053927):8.299281952,Apristurus_macrorhynchus:24.00467365):29.21930485,Apristurus_exsanguis:53.2239785):10.70514451):6.962932256):10.20678597,Apristurus_saldanha:81.09884224):7.065326616,(((((Parmaturus_pilosus:3.863679753,(Parmaturus_campechiensis:3.347089727,Parmaturus_bigus:3.347089727):0.5165900253):9.086203566,(Parmaturus_albimarginatus:3.090710639,Parmaturus_sp.:3.090710639):9.859172679):6.719280529,((Parmaturus_melanobranchus:9.779649067,Parmaturus_albipenis:9.779649067):2.301776576,(Parmaturus_macmillani:0.8769423662,Parmaturus_xaniurus:0.8769423662):11.20448328):7.587738205):14.96656728,Parmaturus_lanatus:34.63573113):29.74324948,(Galeus_eastmani:48.47522869,Galeus_sauteri:48.47522869):15.90375093):23.78518824):7.380294526,(((Apristurus_investigatoris:34.35170453,(Apristurus_stenseni:8.949805272,Apristurus_riveri:8.949805272):25.40189926):20.73477259,((Apristurus_gibbosus:8.072209594,Apristurus_australis:8.072208594):40.36019453,(Apristurus_longicephalus:7.271468006,Apristurus_nasutus:7.271469006):41.16093512):6.654072992):2.202368655,(((Apristurus_macrostomus:8.42071245,(Apristurus_albisoma:3.608672727,Apristurus_bucephalus:3.608672727):4.812039722):21.03470219,((((Apristurus_indicus:3.430665966,Apristurus_fedorovi:3.430665966):6.973771697,Apristurus_kampae:10.40443766):6.466682086,(Apristurus_parvipinnis:4.236678078,Apristurus_aphyodes:4.236678078):12.63444167):4.764503466,(Apristurus_ampliceps:6.462678846,(Apristurus_manis:0.5912123198,Apristurus_microps:0.5912123198):5.871466526):15.17294437):7.819791425):2.653150503,(Apristurus_pinguis:12.64791336,Apristurus_profundorum:12.64791336):19.46065179):25.18028063):38.25561761):4.110116047):8.937149966):18.0806688,(((((Halaelurus_quagga:15.79295174,Halaelurus_maculosus:15.79295174):11.57301442,(Halaelurus_sellus:0.3148469232,Halaelurus_buergeri:0.3148469232):27.05111924):16.97671886,(Halaelurus_boesemani:30.06205835,(Halaelurus_natalensis:21.4140913,Halaelurus_lineatus:21.4140913):8.647967049):14.28062668):4.060016728,(Haploblepharus_fuscus:38.26163983,(Haploblepharus_edwardsii:11.7341343,(Haploblepharus_pictus:4.885439657,Haploblepharus_kistnasamyi:4.885439657):6.848694645):26.52750552):10.14106093):59.87827239,(Pentanchus_profundicolus:27.92186427,(Holohalaelurus_grennian:17.33014992,(Holohalaelurus_melanostigma:12.490452,((Holohalaelurus_favus:2.545224266,Holohalaelurus_regani:2.545224266):0.1340222516,Holohalaelurus_punctatus:2.679246517):9.811205485):4.839697922):10.59171434):80.35910989):18.39142204):1.328183538):21.28008339):18.2347998):10.53420754):17.50647266):28.13965111):14.9125452):15.88753659):22.86254789):98.15290602)Chondrichthyes; diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Holocephali_Inoue2010.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Holocephali_Inoue2010.PHY deleted file mode 100755 index 5c3332ec..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Holocephali_Inoue2010.PHY +++ /dev/null @@ -1,4 +0,0 @@ -[Holocephalans from: Evolutionary Origin and Phylogeny of the Modern Holocephalans (Chondrichthyes: Chimaeriformes): A Mitogenomic Perspective https://doi.org/10.1093/molbev/msq147 - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(Callorhinchidae_ott550645@:170,((Rhinochimaera_ott886233@:50,(Harriotta_ott776014@:50,Neoharriotta_ott195188@:50))Rhinochimaeridae:70,(Chimaera_ott29488@:80,Hydrolagus_ott29492@:80)Chimaeridae:40):50)Holocephali; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Carcharhinicae_minus.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Carcharhinicae_minus.PHY deleted file mode 100755 index 68c2bd62..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Carcharhinicae_minus.PHY +++ /dev/null @@ -1,9 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -The OpenTree sharks (draftversion 3) are hopelessly mixed - -Where possible, I have replaced monophyletic genera with their OToL equivalents, to increase the number of species present. - -To resolve the few polytomies, the opentree has been used. In the case of Triaenodon_obesus it has been shuffled to a new place, as per the OpenTree. - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(((((((((((Apristurus_ampliceps,Apristurus_manis),Apristurus_profundorum),Apristurus_kampae),Apristurus_australis),(((((Apristurus_brunneus,Apristurus_laurussonii),Apristurus_melanoasper),Apristurus_platyrhynchus),(Apristurus_exsanguis,Apristurus_macrorhynchus)),(Galeus_sauteri,Parmaturus_ott821052@))),((Galeus_arae,(Galeus_polli,Galeus_melastomus)),Galeus_murinus)),((Asymbolus_ott296645@,Figaro_ott105538@),Bythaelurus_ott903791@)),((((Halaelurus_lineatus,Halaelurus_natalensis),Haploblepharus_ott201994@),(Halaelurus_maculosus,Halaelurus_sellus)),Holohalaelurus_ott103472@)),Proscyllium_ott821049@),(((((((((((((Carcharhinus_acronotus,Nasolamia_velox),Carcharhinus_isodon),Isogomphodon_oxyrhynchus),((Carcharhinus_dussumieri,Carcharhinus_sealei),(Carcharhinus_borneensis,Carcharhinus_macloti))),((((Carcharhinus_amblyrhynchos,Carcharhinus_wheeleri),((Carcharhinus_falciformis,Prionace_glauca),Carcharhinus_albimarginatus)),(Carcharhinus_altimus,Carcharhinus_plumbeus)),((((((((Carcharhinus_amblyrhynchoides,(Carcharhinus_limbatus,Carcharhinus_tilstoni)),Carcharhinus_leiodon),Carcharhinus_fitzroyensis),(Carcharhinus_cautus,Carcharhinus_melanopterus)),Carcharhinus_signatus),Carcharhinus_sorrah),(((Carcharhinus_amboinensis,Carcharhinus_leucas),Triaenodon_obesus),(Carcharhinus_brachyurus,Carcharhinus_brevipinna))),(((Carcharhinus_galapagensis,Carcharhinus_obscurus),Carcharhinus_longimanus),Carcharhinus_perezii)))),Carcharhinus_porosus),((Glyphis_ott541142@,Lamiopsis_ott19958@),Negaprion_ott450140@)),((Loxodon_macrorhinus,(Scoliodon_laticaudus,Scoliodon_macrorhynchos)),Rhizoprionodon_ott846406@)),((Eusphyra_blochii,(Sphyrna_mokarran,Sphyrna_zygaena)),(((Sphyrna_tiburo,Sphyrna_tudes),Sphyrna_corona),Sphyrna_lewini))),Galeocerdo_cuvier),Leptocharias_smithii),(((Chaenogaleus, Hemigaleus), Paragaleus),Hemipristis_elongata)),(((((Furgaleus_ott401912@,Hemitriakis_ott32028@),(Triakis_scyllium,Triakis_semifasciata)),(((((((Mustelus_albipinnis,Mustelus_canis),Mustelus_henlei),Mustelus_norrisi),Mustelus_californicus),((Mustelus_mosis,(Mustelus_ravidus,Mustelus_widodoi)),Mustelus_mustelus)),Mustelus_lunulatus),(((((Mustelus_antarcticus,Mustelus_lenticulatus),Mustelus_stevensi),((Mustelus_asterias,Mustelus_palumbes),Mustelus_schmitti)),Mustelus_manazo),(Scylliogaleus_quecketti,Triakis_megalopterus)))),(Galeorhinus_ott29487@,Hypogaleus_ott1035202@)),Iago_ott1037426@))),((Gollum_ott1037449@,Pseudotriakis_ott261215@),Eridacnis_ott73188@)); \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Dalatiidae.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Dalatiidae.PHY deleted file mode 100755 index add0050d..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Dalatiidae.PHY +++ /dev/null @@ -1,4 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((Squaliolus_ott956134@,(Euprotomicroides_ott3595317@,Euprotomicrus_ott547469@)),(Dalatias_ott1027234@,Isistius_ott277031@))Dalatiidae; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Etmopteridae.phy b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Etmopteridae.phy deleted file mode 100755 index 60631bea..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Etmopteridae.phy +++ /dev/null @@ -1,5 +0,0 @@ -[Lantern shark arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' https://doi.org/10.1201/B11867-9 (see http://sharksrays.org) -Etmopteridae (esp Etmopterus) is scattered all over the OpenTree, so here we use the subtree from sharksrays.org, which is unfortunately missing many species - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((((((Centroscyllium_fabricii,Centroscyllium_ritteri),(Centroscyllium_granulatum,Centroscyllium_nigrum)),Centroscyllium_excelsum),Aculeola_nigra),Trigonognathus_kabeyai),((((Etmopterus_granulosus,((Etmopterus_spinax,Etmopterus_princeps),Etmopterus_unicolor)),(Etmopterus_gracilispinis,((Etmopterus_virens,Etmopterus_polli),Etmopterus_schultzi))),((Etmopterus_bigelowi,Etmopterus_pusillus),(Etmopterus_splendidus,Etmopterus_sentosus))),(Etmopterus_lucifer,((Etmopterus_molleri,Etmopterus_brachyurus),Etmopterus_sheikoi))))Etmopteridae; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Pristiophoridae.phy b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Pristiophoridae.phy deleted file mode 100755 index bded4d66..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Pristiophoridae.phy +++ /dev/null @@ -1,6 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -Pristiophoridae is non-monophyletic in the Opentree, but this is probably an error. So we use the subtree from sharksrays.org, -but unfortunately this is missing about 5 species of Pristiophorus - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(Pliotrema_warreni,((Pristiophorus_nudipinnis,Pristiophorus_cirratus),Pristiophorus_japonicus))Pristiophoridae; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae2.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae2.PHY deleted file mode 100755 index e445416c..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae2.PHY +++ /dev/null @@ -1,5 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -Note that this was labelled Scyliorhinidae-I in TimeTree. OpenTree has non-monophyletic Atelomycterus (catfish), so here we use the Naylor tree, which unfortunately is missing about 5 spp. - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(((Atelomycterus_marmoratus,Atelomycterus_marnkalha),Aulohalaelurus_ott541139@),Schroederichthys_ott618631@); \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae3.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae3.PHY deleted file mode 100755 index b771d697..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae3.PHY +++ /dev/null @@ -1,5 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -Note that this was labelled Scyliorhinidae-2 in TimeTree. OpenTree has non monophyletic Scyliorhinus (catfish), so here we use the Naylor tree, which unfortunately is missing about 10 spp. - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(Cephaloscyllium_ott481014@,((Poroderma_africanum,Poroderma_pantherinum),(Scyliorhinus_canicula,(Scyliorhinus_capensis,(Scyliorhinus_retifer,Scyliorhinus_stellaris))))); \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Selachimorpha.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Selachimorpha.PHY deleted file mode 100755 index 96f71918..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Selachimorpha.PHY +++ /dev/null @@ -1,37 +0,0 @@ -[Selachimorpha (Selachii) tree, The opentree (draftversion3) is rather screwed up for sharks, especially the Carcharhiniformes, so ths is the most complex of the trees and involved inclusion of many other files, all labelled Naylor2012***, as they are mostly based on Naylor et al (2012): 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' from Biology of Sharks and Their Relatives (https://doi.org/10.1201/B11867-9 see http://sharksrays.org). Relative dates from an earlier Naylor paper in the timetree book (http://hedgeslab.com/pubs/216.pdf), with some polytomies broken using https://doi.org/10.1186/s12862-015-0446-6 - -Then rescaled by path d8, using this: - -#sequence length should be irrelevant here, set it to something high -Sequence length = 1000000; - -I subtract by hand a few misleading species around the great white. - -(Chimaeridae:470.2079,(Rajidae:392.60970000000003,((((((CARCHARHINICAE_MINUS@,SCYLIORHINIDAE2@:178.291):47.1132,SCYLIORHINIDAE3@:225.4042):32.3326,(Lamniformes_minus_Mitsukurinidae__ott~32038-801828-760455@:183.8337,Mitsukurinidae_ott801828@:183.8337)Lamniformes_ott32038:73.903):29.5612,(((((Ginglymostoma_ott400230@:0,Nebrius_ott833444@:0)Ginglymostomatidae:52.6559,((Stegostomatidae_ott696403@:0,Pseudoginglymostoma_ott356286@:0),Rhincodontidae_ott738324@:0):52.6559):52.6559,Hemiscylliidae_ott438189@:105.3118):88.6836,(Brachaeluridae_ott274901@:141.3395,Orectolobidae_ott572732@:141.3395):53.5797):41.5704,Parascylliidae_ott154216@:236.4896)Orectolobiformes:50.8083):29.5612,Heterodontidae_ott335517@:315.9353):32.3326,(((Squalidae_ott856584@:168.1293,((DALATIIDAE@:135.7968,(SOMNIOSIDAEOXYNOTIDAE@:133.9492,ETMOPTERIDAE@:135.7968):0.0):0.0,Centrophoridae_ott852403@:133.9492):35.0):92.3788,(SQUATINIDAE@:213.3949,(Echinorhiniformes_ott340760@:213.3949,PRISTIOPHORIDAE@:213.3949):0):49):62.8175,(Chlamydoselachidae_ott1093534@:233.7182,Notorynchidae_plus_Hexanchidae__ott~32032-1093534@:233.7182)Hexanchiformes_ott32032:89.6074):24.0185):42.4942):78.5219); - - -mrca: Chimaeridae,Carcharhinicae, fixage=420; -mrca: Rajidae,Carcharhinicae, fixage=300; - -#from N.C. Aschliman et al. / Molecular Phylogenetics and Evolution 63 (2012) 28–42 Supp Mat https://doi.org/10.1016/j.ympev.2011.12.012 - -mrca:Squalidae_ott856584@,Heterodontidae_ott335517@, fixage=225; -mrca:Heterodontidae_ott335517@,SCYLIORHINIDAE3@, fixage=200; -mrca:CARCHARHINICAE_MINUS@,SCYLIORHINIDAE3@, fixage=170; - -#from Straube et al. BMC Evolutionary Biology (2015) 15:162 (http://www.biomedcentral.com/content/pdf/s12862-015-0446-6.pdf) Fig 2 https://doi.org/10.1186/s12862-015-0446-6 - -mrca:Squalidae_ott856584@,Chlamydoselachidae_ott1093534, fixage=202.8; -mrca:Squalidae_ott856584@,Centrophoridae_ott852403@, fixage=132.86; -mrca:Centrophoridae_ott852403@,DALATIIDAE@, fixage=126.68; -mrca:DALATIIDAE@,ETMOPTERIDAE@, fixage=116.1; -mrca:ETMOPTERIDAE@,SOMNIOSIDAEOXYNOTIDAE@, fixage=110.51; -mrca:Squalidae_ott856584@,SQUATINIDAE@, fixage=177.34; -mrca:SQUATINIDAE@,Echinorhiniformes_ott340760@, fixage=147.59; - -##this produces (Chimaeridae:420.000000,(Rajidae:300.000000,((((((CARCHARHINICAE_MINUS@:134.467193,SCYLIORHINIDAE2@:134.467193):35.532807,SCYLIORHINIDAE3@:170.000000):3.555331,(Lamniformes_minus_Mitsukurinidae__ott~32038-801828-760455@:116.193063,Mitsukurinidae_ott801828@:116.193063)Lamniformes_ott32038:57.362269):9.696336,(((((Ginglymostoma_ott400230@:0.000000,Nebrius_ott833444@:0.000000)Ginglymostomatidae:33.281440,((Stegostomatidae_ott696403@:0.000000,Pseudoginglymostoma_ott356286@:0.000000):0.000000,Rhincodontidae_ott738324@:0.000000):33.281440):33.281440,Hemiscylliidae_ott438189@:66.562880):56.198912,(Brachaeluridae_ott274901@:89.334379,Orectolobidae_ott572732@:89.334379):33.427413):26.323449,Parascylliidae_ott154216@:149.085241)Orectolobiformes:34.166426):16.748333,Heterodontidae_ott335517@:200.000000):25.000000,(((Squalidae_ott856584@:132.860000,((DALATIIDAE@:116.100000,(SOMNIOSIDAEOXYNOTIDAE@:110.510000,ETMOPTERIDAE@:110.510000):5.590000):10.580000,Centrophoridae_ott852403@:126.680000):6.180000):44.480000,(SQUATINIDAE@:147.590000,(Echinorhiniformes_ott340760@:147.590000,PRISTIOPHORIDAE@:147.590000):0.000000):29.750000):25.460000,(Chlamydoselachidae_ott1093534@:146.024768,Notorynchidae_plus_Hexanchidae__ott~32032-1093534@:146.024768)Hexanchiformes:56.775232):22.200000):75.000000):120.000000) - -From which the Chimaeridae and Rajidae branches have been removed - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((((((CARCHARHINICAE_MINUS@:134.467193,SCYLIORHINIDAE2@:134.467193):35.532807,SCYLIORHINIDAE3@:170.000000):3.555331,(Lamniformes_minus_Mitsukurinidae__ott~32038-801828-760455@:116.193063,Mitsukurinidae_ott801828@:116.193063)Lamniformes_ott32038:57.362269):9.696336,(((((Ginglymostoma_ott400230@:0.000000,Nebrius_ott833444@:0.000000)Ginglymostomatidae:33.281440,((Stegostomatidae_ott696403@:0.000000,Pseudoginglymostoma_ott356286@:0.000000):0.000000,Rhincodontidae_ott738324@:0.000000):33.281440):33.281440,Hemiscylliidae_ott438189@:66.562880):56.198912,(Brachaeluridae_ott274901@:89.334379,Orectolobidae_ott572732@:89.334379):33.427413):26.323449,Parascylliidae_ott154216@:149.085241)Orectolobiformes:34.166426):16.748333,Heterodontidae_ott335517@:200.000000):25.000000,(((Squalidae_ott856584@:132.860000,((DALATIIDAE@:116.100000,(SOMNIOSIDAEOXYNOTIDAE@:110.510000,ETMOPTERIDAE@:110.510000):5.590000):10.580000,Centrophoridae_ott852403@:126.680000):6.180000):44.480000,(SQUATINIDAE@:147.590000,(Echinorhiniformes_ott340760@:147.590000,PRISTIOPHORIDAE@:147.590000):0.000000):29.750000):25.460000,(Chlamydoselachidae_ott1093534@:146.024768,Notorynchidae_plus_Hexanchidae__ott~32032-1093534@:146.024768)Hexanchiformes:56.775232):22.200000)Selachii; diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Somniosidae_Oxynotidae.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Somniosidae_Oxynotidae.PHY deleted file mode 100755 index 275bb60a..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Somniosidae_Oxynotidae.PHY +++ /dev/null @@ -1,6 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org). NB, the Somniosidae/Oxynotidae -clade from sharksrays.org should also include Centroscymnus & Proscymnodon, but these are hard to include -from the opentree, as they are marked as non monophyletic, so have simply been omitted (yuck) - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(Somniosus_ott442056@,(Scymnodalatias_ott3595305@,((Centroselachus_ott756559@,Zameus_ott399864@),(Scymnodon_ott956139@,Oxynotidae_ott250745@))))SomniosidaeOxynotidae; diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Squatinidae.phy b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Squatinidae.phy deleted file mode 100755 index 603406f4..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Squatinidae.phy +++ /dev/null @@ -1,5 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -Angel sharks (Squatinidae) are non monophyletic in the Opentree, but this is probably an error. So we use the subtree from sharksrays.org, with additions from the opentree (based on VélezZuazo), but unfortunately this is missing about 20 species. - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((Squatina_albipunctata,(Squatina_californica,Squatina_dumeril)),(((Squatina_tergocellatoides,Squatina_japonica),(Squatina_legnota,Squatina_formosa)),((Squatina_squatina,Squatina_aculeata),Squatina_oculata)))Squatinidae; \ No newline at end of file diff --git a/oz_tree_build/tree_build/token_to_oz_tree_file_mapping.py b/oz_tree_build/tree_build/token_to_oz_tree_file_mapping.py index f5d53c98..209d8f18 100644 --- a/oz_tree_build/tree_build/token_to_oz_tree_file_mapping.py +++ b/oz_tree_build/tree_build/token_to_oz_tree_file_mapping.py @@ -30,67 +30,15 @@ "GNATHOSTOMATA": {"file": "BonyFishOpenTree.PHY", "edge_length": 65, "taxon": None}, # for fewer species but with dates, try deepfin2, with Concestor 20 @ ~ 430Ma # tree.substitute('GNATHOSTOMATA@', 'BespokeTree/include_files/Deepfin2.phy', 37.6) + # Species-level chondrichthyan tree, crown at 375.511333. 460 - 375.511333 = 84.488667 keeps + # Gnathostomata at 460Mya, agreeing with the Euteleostomi side of BonyFishOpenTree.PHY. + # It is self-contained, so the old Renz2013 / Inoue2010 / Aschliman2012 / Naylor2012* split + # (and their HOLOCEPHALI@, BATOIDEA@, SELACHII@, ... tokens) are no longer needed. "CHONDRICHTHYES": { - "file": "Chondrichthyes_Renz2013.phy", - "edge_length": 40, + "file": "Chondrichthyes_Stein2018.PHY", + "edge_length": 84.488667, "taxon": None, }, - "HOLOCEPHALI": { - "file": "Holocephali_Inoue2010.PHY", - "edge_length": 250, - "taxon": None, - }, - "BATOIDEA": { - "file": "Batoids_Aschliman2012.PHY", - "edge_length": 100, - "taxon": None, - }, - # sharks are problematic in OToL v3 & 4, hence lots of files included here - "SELACHII": { - "file": "Naylor2012Selachimorpha.PHY", - "edge_length": 75, - "taxon": None, - }, - "DALATIIDAE": { - "file": "Naylor2012Dalatiidae.PHY", - "edge_length": 116.1, - "taxon": None, - }, - "SOMNIOSIDAEOXYNOTIDAE": { - "file": "Naylor2012Somniosidae_Oxynotidae.PHY", - "edge_length": 110.51, - "taxon": None, - }, - "ETMOPTERIDAE": { - "file": "Naylor2012Etmopteridae.phy", - "edge_length": 110.51, - "taxon": None, - }, - "SQUATINIDAE": { - "file": "Naylor2012Squatinidae.phy", - "edge_length": 147.59, - "taxon": None, - }, - "PRISTIOPHORIDAE": { - "file": "Naylor2012Pristiophoridae.phy", - "edge_length": 147.59, - "taxon": None, - }, - "SCYLIORHINIDAE3": { - "file": "Naylor2012Scyliorhinidae3.PHY", - "edge_length": 170, - "taxon": None, - }, - "SCYLIORHINIDAE2": { - "file": "Naylor2012Scyliorhinidae2.PHY", - "edge_length": 134.467193, - "taxon": None, - }, - "CARCHARHINICAE_MINUS": { - "file": "Naylor2012Carcharhinicae_minus.PHY", - "edge_length": 134.467193, - "taxon": "Most_Carcharhinicae_", - }, # Choanoflagellates: http://www.pnas.org/content/105/43/16641.short ## NB: to use the original deepfin tree, substitute these text strings back in instead ## # tree.substitute('TETRAPODA@', '(Xenopus_tropicalis:335.4,(Monodelphis_domestica:129,(Mus_musculus:71.12,Homo_sapiens:71.12):57.88):206.4)Tetrapodomorpha:46.5'); # noqa E501 From 0ec94045523f0bcc264dd41b3a6dc56781f92053 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 27 Aug 2026 10:19:08 +0000 Subject: [PATCH 34/62] AllLife/OpenTreeParts: Add copy of rotifers from previous tree #130 Rotifers have gone AWOL, add a snippet from the old tree to get by for now. --- data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/471706.nwk | 1 + 1 file changed, 1 insertion(+) create mode 100644 data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/471706.nwk diff --git a/data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/471706.nwk b/data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/471706.nwk new file mode 100644 index 00000000..8b7f34f4 --- /dev/null +++ b/data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/471706.nwk @@ -0,0 +1 @@ +(((((Adineta_oculata_ott4101,Adineta_barbata_ott122767,Adineta_tuberculosa_ott653951,Adineta_vaga_ott681215,Adineta_steineri_ott743723,Adineta_ricciae_ott743825,Adineta_gracilis_ott991846,Adineta_grandis_ott991851,Adineta_environmental_sample_ott4952810,Adineta_longicornis_ott4952811,Adineta_elongata_ott4952812,Adineta_glauca_ott4952813,Adineta_cuneata_ott4952814,Adineta_bartosi_ott4952815,Adineta_acuticornis_ott4952816,Adineta_editae_ott5974468,Adineta_emsliei_ott5974469,Adineta_fontanetoi_ott5974470,'Adineta_vaga_complex_sp._A_JFF-2016_ott5974472','Adineta_vaga_complex_sp._B_JFF-2016_ott5974473','Adineta_vaga_complex_sp._C_JFF-2016_ott5974474','Adineta_vaga_complex_sp._D_JFF-2016_ott5974475','Adineta_vaga_complex_sp._E_JFF-2016_ott5974476','Adineta_vaga_complex_sp._F_JFF-2016_ott5974477',Adineta_beysunae_ott7506288,Adineta_coatsi_ott7506289)Adineta_ott681218,(Bradyscela_clauda_ott991847,Bradyscela_granulosa_ott4952817,Bradyscela_hoonsooi_ott6367874)Bradyscela_ott991844)Adinetidae_ott787513)Adinetida_ott157975,(Rotaria_tardigrada_ott48499,Rotaria_neptunoida_ott48504,Rotaria_magnacalcarata_ott48505,Rotaria_sordida_ott48506,Rotaria_socialis_ott48507,Rotaria_citrina_ott621261,Rotaria_macrura_ott621262,Rotaria_macroceros_ott755138,Rotaria_rotatoria_ott1018353,Rotaria_neptunia_ott1018354,Rotaria_tridens_ott4953152,Rotaria_elongata_ott4953153,Rotaria_haptica_ott4953154,Rotaria_laticeps_ott4953155,Rotaria_montana_ott4953156,Rotaria_quadrangularis_ott4953157,Rotaria_spicata_ott4953158,Rotaria_trisecata_ott4953159,Rotaria_exoculis_ott4953160,Rotaria_mento_ott4953161,Rotaria_quadrioculata_ott4953162,Rotaria_ovata_ott4953163,Rotaria_murrayi_ott4953164,Rotaria_curtipes_ott4953165,'Rotaria_cf._rotatoria_DDF-2018_ott7506308',Rotaria_megarostris_ott7506309)Rotaria_ott1018360,(((Philodina_plena_ott122765,Philodina_flaviceps_ott124555,Philodina_roseola_ott236073,Philodina_citrina_ott342351,Philodina_megalotrocha_ott400740,Philodina_vorax_ott644416,Philodina_duplicalcar_ott726737,Philodina_rugosa_ott790563,Philodina_quadrata_ott4953037,Philodina_americana_ott4953038,Philodina_jeanneli_ott4953039,Philodina_eurystephana_ott4953040,Philodina_gregaria_ott4953041,Philodina_nitida_ott4953045,Philodina_calceata_ott4953046,Philodina_australis_ott4953047,Philodina_parvicalcar_ott4953048,Philodina_praelonga_ott4953049,Philodina_dobrogensis_ott4953050,Philodina_proterva_ott4953052,Philodina_indica_ott4953053,Philodina_amethystina_ott4953054,Philodina_lepta_ott4953055,Philodina_antarctica_ott4953056,Philodina_grandis_ott4953057,Philodina_alata_ott4953058,Philodina_inopinata_ott4953059,Philodina_scabra_ott4953060,Philodina_erythrophthalma_ott4953061,Philodina_rapida_ott4953062,Philodina_foissneri_ott4953063,Philodina_striata_ott4953064,Philodina_cristata_ott4953065,Philodina_nemoralis_ott4953066,Philodina_brevipes_ott4953067,Philodina_childi_ott4953068,Philodina_tridentata_ott4953069,Philodina_arndti_ott4953070,Philodina_squamosa_ott4953071,Philodina_tenuicalcar_ott4953072,Philodina_morigera_ott4953073,Philodina_patula_ott4953074,Philodina_tranquilla_ott4953075,Philodina_convergens_ott4953076,Philodina_dartnallis_ott5974492,(Philodina_acuticornis_odiosa_ott7506298)Philodina_acuticornis_ott831317,Philodina_koreana_ott7506299,Philodina_shackletoni_ott7506300)Philodina_ott831316,(Macrotrachela_papillosa_ott267232,Macrotrachela_musculosa_ott267233,Macrotrachela_bullata_ott267234,Macrotrachela_multispinosa_ott267236,Macrotrachela_habita_ott267237,Macrotrachela_latior_ott377601,Macrotrachela_ehrenbergi_ott570855,Macrotrachela_quadricornifera_ott681209,Macrotrachela_plicata_ott790558,Macrotrachela_extensa_ott4952961,Macrotrachela_bilfingeri_ott4952962,Macrotrachela_rostrata_ott4952963,Macrotrachela_aculeata_ott4952964,Macrotrachela_zichendrahti_ott4952965,Macrotrachela_nixa_ott4952966,Macrotrachela_ehrenbergii_ott4952967,Macrotrachela_cuthberti_ott4952968,Macrotrachela_smithi_ott4952969,Macrotrachela_angusta_ott4952970,Macrotrachela_herzigana_ott4952971,Macrotrachela_labiata_ott4952972,Macrotrachela_tenuis_ott4952973,Macrotrachela_inermis_ott4952974,Macrotrachela_concinna_ott4952975,Macrotrachela_induta_ott4952977,Macrotrachela_tuberilabris_ott4952978,Macrotrachela_longistyla_ott4952979,Macrotrachela_samali_ott4952980,Macrotrachela_allani_ott4952981,Macrotrachela_insulana_ott4952982,Macrotrachela_mariae_ott4952983,Macrotrachela_punctata_ott4952984,Macrotrachela_formosa_ott4952985,Macrotrachela_pilousi_ott4952986,Macrotrachela_muricata_ott4952987,Macrotrachela_verecunda_ott4952988,Macrotrachela_pacifica_ott4952989,Macrotrachela_compacta_ott4952990,Macrotrachela_hewitti_ott4952991,Macrotrachela_ligulifera_ott4952992,Macrotrachela_plicatula_ott4952993,Macrotrachela_magna_ott4952994,Macrotrachela_murrayi_ott4952995,Macrotrachela_ambigua_ott4952996,Macrotrachela_pinnigera_ott4952997,Macrotrachela_vesicularis_ott4952998,Macrotrachela_canadensis_ott4952999,Macrotrachela_gracillima_ott4953000,Macrotrachela_nana_ott4953001,Macrotrachela_serrulata_ott4953002,Macrotrachela_decora_ott4953003,Macrotrachela_microcornis_ott4953004,Macrotrachela_intermedia_ott4953005,Macrotrachela_macmillani_ott4953006,Macrotrachela_minuta_ott4953007,Macrotrachela_kallosoma_ott4953008,Macrotrachela_speciosa_ott4953009,Macrotrachela_brevilabris_ott4953010,Macrotrachela_sonorensis_ott4953011,Macrotrachela_natans_ott4953012,Macrotrachela_crucicornis_ott4953013,Macrotrachela_insolita_ott4953014,Macrotrachela_timida_ott4953015,Macrotrachela_faveolata_ott4953016,Macrotrachela_oblita_ott4953017,Macrotrachela_vanoyei_ott4953018,Macrotrachela_gunningi_ott4953019,Macrotrachela_petulans_ott4953020,Macrotrachela_obtusa_ott4953021,Macrotrachela_festinans_ott4953022,Macrotrachela_lata_ott4953023,Macrotrachela_cancrophila_ott4953024,Macrotrachela_zickendrahti_ott4953025,Macrotrachela_lepida_ott4953026,Macrotrachela_ornata_ott4953027,Macrotrachela_brachysoma_ott4953028,Macrotrachela_ligulata_ott4953029,Macrotrachela_fungicola_ott4953030,Macrotrachela_libera_ott4953031,Macrotrachela_armillata_ott4953032,Macrotrachela_asperula_ott4953033,Macrotrachela_donneri_ott5974487,Macrotrachela_ioannae_ott5974488,Macrotrachela_jankoi_ott5974489)Macrotrachela_ott681212,(Embata_parasitica_ott267238,Embata_hamata_ott267239,Embata_laticeps_ott400736,Embata_commensalis_ott841828,Embata_environmental_sample_ott4953078,Embata_laticornis_ott4953079)Embata_ott841827,(Anomopus_telphusae_ott368451,Anomopus_chasmagnathi_ott4953092)Anomopus_ott893424,((Dissotrocha_macrostyla_tuberculata_ott436785)Dissotrocha_macrostyla_ott621248,(Dissotrocha_aculeata_crystallina_ott621251,Dissotrocha_aculeata_medioaculeata_ott621252)Dissotrocha_aculeata_ott621254,Dissotrocha_pectinata_ott4952953,Dissotrocha_scutellata_ott4952954,Dissotrocha_hertzogi_ott4952955,Dissotrocha_guyanensis_ott4952956,Dissotrocha_decembullata_ott4952957,Dissotrocha_kostei_ott4952958,Dissotrocha_schlienzi_ott4952959,Dissotrocha_bjoerki_ott4952960,Dissotrocha_spinosa_ott6367877)Dissotrocha_ott621253,(Didymodactylos_carnosus_ott693524)Didymodactylos_ott693526,(Pleuretra_lineata_ott790553,Pleuretra_hystrix_ott790565,Pleuretra_similis_ott4953080,Pleuretra_bovicornis_ott4953081,Pleuretra_proxima_ott4953082,Pleuretra_humerosa_ott4953083,Pleuretra_alpium_ott4953084,Pleuretra_triangularis_ott4953085,Pleuretra_intermedia_ott4953086,Pleuretra_africana_ott4953087,Pleuretra_brycei_ott4953088,Pleuretra_costata_ott4953089,Pleuretra_reticulata_ott4953090,Pleuretra_sulcata_ott4953091)Pleuretra_ott841829,(Mniobia_incrassata_ott790559,Mniobia_russeola_ott961833,Mniobia_dentata_ott4953093,Mniobia_recurvicornis_ott4953094,Mniobia_setifera_ott4953095,Mniobia_mirabilis_ott4953096,Mniobia_granulosa_ott4953097,Mniobia_adhaerens_ott4953098,Mniobia_ocypetes_ott4953099,Mniobia_frankenbergeri_ott4953100,Mniobia_edmondsoni_ott4953101,Mniobia_scarlatina_ott4953102,Mniobia_tentans_ott4953103,Mniobia_obtusicalcar_ott4953104,Mniobia_iurensis_ott4953105,Mniobia_bdelloidea_ott4953106,Mniobia_burgeri_ott4953107,Mniobia_procera_ott4953108,Mniobia_lobata_ott4953109,Mniobia_armata_ott4953110,Mniobia_symbiotica_ott4953111,Mniobia_donneri_ott4953112,Mniobia_placida_ott4953113,Mniobia_incurata_ott4953114,Mniobia_barbatula_ott4953115,Mniobia_tarda_ott4953116,Mniobia_montium_ott4953117,Mniobia_scabrosa_ott4953118,Mniobia_circinata_ott4953119,Mniobia_loxocorona_ott4953120,Mniobia_lamellata_ott4953121,Mniobia_obtusicornis_ott4953122,Mniobia_tetraodon_ott4953123,Mniobia_magna_ott4953124,Mniobia_punctulata_ott4953125,Mniobia_conarus_ott4953126,Mniobia_branchicola_ott4953127,Mniobia_lineata_ott4953128,Mniobia_modesta_ott4953129,Mniobia_storkani_ott4953130,Mniobia_animosa_ott4953131,Mniobia_orta_ott4953132,Mniobia_variabilis_ott4953133,Mniobia_bredensis_ott4953134,Mniobia_discophora_ott4953135,Mniobia_vargai_ott4953136,Mniobia_lenta_ott4953137,Mniobia_brachypoda_ott4953138,Mniobia_punctata_ott4953139)Mniobia_ott961834,(Ceratotrocha_multiserialis_ott4953142,Ceratotrocha_cornigera_ott4953144,Ceratotrocha_franzi_ott4953145,Ceratotrocha_velata_ott4953146,Ceratotrocha_rodewaldi_ott4953147)Ceratotrocha_ott4953141,(Pseudoembata_acutipoda_ott4953149)Pseudoembata_ott4953148,(Zelinkiella_synaptae_ott4953151)Zelinkiella_ott4953150)Philodinidae_ott831315,((Habrotrocha_rosa_ott234664,Habrotrocha_lata_ott267235,Habrotrocha_bidens_ott267241,Habrotrocha_constricta_ott681211,Habrotrocha_modesta_ott4952824,Habrotrocha_pavida_ott4952825,Habrotrocha_komareki_ott4952826,Habrotrocha_gibbosa_ott4952827,Habrotrocha_recumbens_ott4952828,Habrotrocha_crenata_ott4952829,Habrotrocha_visa_ott4952830,Habrotrocha_roperi_ott4952831,Habrotrocha_novemdens_ott4952832,Habrotrocha_vicina_ott4952833,Habrotrocha_solitaria_ott4952834,Habrotrocha_levis_ott4952835,Habrotrocha_incola_ott4952836,Habrotrocha_praelonga_ott4952837,Habrotrocha_fuhrmanni_ott4952838,Habrotrocha_caudata_ott4952839,Habrotrocha_strangulata_ott4952840,Habrotrocha_scabropyga_ott4952841,Habrotrocha_placida_ott4952842,Habrotrocha_baradlana_ott4952843,Habrotrocha_soror_ott4952844,Habrotrocha_microcephala_ott4952845,Habrotrocha_tranquilla_ott4952846,Habrotrocha_insignis_ott4952847,Habrotrocha_leitgebii_ott4952848,Habrotrocha_solida_ott4952849,Habrotrocha_appendiculata_ott4952850,Habrotrocha_thienemanni_ott4952851,Habrotrocha_colliflectens_ott4952852,Habrotrocha_mediocris_ott4952853,Habrotrocha_diarthrantenna_ott4952854,Habrotrocha_valida_ott4952855,Habrotrocha_curvicollis_ott4952856,Habrotrocha_murrayi_ott4952857,Habrotrocha_pusilla_ott4952858,Habrotrocha_tridens_ott4952859,Habrotrocha_minuta_ott4952860,Habrotrocha_subtilis_ott4952861,Habrotrocha_angularis_ott4952862,Habrotrocha_tripus_ott4952863,Habrotrocha_porrecta_ott4952864,Habrotrocha_flexicollis_ott4952865,Habrotrocha_humilis_ott4952866,Habrotrocha_serpens_ott4952867,Habrotrocha_thermalis_ott4952868,Habrotrocha_maculata_ott4952869,Habrotrocha_heinisi_ott4952870,Habrotrocha_alacris_ott4952871,Habrotrocha_gracilis_ott4952872,Habrotrocha_roeperi_ott4952873,Habrotrocha_angusticollis_ott4952874,Habrotrocha_stenochlaena_ott4952875,Habrotrocha_nodosa_ott4952876,Habrotrocha_tridentata_ott4952877,Habrotrocha_flaviformis_ott4952878,Habrotrocha_scepanotrochoides_ott4952879,Habrotrocha_collaris_ott4952880,Habrotrocha_gulosa_ott4952881,Habrotrocha_flava_ott4952882,Habrotrocha_ligula_ott4952883,Habrotrocha_aspera_ott4952884,Habrotrocha_megalocephala_ott4952885,Habrotrocha_crassa_ott4952886,Habrotrocha_parvipes_ott4952887,Habrotrocha_elliptica_ott4952888,Habrotrocha_fuscochlaena_ott4952889,Habrotrocha_reclusa_ott4952890,Habrotrocha_plana_ott4952891,Habrotrocha_longicollis_ott4952892,Habrotrocha_brocklehursti_ott4952893,Habrotrocha_lamellata_ott4952894,Habrotrocha_eremita_ott4952895,Habrotrocha_cucullata_ott4952896,Habrotrocha_quinquedens_ott4952897,Habrotrocha_nodulata_ott4952898,Habrotrocha_trilobata_ott4952899,Habrotrocha_longula_ott4952900,Habrotrocha_sylvestris_ott4952901,Habrotrocha_annulata_ott4952902,Habrotrocha_schultei_ott4952903,Habrotrocha_acornis_ott4952904,Habrotrocha_amphichlaena_ott4952905,Habrotrocha_longicalcarata_ott4952906,Habrotrocha_bulbosa_ott4952907,Habrotrocha_cuneata_ott4952908,Habrotrocha_spicula_ott4952909,Habrotrocha_pulchra_ott4952910,Habrotrocha_perforata_ott4952911,Habrotrocha_minima_ott4952912,Habrotrocha_bartosi_ott4952913,Habrotrocha_rara_ott4952914,Habrotrocha_fusca_ott4952915,Habrotrocha_puella_ott4952916,Habrotrocha_sollicita_ott4952917,Habrotrocha_ampulla_ott4952918,Habrotrocha_iners_ott4952919,Habrotrocha_elegans_ott4952920,Habrotrocha_calosa_ott4952921,Habrotrocha_curva_ott4952922,Habrotrocha_stenostephana_ott4952923,Habrotrocha_longiceps_ott4952924,Habrotrocha_pertinax_ott4952925,Habrotrocha_filum_ott4952926,Habrotrocha_munda_ott4952927,Habrotrocha_granulata_ott4952928,(Habrotrocha_elusa_elusa_ott5706037)Habrotrocha_elusa_ott267240,Habrotrocha_antarctica_ott5974478,Habrotrocha_devetteri_ott5974479,Habrotrocha_vernadskii_ott5974486)Habrotrocha_ott681214,(Scepanotrocha_sp._SceS1_ott749559,Scepanotrocha_setifera_ott4952929,Scepanotrocha_delicata_ott4952930,Scepanotrocha_haueri_ott4952931,Scepanotrocha_impexa_ott4952932,Scepanotrocha_rubra_ott4952933,Scepanotrocha_simplex_ott4952934,Scepanotrocha_galeata_ott4952935,Scepanotrocha_parva_ott4952936,Scepanotrocha_semitecta_ott4952937,Scepanotrocha_corniculata_ott4952938)Scepanotrocha_ott749561,(Otostephanos_donneri_ott790561,Otostephanos_kostei_ott4952939,Otostephanos_torquatus_ott4952940,Otostephanos_monteti_ott4952941,Otostephanos_annulatus_ott4952942,Otostephanos_regalis_ott4952943,Otostephanos_macrantennus_ott4952944,Otostephanos_jersabeki_ott4952945,Otostephanos_mundiformis_ott4952946,Otostephanos_auriculatus_ott4952947,Otostephanos_cuspidilabris_ott4952948,Otostephanos_jolantae_ott5693687)Otostephanos_ott874457,Habrotrochidae_sp._nd_ott4153537)Habrotrochidae_ott681213)Philodinida_ott725949,((Philodinavus_paradoxus_ott633720,Philodinavus_aussiensis_ott4952805)Philodinavus_ott841830,(Abrochtha_meselsoni_ott688001,Abrochtha_sonneborni_ott785906,Abrochtha_kingi_ott785919,'Abrochtha_sp._CWB-2010_ott785920',Abrochtha_carnivora_ott4952803,Abrochtha_intermedia_ott4952804)Abrochtha_ott113845,(Henoceros_falcatus_ott4952807,Henoceros_caudatus_ott4952808)Henoceros_ott2942292)Philodinavidae_ott113846)Bdelloidea_ott662648,(((((Trichotria_tetractis_ott21309,Trichotria_curta_ott2941775,Trichotria_truncata_ott2941776,Trichotria_buchneri_ott2941777,Trichotria_zanclum_ott2941778,Trichotria_pseudocurta_ott2941779,Trichotria_eukosmeta_ott2941780,Trichotria_pocillum_ott2941781,Trichotria_brevidactyla_ott7992165)Trichotria_ott740411,(Macrochaetus_collinsi_ott563703,Macrochaetus_paggiensae_ott2941783,Macrochaetus_aspera_ott2941784,Macrochaetus_collinsii_ott2941785,Macrochaetus_altamirai_ott2941786,Macrochaetus_hauerianus_ott2941787,Macrochaetus_clavicornis_ott2941788,Macrochaetus_sericus_ott2941789,Macrochaetus_philopax_ott2941790,Macrochaetus_americanus_ott2941792,Macrochaetus_longipes_ott2941793,Macrochaetus_longisetus_ott2941795,Macrochaetus_subquadratus_ott2941796,Macrochaetus_aspinus_ott2941797,Macrochaetus_multispinosus_ott2941798,Macrochaetus_dispar_ott2941799,Macrochaetus_kostei_ott2941800,Macrochaetus_danneelae_ott4952788)Macrochaetus_ott563702,(Wolga_spinifera_ott2941803)Wolga_ott2941804,(Pulchritia_dorsicornuta_ott7992164)Pulchritia_ott7992163)Trichotriidae_ott563705,((Keratella_quadrata_ott59521,(Keratella_cochlearis_robusta_ott169321,Keratella_cochlearis_tecta_ott743644,Keratella_cochlearis_faluta_ott1021339)Keratella_cochlearis_ott169319,Keratella_morenoi_ott620072,Keratella_hiemalis_ott743643,Keratella_cochlearis_AEG1_ott820711,Keratella_tropica_ott1023631,Keratella_americana_ott1023633,Keratella_procurva_ott2941872,Keratella_cruciformis_ott2941874,Keratella_stipita_ott2941875,Keratella_mixta_ott2941877,Keratella_eichwaldi_ott2941878,Keratella_maliensis_ott2941879,Keratella_zhugeae_ott2941880,Keratella_crassa_ott2941881,Keratella_lenzi_ott2941882,Keratella_kostei_ott2941883,Keratella_serrulata_ott2941884,Keratella_ahlstromi_ott2941885,Keratella_slacki_ott2941886,Keratella_javana_ott2941887,Keratella_irregularis_ott2941888,Keratella_taurocephala_ott2941890,Keratella_mexicana_ott2941891,Keratella_yamana_ott2941892,Keratella_sinensis_ott2941893,Keratella_taksinensis_ott2941895,Keratella_paludosa_ott2941896,Keratella_reducta_ott2941897,Keratella_shieli_ott2941898,Keratella_earlinae_ott2941899,Keratella_sancta_ott2941900,Keratella_ona_ott2941901,Keratella_valga_ott2941902,Keratella_wangi_ott2941903,Keratella_nhamundaiensis_ott2941904,Keratella_edmondsoni_ott2941905,Keratella_ticinensis_ott2941906,Keratella_trapezoida_ott2941907,Keratella_testudo_ott2941908,Keratella_mongoliana_ott2941909,Keratella_armadura_ott2941910,Keratella_canadensis_ott2941911,Keratella_thomassoni_ott2941912,Keratella_australis_ott2941913,Keratella_tecta_ott7506423,Keratella_valdiviensis_ott7992119)Keratella_ott169316,((Mytilina_ventralis_macracantha_ott98572,Mytilina_ventralis_brevispina_ott98575,Mytilina_ventralis_macracantha_AEG3_ott826220,Mytilina_ventralis_macracantha_AEG4_ott871351,Mytilina_ventralis_macracantha_AEG2_ott871352)Mytilina_ventralis_ott937969,Mytilina_mucronata_ott937971,Mytilina_trigona_ott2941914,Mytilina_bicarinata_ott2941915,Mytilina_macrocera_ott2941916,Mytilina_crassipes_ott2941917,Mytilina_bisulcata_ott2941918,Mytilina_compressa_ott2941919,Mytilina_lobata_ott2941920,Mytilina_carpatica_ott2941922,Mytilina_mutica_ott2941924,Mytilina_macrocerca_ott2941925,Mytilina_unguipes_ott2941926,Mytilina_acanthophora_ott2941927,Mytilina_michelangellii_ott7992121)Mytilina_ott937972,(Brachionus_urceolaris_ott169314,Brachionus_caudatus_ott197558,'Brachionus_aff._plicatilis_LSE-2010_ott255994','Brachionus_aff._dimidiatus_LSE-2010_ott255995',Brachionus_orientalis_ott259976,Brachionus_patulus_ott263248,(Brachionus_quadridentatus_brevispinus_AEG1_ott296318,Brachionus_quadridentatus_cluniorbicularis_ott745906,Brachionus_quadridentatus_brevispinus_AEG2_ott820707,Brachionus_quadridentatus_cluniorbicularis_AEG1_ott820708,Brachionus_quadridentatus_brevispinus_ott890462)Brachionus_quadridentatus_ott1047970,'Brachionus_plicatilis_group_sp._MEG-2012_ott296319',Brachionus_koreanus_ott430689,Brachionus_plicatilis_ott471703,Brachionus_dimidiatus_ott533638,Brachionus_forficula_ott618412,Brachionus_diversicornis_ott618413,Brachionus_bidentatus_ott620076,Brachionus_variabilis_ott620078,Brachionus_havanaensis_ott700604,Brachionus_macracanthus_ott700607,Brachionus_falcatus_ott700608,Brachionus_rubens_ott700609,Brachionus_manjavacas_ott737934,Brachionus_sericus_ott797562,Brachionus_calyciflorus_AEG1_ott820710,Brachionus_leydigi_ott837283,(Brachionus_angularis_pseudodolabratus_ott890473)Brachionus_angularis_ott837275,Brachionus_calyciflorus_ott939320,Brachionus_rotundiformis_ott1009399,Brachionus_ibericus_ott1047966,'Brachionus_cf._urceolaris_MEG-2012_ott1094691',Brachionus_murphyi_ott2941942,Brachionus_sessilis_ott2941943,Brachionus_durgae_ott2941944,Brachionus_pseudonilsoni_ott2941945,Brachionus_ahlstromi_ott2941946,Brachionus_pterodinoides_ott2941947,Brachionus_mirus_ott2941948,Brachionus_keikoa_ott2941949,Brachionus_spatiosus_ott2941950,Brachionus_dolabratus_ott2941951,Brachionus_amsterdamensis_ott2941952,Brachionus_schwoerbeli_ott2941953,Brachionus_satanicus_ott2941955,Brachionus_zahniseri_ott2941956,Brachionus_adisi_ott2941957,Brachionus_srisumonae_ott2941959,Brachionus_postcurvatus_ott2941960,Brachionus_lyratus_ott2941961,Brachionus_kostei_ott2941962,Brachionus_donneri_ott2941963,Brachionus_incertus_ott2941964,Brachionus_austrogenitus_ott2941965,Brachionus_huangi_ott2941966,Brachionus_amazonicus_ott2941967,Brachionus_pinneenaus_ott2941968,Brachionus_budapestinensis_ott2941969,Brachionus_nilsoni_ott2941970,Brachionus_novaezealandiae_ott2941971,Brachionus_leydigii_ott2941972,Brachionus_africanus_ott2941973,Brachionus_bennini_ott2941974,Brachionus_baylyi_ott2941975,Brachionus_charini_ott2941976,Brachionus_kultrum_ott2941977,Brachionus_josefinae_ott2941978,Brachionus_asplanchnoides_ott2941979,Brachionus_dichotomus_ott2941980,Brachionus_araceliae_ott4952777,Brachionus_mirabilis_ott4952778,Brachionus_plicatilis_complex_sp._1CM13_ott5974504,'Brachionus_plicatilis_complex_sp._28_FCOIROT_ott5974505','Brachionus_plicatilis_complex_sp._AF3_6_Turkana_Lake_Kenya_ott5974506','Brachionus_plicatilis_complex_sp._AL_1_1_ott5974507','Brachionus_plicatilis_complex_sp._AL_1_10_ott5974508','Brachionus_plicatilis_complex_sp._AL_1_11_ott5974509','Brachionus_plicatilis_complex_sp._AL_1_2_ott5974510','Brachionus_plicatilis_complex_sp._AL_1_3_ott5974511','Brachionus_plicatilis_complex_sp._AL_1_4_ott5974512','Brachionus_plicatilis_complex_sp._AL_1_5_ott5974513','Brachionus_plicatilis_complex_sp._AL_1_6_ott5974514','Brachionus_plicatilis_complex_sp._AL_1_7_ott5974515','Brachionus_plicatilis_complex_sp._AL_1_8_ott5974516','Brachionus_plicatilis_complex_sp._AL_1_9_ott5974517',Brachionus_plicatilis_complex_sp._AUBUS001_ott5974518,Brachionus_plicatilis_complex_sp._AUCOL051_ott5974519,Brachionus_plicatilis_complex_sp._AUCOL075_ott5974520,Brachionus_plicatilis_complex_sp._AUCOL149_ott5974521,Brachionus_plicatilis_complex_sp._AUCOL155_ott5974522,Brachionus_plicatilis_complex_sp._AUDAM003_ott5974523,Brachionus_plicatilis_complex_sp._AUDAM004_ott5974524,Brachionus_plicatilis_complex_sp._AUDAM006_ott5974525,Brachionus_plicatilis_complex_sp._AUDAM007_ott5974526,Brachionus_plicatilis_complex_sp._AUDAM008_ott5974527,Brachionus_plicatilis_complex_sp._AUDAM009_ott5974528,Brachionus_plicatilis_complex_sp._AUDAM010_ott5974529,Brachionus_plicatilis_complex_sp._AUDAM011_ott5974530,Brachionus_plicatilis_complex_sp._AUDAM012_ott5974531,Brachionus_plicatilis_complex_sp._AUDAM013_ott5974532,Brachionus_plicatilis_complex_sp._AUDAM015_ott5974533,Brachionus_plicatilis_complex_sp._AUDAM018_ott5974534,Brachionus_plicatilis_complex_sp._AUDAM019_ott5974535,Brachionus_plicatilis_complex_sp._AUDAM021_ott5974536,Brachionus_plicatilis_complex_sp._AUDAM025_ott5974537,Brachionus_plicatilis_complex_sp._AUDAM028_ott5974538,Brachionus_plicatilis_complex_sp._AUDAM029_ott5974539,Brachionus_plicatilis_complex_sp._AUDAM030_ott5974540,Brachionus_plicatilis_complex_sp._AUDAM033_ott5974541,Brachionus_plicatilis_complex_sp._AUDAM034A_ott5974542,Brachionus_plicatilis_complex_sp._AUDAM034B_ott5974543,Brachionus_plicatilis_complex_sp._AUDAM040_ott5974544,Brachionus_plicatilis_complex_sp._AUDAM041_ott5974545,Brachionus_plicatilis_complex_sp._AUDAM042_ott5974546,Brachionus_plicatilis_complex_sp._AUDAM045_ott5974547,Brachionus_plicatilis_complex_sp._AUDAM046_ott5974548,Brachionus_plicatilis_complex_sp._AUDAM047_ott5974549,Brachionus_plicatilis_complex_sp._AUDAM048_ott5974550,Brachionus_plicatilis_complex_sp._AUDAM049_ott5974551,Brachionus_plicatilis_complex_sp._AUDAM056_ott5974552,Brachionus_plicatilis_complex_sp._AUDAM057_ott5974553,Brachionus_plicatilis_complex_sp._AUDAM058_ott5974554,Brachionus_plicatilis_complex_sp._AUDAM059_ott5974555,Brachionus_plicatilis_complex_sp._AUDAM060_ott5974556,Brachionus_plicatilis_complex_sp._AUDAM061_ott5974557,Brachionus_plicatilis_complex_sp._AUDUN001_ott5974558,Brachionus_plicatilis_complex_sp._AUDUN003_ott5974559,Brachionus_plicatilis_complex_sp._AULAT006_ott5974560,Brachionus_plicatilis_complex_sp._AULAT007_ott5974561,Brachionus_plicatilis_complex_sp._AULAT013_ott5974562,Brachionus_plicatilis_complex_sp._AULAT017_ott5974563,Brachionus_plicatilis_complex_sp._AULAT019_ott5974564,Brachionus_plicatilis_complex_sp._AULAT024_ott5974565,Brachionus_plicatilis_complex_sp._AULAT042_ott5974566,Brachionus_plicatilis_complex_sp._AUPEA002_ott5974567,Brachionus_plicatilis_complex_sp._AUPEA008_ott5974568,Brachionus_plicatilis_complex_sp._AUPEA011_ott5974569,Brachionus_plicatilis_complex_sp._AUPEA013_ott5974570,Brachionus_plicatilis_complex_sp._AUPEA015_ott5974571,Brachionus_plicatilis_complex_sp._AUPEA020_ott5974572,Brachionus_plicatilis_complex_sp._AUPEA021_ott5974573,Brachionus_plicatilis_complex_sp._AUPEA022_ott5974574,Brachionus_plicatilis_complex_sp._AUPEA025_ott5974575,Brachionus_plicatilis_complex_sp._AUPEA028_ott5974576,Brachionus_plicatilis_complex_sp._AUPEA030_ott5974577,Brachionus_plicatilis_complex_sp._AUPEA031_ott5974578,Brachionus_plicatilis_complex_sp._AUPIP011_ott5974579,Brachionus_plicatilis_complex_sp._AUTOW002_ott5974580,Brachionus_plicatilis_complex_sp._AUTOW003_ott5974581,Brachionus_plicatilis_complex_sp._AUTYEN080_ott5974582,Brachionus_plicatilis_complex_sp._AUWAR001_ott5974583,Brachionus_plicatilis_complex_sp._AUWAR002_ott5974584,Brachionus_plicatilis_complex_sp._AUWARCL_ott5974585,Brachionus_plicatilis_complex_sp._AUYEL003_ott5974586,Brachionus_plicatilis_complex_sp._AUYEL004_ott5974587,Brachionus_plicatilis_complex_sp._AUYEL005_ott5974588,Brachionus_plicatilis_complex_sp._AUYEN005_ott5974589,Brachionus_plicatilis_complex_sp._AUYEN010_ott5974590,Brachionus_plicatilis_complex_sp._AUYEN016_ott5974591,Brachionus_plicatilis_complex_sp._AUYEN020_ott5974592,Brachionus_plicatilis_complex_sp._AUYEN075_ott5974593,Brachionus_plicatilis_complex_sp._AUYEN076_ott5974594,Brachionus_plicatilis_complex_sp._AUYEN077_ott5974595,Brachionus_plicatilis_complex_sp._AUYEN078_ott5974596,Brachionus_plicatilis_complex_sp._AUYEN079_ott5974597,Brachionus_plicatilis_complex_sp._AUYEN080_ott5974598,Brachionus_plicatilis_complex_sp._AUYEN081_ott5974599,Brachionus_plicatilis_complex_sp._AUYEN082_ott5974600,Brachionus_plicatilis_complex_sp._AUYEN083_ott5974601,Brachionus_plicatilis_complex_sp._Almenara_ott5974602,Brachionus_plicatilis_complex_sp._Almenara2_ott5974603,Brachionus_plicatilis_complex_sp._Almenara3_ott5974604,Brachionus_plicatilis_complex_sp._Alvarado_ott5974605,Brachionus_plicatilis_complex_sp._BEARC001_ott5974606,Brachionus_plicatilis_complex_sp._BEARC002_ott5974607,Brachionus_plicatilis_complex_sp._BEARC003_ott5974608,Brachionus_plicatilis_complex_sp._BEARC004_ott5974609,Brachionus_plicatilis_complex_sp._BEARC005_ott5974610,Brachionus_plicatilis_complex_sp._BEARC006_ott5974611,Brachionus_plicatilis_complex_sp._BEARC007_ott5974612,Brachionus_plicatilis_complex_sp._BEARC008_ott5974613,Brachionus_plicatilis_complex_sp._BEARC009_ott5974614,Brachionus_plicatilis_complex_sp._BEARC010_ott5974615,Brachionus_plicatilis_complex_sp._BEARC011_ott5974616,Brachionus_plicatilis_complex_sp._BEARC012_ott5974617,Brachionus_plicatilis_complex_sp._BEARC013_ott5974618,Brachionus_plicatilis_complex_sp._BEARC014_ott5974619,Brachionus_plicatilis_complex_sp._BEARC015_ott5974620,Brachionus_plicatilis_complex_sp._BEARC016_ott5974621,Brachionus_plicatilis_complex_sp._BEARC017_ott5974622,Brachionus_plicatilis_complex_sp._BEARC018_ott5974623,Brachionus_plicatilis_complex_sp._BEARC019_ott5974624,Brachionus_plicatilis_complex_sp._BEARC020_ott5974625,Brachionus_plicatilis_complex_sp._BUL01_ott5974626,Brachionus_plicatilis_complex_sp._BUL03_ott5974627,Brachionus_plicatilis_complex_sp._BUL04_ott5974628,Brachionus_plicatilis_complex_sp._BUL05_ott5974629,Brachionus_plicatilis_complex_sp._BULO4_ott5974630,Brachionus_plicatilis_complex_sp._BUS06_ott5974631,Brachionus_plicatilis_complex_sp._BUS08_ott5974632,Brachionus_plicatilis_complex_sp._BUS20_ott5974633,Brachionus_plicatilis_complex_sp._BUSCL_ott5974634,Brachionus_plicatilis_complex_sp._COL01_ott5974635,Brachionus_plicatilis_complex_sp._COL03_ott5974636,Brachionus_plicatilis_complex_sp._COL05_ott5974637,Brachionus_plicatilis_complex_sp._COL07_ott5974638,Brachionus_plicatilis_complex_sp._COY01_ott5974639,Brachionus_plicatilis_complex_sp._COY03_ott5974640,Brachionus_plicatilis_complex_sp._COY05_ott5974641,Brachionus_plicatilis_complex_sp._COY08_ott5974642,Brachionus_plicatilis_complex_sp._COYCL_ott5974643,Brachionus_plicatilis_complex_sp._DAM01_ott5974644,Brachionus_plicatilis_complex_sp._DAM02_ott5974645,Brachionus_plicatilis_complex_sp._DAM03_ott5974646,Brachionus_plicatilis_complex_sp._DAM04_ott5974647,Brachionus_plicatilis_complex_sp._DAMCL_ott5974648,Brachionus_plicatilis_complex_sp._DUN01_ott5974649,Brachionus_plicatilis_complex_sp._DUN05_ott5974650,Brachionus_plicatilis_complex_sp._DUN08_ott5974651,Brachionus_plicatilis_complex_sp._DUN09_ott5974652,Brachionus_plicatilis_complex_sp._DUN13_ott5974653,Brachionus_plicatilis_complex_sp._DUNCL_ott5974654,Brachionus_plicatilis_complex_sp._ESUVE001_ott5974655,Brachionus_plicatilis_complex_sp._Esk_ott5974656,Brachionus_plicatilis_complex_sp._Figure8_ott5974657,Brachionus_plicatilis_complex_sp._GRKOR003_ott5974658,Brachionus_plicatilis_complex_sp._GRKOR021_ott5974659,Brachionus_plicatilis_complex_sp._GRKOR044_ott5974660,Brachionus_plicatilis_complex_sp._HAR01_ott5974661,Brachionus_plicatilis_complex_sp._HAR04_ott5974662,Brachionus_plicatilis_complex_sp._HAR11_ott5974663,Brachionus_plicatilis_complex_sp._HOT02_ott5974664,Brachionus_plicatilis_complex_sp._HOTCL_ott5974665,Brachionus_plicatilis_complex_sp._JPNAG001_ott5974666,Brachionus_plicatilis_complex_sp._JPNAG002_ott5974667,Brachionus_plicatilis_complex_sp._JPNAG003_ott5974668,Brachionus_plicatilis_complex_sp._JPNAG004_ott5974669,Brachionus_plicatilis_complex_sp._JPNAG005_ott5974670,Brachionus_plicatilis_complex_sp._JPNAG006_ott5974671,Brachionus_plicatilis_complex_sp._JPNAG007_ott5974672,Brachionus_plicatilis_complex_sp._JPNAG008_ott5974673,Brachionus_plicatilis_complex_sp._JPNAG009_ott5974674,Brachionus_plicatilis_complex_sp._JPNAG010_ott5974675,Brachionus_plicatilis_complex_sp._JPNAG011_ott5974676,Brachionus_plicatilis_complex_sp._JPNAG012_ott5974677,Brachionus_plicatilis_complex_sp._JPNAG013_ott5974678,Brachionus_plicatilis_complex_sp._JPNAG014_ott5974679,Brachionus_plicatilis_complex_sp._JPNAG015_ott5974680,Brachionus_plicatilis_complex_sp._JPNAG016_ott5974681,Brachionus_plicatilis_complex_sp._JPNAG017_ott5974682,Brachionus_plicatilis_complex_sp._JPNAG018_ott5974683,Brachionus_plicatilis_complex_sp._JPNAG019_ott5974684,Brachionus_plicatilis_complex_sp._JPNAG020_ott5974685,Brachionus_plicatilis_complex_sp._JPNAG022_ott5974686,Brachionus_plicatilis_complex_sp._JPNAG023_ott5974687,Brachionus_plicatilis_complex_sp._JPNAG024_ott5974688,Brachionus_plicatilis_complex_sp._JPNAG028_ott5974689,Brachionus_plicatilis_complex_sp._JPNAG030_ott5974690,Brachionus_plicatilis_complex_sp._JPNAG032_ott5974691,Brachionus_plicatilis_complex_sp._JPNAG033_ott5974692,Brachionus_plicatilis_complex_sp._JPNAG034_ott5974693,Brachionus_plicatilis_complex_sp._JPNAG035_ott5974694,Brachionus_plicatilis_complex_sp._JPNAG036_ott5974695,Brachionus_plicatilis_complex_sp._JPNAG037_ott5974696,Brachionus_plicatilis_complex_sp._JPNAG038_ott5974697,Brachionus_plicatilis_complex_sp._JPNAG039_ott5974698,Brachionus_plicatilis_complex_sp._JPNAG041_ott5974699,Brachionus_plicatilis_complex_sp._JPNAG042_ott5974700,Brachionus_plicatilis_complex_sp._JPNAG043_ott5974701,Brachionus_plicatilis_complex_sp._JPNAG044_ott5974702,Brachionus_plicatilis_complex_sp._JPNAG046_ott5974703,Brachionus_plicatilis_complex_sp._JPNAG047_ott5974704,Brachionus_plicatilis_complex_sp._JPNAG048_ott5974705,Brachionus_plicatilis_complex_sp._JPNAG049_ott5974706,Brachionus_plicatilis_complex_sp._JPNAG050_ott5974707,Brachionus_plicatilis_complex_sp._JPNAG051_ott5974708,Brachionus_plicatilis_complex_sp._JPNAG052_ott5974709,Brachionus_plicatilis_complex_sp._JPNAG053_ott5974710,Brachionus_plicatilis_complex_sp._JPNAG054_ott5974711,Brachionus_plicatilis_complex_sp._JPNAG055_ott5974712,Brachionus_plicatilis_complex_sp._JPNAG056_ott5974713,Brachionus_plicatilis_complex_sp._JPNAG057_ott5974714,Brachionus_plicatilis_complex_sp._JPNAG058_ott5974715,Brachionus_plicatilis_complex_sp._JPNAG059_ott5974716,Brachionus_plicatilis_complex_sp._JPNAG060_ott5974717,Brachionus_plicatilis_complex_sp._JPNAG061_ott5974718,Brachionus_plicatilis_complex_sp._JPNAG062_ott5974719,Brachionus_plicatilis_complex_sp._JPNAG063_ott5974720,Brachionus_plicatilis_complex_sp._JPNAG064_ott5974721,Brachionus_plicatilis_complex_sp._JPNAG065_ott5974722,Brachionus_plicatilis_complex_sp._JPNAG066_ott5974723,Brachionus_plicatilis_complex_sp._JPNAG067_ott5974724,Brachionus_plicatilis_complex_sp._JPNAG068_ott5974725,Brachionus_plicatilis_complex_sp._JPNAG069_ott5974726,Brachionus_plicatilis_complex_sp._JPNAG070_ott5974727,Brachionus_plicatilis_complex_sp._JPNAG071_ott5974728,Brachionus_plicatilis_complex_sp._JUS02_ott5974729,Brachionus_plicatilis_complex_sp._JUS04_ott5974730,'Brachionus_plicatilis_complex_sp._KS_1_1_ott5974731','Brachionus_plicatilis_complex_sp._KS_1_2_ott5974732','Brachionus_plicatilis_complex_sp._KS_1_3_ott5974733','Brachionus_plicatilis_complex_sp._KS_1_4_ott5974734','Brachionus_plicatilis_complex_sp._KS_1_5_ott5974735','Brachionus_plicatilis_complex_sp._KS_1_6_ott5974736','Brachionus_plicatilis_complex_sp._KS_1_7_ott5974737',Brachionus_plicatilis_complex_sp._LAT01_ott5974738,Brachionus_plicatilis_complex_sp._LAT02_ott5974739,Brachionus_plicatilis_complex_sp._LAT03_ott5974740,Brachionus_plicatilis_complex_sp._LAT04_ott5974741,Brachionus_plicatilis_complex_sp._MNCHU002_ott5974742,Brachionus_plicatilis_complex_sp._MNCHU003_ott5974743,Brachionus_plicatilis_complex_sp._MNCHU008_ott5974744,Brachionus_plicatilis_complex_sp._MNCHU010_ott5974745,Brachionus_plicatilis_complex_sp._MNCHU012_ott5974746,Brachionus_plicatilis_complex_sp._MNCHU020_ott5974747,Brachionus_plicatilis_complex_sp._MNCHU024_ott5974748,Brachionus_plicatilis_complex_sp._MNCHU031_ott5974749,Brachionus_plicatilis_complex_sp._MNCHU035_ott5974750,Brachionus_plicatilis_complex_sp._MNTSA011_ott5974751,Brachionus_plicatilis_complex_sp._MNTSA012_ott5974752,Brachionus_plicatilis_complex_sp._MOF01_ott5974753,Brachionus_plicatilis_complex_sp._MOR03_ott5974754,Brachionus_plicatilis_complex_sp._MOR05_ott5974755,Brachionus_plicatilis_complex_sp._MOR07_ott5974756,Brachionus_plicatilis_complex_sp._MOR10_ott5974757,Brachionus_plicatilis_complex_sp._MORCL_ott5974758,Brachionus_plicatilis_complex_sp._MUL02_ott5974759,Brachionus_plicatilis_complex_sp._MUL06_ott5974760,Brachionus_plicatilis_complex_sp._MUL08_ott5974761,Brachionus_plicatilis_complex_sp._MUL15_ott5974762,Brachionus_plicatilis_complex_sp._MULCL_ott5974763,Brachionus_plicatilis_complex_sp._MXALC001_ott5974764,Brachionus_plicatilis_complex_sp._MXALC002_ott5974765,Brachionus_plicatilis_complex_sp._MXALC004_ott5974766,Brachionus_plicatilis_complex_sp._MXALC008_ott5974767,Brachionus_plicatilis_complex_sp._MXALC010_ott5974768,Brachionus_plicatilis_complex_sp._MXALC013_ott5974769,Brachionus_plicatilis_complex_sp._MXALC018_ott5974770,Brachionus_plicatilis_complex_sp._MXALC021_ott5974771,Brachionus_plicatilis_complex_sp._MXALC022_ott5974772,Brachionus_plicatilis_complex_sp._MXALC024_ott5974773,Brachionus_plicatilis_complex_sp._MXALC025_ott5974774,Brachionus_plicatilis_complex_sp._MXALC026_ott5974775,Brachionus_plicatilis_complex_sp._MXALC028_ott5974776,Brachionus_plicatilis_complex_sp._MXALC029_ott5974777,Brachionus_plicatilis_complex_sp._MXALC031_ott5974778,Brachionus_plicatilis_complex_sp._MXALC033_ott5974779,Brachionus_plicatilis_complex_sp._MXALC034_ott5974780,Brachionus_plicatilis_complex_sp._MXALC036_ott5974781,Brachionus_plicatilis_complex_sp._MXALC037_ott5974782,Brachionus_plicatilis_complex_sp._MXALC038_ott5974783,Brachionus_plicatilis_complex_sp._MXALC039_ott5974784,Brachionus_plicatilis_complex_sp._MXALC042_ott5974785,Brachionus_plicatilis_complex_sp._MXALC045_ott5974786,Brachionus_plicatilis_complex_sp._MXALC046_ott5974787,Brachionus_plicatilis_complex_sp._MXALC047_ott5974788,Brachionus_plicatilis_complex_sp._MXALC049_ott5974789,Brachionus_plicatilis_complex_sp._MXALC054_ott5974790,Brachionus_plicatilis_complex_sp._MXATE005_ott5974791,Brachionus_plicatilis_complex_sp._MXPRE001_ott5974792,Brachionus_plicatilis_complex_sp._MXPRE004_ott5974793,Brachionus_plicatilis_complex_sp._MXPRE005_ott5974794,Brachionus_plicatilis_complex_sp._NOCCN001_ott5974795,Brachionus_plicatilis_complex_sp._Nakuru10_ott5974796,Brachionus_plicatilis_complex_sp._Nakuru11_ott5974797,Brachionus_plicatilis_complex_sp._Nakuru12_ott5974798,Brachionus_plicatilis_complex_sp._Nakuru13_ott5974799,Brachionus_plicatilis_complex_sp._Nakuru14_ott5974800,Brachionus_plicatilis_complex_sp._Nakuru15_ott5974801,Brachionus_plicatilis_complex_sp._Nakuru16_ott5974802,Brachionus_plicatilis_complex_sp._Nakuru3_ott5974803,Brachionus_plicatilis_complex_sp._Nakuru4_ott5974804,Brachionus_plicatilis_complex_sp._Nakuru5_ott5974805,Brachionus_plicatilis_complex_sp._Nakuru6_ott5974806,Brachionus_plicatilis_complex_sp._Nakuru7_ott5974807,Brachionus_plicatilis_complex_sp._Nakuru8_ott5974808,Brachionus_plicatilis_complex_sp._Nakuru9_ott5974809,Brachionus_plicatilis_complex_sp._OHJ10_ott5974810,Brachionus_plicatilis_complex_sp._OHJ100_ott5974811,Brachionus_plicatilis_complex_sp._OHJ101_ott5974812,Brachionus_plicatilis_complex_sp._OHJ102_ott5974813,Brachionus_plicatilis_complex_sp._OHJ103_ott5974814,Brachionus_plicatilis_complex_sp._OHJ104_ott5974815,Brachionus_plicatilis_complex_sp._OHJ105_ott5974816,Brachionus_plicatilis_complex_sp._OHJ11_ott5974817,Brachionus_plicatilis_complex_sp._OHJ13_ott5974818,Brachionus_plicatilis_complex_sp._OHJ19_ott5974819,Brachionus_plicatilis_complex_sp._OHJ1new_ott5974820,Brachionus_plicatilis_complex_sp._OHJ2_ott5974821,Brachionus_plicatilis_complex_sp._OHJ21_ott5974822,Brachionus_asplanchnoidis_ott5974823,Brachionus_plicatilis_complex_sp._OHJ23_ott5974824,Brachionus_plicatilis_complex_sp._OHJ24_ott5974825,Brachionus_plicatilis_complex_sp._OHJ25_ott5974826,Brachionus_plicatilis_complex_sp._OHJ26_ott5974827,Brachionus_plicatilis_complex_sp._OHJ28_ott5974828,Brachionus_plicatilis_complex_sp._OHJ29_ott5974829,Brachionus_plicatilis_complex_sp._OHJ30_ott5974830,Brachionus_plicatilis_complex_sp._OHJ31_ott5974831,Brachionus_plicatilis_complex_sp._OHJ32_ott5974832,Brachionus_plicatilis_complex_sp._OHJ34_ott5974833,Brachionus_plicatilis_complex_sp._OHJ36_ott5974834,Brachionus_plicatilis_complex_sp._OHJ38_ott5974835,Brachionus_plicatilis_complex_sp._OHJ40_ott5974836,Brachionus_plicatilis_complex_sp._OHJ41_ott5974837,Brachionus_plicatilis_complex_sp._OHJ42_ott5974838,Brachionus_plicatilis_complex_sp._OHJ43_ott5974839,Brachionus_plicatilis_complex_sp._OHJ44_ott5974840,Brachionus_plicatilis_complex_sp._OHJ45_ott5974841,Brachionus_plicatilis_complex_sp._OHJ46_ott5974842,Brachionus_plicatilis_complex_sp._OHJ47_ott5974843,Brachionus_plicatilis_complex_sp._OHJ48_ott5974844,Brachionus_plicatilis_complex_sp._OHJ49_ott5974845,Brachionus_plicatilis_complex_sp._OHJ4new_ott5974846,Brachionus_plicatilis_complex_sp._OHJ50_ott5974847,Brachionus_plicatilis_complex_sp._OHJ51_ott5974848,Brachionus_plicatilis_complex_sp._OHJ53_ott5974849,Brachionus_plicatilis_complex_sp._OHJ54_ott5974850,Brachionus_plicatilis_complex_sp._OHJ6_ott5974851,Brachionus_plicatilis_complex_sp._OHJ60_ott5974852,Brachionus_plicatilis_complex_sp._OHJ61_ott5974853,Brachionus_plicatilis_complex_sp._OHJ62_ott5974854,Brachionus_plicatilis_complex_sp._OHJ64_ott5974855,Brachionus_plicatilis_complex_sp._OHJ65_ott5974856,Brachionus_plicatilis_complex_sp._OHJ66_ott5974857,Brachionus_plicatilis_complex_sp._OHJ67_ott5974858,Brachionus_plicatilis_complex_sp._OHJ68_ott5974859,Brachionus_plicatilis_complex_sp._OHJ69_ott5974860,Brachionus_plicatilis_complex_sp._OHJ7_ott5974861,Brachionus_plicatilis_complex_sp._OHJ70_ott5974862,Brachionus_plicatilis_complex_sp._OHJ71_ott5974863,Brachionus_plicatilis_complex_sp._OHJ72_ott5974864,Brachionus_plicatilis_complex_sp._OHJ73_ott5974865,Brachionus_plicatilis_complex_sp._OHJ74_ott5974866,Brachionus_plicatilis_complex_sp._OHJ75_ott5974867,Brachionus_plicatilis_complex_sp._OHJ76_ott5974868,Brachionus_plicatilis_complex_sp._OHJ77_ott5974869,Brachionus_plicatilis_complex_sp._OHJ78_ott5974870,Brachionus_plicatilis_complex_sp._OHJ79_ott5974871,Brachionus_plicatilis_complex_sp._OHJ80_ott5974872,Brachionus_plicatilis_complex_sp._OHJ81_ott5974873,Brachionus_plicatilis_complex_sp._OHJ82_ott5974874,Brachionus_plicatilis_complex_sp._OHJ83_ott5974875,Brachionus_plicatilis_complex_sp._OHJ84_ott5974876,Brachionus_plicatilis_complex_sp._OHJ85_ott5974877,Brachionus_plicatilis_complex_sp._OHJ86_ott5974878,Brachionus_plicatilis_complex_sp._OHJ87_ott5974879,Brachionus_plicatilis_complex_sp._OHJ88_ott5974880,Brachionus_plicatilis_complex_sp._OHJ89_ott5974881,Brachionus_plicatilis_complex_sp._OHJ9_ott5974882,Brachionus_plicatilis_complex_sp._OHJ90_ott5974883,Brachionus_plicatilis_complex_sp._OHJ91_ott5974884,Brachionus_plicatilis_complex_sp._OHJ92_ott5974885,Brachionus_plicatilis_complex_sp._OHJ93_ott5974886,Brachionus_plicatilis_complex_sp._OHJ94_ott5974887,Brachionus_plicatilis_complex_sp._OHJ95_ott5974888,Brachionus_plicatilis_complex_sp._OHJ96_ott5974889,Brachionus_plicatilis_complex_sp._OHJ97_ott5974890,Brachionus_plicatilis_complex_sp._OHJ98_ott5974891,Brachionus_plicatilis_complex_sp._OHJ99_ott5974892,Brachionus_plicatilis_complex_sp._PEA02_ott5974893,Brachionus_plicatilis_complex_sp._PEA05_ott5974894,Brachionus_plicatilis_complex_sp._PEA09_ott5974895,Brachionus_plicatilis_complex_sp._PEA15_ott5974896,Brachionus_plicatilis_complex_sp._PEA18_ott5974897,Brachionus_plicatilis_complex_sp._PEACL_ott5974898,Brachionus_plicatilis_complex_sp._PIP02_ott5974899,Brachionus_plicatilis_complex_sp._PIP03_ott5974900,Brachionus_plicatilis_complex_sp._PIP04_ott5974901,'Brachionus_plicatilis_complex_sp._Poza_sur_ott5974902','Brachionus_plicatilis_complex_sp._Qo_L2_ott5974903','Brachionus_plicatilis_complex_sp._Qo_S1_ott5974904',Brachionus_plicatilis_complex_sp._SAN02_ott5974905,Brachionus_plicatilis_complex_sp._SAN07_ott5974906,Brachionus_plicatilis_complex_sp._SAN10_ott5974907,Brachionus_plicatilis_complex_sp._SAN11_ott5974908,Brachionus_plicatilis_complex_sp._SAN19_ott5974909,'Brachionus_plicatilis_complex_sp._Segu_L1_ott5974910','Brachionus_plicatilis_complex_sp._Sht_L1_ott5974911','Brachionus_plicatilis_complex_sp._Sht_L2_ott5974912','Brachionus_plicatilis_complex_sp._Sht_S1_ott5974913','Brachionus_plicatilis_complex_sp._Sht_S2_ott5974914',Brachionus_plicatilis_complex_sp._TOW01_ott5974915,Brachionus_plicatilis_complex_sp._TOW02_ott5974916,Brachionus_plicatilis_complex_sp._TOW15_ott5974917,Brachionus_plicatilis_complex_sp._TOWCL_ott5974918,Brachionus_plicatilis_complex_sp._USGET002_ott5974919,Brachionus_plicatilis_complex_sp._USGET003_ott5974920,Brachionus_plicatilis_complex_sp._USGET004_ott5974921,Brachionus_plicatilis_complex_sp._USGET005_ott5974922,Brachionus_plicatilis_complex_sp._USGET006_ott5974923,Brachionus_plicatilis_complex_sp._USGET007_ott5974924,Brachionus_plicatilis_complex_sp._USIND002_ott5974925,Brachionus_plicatilis_complex_sp._USIND092_ott5974926,Brachionus_plicatilis_complex_sp._USIND125_ott5974927,Brachionus_plicatilis_complex_sp._USIND168_ott5974928,Brachionus_plicatilis_complex_sp._USIND172_ott5974929,Brachionus_plicatilis_complex_sp._USIND182_ott5974930,Brachionus_plicatilis_complex_sp._USIND190_ott5974931,Brachionus_plicatilis_complex_sp._USIND237_ott5974932,Brachionus_plicatilis_complex_sp._USSAL020_ott5974933,Brachionus_plicatilis_complex_sp._USSAL024_ott5974934,Brachionus_plicatilis_complex_sp._USSAL033_ott5974935,Brachionus_plicatilis_complex_sp._USSAL042_ott5974936,Brachionus_plicatilis_complex_sp._USSAL049_ott5974937,Brachionus_plicatilis_complex_sp._WAN02_ott5974938,Brachionus_plicatilis_complex_sp._WAN11_ott5974939,Brachionus_plicatilis_complex_sp._WAN16_ott5974940,Brachionus_plicatilis_complex_sp._WANCL_ott5974941,Brachionus_plicatilis_complex_sp._WAR02_ott5974942,Brachionus_plicatilis_complex_sp._WAR05_ott5974943,Brachionus_plicatilis_complex_sp._WAR06_ott5974944,Brachionus_plicatilis_complex_sp._WAR07_ott5974945,Brachionus_plicatilis_complex_sp._WARCL_ott5974946,Brachionus_plicatilis_complex_sp._YEL02_ott5974947,Brachionus_plicatilis_complex_sp._YEL05_ott5974948,Brachionus_plicatilis_complex_sp._YEL11_ott5974949,Brachionus_plicatilis_complex_sp._YELCL_ott5974950,Brachionus_plicatilis_complex_sp._YEN01_ott5974951,Brachionus_plicatilis_complex_sp._YEN02_ott5974952,Brachionus_plicatilis_complex_sp._YEN03_ott5974953,Brachionus_plicatilis_complex_sp._YEN05_ott5974954,Brachionus_plicatilis_complex_sp._YEN06_ott5974955,Brachionus_plicatilis_complex_sp._YEN16_ott5974956,Brachionus_plicatilis_complex_sp._YENCL_ott5974957,'Brachionus_plicatilis_complex_sp._Zbl1_L_ott5974958','Brachionus_plicatilis_complex_sp._Zbl3_1_ott5974959','Brachionus_plicatilis_complex_sp._Zbl4_L_ott5974960',Brachionus_budapestensis_ott6367884,'Brachionus_cf._plicatilis_MEG-2019_ott7506334','Brachionus_plicatilis_complex_sp._CHILE_clone_1_ott7506335','Brachionus_plicatilis_complex_sp._CHILE_clone_12_ott7506336','Brachionus_plicatilis_complex_sp._CHILE_clone_13_ott7506337','Brachionus_plicatilis_complex_sp._CHILE_clone_14_ott7506338','Brachionus_plicatilis_complex_sp._CHILE_clone_16_ott7506339','Brachionus_plicatilis_complex_sp._CHILE_clone_17_ott7506340','Brachionus_plicatilis_complex_sp._CHILE_clone_18_ott7506341','Brachionus_plicatilis_complex_sp._CHILE_clone_1a_ott7506342','Brachionus_plicatilis_complex_sp._CHILE_clone_2_ott7506343','Brachionus_plicatilis_complex_sp._CHILE_clone_21_ott7506344','Brachionus_plicatilis_complex_sp._CHILE_clone_22_ott7506345','Brachionus_plicatilis_complex_sp._CHILE_clone_26_ott7506346','Brachionus_plicatilis_complex_sp._CHILE_clone_28_ott7506347','Brachionus_plicatilis_complex_sp._CHILE_clone_29_ott7506348','Brachionus_plicatilis_complex_sp._CHILE_clone_2a_ott7506349','Brachionus_plicatilis_complex_sp._CHILE_clone_3_ott7506350','Brachionus_plicatilis_complex_sp._CHILE_clone_3a_ott7506351','Brachionus_plicatilis_complex_sp._CHILE_clone_4_ott7506352','Brachionus_plicatilis_complex_sp._CHILE_clone_4a_ott7506353','Brachionus_plicatilis_complex_sp._CHILE_clone_5_ott7506354','Brachionus_plicatilis_complex_sp._CHILE_clone_6_ott7506355','Brachionus_plicatilis_complex_sp._CHILE_clone_7_ott7506356','Brachionus_plicatilis_complex_sp._CHILE_clone_7a_ott7506357','Brachionus_plicatilis_complex_sp._CHILE_clone_8_ott7506358','Brachionus_plicatilis_complex_sp._CHILE_clone_9_ott7506359',Brachionus_plicatilis_complex_sp._Chile_clone_1_ott7506360,Brachionus_plicatilis_complex_sp._Chile_clone_12_ott7506361,Brachionus_plicatilis_complex_sp._Chile_clone_13_ott7506362,Brachionus_plicatilis_complex_sp._Chile_clone_14_ott7506363,Brachionus_plicatilis_complex_sp._Chile_clone_15_ott7506364,Brachionus_plicatilis_complex_sp._Chile_clone_16_ott7506365,Brachionus_plicatilis_complex_sp._Chile_clone_17_ott7506366,Brachionus_plicatilis_complex_sp._Chile_clone_18_ott7506367,Brachionus_plicatilis_complex_sp._Chile_clone_1a_ott7506368,Brachionus_plicatilis_complex_sp._Chile_clone_2_ott7506369,Brachionus_plicatilis_complex_sp._Chile_clone_21_ott7506370,Brachionus_plicatilis_complex_sp._Chile_clone_24_ott7506371,Brachionus_plicatilis_complex_sp._Chile_clone_25_ott7506372,Brachionus_plicatilis_complex_sp._Chile_clone_26_ott7506373,Brachionus_plicatilis_complex_sp._Chile_clone_27_ott7506374,Brachionus_plicatilis_complex_sp._Chile_clone_28_ott7506375,Brachionus_plicatilis_complex_sp._Chile_clone_29_ott7506376,Brachionus_plicatilis_complex_sp._Chile_clone_2a_ott7506377,Brachionus_plicatilis_complex_sp._Chile_clone_3_ott7506378,Brachionus_plicatilis_complex_sp._Chile_clone_3a_ott7506379,Brachionus_plicatilis_complex_sp._Chile_clone_4a_ott7506380,Brachionus_plicatilis_complex_sp._Chile_clone_5_ott7506381,Brachionus_plicatilis_complex_sp._Chile_clone_6_ott7506382,Brachionus_plicatilis_complex_sp._Chile_clone_7_ott7506383,Brachionus_plicatilis_complex_sp._Chile_clone_7a_ott7506384,Brachionus_plicatilis_complex_sp._Chile_clone_8_ott7506385,Brachionus_plicatilis_complex_sp._Chile_clone_9_ott7506386,Brachionus_amphiceras_ott7992110,Brachionus_budapestiensis_ott7992111,Brachionus_tetracanthus_ott7992112,Brachionus_urcealaris_ott7992113,Brachionus_urceus_ott7992114)Brachionus_ott471702,(Notholca_acuminata_ott492241,Notholca_squamula_ott2941805,Notholca_bipalium_ott2941806,Notholca_triarthroides_ott2941807,Notholca_orbiculata_ott2941808,Notholca_verae_ott2941809,Notholca_ikaitophila_ott2941811,Notholca_japonica_ott2941815,Notholca_laurentiae_ott2941816,Notholca_foliacea_ott2941817,Notholca_jugosa_ott2941818,Notholca_guidoi_ott2941819,Notholca_labis_ott2941820,Notholca_olchonensis_ott2941821,Notholca_michiganensis_ott2941822,Notholca_lyrata_ott2941823,Notholca_latistyla_ott2941824,Notholca_angulata_ott2941826,Notholca_cornuta_ott2941827,Notholca_hollowdayi_ott2941828,Notholca_haueri_ott2941830,Notholca_beta_ott2941831,Notholca_lamellifera_ott2941832,Notholca_walterkostei_ott2941834,Notholca_cinetura_ott2941835,Notholca_liepetterseni_ott2941836,Notholca_kozhovi_ott2941837,Notholca_marina_ott2941839,Notholca_striata_ott2941840,Notholca_tibetica_ott2941841,Notholca_gaigalasi_ott2941842,Notholca_angakkoq_ott2941843,Notholca_psammarina_ott2941844,Notholca_caudata_ott2941845,Notholca_kostei_ott2941846,Notholca_jasnitskii_ott4952779,Notholca_rectospina_ott4952780,Notholca_baicalensis_ott4952781,Notholca_bythonoma_ott6367886)Notholca_ott492249,(Kellicottia_bostoniensis_ott672796,Kellicottia_longispina_ott2941812)Kellicottia_ott672795,(Platyias_quadricornis_ott937975,Platyias_leloupi_ott2941931,Platyias_latiscapularis_ott2941932)Platyias_ott937963,(Euchlanis_dilatata_ott1018357,Euchlanis_alata_ott1037814,Euchlanis_perpusilla_ott2941847,Euchlanis_mikropous_ott2941848,Euchlanis_mamorokaensis_ott2941849,Euchlanis_ligulata_ott2941850,Euchlanis_callimorpha_ott2941852,Euchlanis_phryne_ott2941853,Euchlanis_hyphidactyla_ott2941854,Euchlanis_semicarinata_ott2941855,Euchlanis_dactyliseta_ott2941856,Euchlanis_triquetra_ott2941857,Euchlanis_meneta_ott2941858,Euchlanis_incisa_ott2941859,Euchlanis_pyriformis_ott2941860,Euchlanis_callysta_ott2941861,Euchlanis_contorta_ott2941862,Euchlanis_lucksiana_ott2941863,Euchlanis_dapidula_ott2941864,Euchlanis_arenosa_ott2941865,Euchlanis_parameneta_ott2941866,Euchlanis_calpidia_ott2941867,Euchlanis_deflexa_ott2941868,Euchlanis_lyra_ott2941869,Euchlanis_parva_ott2941870)Euchlanis_ott492240,(Anuraeopsis_fissa_ott1023628,Anuraeopsis_urawensis_ott2941933,Anuraeopsis_quadriantennata_ott2941934,Anuraeopsis_lata_ott2941935,Anuraeopsis_navicula_ott2941936,Anuraeopsis_miracleae_ott2941937,Anuraeopsis_wulferti_ott2941938,Anuraeopsis_coelata_ott2941939,Anuraeopsis_cristata_ott2941940,Anuraeopsis_siolii_ott2941941,'Anuraeopsis_sp._WM-2017a_ott7506333')Anuraeopsis_ott1023630,((Plationus_patulus_macracanthus_ott1037810)Plationus_patulus_ott937964,Plationus_polyacanthus_ott2941928,Plationus_felicitas_ott2941929)Plationus_ott937966,(Lophocharis_oxysternon_ott2942212,Lophocharis_kutikovae_ott2942216,Lophocharis_rubens_ott2942217,Lophocharis_tutiurensis_ott2942219,Lophocharis_turanica_ott2942220,Lophocharis_parva_ott2942222,Lophocharis_curvata_ott2942223,Lophocharis_hutchinsoni_ott2942225,Lophocharis_salpina_ott2942334,Lophocharis_naias_ott2942335,Lophocharis_ambidentata_ott2942347)Lophocharis_ott2942213)Brachionidae_ott471705,((Eosphora_ehrenbergi_ott107179,Eosphora_therina_ott2940733,Eosphora_thoides_ott2940734,Eosphora_gibba_ott2940735,Eosphora_najas_ott2940736,Eosphora_anthadis_ott2940737,Eosphora_thoa_ott2940738)Eosphora_ott107180,(Notommata_copeus_ott171571,Notommata_cordonella_ott937967,Notommata_allantois_ott937968,Notommata_torulosa_ott2940662,Notommata_pachyura_ott2940664,Notommata_voigti_ott2940665,Notommata_grandis_ott2940666,Notommata_avena_ott2940667,Notommata_codonella_ott2940668,Notommata_cyrtopus_ott2940669,Notommata_megaladena_ott2940670,Notommata_weberi_ott2940671,Notommata_collaris_ott2940672,Notommata_omentata_ott2940673,Notommata_diasema_ott2940674,Notommata_peridia_ott2940675,Notommata_groenlandica_ott2940676,Notommata_myrmeleo_ott2940677,Notommata_veroleti_ott2940678,Notommata_reinhardti_ott2940679,Notommata_venusta_ott2940681,Notommata_brachyota_ott2940682,Notommata_falcinella_ott2940683,Notommata_potamis_ott2940684,Notommata_doneta_ott2940685,Notommata_angusta_ott2940686,Notommata_sulcata_ott2940687,Notommata_meganglena_ott2940688,Notommata_syrinx_ott2940689,Notommata_cherada_ott2940690,Notommata_rugosa_ott2940691,Notommata_haueri_ott2940692,Notommata_glyphura_ott2940693,Notommata_apochaeta_ott2940694,Notommata_aurita_ott2940695,Notommata_lenis_ott2940696,Notommata_mera_ott2940697,Notommata_saccigera_ott2940698,Notommata_fasciola_ott2940699,Notommata_tripus_ott2940700,Notommata_aethis_ott2940701,Notommata_stitista_ott2940702,Notommata_silphoides_ott2940703,Notommata_placida_ott2940704,Notommata_bennetchi_ott2940705,Notommata_thopica_ott2940708,Notommata_ovulum_ott2940709,Notommata_galena_ott2940710,Notommata_gisleni_ott2940711,Notommata_spinata_ott2940712,Notommata_longina_ott2940713,Notommata_contorta_ott2940714,Notommata_pseudocerberus_ott2940715,Notommata_endoxa_ott2940716,Notommata_prodota_ott2940717,Notommata_cerberus_ott2940718,Notommata_pygmaea_ott2940719,Notommata_cerebrus_ott2940720,Notommata_silpha_ott2940721,Notommata_bennetschi_ott2940722,Notommata_onisciformis_ott2940723,Notommata_limax_ott2940724,Notommata_paracyrtopus_ott4952790)Notommata_ott171572,('Cephalodella_cf._gibba_MEG-2012_ott296322',Cephalodella_gibba_ott492250,Cephalodella_forficula_ott513188,Cephalodella_brandorffi_ott2940502,Cephalodella_curiculata_ott2940503,Cephalodella_balatonica_ott2940504,Cephalodella_segersi_ott2940505,Cephalodella_boettgeri_ott2940506,Cephalodella_dora_ott2940507,Cephalodella_intuta_ott2940508,Cephalodella_maior_ott2940509,Cephalodella_mira_ott2940510,Cephalodella_mus_ott2940511,Cephalodella_ungulata_ott2940512,Cephalodella_decidua_ott2940513,Cephalodella_tantilla_ott2940514,Cephalodella_elegans_ott2940515,Cephalodella_hoodii_ott2940518,Cephalodella_monica_ott2940519,Cephalodella_wrighti_ott2940520,Cephalodella_panarista_ott2940521,Cephalodella_mucronata_ott2940522,Cephalodella_forficata_ott2940523,Cephalodella_irisae_ott2940524,Cephalodella_eva_ott2940525,Cephalodella_auriculata_ott2940526,Cephalodella_gibboides_ott2940527,Cephalodella_akrobeles_ott2940528,Cephalodella_rostrum_ott2940529,Cephalodella_hollowdayi_ott2940530,Cephalodella_collactea_ott2940531,Cephalodella_planera_ott2940532,Cephalodella_apocolea_ott2940533,Cephalodella_jakubskii_ott2940534,Cephalodella_anebodica_ott2940535,Cephalodella_carina_ott2940536,Cephalodella_cyclops_ott2940537,Cephalodella_tincaformis_ott2940538,Cephalodella_hiulca_ott2940539,Cephalodella_subsecunda_ott2940540,Cephalodella_celeris_ott2940541,Cephalodella_tenuis_ott2940542,Cephalodella_limosa_ott2940543,Cephalodella_angusta_ott2940544,Cephalodella_arcuata_ott2940545,Cephalodella_obvia_ott2940546,Cephalodella_paxi_ott2940547,Cephalodella_pentaplax_ott2940548,Cephalodella_inquilina_ott2940549,Cephalodella_vitella_ott2940550,Cephalodella_gusuleaci_ott2940551,Cephalodella_mineri_ott2940552,Cephalodella_derbyi_ott2940553,Cephalodella_volvocicola_ott2940554,Cephalodella_gobio_ott2940555,Cephalodella_latifulcrum_ott2940556,Cephalodella_elongata_ott2940557,Cephalodella_incila_ott2940558,Cephalodella_unguitata_ott2940559,Cephalodella_belone_ott2940560,Cephalodella_poitera_ott2940561,Cephalodella_delicata_ott2940562,Cephalodella_praelonga_ott2940563,Cephalodella_tantilloides_ott2940564,Cephalodella_hyalina_ott2940565,Cephalodella_gigantea_ott2940566,Cephalodella_bertonicensis_ott2940567,Cephalodella_glypha_ott2940568,Cephalodella_forceps_ott2940569,Cephalodella_compacta_ott2940570,Cephalodella_harringi_ott2940571,Cephalodella_friebei_ott2940572,Cephalodella_tachyphora_ott2940573,Cephalodella_catellina_ott2940574,Cephalodella_lindamayae_ott2940575,Cephalodella_pseudeva_ott2940576,Cephalodella_vacuna_ott2940577,Cephalodella_zeteta_ott2940578,Cephalodella_biungulata_ott2940579,Cephalodella_songkhlaensis_ott2940580,Cephalodella_paxilla_ott2940581,Cephalodella_dixonnuttalli_ott2940582,Cephalodella_trigona_ott2940583,Cephalodella_eupoda_ott2940584,Cephalodella_conica_ott2940585,Cephalodella_retusa_ott2940586,Cephalodella_licinia_ott2940587,Cephalodella_dorseyi_ott2940588,Cephalodella_speciosa_ott2940589,Cephalodella_eurynota_ott2940590,Cephalodella_ablusa_ott2940591,Cephalodella_abstrusa_ott2940592,Cephalodella_stenroosi_ott2940593,Cephalodella_qionghaiensis_ott2940594,Cephalodella_sterea_ott2940595,Cephalodella_megalotrocha_ott2940596,Cephalodella_lipara_ott2940597,Cephalodella_pheloma_ott2940598,Cephalodella_gisleni_ott2940599,Cephalodella_vittata_ott2940600,Cephalodella_licina_ott2940601,Cephalodella_montana_ott2940602,Cephalodella_tecta_ott2940603,Cephalodella_asarcia_ott2940604,Cephalodella_euderbyi_ott2940605,Cephalodella_tenuiseta_ott2940606,Cephalodella_rigida_ott2940607,Cephalodella_misgurnus_ott2940608,Cephalodella_exigua_ott2940609,Cephalodella_unquitara_ott2940610,Cephalodella_rotunda_ott2940611,Cephalodella_dentata_ott2940612,Cephalodella_melia_ott2940613,Cephalodella_pachyodon_ott2940614,Cephalodella_papillosa_ott2940615,Cephalodella_glandulosa_ott2940616,Cephalodella_astricta_ott2940617,Cephalodella_strigosa_ott2940618,Cephalodella_globata_ott2940619,Cephalodella_fluviatilis_ott2940620,Cephalodella_calosa_ott2940621,Cephalodella_xenica_ott2940622,Cephalodella_physalis_ott2940623,Cephalodella_minora_ott2940624,Cephalodella_clara_ott2940625,Cephalodella_edax_ott2940626,Cephalodella_tinca_ott2940627,Cephalodella_eunoma_ott2940628,Cephalodella_macrodactyla_ott2940629,Cephalodella_crassipes_ott2940630,Cephalodella_tenuior_ott2940631,Cephalodella_graciosa_ott2940632,Cephalodella_asta_ott2940633,Cephalodella_nelitis_ott2940634,Cephalodella_mucosa_ott2940635,Cephalodella_reimanni_ott2940636,Cephalodella_innesi_ott2940637,Cephalodella_somniculosa_ott2940638,Cephalodella_gracilis_ott2940639,Cephalodella_euknema_ott2940640,Cephalodella_parasitica_ott2940641,Cephalodella_tempesta_ott2940642,Cephalodella_pachydactyla_ott2940643,Cephalodella_nana_ott2940644,Cephalodella_paggiae_ott2940645,Cephalodella_compressa_ott2940646,Cephalodella_psammophila_ott2940647,Cephalodella_plicata_ott2940648,Cephalodella_megalocephala_ott2940649,Cephalodella_marina_ott2940650,Cephalodella_doryphora_ott2940651,Cephalodella_labiosa_ott2940652,Cephalodella_dorystoma_ott2940653,Cephalodella_theodora_ott2940654,Cephalodella_conjuncta_ott2940655,Cephalodella_lepida_ott2940656,Cephalodella_evabroedae_ott2940657,Cephalodella_ventripes_ott2940658,Cephalodella_obesa_ott2940659,Cephalodella_oxydactyla_ott2940660,Cephalodella_laisi_ott4952792,Cephalodella_inquila_ott5705992,Cephalodella_jersabeki_ott6367900,Cephalodella_teniuseta_ott6367901,Cephalodella_elouenteita_ott7992147,Cephalodella_promta_ott7992148,Cephalodella_pseudocuneata_ott7992149,Cephalodella_sasquatcha_ott7992150,Cephalodella_symbiotica_ott7992151)Cephalodella_ott492248,(Eothinia_elongata_ott460955,Eothinia_euklopa_ott2940725,Eothinia_eukolpa_ott2940726,Eothinia_triphaea_ott2940727,Eothinia_striata_ott2940729,Eothinia_argus_ott2940730,Eothinia_poitera_ott2940731,Eothinia_carogaensis_ott2940732)Eothinia_ott576380,(Monommata_maculata_ott563701,Monommata_actices_ott2940739,Monommata_pseudophoxa_ott2940740,Monommata_phoxa_ott2940741,Monommata_appendiculata_ott2940742,Monommata_enedra_ott2940743,Monommata_diaphora_ott2940744,Monommata_caeca_ott2940745,Monommata_dissimilis_ott2940746,Monommata_longiseta_ott2940747,Monommata_caudata_ott2940748,Monommata_viridis_ott2940749,Monommata_grandis_ott2940750,Monommata_astia_ott2940751,Monommata_aequalis_ott2940752,Monommata_aeschyna_ott2940753,Monommata_dentata_ott2940754,Monommata_hyalina_ott2940755)Monommata_ott563700,(Pleurotrocha_petromyzon_ott995259,Pleurotrocha_aurea_ott2940756,Pleurotrocha_thrua_ott2940758,Pleurotrocha_altila_ott2940761,Pleurotrocha_daphnicola_ott2940762,Pleurotrocha_chalicodis_ott2940763,Pleurotrocha_altanica_ott2940764,Pleurotrocha_channa_ott2940765,Pleurotrocha_atlantica_ott2940766,Pleurotrocha_larvarum_ott2940767,Pleurotrocha_elegans_ott2940768,Pleurotrocha_sigmoidea_ott4952791,Pleurotrocha_fontanetoi_ott6367902,Pleurotrocha_altilis_ott7992153)Pleurotrocha_ott418185,(Pleurata_vernalis_ott2940757,Pleurata_trypeta_ott2940769,Pleurata_tyleri_ott4153445,Pleurata_uroglenae_ott4153446,Pleurata_thura_ott4153447,Pleurata_tithasa_ott4153448,Pleurata_chalicodes_ott4153449)Pleurata_ott4153443,(Rousseletia_corniculata_ott2940770)Rousseletia_ott2940771,(Taphrocampa_levinseni_ott2940772,Taphrocampa_lemurensis_ott2940781,Taphrocampa_selenura_ott2940802,Taphrocampa_clavigera_ott2940803,Taphrocampa_annulosa_ott2940809)Taphrocampa_ott2940773,(Tylotrocha_monopus_ott2940774)Tylotrocha_ott2940775,(Pourriotia_carcharodonta_ott2940777,Pourriotia_werneckii_ott4952796)Pourriotia_ott2940776,(Drilophaga_delagei_ott2940778,Drilophaga_bucephalus_ott2940780,Drilophaga_judayi_ott2940786)Drilophaga_ott2940779,(Pleurotrochopsis_multispinosa_ott2940782)Pleurotrochopsis_ott2940783,(Resticula_vermiculus_ott2940784,Resticula_gelida_ott2940787,Resticula_nyssa_ott2940788,Resticula_plicata_ott2940791,Resticula_anceps_ott2940792,Resticula_vermisculus_ott2940797,Resticula_lestes_ott2940804,Resticula_melandocus_ott2940805)Resticula_ott2940785,(Dorystoma_caudata_ott2940789,Dorystoma_furcata_ott2940795)Dorystoma_ott2940790,(Enteroplea_lacustris_ott2940793,Enteroplea_similis_ott2940801)Enteroplea_ott2940794,(Sphyrias_lophuana_ott2940800,Sphyrias_lofuana_ott2940808)Sphyrias_ott2940798,(Pseudoharringia_similis_ott4153441,Pseudoharringia_romanica_ott4153442)Pseudoharringia_ott4153440,Notommatidae_environmental_sample_ott4952795)Notommatidae_ott71917,((Trichocerca_elongata_ott128964,Trichocerca_rattus_ott128965,'Trichocerca_cf._capucina_MEG-2012_ott296328',Trichocerca_longiseta_ott352613,Trichocerca_stylata_ott672791,Trichocerca_tenuior_ott740398,Trichocerca_murchiearum_ott2941163,Trichocerca_weberi_ott2941164,Trichocerca_iernis_ott2941165,Trichocerca_catellus_ott2941166,Trichocerca_scipio_ott2941167,Trichocerca_inermis_ott2941168,Trichocerca_tenuidens_ott2941170,Trichocerca_uncinata_ott2941171,Trichocerca_compressa_ott2941172,Trichocerca_gracilis_ott2941173,Trichocerca_intermedia_ott2941174,Trichocerca_minuta_ott2941175,Trichocerca_bambekei_ott2941176,Trichocerca_ripli_ott2941177,Trichocerca_rousseleti_ott2941178,Trichocerca_dixonnuttalli_ott2941179,Trichocerca_myersi_ott2941180,Trichocerca_sulcata_ott2941181,Trichocerca_bidens_ott2941182,Trichocerca_voluta_ott2941183,Trichocerca_porcellus_ott2941184,Trichocerca_cavia_ott2941185,Trichocerca_capucina_ott2941186,Trichocerca_similis_ott2941187,Trichocerca_agnatha_ott2941188,Trichocerca_pustilla_ott2941189,Trichocerca_maior_ott2941190,Trichocerca_euodonta_ott2941191,Trichocerca_cimolia_ott2941192,Trichocerca_brachydactyla_ott2941193,Trichocerca_pygocera_ott2941196,Trichocerca_abilioi_ott2941197,Trichocerca_macera_ott2941198,Trichocerca_brevidactyla_ott2941199,Trichocerca_obtusidens_ott2941200,Trichocerca_bicurvicornis_ott2941201,Trichocerca_ruttneri_ott2941202,Trichocerca_caspica_ott2941203,Trichocerca_sejunctipes_ott2941204,Trichocerca_platessa_ott2941205,Trichocerca_bicuspes_ott2941206,Trichocerca_multicrinis_ott2941207,Trichocerca_longistyla_ott2941208,Trichocerca_wanarra_ott2941209,Trichocerca_taurocephala_ott2941210,Trichocerca_edmondsoni_ott2941211,Trichocerca_unidens_ott2941212,Trichocerca_gillardi_ott2941213,Trichocerca_microstyla_ott2941214,Trichocerca_rectangularis_ott2941215,Trichocerca_braziliensis_ott2941216,Trichocerca_nitida_ott2941219,Trichocerca_lophoessa_ott2941220,Trichocerca_insolens_ott2941221,Trichocerca_flagellata_ott2941222,Trichocerca_antilopaea_ott2941223,Trichocerca_tigris_ott2941224,Trichocerca_chattoni_ott2941225,Trichocerca_collaris_ott2941226,Trichocerca_siamensis_ott2941227,Trichocerca_flava_ott2941228,Trichocerca_bilunaris_ott2941229,Trichocerca_insulana_ott2941230,Trichocerca_pusilla_ott2941231,Trichocerca_mus_ott2941232,Trichocerca_cylindrica_ott2941233,Trichocerca_heterodactyla_ott2941234,Trichocerca_vernalis_ott2941235,Trichocerca_lata_ott2941236,Trichocerca_cryptodus_ott2941237,Trichocerca_mucripes_ott2941238,Trichocerca_plaka_ott2941239,Trichocerca_pediculus_ott2941240,Trichocerca_parva_ott2941242,Trichocerca_insignis_ott2941244,Trichocerca_marina_ott2941245,Trichocerca_harveyensis_ott2941246,Trichocerca_brevistyla_ott2941247,Trichocerca_ornata_ott2941248,Trichocerca_mollis_ott2941249,Trichocerca_calypta_ott2941250,Trichocerca_vargai_ott2941251,Trichocerca_cuspidata_ott2941252,Trichocerca_vassilijevae_ott2941254,Trichocerca_musculus_ott2941255,Trichocerca_orca_ott2941256,Trichocerca_bicristata_ott2941257,Trichocerca_simoneae_ott2941258,Trichocerca_barsica_ott2941259,Trichocerca_kostei_ott2941260,Trichocerca_rotundata_ott2941261,Trichocerca_mucosa_ott2941262,Trichocerca_hollaerti_ott2941263,Trichocerca_helminthodes_ott2941265,Trichocerca_valga_ott2941266,Trichocerca_rosea_ott2941267,Trichocerca_brachyura_ott2941268,Trichocerca_joblotii_ott4952764,Trichocerca_bauthiemensis_ott6367906)Trichocerca_ott128969,(Elosa_worrallii_ott2941269,Elosa_spinifera_ott2941271)Elosa_ott2941270,(Ascomorphella_volvocicola_ott2941273)Ascomorphella_ott2941272,Trichocercidae_environmental_sample_ott4952766,Trichocercidae_sp._SHCX151203c_ott5974973)Trichocercidae_ott128966,((Scaridium_longicaudum_ott128980,Scaridium_grande_ott2941275,Scaridium_montanum_ott2941276,Scaridium_neglectum_ott2941277,Scaridium_bostjani_ott2941278,Scaridium_elegans_ott2941279,Scaridium_elongatum_ott2941280)Scaridium_ott128971)Scaridiidae_ott412239,((Synchaeta_grandis_ott154828,Synchaeta_kitina_ott390795,Synchaeta_lakowitziana_ott390797,Synchaeta_tremula_ott418177,Synchaeta_pectinata_ott418179,'Synchaeta_cf._tremula/oblonga_UO-2012_ott645986','Synchaeta_cf._cecilia_UO-2012_ott645987',Synchaeta_cf._pectinata_ott743639,Synchaeta_longipes_ott2940810,Synchaeta_neapolitana_ott2940811,Synchaeta_fennica_ott2940812,Synchaeta_gyrina_ott2940813,Synchaeta_oblonga_ott2940814,Synchaeta_curvata_ott2940815,Synchaeta_triophthalma_ott2940816,Synchaeta_cecilia_ott2940817,Synchaeta_lackowitziana_ott2940818,Synchaeta_bicornis_ott2940819,Synchaeta_atlantica_ott2940820,Synchaeta_bacillifera_ott2940822,Synchaeta_pachypoida_ott2940823,Synchaeta_stylata_ott2940824,Synchaeta_vorax_ott2940825,Synchaeta_elsteri_ott2940826,Synchaeta_tremuloida_ott2940827,Synchaeta_jollyae_ott2940828,Synchaeta_glacialis_ott2940829,Synchaeta_arcifera_ott2940830,Synchaeta_baltica_ott2940831,Synchaeta_rousseleti_ott2940832,Synchaeta_pachypoda_ott2940833,Synchaeta_littoralis_ott2940834,Synchaeta_tamara_ott2940835,Synchaeta_tavina_ott2940836,Synchaeta_prominula_ott2940837,Synchaeta_rufina_ott2940838,Synchaeta_hutchingsi_ott2940839,Synchaeta_squamadigitata_ott2940840,Synchaeta_hyperborea_ott2940841,Synchaeta_johanseni_ott2940842,Synchaeta_calva_ott2940843,Synchaeta_monopus_ott2940844,Synchaeta_verrucosa_ott2940845,Synchaeta_cylindrica_ott2940846,Synchaeta_grimpei_ott2940847,'Synchaeta_cf._tremula/oblonga_CQT-2013_ott5454060')Synchaeta_ott743640,(Ploesoma_hudsoni_ott412240,Ploesoma_truncatus_ott412241,Ploesoma_lenticulare_ott2940862,Ploesoma_molle_ott2940864,Ploesoma_triacanthum_ott2940866,Ploesoma_murrayi_ott2940867,Ploesoma_truncatum_ott2940868,Ploesoma_multispinatum_ott2940869,Ploesoma_africanum_ott2940870,Ploesoma_peipsiense_ott2940871)Ploesoma_ott937974,(Polyarthra_remata_ott412243,Polyarthra_dolichoptera_ott967150,Polyarthra_major_ott2940848,Polyarthra_dissimulans_ott2940849,Polyarthra_bicera_ott2940850,Polyarthra_hexaptera_ott2940851,Polyarthra_trigla_ott2940852,Polyarthra_platyptera_ott2940853,Polyarthra_vulgaris_ott2940854,Polyarthra_minor_ott2940855,Polyarthra_bicerca_ott2940856,Polyarthra_luminosa_ott2940857,Polyarthra_longiremis_ott2940858,Polyarthra_leleki_ott2940859,Polyarthra_indica_ott2940860,Polyarthra_euryptera_ott2940861,'Polyarthra_dolichoptera_complex_sp._UO-2013_ott5223858',Polyarthra_platensis_ott6367905)Polyarthra_ott412242,(Pseudoploesoma_formosum_ott2940865,Pseudoploesoma_greeni_ott2940874)Pseudoploesoma_ott2940872)Synchaetidae_ott743641,((Asplanchna_silvestrii_ott245637,Asplanchna_sieboldi_ott251968,'Asplanchna_cf._sieboldi_MEG-2012_ott871348',Asplanchna_silvestris_ott2940875,Asplanchna_herricki_ott2940876,Asplanchna_terminalis_ott2940877,Asplanchna_amphora_ott2940878,Asplanchna_sieboldii_ott2940879,Asplanchna_herrickii_ott2940880,Asplanchna_girodi_ott2940881,Asplanchna_intermedia_ott2940882,Asplanchna_seiboldi_ott2940883,Asplanchna_brightwellii_ott2940884,Asplanchna_priodonta_ott2940885,Asplanchna_tropica_ott2940886,Asplanchna_asymmetrica_ott2940887)Asplanchna_ott251970,(Asplanchnopus_dahlgreni_ott1037813,Asplanchnopus_multiceps_ott2940888,Asplanchnopus_syrinx_ott2940889,Asplanchnopus_hyalimus_ott2940891,Asplanchnopus_hyalinus_ott2940892,Asplanchnopus_bhimavaramensis_ott2940893)Asplanchnopus_ott1037806,(Harringia_rousseleti_ott2940894,Harringia_eupoda_ott2940896)Harringia_ott2940895)Asplanchnidae_ott251965,((Lecane_lunaris_AEG1_ott245955,Lecane_hastata_AEG1_ott245956,Lecane_ohioensis_ott296324,Lecane_luna_ott316622,Lecane_rhenana_ott349144,Lecane_lunaris_ott349181,Lecane_cornuta_ott429869,Lecane_crepida_ott429871,Lecane_hamata_ott429874,Lecane_bulla_ott532775,Lecane_quadridentata_ott581923,Lecane_spinulifera_ott581939,Lecane_papuana_ott581949,Lecane_monostyla_ott581951,Lecane_closterocerca_ott620074,Lecane_curvicornis_ott620080,Lecane_grandis_ott701403,Lecane_hastata_ott701563,Lecane_crepida_AEG1_ott820705,Lecane_curvicornis_AEG2_ott820706,Lecane_bulla_AEG10_ott820712,Lecane_bulla_AEG9_ott820713,Lecane_cornuta_AEG2_ott820714,Lecane_elsa_ott1090174,Lecane_leontina_ott1090177,Lecane_affinis_ott2940904,Lecane_boorali_ott2940905,Lecane_carpatica_ott2940906,Lecane_elliptoides_ott2940907,Lecane_opias_ott2940908,Lecane_pyrrha_ott2940909,Lecane_sympoda_ott2940910,Lecane_gallagherorum_ott2940911,Lecane_stichaea_ott2940912,Lecane_styrax_ott2940913,Lecane_levistyla_ott2940914,Lecane_tryphema_ott2940915,Lecane_thalera_ott2940916,Lecane_sinuosa_ott2940917,Lecane_superaculeata_ott2940918,Lecane_crenata_ott2940919,Lecane_verecunda_ott2940920,Lecane_flabellata_ott2940921,Lecane_noobijupi_ott2940922,Lecane_niwati_ott2940923,Lecane_subulata_ott2940924,Lecane_schraederi_ott2940925,Lecane_bidactyla_ott2940926,Lecane_rudescui_ott2940927,Lecane_venusta_ott2940928,Lecane_herzigi_ott2940929,Lecane_arcuata_ott2940930,Lecane_romeroi_ott2940931,Lecane_infula_ott2940932,Lecane_ungulata_ott2940933,Lecane_psammophila_ott2940934,Lecane_namatai_ott2940935,Lecane_pyriformis_ott2940936,Lecane_galeata_ott2940937,Lecane_jaintiaensis_ott2940938,Lecane_marshi_ott2940939,Lecane_braumi_ott2940940,Lecane_jessupi_ott2940941,Lecane_hornemanni_ott2940942,Lecane_tabida_ott2940943,Lecane_inconspicua_ott2940944,Lecane_donneri_ott2940945,Lecane_latissima_ott2940946,Lecane_myersi_ott2940947,Lecane_elongata_ott2940948,Lecane_ordwayi_ott2940949,Lecane_perpusilla_ott2940950,Lecane_abanica_ott2940951,Lecane_scutata_ott2940952,Lecane_eylesi_ott2940953,Lecane_bifastigata_ott2940954,Lecane_nitida_ott2940955,Lecane_althausi_ott2940956,Lecane_subtilis_ott2940957,Lecane_copeis_ott2940958,Lecane_ligona_ott2940960,Lecane_inquieta_ott2940961,Lecane_pomiformis_ott2940964,Lecane_kutikowa_ott2940965,Lecane_solfatara_ott2940966,Lecane_doryssa_ott2940967,Lecane_bryophila_ott2940968,Lecane_dysorata_ott2940969,Lecane_mitis_ott2940970,Lecane_aspasia_ott2940971,Lecane_pelatis_ott2940972,Lecane_thailandensis_ott2940973,Lecane_lungae_ott2940974,Lecane_sverigis_ott2940975,Lecane_kunthuleensis_ott2940976,Lecane_ruttneri_ott2940977,Lecane_sola_ott2940978,Lecane_nana_ott2940979,Lecane_mira_ott2940980,Lecane_imbricata_ott2940981,Lecane_pawlowskii_ott2940982,Lecane_plesia_ott2940983,Lecane_tenua_ott2940984,Lecane_haliclysta_ott2940985,Lecane_tuxeni_ott2940986,Lecane_rhytida_ott2940987,Lecane_pertica_ott2940988,Lecane_goniata_ott2940989,Lecane_sinuata_ott2940992,Lecane_depressa_ott2940993,Lecane_furcata_ott2940994,Lecane_broaensis_ott2940995,Lecane_acus_ott2940996,Lecane_lauterborni_ott2940997,Lecane_thienemanni_ott2940998,Lecane_pycina_ott2940999,Lecane_enowi_ott2941000,Lecane_climacois_ott2941001,Lecane_lamellata_ott2941002,Lecane_segersi_ott2941003,Lecane_deridderae_ott2941004,Lecane_candida_ott2941005,Lecane_stephensae_ott2941006,Lecane_matsaluensis_ott2941009,Lecane_flexilis_ott2941010,Lecane_asymmetrica_ott2941013,Lecane_pluto_ott2941014,Lecane_satyrus_ott2941016,Lecane_ludwigii_ott2941018,Lecane_bifurca_ott2941019,Lecane_insulaconae_ott2941020,Lecane_pusilla_ott2941021,Lecane_nwadiaroi_ott2941022,Lecane_pumila_ott2941023,Lecane_inopinata_ott2941024,Lecane_eutarsa_ott2941025,Lecane_stenroosi_ott2941026,Lecane_margalefi_ott2941027,Lecane_boettgeri_ott2941028,Lecane_isanensis_ott2941029,Lecane_whitfordi_ott2941030,Lecane_formosa_ott2941031,Lecane_gwileti_ott2941032,Lecane_calcaria_ott2941033,Lecane_paradoxa_ott2941034,Lecane_obtusa_ott2941035,Lecane_pustulosa_ott2941036,Lecane_donyanaensis_ott2941037,Lecane_tanganyikae_ott2941038,Lecane_blachei_ott2941039,Lecane_intrasinuata_ott2941040,Lecane_sibina_ott2941042,Lecane_arcula_ott2941043,Lecane_sulcata_ott2941044,Lecane_stokesii_ott2941045,Lecane_hospes_ott2941046,Lecane_eswari_ott2941047,Lecane_gracilis_ott2941048,Lecane_unguitata_ott2941049,Lecane_nigeriensis_ott2941050,Lecane_fadeevi_ott2941051,Lecane_armata_ott2941052,Lecane_rugosa_ott2941053,Lecane_fusilis_ott2941054,Lecane_clara_ott2941055,Lecane_mitella_ott2941056,Lecane_dumonti_ott2941058,Lecane_tabulifera_ott2941059,Lecane_symoensi_ott2941060,Lecane_melini_ott2941061,Lecane_punctata_ott2941062,Lecane_braziliensis_ott2941063,Lecane_tenuiseta_ott2941064,Lecane_junki_ott2941065,Lecane_paxiana_ott2941066,Lecane_remanei_ott2941067,Lecane_halsei_ott2941068,Lecane_acanthinula_ott2941069,Lecane_amazonica_ott2941070,Lecane_mucronata_ott2941071,Lecane_elegans_ott2941072,Lecane_perplexa_ott2941073,Lecane_palinacis_ott2941074,Lecane_gillardi_ott2941075,Lecane_undulata_ott2941076,Lecane_agilis_ott2941077,Lecane_spiniventris_ott2941078,Lecane_stichoclysta_ott2941079,Lecane_nelsoni_ott2941080,Lecane_elasma_ott2941081,Lecane_baimaii_ott2941082,Lecane_shieli_ott2941083,Lecane_margarethae_ott2941084,Lecane_ivli_ott2941085,Lecane_boliviana_ott2941086,Lecane_sagula_ott2941087,Lecane_leura_ott2941089,Lecane_robertsonae_ott2941090,Lecane_lateralis_ott2941091,Lecane_rhopalura_ott2941092,Lecane_aeganea_ott2941093,Lecane_balatonica_ott2941094,Lecane_rhacois_ott2941095,Lecane_aculeata_ott2941096,Lecane_serrata_ott2941098,Lecane_marchantaria_ott2941099,Lecane_ercodes_ott2941100,Lecane_minuta_ott2941101,Lecane_niothis_ott2941102,Lecane_simonneae_ott2941103,Lecane_kluchor_ott2941104,Lecane_uenoi_ott2941105,Lecane_proiecta_ott2941106,Lecane_signifera_ott2941107,Lecane_inermis_ott2941108,Lecane_batillifer_ott2941109,Lecane_branchicola_ott2941110,Lecane_difficilis_ott2941111,Lecane_syngenes_ott2941112,Lecane_sylviae_ott2941113,Lecane_decipiens_ott2941114,Lecane_urna_ott2941115,Lecane_martensi_ott4952772,Lecane_yatseni_ott4952773,Lecane_chinesensis_ott4952774,Lecane_dorysimilis_ott6367894,Lecane_phapi_ott6367895,Lecane_compta_ott7992133,Lecane_flexlils_ott7992134,Lecane_ohiensis_ott7992135,Lecane_zhanjiangensis_ott7992137)Lecane_ott236072,Monostyla_ott488319,Lecanidae_environmental_sample_ott4952771)Lecanidae_ott404255,(('Dicranophorus_sp._MEG-2012_ott296321',Dicranophorus_forcipatus_ott513197,Dicranophorus_difflugiarum_ott2941638,Dicranophorus_colastes_ott2941639,Dicranophorus_aspondus_ott2941641,Dicranophorus_edestes_ott2941642,Dicranophorus_myriophylli_ott2941643,Dicranophorus_grypus_ott2941644,Dicranophorus_ponerus_ott2941645,Dicranophorus_hauerianus_ott2941646,Dicranophorus_scotius_ott2941647,Dicranophorus_halbachi_ott2941648,Dicranophorus_thysanus_ott2941649,Dicranophorus_esox_ott2941650,Dicranophorus_sigmoides_ott2941651,Dicranophorus_corystis_ott2941652,Dicranophorus_capucinus_ott2941653,Dicranophorus_pennatus_ott2941654,Dicranophorus_robustus_ott2941655,Dicranophorus_strigosus_ott2941656,Dicranophorus_cambari_ott2941657,Dicranophorus_kostei_ott2941658,Dicranophorus_stultus_ott2941659,Dicranophorus_lenapensis_ott2941661,Dicranophorus_dolerus_ott2941662,Dicranophorus_rostratus_ott2941663,Dicranophorus_luetkeni_ott2941664,Dicranophorus_alcimus_ott2941665,Dicranophorus_siedleckii_ott2941666,Dicranophorus_facinus_ott2941668,Dicranophorus_minutes_ott2941669,Dicranophorus_isothes_ott2941670,Dicranophorus_pauliani_ott2941671,Dicranophorus_mesotis_ott2941672,Dicranophorus_haueri_ott2941673,Dicranophorus_tegillus_ott2941674,Dicranophorus_proclestes_ott2941675,Dicranophorus_facilis_ott2941676,Dicranophorus_leptodon_ott2941678,Dicranophorus_epicharis_ott2941679,Dicranophorus_macrostyla_ott2941680,Dicranophorus_hercules_ott2941682,Dicranophorus_proclastes_ott2941683,Dicranophorus_spiculatus_ott2941684,Dicranophorus_bulgaricus_ott2941685,Dicranophorus_grandis_ott2941686,Dicranophorus_artamus_ott2941687,Dicranophorus_sebastus_ott2941689,Dicranophorus_biastis_ott2941690,Dicranophorus_prionacis_ott2941691,Dicranophorus_riparius_ott2941692,Dicranophorus_saevus_ott2941693,Dicranophorus_semnus_ott2941694)Dicranophorus_ott513196,(Encentrum_astridae_ott1090172,Encentrum_tectipes_ott1090175,Encentrum_barti_ott2941527,Encentrum_stechlinensis_ott2941528,Encentrum_mariae_ott2941529,Encentrum_gibbosum_ott2941530,Encentrum_salsum_ott2941531,Encentrum_elongatum_ott2941532,Encentrum_permolle_ott2941533,Encentrum_sutoroides_ott2941534,Encentrum_goldschmidi_ott2941535,Encentrum_carlini_ott2941536,Encentrum_valkanovi_ott2941537,Encentrum_eulitorale_ott2941538,Encentrum_kulmatyckii_ott2941539,Encentrum_boreale_ott2941540,Encentrum_tobyhannaense_ott2941541,Encentrum_desmeti_ott2941542,Encentrum_parime_ott2941543,Encentrum_murrayi_ott2941544,Encentrum_longirostrum_ott2941545,Encentrum_umbonatum_ott2941546,Encentrum_insolitum_ott2941547,Encentrum_stechlinense_ott2941548,Encentrum_flexile_ott2941549,Encentrum_pachypus_ott2941550,Encentrum_fluviatile_ott2941551,Encentrum_uncinatum_ott2941553,Encentrum_lupus_ott2941554,Encentrum_striatum_ott2941555,Encentrum_rousseleti_ott2941556,Encentrum_belluinum_ott2941557,Encentrum_dieteri_ott2941558,Encentrum_nesites_ott2941559,Encentrum_moldavicum_ott2941560,Encentrum_enteromorphae_ott2941561,Encentrum_permutandum_ott2941562,Encentrum_sacculiforme_ott2941563,Encentrum_kostei_ott2941564,Encentrum_saundersiae_ott2941565,Encentrum_mustela_ott2941566,Encentrum_kutikovae_ott2941567,Encentrum_forcipatum_ott2941568,Encentrum_pornsilpi_ott2941569,Encentrum_eristes_ott2941570,Encentrum_bidentatum_ott2941571,Encentrum_algente_ott2941572,Encentrum_torvitum_ott2941573,Encentrum_nikor_ott2941574,Encentrum_villosum_ott2941575,Encentrum_glaucum_ott2941576,Encentrum_lacidum_ott2941577,Encentrum_acrodon_ott2941578,Encentrum_marinum_ott2941579,Encentrum_tenuidigitatum_ott2941580,Encentrum_oxyodon_ott2941582,Encentrum_asellicola_ott2941583,Encentrum_cruentum_ott2941584,Encentrum_remanei_ott2941585,Encentrum_arvicola_ott2941586,Encentrum_mucronatum_ott2941587,Encentrum_hofsteni_ott2941588,Encentrum_incisum_ott2941589,Encentrum_tyrphos_ott2941590,Encentrum_longidens_ott2941591,Encentrum_felis_ott2941592,Encentrum_spinosum_ott2941593,Encentrum_martoides_ott2941594,Encentrum_porsildi_ott2941595,Encentrum_walterkostei_ott2941596,Encentrum_listensoides_ott2941597,Encentrum_longipes_ott2941598,Encentrum_arenarium_ott2941599,Encentrum_gulo_ott2941600,Encentrum_zetetum_ott2941601,Encentrum_listense_ott2941602,Encentrum_simillimum_ott2941603,Encentrum_rapax_ott2941604,Encentrum_aquila_ott2941605,Encentrum_torvitoides_ott2941607,Encentrum_otois_ott2941608,Encentrum_eurycephalum_ott2941609,Encentrum_putorius_ott2941610,Encentrum_orthodactylum_ott2941611,Encentrum_spatiatum_ott2941612,Encentrum_matthesi_ott2941613,Encentrum_wiszniewskii_ott2941614,Encentrum_sorex_ott2941615,Encentrum_psammophilum_ott2941616,Encentrum_caratum_ott2941617,Encentrum_oculatum_ott2941618,Encentrum_sutor_ott2941619,Encentrum_kozminskii_ott2941620,Encentrum_minax_ott2941621,Encentrum_myersi_ott2941622,Encentrum_axi_ott2941623,Encentrum_ussuriensis_ott2941624,Encentrum_lutra_ott2941625,Encentrum_frenoti_ott2941627,Encentrum_parvum_ott2941628,Encentrum_diglandula_ott2941629,Encentrum_graingeri_ott2941631,Encentrum_alpinum_ott2941632,Encentrum_martes_ott2941633,Encentrum_voigti_ott2941634,Encentrum_limicola_ott2941635,Encentrum_semiplicatum_ott2941636,Encentrum_obesum_ott2941637,Encentrum_armatum_ott4952785,Encentrum_liepolti_ott4952786,Encentrum_aluligerum_ott6367887,Encentrum_foroiuliense_ott6367888,Encentrum_loefgreni_ott6367889,Encentrum_pugiodigitatum_ott6367890,Encentrum_spatitium_ott6367891,Encentrum_uncinatoides_ott6367892)Encentrum_ott1090173,(Paradicranophorus_sordidus_ott2941606,Paradicranophorus_wesenberglundi_ott2941756,Paradicranophorus_sinus_ott2941759,Paradicranophorus_aculeatus_ott2941760,Paradicranophorus_verae_ott2941761,Paradicranophorus_hudsoni_ott2941763,Paradicranophorus_halophilus_ott6367893)Paradicranophorus_ott2941757,(Dicranophoroides_caudatus_ott2941667,Dicranophoroides_venezueliensis_ott2941720,Dicranophoroides_australiensis_ott2941722,Dicranophoroides_claviger_ott2941723)Dicranophoroides_ott2941717,(Wierzejskiella_subulosa_ott2941696,Wierzejskiella_marina_ott2941724,Wierzejskiella_ricciae_ott2941730,Wierzejskiella_subterranea_ott2941731,Wierzejskiella_ambigua_ott2941732,Wierzejskiella_elongata_ott2941733,Wierzejskiella_velox_ott2941734,Wierzejskiella_sabulosa_ott2941735,Wierzejskiella_vagneri_ott2941737)Wierzejskiella_ott2941697,(Erignatha_longidentata_ott2941698,Erignatha_capula_ott2941712,Erignatha_sagittoides_ott2941716,Erignatha_tenuidens_ott2941718,Erignatha_clastopis_ott2941719,Erignatha_sagitta_ott2941736,Erignatha_thienemanni_ott7992128)Erignatha_ott2941699,(Albertia_woronkowi_ott2941701,Albertia_reicheltae_ott2941703,Albertia_naidis_ott2941704,Albertia_ovagranulata_ott2941705,Albertia_vermiculus_ott2941708,Albertia_crystallina_ott2941709,Albertia_typhlina_ott2941711,Albertia_vermisculus_ott7992125)Albertia_ott2941702,(Balatro_calvus_ott2941706,Balatro_anguiformis_ott2941710,Balatro_aciliatus_ott2941738,Balatro_fridericiae_ott2941739)Balatro_ott2941707,(Aspelta_imputa_ott2941715,Aspelta_reibischi_ott2941725,Aspelta_imbuta_ott2941726,Aspelta_psitta_ott2941727,Aspelta_egregia_ott2941728,Aspelta_tilba_ott2941729,Aspelta_clydona_ott2941750,Aspelta_curvidactyla_ott2941751,Aspelta_angusta_ott2941752,Aspelta_chorista_ott2941753,Aspelta_pachida_ott2941754,Aspelta_circinator_ott2941755,Aspelta_labri_ott2941764,Aspelta_intradentata_ott2941765,Aspelta_europaea_ott2941766,Aspelta_alastor_ott2941767,Aspelta_beltista_ott2941768,Aspelta_aper_ott2941769,Aspelta_lestes_ott2941770,Aspelta_bidentata_ott2941771,Aspelta_macra_ott2941772,Aspelta_harringi_ott2941773,Aspelta_secreta_ott4952787)Aspelta_ott2941714,(Parencentrum_lutetiae_ott2941741,Parencentrum_plicatum_ott2941743)Parencentrum_ott2941742,(Dorria_dalecarlica_ott2941744)Dorria_ott2941745,(Wigrella_depressa_ott2941746,Wigrella_amphora_ott2941774)Wigrella_ott2941700,(Pedipartia_gracilis_ott2941747)Pedipartia_ott2941748,(Streptognatha_lepta_ott2941749)Streptognatha_ott2941740,(Donneria_sudzukii_ott2941762)Donneria_ott2941695,(Kostea_wockei_ott4153456)Kostea_ott4153455,(Myersinella_tetraglena_ott4153458,Myersinella_uncodonta_ott4153459,Myersinella_belodon_ott4153460,Myersinella_longiforceps_ott4153461)Myersinella_ott4153457,(Inflatana_pomazkovae_ott4153463)Inflatana_ott4153462,(Glaciera_schabetsbergeri_ott4153466)Glaciera_ott4153465)Dicranophoridae_ott513189,(('Lepadella_cf._ovalis_MEG-2012_ott296329',(Lepadella_patella_oblonga_ott630011)Lepadella_patella_ott1018359,Lepadella_triba_ott630013,Lepadella_rhomboides_ott1090179,Lepadella_astacicola_ott2941342,Lepadella_cornuta_ott2941343,Lepadella_vitrea_ott2941344,Lepadella_rhodesiana_ott2941345,Lepadella_neboissi_ott2941346,Lepadella_strepta_ott2941347,Lepadella_apsicora_ott2941348,Lepadella_branchicola_ott2941349,Lepadella_decora_ott2941350,Lepadella_quadricarinata_ott2941351,Lepadella_pseudosimilis_ott2941352,Lepadella_discoidea_ott2941353,Lepadella_ovalis_ott2941354,Lepadella_latusinus_ott2941355,Lepadella_amphitropis_ott2941356,Lepadella_monodactyla_ott2941357,Lepadella_mica_ott2941358,Lepadella_triprojectus_ott2941359,Lepadella_minuscula_ott2941360,Lepadella_dorsalis_ott2941361,Lepadella_deridderae_ott2941362,Lepadella_latusimus_ott2941363,Lepadella_pterygoida_ott2941364,Lepadella_duvigneaudi_ott2941365,Lepadella_hyalina_ott2941366,Lepadella_salisburii_ott2941367,Lepadella_heterodactyla_ott2941368,Lepadella_pterygoides_ott2941369,Lepadella_pseudoacuminata_ott2941370,Lepadella_dactyliseta_ott2941371,Lepadella_parasitica_ott2941372,Lepadella_vanoyei_ott2941373,Lepadella_psammophila_ott2941374,Lepadella_angusta_ott2941375,Lepadella_vandenbrandei_ott2941376,Lepadella_zigzag_ott2941377,Lepadella_donneri_ott2941378,Lepadella_cryphaea_ott2941379,Lepadella_acuminata_ott2941380,Lepadella_pyriformis_ott2941381,Lepadella_minorui_ott2941382,Lepadella_pumilo_ott2941383,Lepadella_tenella_ott2941384,Lepadella_gelida_ott2941385,Lepadella_rhomboidula_ott2941386,Lepadella_adjuncta_ott2941387,Lepadella_canadaensis_ott2941388,Lepadella_intermedia_ott2941389,Lepadella_sali_ott2941390,Lepadella_longiseta_ott2941391,Lepadella_borealis_ott2941392,Lepadella_wrighti_ott2941393,Lepadella_berzinsi_ott2941394,Lepadella_bidentata_ott2941395,Lepadella_pejleri_ott2941396,Lepadella_minuta_ott2941397,Lepadella_monodi_ott2941398,Lepadella_venefica_ott2941399,Lepadella_serrata_ott2941400,Lepadella_heterostyla_ott2941401,Lepadella_crytopus_ott2941402,Lepadella_mascarensis_ott2941403,Lepadella_desmeti_ott2941404,Lepadella_favorita_ott2941405,Lepadella_obtusa_ott2941406,Lepadella_beyensi_ott2941408,Lepadella_cristata_ott2941409,Lepadella_abbei_ott2941410,Lepadella_pontica_ott2941411,Lepadella_eurysterna_ott2941412,Lepadella_glossa_ott2941413,Lepadella_parvula_ott2941414,Lepadella_ehrenbergii_ott2941415,Lepadella_princisi_ott2941416,Lepadella_biloba_ott2941417,Lepadella_cyrtopus_ott2941418,Lepadella_curvicaudata_ott2941419,Lepadella_lata_ott2941420,Lepadella_bicornis_ott2941421,Lepadella_amazonica_ott2941422,Lepadella_xenica_ott2941423,Lepadella_nympha_ott2941424,Lepadella_tricostata_ott2941425,Lepadella_quadricurvata_ott2941426,Lepadella_kostei_ott2941427,Lepadella_haueri_ott2941428,Lepadella_akrobeles_ott2941429,Lepadella_rottenburgi_ott2941430,Lepadella_tyleri_ott2941431,Lepadella_evaginata_ott2941432,Lepadella_costatoides_ott2941433,Lepadella_koniari_ott2941434,Lepadella_punctata_ott2941435,Lepadella_whitfordi_ott2941436,Lepadella_costata_ott2941437,Lepadella_chengalathi_ott2941438,Lepadella_margalefi_ott2941439,Lepadella_benjamini_ott2941440,Lepadella_myersi_ott2941441,Lepadella_quinquecostata_ott2941442,Lepadella_elongata_ott2941444,Lepadella_neglecta_ott2941445,Lepadella_degreefi_ott2941446,Lepadella_elliptica_ott2941447,Lepadella_persimilis_ott2941448,Lepadella_imbricata_ott2941449,Lepadella_tana_ott2941450,Lepadella_mataca_ott2941451,Lepadella_minoruoides_ott2941452,Lepadella_apsida_ott2941453,Lepadella_ptilota_ott2941454,Lepadella_riedeli_ott2941455,Lepadella_lindaui_ott2941456,Lepadella_triptera_ott2941457,Lepadella_nartiangensis_ott2941458,Lepadella_visenda_ott2941459,Lepadella_paparoa_ott2941460,'Lepadella_cf._quadricarinata_AGROT318-10_ott7506432','Lepadella_cf._quadricarinata_AGROT327-10_ott7506433','Lepadella_cf._quadricarinata_AGROT337-10_ott7506434','Lepadella_cf._quadricarinata_AGROT338-10_ott7506435')Lepadella_ott1018358,(Squatinella_rostrum_ott995274,Squatinella_retrospina_ott2941461,Squatinella_lamellaris_ott2941462,Squatinella_leydigii_ott2941463,Squatinella_lunata_ott2941464,Squatinella_macrodactyla_ott2941465,Squatinella_geleii_ott2941467,Squatinella_stylata_ott2941468,Squatinella_longispinata_ott2941469,Squatinella_pseudorostrum_ott2941470,Squatinella_microdactyla_ott2941472,Squatinella_bifurca_ott2941473,Squatinella_bisetata_ott7992143)Squatinella_ott995261,(Colurella_unicauda_ott2941474,Colurella_collaris_ott2941476,Colurella_denticauda_ott2941477,Colurella_halophila_ott2941478,Colurella_hindenburgi_ott2941479,Colurella_obtusa_ott2941481,Colurella_marinovi_ott2941482,Colurella_dicentra_ott2941483,Colurella_adriatica_ott2941484,Colurella_mucronulata_ott2941485,Colurella_psammophila_ott2941486,Colurella_sulcata_ott2941490,Colurella_geophila_ott2941493,Colurella_tesselata_ott2941494,Colurella_sanoamuangae_ott2941495,Colurella_oblonga_ott2941496,Colurella_colurus_ott2941497,Colurella_aquaeducti_ott2941498,Colurella_anodonta_ott2941499,Colurella_paludosa_ott2941500,Colurella_sinistra_ott2941501,Colurella_salina_ott2941502,Colurella_uncinata_ott2941503,Colurella_unicaudata_ott6367897,Colurella_aquaducti_ott7992140,Colurella_ovalis_ott7992141,Colurella_oxycauda_ott7992142)Colurella_ott2941475,(Paracolurella_aemula_ott2941487,Paracolurella_logima_ott2941491)Paracolurella_ott2941489,Lepadellidae_environmental_sample_ott4952798,Lepadellidae_sp._SHQC150123a_ott5974969)Lepadellidae_ott1090176,((Epiphanes_senta_ott368450,Epiphanes_pelagica_ott2941139,Epiphanes_clavatula_ott2941140,Epiphanes_clavulata_ott2941141,Epiphanes_brachionus_ott2941142,Epiphanes_macroura_ott2941143,Epiphanes_desmeti_ott2941144,Epiphanes_chihuahuaensis_ott4952767,Epiphanes_ukera_ott4952768,Epiphanes_hawaiensis_ott4952769)Epiphanes_ott368449,(Cyrtonia_tuba_ott958183)Cyrtonia_ott958180,(Mikrocodides_robustus_ott2940759,Mikrocodides_chlaena_ott2941158,Mikrocodides_hertha_ott2941162)Mikrocodides_ott2941152,(Rhinoglena_frontalis_ott2941146,Rhinoglena_tokioensis_ott2941150,Rhinoglena_fertoeensis_ott2941155,Rhinoglena_kutikovae_ott2941156,Rhinoglena_tokidensis_ott2941161,Rhinoglena_ovigera_ott7992131,Rhinoglena_texana_ott7992132)Rhinoglena_ott2941147,(Proalides_subtilis_ott2941148,Proalides_digitus_ott2941153,Proalides_tentaculatus_ott2941159)Proalides_ott2941149,(Microcodides_hertha_ott7992130)Microcodides_ott7992129)Epiphanidae_ott368452,((Proales_similis_ott412238,Proales_doliaris_ott412245,Proales_reinhardti_ott412247,Proales_daphnicola_ott958185,Proales_theodora_ott958188,Proales_fallaciosa_ott958190,Proales_werneckii_ott2940661,Proales_gonothyraeae_ott2941281,Proales_prehensor_ott2941282,Proales_palimmeka_ott2941283,Proales_othodon_ott2941284,Proales_sordida_ott2941285,Proales_minima_ott2941286,Proales_granulosa_ott2941287,Proales_adenodis_ott2941289,Proales_gladia_ott2941290,Proales_halophila_ott2941291,Proales_lenta_ott2941293,Proales_globulifera_ott2941294,Proales_phaeopis_ott2941295,Proales_gigantea_ott2941296,Proales_parasita_ott2941297,Proales_cryptopus_ott2941298,Proales_litoralis_ott2941299,Proales_commutata_ott2941300,Proales_alba_ott2941301,Proales_baradlana_ott2941302,Proales_syltensis_ott2941303,Proales_micropus_ott2941304,Proales_macrura_ott2941305,Proales_indirae_ott2941306,Proales_oculata_ott2941308,Proales_kostei_ott2941309,Proales_christinae_ott2941310,Proales_coryneger_ott2941311,Proales_paguri_ott2941312,Proales_wesenbergi_ott2941313,Proales_ornata_ott2941314,Proales_decipiens_ott2941315,Proales_simplex_ott2941316,Proales_germanica_ott2941317,Proales_provida_ott2941318,Proales_segnis_ott2941319,Proales_bemata_ott2941320,Proales_cognita_ott2941321,Proales_pugio_ott2941322,Proales_ardechensis_ott4952775,Proales_laticauda_ott4952776,Proales_francescae_ott6367903,Proales_gammaricola_ott6367904,'Proales_sp._EM-2017_ott7506438',Proales_tillyensis_ott7992156)Proales_ott412244,(Proalinopsis_caudatus_ott2940706,Proalinopsis_squamipes_ott2941331,Proalinopsis_lobatus_ott2941333,Proalinopsis_selene_ott2941334,Proalinopsis_gracilis_ott2941335,Proalinopsis_staurus_ott2941336,Proalinopsis_phacus_ott2941338,Proalinopsis_phagus_ott2941339,Proalinopsis_pellucida_ott2941340)Proalinopsis_ott2941330,(Wulfertia_ornata_ott2941323,Wulfertia_kindensis_ott2941325,Wulfertia_kivuensis_ott2941329)Wulfertia_ott2941324,(Bryceella_tenella_ott2941327,Bryceella_stylata_ott2941328,Bryceella_perpusilla_ott7992154,Bryceella_voigtii_ott7992155)Bryceella_ott2941326)Proalidae_ott412246,((Ascomorpha_ovalis_ott513186,Ascomorpha_ovalis_AEG1_ott820709,Ascomorpha_agilis_ott2941519,Ascomorpha_minima_ott2941520,Ascomorpha_ecaudis_ott2941521,Ascomorpha_saltans_ott2941522,Ascomorpha_dumonti_ott2941523,Ascomorpha_tundisii_ott2941524,Ascomorpha_minuta_ott2941525,Ascomorpha_klementi_ott2941526)Ascomorpha_ott513185)Gastropidae_ott513193,((Lindia_torulosa_ott563704,(Lindia_tecusa_ott1090165,Lindia_elsae_ott2941510,Lindia_gravitata_ott2941517)Halolindia_ott2941518,Lindia_gracilis_ott2941504,Lindia_producta_ott2941505,Lindia_janickii_ott2941506,Lindia_truncata_ott2941507,Lindia_annecta_ott2941508,Lindia_caerulea_ott2941509,Lindia_ecela_ott2941511,Lindia_fulva_ott2941512,Lindia_candida_ott2941513,Lindia_pallida_ott2941514,Lindia_deridderae_ott2941515,Lindia_euchromatica_ott2941516,Lindia_aequorea_ott6367898,Lindia_anebodica_ott7992144,Lindia_sphagnophila_ott7992145,Lindia_virgata_ott7992146)Lindia_ott1090166)Lindiidae_ott1090178,((Microcodon_clavus_ott563707)Microcodon_ott563706)Microcodonidae_ott563711,((Beauchampiella_eudactylota_ott738316)Beauchampiella_ott672794,(Tripleuchlanis_plicata_ott1096272)Tripleuchlanis_ott352056,(Dipleuchlanis_conradi_ott2942183,Dipleuchlanis_ornata_ott2942298,Dipleuchlanis_elegans_ott2942299,Dipleuchlanis_propatula_ott2942300)Dipleuchlanis_ott2942184,(Diplois_daviesiae_ott2942344)Diplois_ott2942343,(Pseudoeuchlanis_longipedes_ott4153481)Pseudoeuchlanis_ott4153480)Euchlanidae_ott672793,((Asciaporrecta_hyalina_ott2940760,Asciaporrecta_arcellicola_ott4153523,Asciaporrecta_difflugicola_ott4153524)Asciaporrecta_ott4153521)Asciaporrectidae_ott2942284,((Itura_globata_ott2942178,Itura_aurita_ott2942266,Itura_myersi_ott2942267,Itura_chamadis_ott2942268,Itura_viridis_ott2942269,Itura_symmetrica_ott2942270,Itura_deridderae_ott2942271)Itura_ott2942179)Ituridae_ott2942180,((Gastropus_stylifer_ott2942247,Gastropus_minor_ott2942293,Gastropus_hyptopus_ott2942342)Gastropus_ott2942248)Gastropodidae_ott5693675,((Tetrasiphon_hydrocora_ott2942255)Tetrasiphon_ott2942256)Tetrasiphonidae_ott2942257,((Birgea_enantia_ott2942261)Birgea_ott2942262)Birgeidae_ott2942263,((Cotylegaleata_perplexa_ott4153520,Cotylegaleata_iskenderunensis_ott7992124)Cotylegaleata_ott4153519)Cotylegaleatidae_ott2942286,((Claria_segmentata_ott4153526)Claria_ott4153525)Clariaidae_ott2942336)Ploima_ott251966)Pseudotrocha_ott5673589,((((Sinantherina_socialis_ott107177,Sinantherina_ariprepes_ott513192,Sinantherina_triglandularis_ott2942057,Sinantherina_spinosa_ott2942058,Sinantherina_procera_ott2942059,Sinantherina_semibullata_ott2942060)Sinantherina_ott107178,(Floscularia_melicerta_ott418175,Floscularia_decora_ott426467,Floscularia_longicauda_ott2942045,Floscularia_wallacei_ott2942046,Floscularia_bifida_ott2942047,Floscularia_conifera_ott2942048,Floscularia_janus_ott2942049,Floscularia_pedunculata_ott2942052,Floscularia_noodti_ott2942053,Floscularia_curvicornis_ott2942054,Floscularia_armata_ott2942055,Floscularia_rigens_ott2942056,Floscularia_environmental_sample_ott4952751)Floscularia_ott435984,(Ptygura_libera_ott513191,Ptygura_kostei_ott2942061,Ptygura_elsteri_ott2942062,Ptygura_brevis_ott2942063,Ptygura_cristata_ott2942064,Ptygura_pilula_ott2942065,Ptygura_crystallina_ott2942067,Ptygura_beauchampi_ott2942068,Ptygura_stygis_ott2942070,Ptygura_agassizi_ott2942071,Ptygura_longicornis_ott2942072,Ptygura_tacita_ott2942073,Ptygura_velata_ott2942074,Ptygura_tihanyensis_ott2942075,Ptygura_wilsonii_ott2942076,Ptygura_mucicola_ott2942077,Ptygura_pedunculata_ott2942078,Ptygura_intermedia_ott2942079,Ptygura_barbata_ott2942080,Ptygura_rotifer_ott2942081,Ptygura_tridorsicornis_ott2942082,Ptygura_stephanion_ott2942083,Ptygura_furcillata_ott2942084,Ptygura_socialis_ott2942085,Ptygura_brachiata_ott2942086,Ptygura_seminatans_ott2942087,Ptygura_spongicola_ott2942088,Ptygura_linguata_ott2942089,Ptygura_noodti_ott6367878,Ptygura_melicerta_ott7506321)Ptygura_ott513190,(Collotheca_campanulata_ott1039706,Collotheca_trilobata_ott2942050,Collotheca_vargai_ott2942090,Collotheca_moselii_ott2942092,Collotheca_tenuilobata_ott2942093,Collotheca_libera_ott2942094,Collotheca_crateriformis_ott2942095,Collotheca_ornata_ott2942096,Collotheca_annulata_ott2942097,Collotheca_calva_ott2942098,Collotheca_lettevalli_ott2942099,Collotheca_sessilis_ott2942100,Collotheca_heptabrachiata_ott2942101,Collotheca_ferox_ott2942102,Collotheca_quadrilobata_ott2942105,Collotheca_triloba_ott2942106,Collotheca_bilfingeri_ott2942107,Collotheca_cucullata_ott2942108,Collotheca_coronetta_ott2942110,Collotheca_tenera_ott2942111,Collotheca_riverai_ott2942112,Collotheca_atrochoides_ott2942113,Collotheca_torquilobata_ott2942114,Collotheca_undulata_ott2942115,Collotheca_monoceros_ott2942116,Collotheca_evansonii_ott2942117,Collotheca_spinata_ott2942118,Collotheca_pelagica_ott2942119,Collotheca_ambigua_ott2942120,Collotheca_discophora_ott2942121,Collotheca_mutabilis_ott2942122,Collotheca_balatonica_ott2942123,Collotheca_stephanochaeta_ott2942124,Collotheca_bulbosa_ott2942125,Collotheca_judayi_ott2942126,Collotheca_quadrinodosa_ott2942127,Collotheca_trifidlobata_ott2942128,Collotheca_edentata_ott2942129,Collotheca_paradoxa_ott2942130,Collotheca_wiszniewskii_ott2942131,Collotheca_thunmarki_ott2942132,Collotheca_edmondsoni_ott2942134,Collotheca_hepatabrachia_ott2942135,Collotheca_minuta_ott2942136,Collotheca_epizootica_ott2942138,Collotheca_rasmae_ott2942139,Collotheca_vargae_ott2942141,Collotheca_gosseii_ott4952754,Collotheca_polyphemus_ott4952755,Collotheca_tetralobata_ott4952756,Collotheca_hexalobata_ott4952757,Collotheca_hoodii_ott4952758)Collotheca_ott1039707,(Lacinularia_reticulata_ott2942145,Lacinularia_racemosa_ott2942147,Lacinularia_flosculosa_ott2942155,Lacinularia_ismailoviensis_ott2942156,Lacinularia_elongata_ott2942158,Lacinularia_striolata_ott2942159,Lacinularia_megalotrocha_ott2942160,Lacinularia_pedunculata_ott2942161,Lacinularia_elliptica_ott2942162,Lacinularia_ismaloviensis_ott2942166,Lacinularia_causeyae_ott7992099)Lacinularia_ott2942146,(Limnias_shiawasseensis_ott2942148,Limnias_melicerta_ott2942157,Limnias_cornuella_ott2942163,Limnias_nymphaea_ott2942164,Limnias_ceratophylli_ott2942165,Limnias_myriophylli_ott2942167,'Limnias_sp._PM-2016_ott7506320')Limnias_ott2942149,(Beauchampia_crucigera_ott2942150)Beauchampia_ott2942151,(Octotrocha_speciosa_ott2942153)Octotrocha_ott2942154,(Pentatrocha_gigantea_ott4153435)Pentatrocha_ott4153434,(Lacinularoides_coloniensis_ott5706031)Lacinularoides_ott5693683)Flosculariidae_ott681210,((Testudinella_patina_AEG5_ott245957,Testudinella_caeca_ott519380,(Testudinella_patina_intermedia_ott890418,Testudinella_patina_dendradena_ott2942015)Testudinella_patina_ott995276,Testudinella_clypeata_ott958181,Testudinella_berzinsi_ott2941982,Testudinella_truncata_ott2941983,Testudinella_brevicaudata_ott2941984,Testudinella_wuhanensis_ott2941985,Testudinella_striata_ott2941986,Testudinella_aspis_ott2941987,Testudinella_magna_ott2941988,Testudinella_robertsonae_ott2941989,Testudinella_angulata_ott2941990,Testudinella_reflexa_ott2941991,Testudinella_husseyi_ott2941992,Testudinella_obscura_ott2941993,Testudinella_haueriensis_ott2941994,Testudinella_unicornuta_ott2941995,Testudinella_triangularis_ott2941996,Testudinella_mucronata_ott2941997,Testudinella_subdiscoidea_ott2941998,Testudinella_andranomenensis_ott2941999,Testudinella_elliptica_ott2942000,Testudinella_neboisi_ott2942001,Testudinella_sphagnicola_ott2942002,Testudinella_parva_ott2942003,Testudinella_ahlstromi_ott2942004,Testudinella_carlini_ott2942005,Testudinella_tridentata_ott2942006,Testudinella_discoidea_ott2942007,Testudinella_kostei_ott2942008,Testudinella_walkeri_ott2942009,Testudinella_gillardi_ott2942010,Testudinella_epicopta_ott2942011,Testudinella_greeni_ott2942012,Testudinella_panonica_ott2942013,Testudinella_emarginula_ott2942014,Testudinella_stappersi_ott2942016,Testudinella_amphora_ott2942017,Testudinella_munda_ott2942018,Testudinella_brycei_ott2942019,Testudinella_vanoyei_ott2942020,Testudinella_ohlei_ott2942021,Testudinella_incisa_ott2942022,Testudinella_ovata_ott2942023,Testudinella_bicorniculata_ott4952759,Testudinella_zhujiangensis_ott4952760,Testudinella_elongata_ott4952761,Testudinella_crassa_ott6367881,Testudinella_pseudobscura_ott6367882,Testudinella_quadrilobata_ott6367883,Testudinella_bonneri_ott7992105,Testudinella_clypleata_ott7992106)Testudinella_ott995263,(Pompholyx_sulcata_ott2942024,Pompholyx_triloba_ott2942028,Pompholyx_complanata_ott2942029)Pompholyx_ott2942025,(Anchitestudinella_mekongensis_ott2942027)Anchitestudinella_ott2942026,'Testudinella_sp._GG-2003_ott4153429',(Pompholys_sulcata_ott7992104)Pompholys_ott7992103)Testudinellidae_ott124556,((Filinia_longiseta_ott424470,Filinia_brachiata_ott2942042,Filinia_terminalis_ott2942043,Filinia_cornuta_ott2942044)Filinia_ott424467,(Horaella_brehmi_ott2942227,Horaella_thomassoni_ott2942341)Horaella_ott2942204,(Trochosphaera_aequatorialis_ott2942250,Trochosphaera_solstitialis_ott2942252)Trochosphaera_ott2942251)Filinidae_ott1026312,((Conochilus_hippocrepis_ott1039708,Conochilus_unicornis_ott1039709,Conochilus_exiguus_ott2942169,Conochilus_deltaicus_ott2942170,Conochilus_coenobasis_ott2942171,Conochilus_natans_ott2942172)Conochilus_ott1039704,(Conochiloides_dossuarius_ott2942168,Conochiloides_exiguus_ott2942175,'Conochiloides_sp._WM-2017a_ott7506313')Conochiloides_ott2942174,(Conochilopsis_causeyae_ott4153438)Conochilopsis_ott4153437)Conochilidae_ott1039705,((Hexarthra_polydonta_ott2942200,Hexarthra_jenkinae_ott2942203,Hexarthra_oxyuris_ott2942208,Hexarthra_polychaeta_ott2942228,Hexarthra_mira_ott2942246,Hexarthra_longicornicula_ott2942253,Hexarthra_mollis_ott2942272,Hexarthra_propinqua_ott2942288,Hexarthra_bulgarica_ott2942289,Hexarthra_reducens_ott2942320,Hexarthra_brandorffi_ott2942322,Hexarthra_fennica_ott2942329,Hexarthra_polyodonta_ott2942330,Hexarthra_libica_ott2942346,(Hexarthra_intermedia_brasiliensis_ott5723812)Hexarthra_intermedia_ott2942196,Hexarthra_polyptera_ott6367880,'Hexarthra_sp._WM-2017a_ott7506323',Hexarthra_oxyure_ott7992101)Hexarthra_ott491882)Hexarthridae_ott2942198,Flosculariaceae_environmental_sample_ott7506316)Flosculariaceae_ott107174,(((Acyclus_inquietus_ott2942237,Acyclus_trilobus_ott2942243)Acyclus_ott2942238,(Atrochus_tentaculatus_ott2942241)Atrochus_ott2942242,(Cupelopagis_vorax_ott2942244,Cupelopagis_bipera_ott2942332)Cupelopagis_ott2942245)Atrochidae_ott2942239,(Stephanoceros_fimbriatus_ott2942283,Stephanoceros_vulgaris_ott2942294,Stephanoceros_millsii_ott7992096)Stephanoceros_ott2942282,'Collothecaceae_sp._EM-2017_ott7506311')Collothecaceae_ott5677241)Gnesiotrocha_ott5673590,'Monogononta_sp._R35_CoastL_ott7506325')Monogononta_ott641254,((((Seison_nebaliae_ott157976,Seison_africanus_ott2940501)Seison_ott779396,(Paraseison_annulatus_ott2940500)Paraseison_ott4153539)Seisonidae_ott5673588)Seisonacea_ott2942226)Seisonidea_ott779397)Rotifera_ott471706; From 3a3591b4185ef7cc733c614e8875c39476cc7ae2 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 24 Aug 2026 16:49:44 +0000 Subject: [PATCH 35/62] newick/format_newick: inline trim_tree build_oz_tree is now redundant, so copy the code here. --- oz_tree_build/newick/format_newick.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/oz_tree_build/newick/format_newick.py b/oz_tree_build/newick/format_newick.py index d1e2b08e..56151e2d 100755 --- a/oz_tree_build/newick/format_newick.py +++ b/oz_tree_build/newick/format_newick.py @@ -22,16 +22,26 @@ import re import sys -from ..tree_build.build_oz_tree import trim_tree - __author__ = "David Ebbo" # Token may be quoted or not whole_token_regex = re.compile("('[^']*'|[^(),;[]+)(:[0-9.]+)?") +def trim_tree(tree): + # Trim any whitespace + tree = tree.strip() + + # Skip the comment block at the start of the file, if any + if tree[0] == "[": + tree = tree[tree.index("]") + 1 :] + tree = tree.lstrip() + + return tree + + def format_nwk(newick_tree, output_stream, indent_spaces=2): - newick_tree = trim_tree(newick_tree, strip_semicolon=False) + newick_tree = trim_tree(newick_tree) indent_string = " " * indent_spaces From 8c1278a42307e336da1a1c7ca11fb57810739e0d Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 24 Aug 2026 16:57:29 +0000 Subject: [PATCH 36/62] utilities/make_js_treefiles: Remove make_js_treefiles is superseded by tree_build/step_jsnewick & versioned_outputs. Also drop the equivalence tests in test_tree_build_step_jsnewick that pinned the new implementation against this one as an oracle. --- oz_tree_build/utilities/make_js_treefiles.py | 248 ------------------- pyproject.toml | 1 - tests/test_make_js_treefiles.py | 20 -- tests/test_tree_build_step_jsnewick.py | 61 ----- 4 files changed, 330 deletions(-) delete mode 100755 oz_tree_build/utilities/make_js_treefiles.py delete mode 100644 tests/test_make_js_treefiles.py diff --git a/oz_tree_build/utilities/make_js_treefiles.py b/oz_tree_build/utilities/make_js_treefiles.py deleted file mode 100755 index fb33af22..00000000 --- a/oz_tree_build/utilities/make_js_treefiles.py +++ /dev/null @@ -1,248 +0,0 @@ -import argparse -import fileinput -import json -import os -import re -import shutil -from subprocess import call - -from ..utilities.debug_util import parse_args_and_add_logging_switch - - -# string -> string -# Given newick filepath(string), return a string without comma and semi comma -# Input: '../../data/output_files/ordered_tree_test.nwk' -> '((,),)' -# Output: '(())' -def tidy_newick(newick_filepath): - res = "" - for line in fileinput.input(files=(newick_filepath)): - res += line.replace(",", "").replace(";", "").replace("\n", "") - return res - - -# String -> String -# Given tidied newick string, return rawData in completetree.js -# Input: (()) -# Output:var rawData = '(())'; -def generate_completetree_js(newick_str): - return "var rawData = '" + newick_str + "';" - - -# String, Number -> String -# Given tidied newick(polytomy) string, return stringified cut position map for -# binary tree and polytomy tree -def generate_cut_position_map(newick_str, threshold): - binary_cut_map = generate_binary_cut_position_map(newick_str, threshold) - polytomy_cut_map = generate_polytomy_cut_position_map(newick_str, threshold) - cut_threshold = "var cut_threshold = " + str(threshold) + ";" - return binary_cut_map + "\n\n" + polytomy_cut_map + "\n\n" + cut_threshold - - -# String, Number -> String -# Given tidied newick string, return stringified cut_position_map object. -# Output example: -# '{ -# "4203700":1302201,"4203701":685684,"4203702":685609,"4203703":683568, -# "4203704":7901,"4203705":7900,"4203706":6417,"4203707":6396 -# }' -def generate_binary_cut_position_map(newick_str, threshold): - count_arr = [None] * len(newick_str) - count = 0 - for index, c in enumerate(reversed(newick_str)): - index = len(newick_str) - index - 1 - if c == "(" or c == "{": - count = count - 1 - elif c == ")" or c == "}": - count = count + 1 - else: - raise ValueError("newick str contains non bracket character: " + c) - count_arr[index] = count - - start_end_arr = [0, len(count_arr) - 1] - cut_position_map = {} - while len(start_end_arr) > 0: - start = start_end_arr.pop(0) - end = start_end_arr.pop(0) - build_cut_position_map(start, end, start_end_arr, count_arr, cut_position_map, threshold) - cut_position_map = json.dumps(cut_position_map) - cut_position_map = "var cut_position_map_json_str = '" + cut_position_map + "';" - return cut_position_map - - -# String, Number -> String -# Given tidied newick(polytomy) string, return stringified cut_position_map object. -# Output example: -# '{ -# "4203700":{685684, 79999, 1302201, 4203701}, -# "4203702":{685609, 4203703}, -# "4203704": {7901,4203705,7900,4203706} -# }' -# The key of the output json string is the end position of a string in the newick_str, the -# value is an array: [start_sub1, end_sub1, start_sub2, end_sub2, ..., start_subN, end_subN]. -# start_subN is the start pos of its nth child, end_subN is the end pos of its nth child. -def generate_polytomy_cut_position_map(newick_str, threshold): - start_end_arr = [0, len(newick_str) - 1] - cut_position_map = {} - while len(start_end_arr) > 0: - start = start_end_arr.pop(0) - end = start_end_arr.pop(0) - cut_position_map[end] = get_polytomy_substring_pos(start, end, start_end_arr, threshold, newick_str) - cut_position_map = json.dumps(cut_position_map) - cut_position_map = "var polytomy_cut_position_map_json_str = '" + cut_position_map + "';" - return cut_position_map - - -# Number, Number, Array, Array, Map, Number -# start, end represent indices of a node A on rawData. -# this function finds cut position of node A on rawData, then store it in cut_position_map -# and put its children start and end position in start_end_arr -def build_cut_position_map(start, end, start_end_arr, count_arr, cut_position_map, threshold): - endValue = count_arr[end] - for index in reversed(range(start, end)): - if count_arr[index] == endValue: - cut_position_map[end] = index - 1 - if (index - start - 2) >= threshold: - start_end_arr.append(start + 1) - start_end_arr.append(index - 1) - if (end - index - 1) >= threshold: - start_end_arr.append(index) - start_end_arr.append(end - 1) - break - - -# Find substring start & end position given a string representing a polytomous tree. -# The start and end position is pushed into start_end_arr if its distance is > than threshold -def get_polytomy_substring_pos(start, end, start_end_arr, threshold, newick_str, called_by_self=False): - res = [] - if end <= start or (called_by_self and newick_str[end] == ")"): - res += [start, end] - if (end - start) > threshold: - start_end_arr.append(start) - start_end_arr.append(end) - return res - - cut_point = None - bracket_count = 0 - for index in reversed(range(start, end + 1)): - c = newick_str[index] - if c == ")" or c == "}": - bracket_count = bracket_count + 1 - elif c == "(" or c == "{": - bracket_count = bracket_count - 1 - if bracket_count == 1: - cut_point = index - 1 - break - if cut_point is not None: - res = res + get_polytomy_substring_pos(start + 1, cut_point, start_end_arr, threshold, newick_str, True) - res = res + get_polytomy_substring_pos(cut_point + 1, end - 1, start_end_arr, threshold, newick_str, True) - else: - res += [start, start, end, end] - return res - - -def write_js_file(outdir, input_path, version_number, args): - # Output to versioned path - input_name = os.path.basename(input_path) - output_path = os.path.join( - outdir, - re.sub( - # Extract any existing version number / extension from filename - r"(_\d+)?(\.[a-zA-Z]+)$", - # Replace with verison number / extension - "_" + str(version_number) + r"\2", - input_name, - ), - ) - - if input_name.startswith("ordered_tree_"): - output_path = re.sub(r"ordered_tree_", "completetree_", output_path) - output_path = re.sub(r"\.(nwk|poly)$", ".js", output_path) - - print(f"{input_path} -> {output_path}") - newick_str = tidy_newick(input_path) - with open(output_path, "w") as out_f: - out_f.write(generate_completetree_js(newick_str)) - - # Generate derived cut-position-map - cut_path = re.sub(r"completetree_", r"cut_position_map_", output_path) - with open(cut_path, "w") as out_f: - out_f.write(generate_cut_position_map(newick_str, args.threshold)) - # Trigger write_js_file for cut map so we gzip it - write_js_file(outdir, cut_path, version_number, args) - - elif input_path == output_path: - # Nothing to do, already in output_path - pass - else: - # By default we just copy file - print(f"{input_path} -> {output_path}") - shutil.copyfile(input_path, output_path) - print(f"{output_path} -> {output_path}.gz") - call(["gzip", "-9fk", output_path]) - - -def main(): - # rawData string + metadata -> output result into file. - - # produce cut_position_map.js and completetree.js given newick tree. - parser = argparse.ArgumentParser( - description="Generate rawData, metadata and cut_position_map given newick string", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - - # pick the most recent ordered_tree_XXX.nwk file - import re - - parser.add_argument( - "--outdir", - "-o", - default=os.path.join( - os.path.dirname(__file__), - "..", - "..", - "..", - "OZtree", - "static", - "FinalOutputs", - "data", - ), - help="output filepath of cut_position_map", - ) - parser.add_argument( - "in_files", - nargs="+", - metavar="FILE", - help="Files to move to outdir, with versions appended if not present", - ) - parser.add_argument( - "--threshold", - default=10000, - type=int, - help=("Threshold for deciding if a node and its descendants needs to be" "recorded in cut_position_map"), - ) - parser.add_argument( - "--version", - type=int, - help=("Version number / serial to append to file names, if not provided assume present on at least one file"), - ) - - args = parse_args_and_add_logging_switch(parser) - - if args.version: - version_number = args.version - else: - # Find higest version number in files present, use that as version - version_number = 0 - for f in args.in_files: - m = re.search(r"_(\d+)\.(\w+)$", f) - if m and int(m.group(1)) > version_number: - version_number = int(m.group(1)) - - for f in args.in_files: - write_js_file(args.outdir, f, version_number, args) - - print("Done") - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 1c586c15..1ff482d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,6 @@ CSV_base_table_creator = "oz_tree_build.taxon_mapping_and_popularity.CSV_base_ta get_wiki_images = "oz_tree_build.images.get_wiki_images:main" get_wiki_vernaculars = "oz_tree_build.vernaculars.get_wiki_vernaculars:main" process_image_bits = "oz_tree_build.images.process_image_bits:main" -make_js_treefiles = "oz_tree_build.utilities.make_js_treefiles:main" format_newick = "oz_tree_build.newick.format_newick:main" extract_minimal_tree = "oz_tree_build.newick.extract_minimal_tree:main" extract_trees = "oz_tree_build.newick.extract_trees:main" diff --git a/tests/test_make_js_treefiles.py b/tests/test_make_js_treefiles.py deleted file mode 100644 index 9814764f..00000000 --- a/tests/test_make_js_treefiles.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Unit tests for make_js_treefiles -""" - -from oz_tree_build.utilities import make_js_treefiles - - -def test_generate_completetree_js(): - assert ( - make_js_treefiles.generate_completetree_js("(())") - == """ -var rawData = '(())'; - """.strip() - ) - assert ( - make_js_treefiles.generate_completetree_js("((()()))") - == """ -var rawData = '((()()))'; - """.strip() - ) diff --git a/tests/test_tree_build_step_jsnewick.py b/tests/test_tree_build_step_jsnewick.py index 3e25848a..84706769 100644 --- a/tests/test_tree_build_step_jsnewick.py +++ b/tests/test_tree_build_step_jsnewick.py @@ -1,6 +1,3 @@ -import json -import random - import ete4 from oz_tree_build.tree_build.step_jsnewick import ( @@ -8,7 +5,6 @@ jsnewick_cutpositionmap_binary, jsnewick_cutpositionmap_polytomy, ) -from oz_tree_build.utilities import make_js_treefiles ######################################## # jsnewick_brief_newick @@ -113,21 +109,6 @@ def test_cutmap_binary_threshold_skips_small_subtrees(): assert jsnewick_cutpositionmap_binary(t, threshold=6) == {7: 0} -def test_cutmap_binary_matches_legacy_on_ladderized_tree(): - """On a tree ladderized smallest-subtree-first, the new map matches the legacy - string-based generator byte-for-byte. The legacy algorithm assumes leaf-first - child ordering — ladderize(ascending=True) is what the OZ pipeline runs to - enforce that.""" - random.seed(1234) - nwk = _random_binary_newick(25) - t = ete4.Tree(nwk, parser=1) - t.ladderize() # ete4 ladderize is ascending by default - - brief = jsnewick_brief_newick(t) - new_map = jsnewick_cutpositionmap_binary(t, threshold=0) - assert new_map == _legacy_binary_map(brief, 0) - - ######################################## # jsnewick_cutpositionmap_polytomy ######################################## @@ -187,45 +168,3 @@ def test_cutmap_polytomy_threshold_off_by_one_vs_binary(): 7: [1, 0, 1, 6], 6: [2, 1, 2, 5], } - - -def test_cutmap_polytomy_matches_legacy_on_ladderized_tree(): - """Matches the legacy polytomy generator on a leaf-first-ordered tree.""" - random.seed(5678) - nwk = _random_binary_newick(25) - t = ete4.Tree(nwk, parser=1) - t.ladderize() - - brief = jsnewick_brief_newick(t) - new_map = jsnewick_cutpositionmap_polytomy(t, threshold=0) - assert new_map == _legacy_polytomy_map(brief, 0) - - -######################################## -# helpers -######################################## - - -def _random_binary_newick(n_leaves): - """Generate a random binary newick string with ``n_leaves`` leaves.""" - - def rec(i, j): - if j - i == 1: - return f"L{i}" - m = random.randint(i + 1, j - 1) - return f"({rec(i, m)},{rec(m, j)})" - - return rec(0, n_leaves) + ";" - - -def _legacy_binary_map(brief, threshold): - """Run the legacy generator and parse the embedded JSON back into a dict.""" - js = make_js_treefiles.generate_binary_cut_position_map(brief, threshold) - payload = js.split("'", 2)[1] - return {int(k): v for k, v in json.loads(payload).items()} - - -def _legacy_polytomy_map(brief, threshold): - js = make_js_treefiles.generate_polytomy_cut_position_map(brief, threshold) - payload = js.split("'", 2)[1] - return {int(k): v for k, v in json.loads(payload).items()} From 1a2c5283e777c4d8fd3e23929727421630513099 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 24 Aug 2026 17:00:28 +0000 Subject: [PATCH 37/62] tree_build/build_oz_tree: Remove This has been superseded by tree_build/tree_build, and now unused. --- oz_tree_build/tree_build/build_oz_tree.py | 164 ---------------------- pyproject.toml | 1 - 2 files changed, 165 deletions(-) delete mode 100644 oz_tree_build/tree_build/build_oz_tree.py diff --git a/oz_tree_build/tree_build/build_oz_tree.py b/oz_tree_build/tree_build/build_oz_tree.py deleted file mode 100644 index ddb41dcb..00000000 --- a/oz_tree_build/tree_build/build_oz_tree.py +++ /dev/null @@ -1,164 +0,0 @@ -""" -Build the entire OneZoom tree from the saved parts. - -It does this in one pass, by starting with the base file (e.g. base.PHY) and -recursively expanding any OneZoom tokens it finds. -""" - -import argparse -import logging -import os -import sys - -from ..utilities.debug_util import parse_args_and_add_logging_switch -from .oz_tokens import enumerate_one_zoom_tokens - -__author__ = "David Ebbo" - - -def trim_tree(tree, strip_semicolon=True): - # Trim any whitespace - tree = tree.strip() - - # Skip the comment block at the start of the file, if any - if tree[0] == "[": - tree = tree[tree.index("]") + 1 :] - tree = tree.lstrip() - - # Strip the trailing semicolon - if strip_semicolon and tree[-1] == ";": - tree = tree[:-1] - - return tree - - -def build_oz_tree(base_file, ot_parts_folder, output_stream, print_file_tree): - """ - Do all the token replacement, starting with the base file - """ - - depth = 0 - - def process_newick( - file, - node_name_in_parent=None, - edge_length_in_parent=None, - override_edge_length=None, - override_taxon=None, - expand_nodes=False, - ): - """ - Copy the input file to the output file, recursively expanding any OneZoom tokens - """ - nonlocal depth - - logging.debug(f"Processing {file}") - - # If we're printing the file tree, print the current file - if print_file_tree and expand_nodes: - print(f"{' ' * depth}{node_name_in_parent}: {edge_length_in_parent} {override_edge_length or 0}") - - if not os.path.exists(file): - logging.warning(f"Subtree file {file} does not exist") - return False - - with open(file, encoding="utf8") as stream: - tree = stream.read() - - tree = trim_tree(tree) - index = 0 - - # We only need to look for children if it's a OneZoom file (i.e. .PHY extension) - if expand_nodes: - for result in enumerate_one_zoom_tokens( - tree, - dict( - ot=ot_parts_folder, - oz=oz_parts_folder, - ot_required=os.path.join(os.path.dirname(os.path.dirname(ot_parts_folder)), "OT_required"), - ), - ): - # Write the part of the tree before the child - output_stream.write(tree[index : result["start"]]) - - depth += 1 - if process_newick( - file=result["file"], - node_name_in_parent=result["node_name_in_parent"], - edge_length_in_parent=result["edge_length_in_parent"], - override_edge_length=result["override_edge_length"], - override_taxon=result["override_taxon"], - expand_nodes=result["expand_nodes"], - ): - index = result["end"] - else: - # If child file absent, we'll need to write the child token as-is - index = result["start"] - depth -= 1 - - # We've processed all the children, and we need to write the rest of the tree - last_chunk = tree[index:] - - # Write the last chunk, but exclude the last name:edge_length, - # which needs special handling - last_closed_bracket = last_chunk.rfind(")") - output_stream.write(last_chunk[: last_closed_bracket + 1]) - - # Parse the last token into the node name and edge length - last_token = last_chunk[last_closed_bracket + 1 :] - last_token_segments = last_token.split(":") - last_token_name = last_token_segments[0] - last_token_edge_length = last_token_segments[1] if len(last_token_segments) > 1 else None - - # Always favor the length from our mapping, falling back to the last token in the file - # Note that we never fall back to edge_length_in_parent here, following old code logic - # DISCUSS: should we? - edge_length = override_edge_length or last_token_edge_length - - if expand_nodes: - # Three levels of fallback for .PHY files: mapping, last token, parent - node_name = override_taxon or last_token_name or node_name_in_parent - else: - # NB: following old code logic, the above parent vs last logic is reversed here - # DISCUSS: is there a logical reason for this? - node_name = node_name_in_parent or last_token_name - - output_stream.write(node_name) - if edge_length: - output_stream.write(f":{edge_length}") - - return True - - # Assume that the base file is in the same folder as the OneZoom parts - oz_parts_folder = os.path.dirname(base_file) - - process_newick(base_file, expand_nodes=True) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--printfiletree", - action="store_true", - help="Print a tree of all the OneZoom included files", - ) - parser.add_argument("treefile", help="The base tree file in newick form") - parser.add_argument("ot_parts_folder", help="The folder containing the Open Tree parts") - parser.add_argument( - "outfile", - type=argparse.FileType("w"), - nargs="?", - default=sys.stdout, - help="The output tree file", - ) - args = parse_args_and_add_logging_switch(parser) - - build_oz_tree(args.treefile, args.ot_parts_folder, args.outfile, args.printfiletree) - - # Write out the ending semi-colon and flush the stream - args.outfile.write(";") - args.outfile.flush() - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 1ff482d6..a1e1e81e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ dev = [ [project.scripts] add_ott_numbers_to_trees = "oz_tree_build.tree_build.ott_mapping.add_ott_numbers_to_trees:main" -build_oz_tree = "oz_tree_build.tree_build.build_oz_tree:main" get_open_trees_from_one_zoom = "oz_tree_build.tree_build.get_open_trees_from_one_zoom:main" generate_filtered_files = "oz_tree_build.utilities.generate_filtered_files:main" filter_eol = "oz_tree_build.utilities.filter_eol:main" From f1a443be654011e8dfb2e61f294dea5038bb1f09 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 24 Aug 2026 17:00:37 +0000 Subject: [PATCH 38/62] tree_build/get_open_trees_from_one_zoom: Remove This has been superseded by tree_build/step_graft, which performs the splicing on an in-memory ete4 tree. --- .../get_open_trees_from_one_zoom.py | 131 ------------------ pyproject.toml | 1 - 2 files changed, 132 deletions(-) delete mode 100644 oz_tree_build/tree_build/get_open_trees_from_one_zoom.py diff --git a/oz_tree_build/tree_build/get_open_trees_from_one_zoom.py b/oz_tree_build/tree_build/get_open_trees_from_one_zoom.py deleted file mode 100644 index 9dc42661..00000000 --- a/oz_tree_build/tree_build/get_open_trees_from_one_zoom.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -Create subtrees from the OpenTree, on the basis of ott numbers in a set of newick files. - -Usage: getOpenTreesFromOneZoom.py OpenTreeFile.tre output_dir file1.PHY file2.PHY ... - -This script places a set of inclusion files into output_dir, based on the names of nodes -in the input .PHY files. The input files should contain one or more node names in the OneZoom -@include format which is the scientific name + '_ott' + (an OTT id, optionally a ~ sign, and -optionally other OTT numbers separated by an minus sign) + '@', e.g. Brachiopoda_ott826261@ -This specifies that the node should be replaced with part of the OpenTree: namely the subtree -starting at ott node 826261. - -E.g. - foobar_ott123@ - create a node named foobar with ott 123, consisting of all descendants of - ott 123 in the opentree. - - foobar_ott123~456-789-111@ - create a node named foobar with ott 123, using ott456 minus the descendant - subtrees with ott 789 and 111 (the tilde sign can be read an a equals, used - as Dendropy doesn't like equals signs in taxon names. - - foobar_ott123~-789-111@ - shorthand for foobar_ott123~123-789-111@ - - foobar_ott~456-789-111@ - create a node named foobar without any OTT number, - using ott456 minus the descendant subtrees 789 and 111 - -The actual inclusion is done by the build_oz_tree.py. This script merely creates the -files to include. It does this by extracting the relevant subtree from the full OpenTree -""" - -import argparse -import logging -import os -import sys -import time - -from ..newick.extract_trees import extract_trees -from .oz_tokens import enumerate_one_zoom_tokens - -__author__ = "David Ebbo" - - -def get_inclusions_and_exclusions_from_one_zoom_file(file, all_included_otts, all_excluded_otts): - """ - Find all the included and excluded ott numbers in a OneZoom files & add them to the sets - """ - - with open(file, encoding="utf8") as stream: - tree = stream.read() - - for result in enumerate_one_zoom_tokens(tree): - # Check if the result has a base ott (won't have it if it's inserting another OZ file) - if result.get("base_ott") is not None: - all_included_otts.add(result["base_ott"]) - all_excluded_otts.update(result["excluded_otts"]) - - -def extract_trees_from_open_tree_file(open_tree_file, output_dir, all_included_otts, all_excluded_otts): - """ - Extract the subtrees from the Open Tree file, based on the list of included/excluded otts - """ - - # Read the contents of the open tree file into a string - with open(open_tree_file, encoding="utf8") as f: - fulltree = f.read() - - trees = extract_trees(fulltree, all_included_otts, excluded_taxa=all_excluded_otts) - - logging.info(f"Extracted {len(trees)} trees from Open Tree file") - - # Save each tree to a file named after the taxon - os.makedirs(output_dir, exist_ok=True) - for ott, tree in trees.items(): - file = os.path.join(output_dir, ott + ".phy") - logging.debug(f"Writing file: {file}") - with open(file, "w", encoding="utf8") as f: - f.write(tree) - f.write(";\n") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - parser.add_argument( - "--verbosity", - "-v", - action="count", - default=0, - help="verbosity level: output extra non-essential info", - ) - parser.add_argument("open_tree_file", help="Path to the Open Tree newick file") - parser.add_argument( - "output_dir", - help="Path to the directory in which to save the OpenTree subtrees", - ) - parser.add_argument( - "parse_files", - nargs="+", - help="A list of newick files to parse for OTT numbers, giving the subtrees to extract", - ) - args = parser.parse_args() - - if args.verbosity == 0: - logging.basicConfig(stream=sys.stderr, level=logging.WARNING) - elif args.verbosity == 1: - logging.basicConfig(stream=sys.stderr, level=logging.INFO) - elif args.verbosity == 2: - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - - start = time.time() - if not os.path.isfile(args.open_tree_file): - logging.warning(f"Could not find the OpenTree file {args.open_tree_file}") - - # Go through all the OneZoom files, and gather all the ott numbers to include and exclude. - # NB: excluded ott numbers don't need to be specifically given an included ott number - included_otts = set() - excluded_otts = set() - for file in args.parse_files: - logging.info(f"== Processing One Zoom file {file}") - get_inclusions_and_exclusions_from_one_zoom_file(file, included_otts, excluded_otts) - - extract_trees_from_open_tree_file(args.open_tree_file, args.output_dir, included_otts, excluded_otts) - - end = time.time() - logging.debug(f"Time taken: {end - start} seconds") - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index a1e1e81e..cbdc8f97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ dev = [ [project.scripts] add_ott_numbers_to_trees = "oz_tree_build.tree_build.ott_mapping.add_ott_numbers_to_trees:main" -get_open_trees_from_one_zoom = "oz_tree_build.tree_build.get_open_trees_from_one_zoom:main" generate_filtered_files = "oz_tree_build.utilities.generate_filtered_files:main" filter_eol = "oz_tree_build.utilities.filter_eol:main" filter_wikidata = "oz_tree_build.utilities.filter_wikidata:main" From 9d1bdfa890d4efd440d201ae61707aa32b4c257e Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Mon, 24 Aug 2026 17:00:44 +0000 Subject: [PATCH 39/62] tree_build/oz_tokens: Remove enumerate_one_zoom_tokens was only imported by build_oz_tree and get_open_trees_from_one_zoom, both removed in the preceding commits. Its own dependency, token_to_oz_tree_file_mapping, is still used by tree_build/step_parse and stays. --- oz_tree_build/tree_build/oz_tokens.py | 104 -------------------------- 1 file changed, 104 deletions(-) delete mode 100644 oz_tree_build/tree_build/oz_tokens.py diff --git a/oz_tree_build/tree_build/oz_tokens.py b/oz_tree_build/tree_build/oz_tokens.py deleted file mode 100644 index 65242aec..00000000 --- a/oz_tree_build/tree_build/oz_tokens.py +++ /dev/null @@ -1,104 +0,0 @@ -__author__ = "David Ebbo" - -import logging -import os.path -import re - -from .token_to_oz_tree_file_mapping import token_to_file_map - -__author__ = "David Ebbo" - -full_ott_token = re.compile(r"'?([\w\-~]+)@'?(?::([\d\.]+))?") -ott_details = re.compile(r"(\w+)_ott(\d*)~?([-\d]*)$") - - -def parse_one_zoom_token(node_label, parts_folders=None): - """ - Parse a single OneZoom token from label name - """ - if parts_folders is None: - parts_folders = {} - - if not node_label: - return None - try: - return next(enumerate_one_zoom_tokens(node_label, parts_folders)) - except StopIteration: - return None - - -def enumerate_one_zoom_tokens(tree, parts_folders=None): - """ - Enumerates all the OneZoom tokens in a tree string (e.g. foobar_ott123~-789-111) - - Yields dicts with the keys: - - - start: Position in in string the match was found - - end: End of match - - node_name_in_parent: Node name from inclusion node, ignoring OZ inclusion syntax - - edge_length_in_parent: Edge length from inclusion node - - file: File path pointing to tree to substitute - - base_ott: OTT of root, if subtree is a OT tree - - excluded_otts: OTTs to exclude from subtree (as strings not ints) - - expand_nodes: Should we recurse and apply OZ inclusion rules to subtree? - - override_edge_length: Replace edge length from root node with this value - - override_taxon: Replace root node name with this value - """ - if parts_folders is None: - parts_folders = {} - - # Skip the comment block at the start of the file - start_index = tree.index("]") if "[" in tree else 0 - - for full_match in full_ott_token.finditer(tree, start_index): - result = { - "start": full_match.start(), - "end": full_match.end(), - "node_name_in_parent": full_match.group(1), - "edge_length_in_parent": float(full_match.group(2)) if full_match.group(2) else None, - } - - # Check if it matches our tilde (aka 'equal') exclusion syntax - match = ott_details.match(result["node_name_in_parent"]) - base_ott = None - if match: - # split by minus signs - result["excluded_otts"] = (match.group(3) or "").split("-") - - # If present, the first number after '=' is the tree to extract. - first_number_after_equal = result["excluded_otts"].pop(0) - base_ott = first_number_after_equal or match.group(2) - - # Note that we don't append the ott in the name if it came after the '=' - result["node_name_in_parent"] = match.group(1) - if not first_number_after_equal: - result["node_name_in_parent"] += f"_ott{base_ott}" - - # Check if OZ token has a base ott (e.g. 123 in foobar_ott123~456-789) - if base_ott is not None: - # It's an extracted Open Tree file, e.g. 123.phy - # NB: We can't make a valid path without parts_folder["ot"], but we probably don't care in this case - result["base_ott"] = base_ott - if os.path.exists( - os.path.join(parts_folders.get("ot_required", "/unconfiguredpath/ot_requried/"), f"{base_ott}.nwk") - ): - # An ot_required orphan OT file exists, use that - result["file"] = os.path.join( - parts_folders.get("ot_required", "/unconfiguredpath/ot_requried/"), f"{base_ott}.nwk" - ) - else: - result["file"] = os.path.join(parts_folders.get("ot") or ".", f"{base_ott}.phy") - result["override_edge_length"] = None - result["override_taxon"] = None - result["expand_nodes"] = False - else: - # Otherwise, it's a OneZoom file, e.g. AMORPHEA@ --> Amorphea.PHY - child_mapping_entry = token_to_file_map[result["node_name_in_parent"]] - result["base_ott"] = None - result["file"] = os.path.join(parts_folders.get("oz") or ".", child_mapping_entry["file"]) - result["override_edge_length"] = child_mapping_entry.get("edge_length", None) - result["override_taxon"] = child_mapping_entry.get("taxon", None) - result["expand_nodes"] = True - - logging.debug(result) - yield result From e66b8c3877ac24e32ff815476989e95bd67d5968 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 25 Aug 2026 08:23:38 +0000 Subject: [PATCH 40/62] tree_build: Wire up exclude_taxa The original option wasn't psased through to popularity_add_prop, put it back. --- dvc.yaml | 1 + oz_tree_build/tree_build/tree_build.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/dvc.yaml b/dvc.yaml index 79578477..13067145 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -63,6 +63,7 @@ stages: --orphan_dir data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ --opentree data/dated_tree/dated_tree_pre.tre --taxon_map data/taxon_map.csv + --exclude ${exclude_from_popularity} --out_dir data/out deps: - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index d291fd0c..519f62ce 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -51,6 +51,16 @@ def main(): default="data/taxon_map.csv", help=("Taxon map CSV as generated by taxon_mapping_and_popularity.taxon_map"), ) + parser.add_argument( + "--exclude", + "-x", + nargs="*", + default=[], + help=( + "(Optional) taxa to exclude from calculation of phylogenetic popularities, " + "such as Dinosauria_ott90215, Archosauria_ott335588" + ), + ) parser.add_argument( "--out_dir", default="data/out", @@ -91,7 +101,7 @@ def main(): "and just apply them post-resolution). Apply popularity based on OTT -> popularity map, percolate " "using existing rules (which preserves popularity from removed subspecies)" ) - popularity_add_prop(base_t) + popularity_add_prop(base_t, exclude_taxa=args.exclude) logger.info("Remove subspecies (now popularity has percolated)") # tidy_remove_subspecies(base_t) From 5cabe5fd36ec2ea8c5f44d8c09b4c50c2fbb919c Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 25 Aug 2026 08:57:16 +0000 Subject: [PATCH 41/62] taxon_map: Support --extra_source_file It got lost along the way, add the option back and read in checks here too. --- dvc.yaml | 3 + .../taxon_mapping_and_popularity/taxon_map.py | 98 +++++--- tests/test_taxon_map.py | 209 ++++++++++++++++++ 3 files changed, 285 insertions(+), 25 deletions(-) create mode 100644 tests/test_taxon_map.py diff --git a/dvc.yaml b/dvc.yaml index 13067145..f224eb31 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -193,6 +193,7 @@ stages: --wikipediaSQLDumpFile data/filtered/OneZoom_enwiki-latest-page.sql --wikipedia_totals_bz2_pageviews data/filtered/pageviews/ --EOLidentifiers data/filtered/OneZoom_provider_ids.csv + --extra_source_file data/OZTreeBuild/${oz_tree}/BespokeTree/SupplementaryTaxonomy.tsv -o data/taxon_map.csv deps: - data/OpenTree/${ot_version}/taxonomy.tsv @@ -200,7 +201,9 @@ stages: - data/filtered/OneZoom_enwiki-latest-page.sql - data/filtered/pageviews/ - data/filtered/OneZoom_provider_ids.csv + - data/OZTreeBuild/${oz_tree}/BespokeTree/SupplementaryTaxonomy.tsv params: + - oz_tree - ot_version outs: - data/taxon_map.csv diff --git a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py index c6c6dedf..b5f82930 100644 --- a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py +++ b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py @@ -324,6 +324,17 @@ def populate_iucn(OTT_ptrs, identifiers_filename, verbosity=0, iucn_num=5): logger.info(f" > Increased IUCN coverage to {used} taxa using wikidata") +def parse_sourceinfo(sourceinfo): + """Deparse a sourceinfo string, e.g. "ncbi:1274384,gbif:8094325", into a dict of IDs""" + out = {} + for i in sourceinfo.split(","): + if not i: + continue + k, v = i.split(":", 1) + out[k] = int(v) if v.isdigit() else v + return out + + def read_ot_taxonomy(path="./data/OpenTree/v16.1/taxonomy.tsv"): """Yield each row of an OpenTree taxonomy file as a dict keyed by header. @@ -338,17 +349,51 @@ def read_ot_taxonomy(path="./data/OpenTree/v16.1/taxonomy.tsv"): out["uid"] = int(out["uid"]) out["parent_uid"] = None if out["parent_uid"] == "" else int(out["parent_uid"]) - - # Deparse sourceinfo to a dict of IDs - sourceinfo = {} - for i in out["sourceinfo"].split(","): - k, v = i.split(":", 1) - sourceinfo[k] = int(v) if v.isdigit() else v - out["sourceinfo"] = sourceinfo + out["sourceinfo"] = parse_sourceinfo(out["sourceinfo"]) yield out +def read_extra_source_file(path): + """Yield each row of an extra source file, in the same form as read_ot_taxonomy(). + + Unlike the OpenTree taxonomy this is a plain TSV, requiring only "uid" and + "sourceinfo" columns. The uid need not be a number (e.g. "mrcaott409215ott616649"). + """ + try: + with open(path, encoding="utf-8", newline="") as f: + for row in csv.DictReader(f, delimiter="\t"): + out = dict(row) + out["uid"] = int(row["uid"]) if row["uid"].isdigit() else row["uid"] + out["sourceinfo"] = parse_sourceinfo(row["sourceinfo"]) + yield out + except FileNotFoundError: + logger.warning(f" Extra source file '{path}' not found, so ignored") + + +def add_taxon_sources(OTT_ptrs, source_ptrs, ott, sourceinfo, rank=None): + """ + Point OTT_ptrs[ott] at an entry in source_ptrs for each source in sourceinfo, + adding the OTT if not already there, and overwriting any existing source of + the same name. + """ + ott_data = OTT_ptrs.setdefault(ott, {"ott": ott, "sources": {}}) + if rank is not None: + ott_data["rank"] = rank + + has_ncbi = False + for src in reversed(sourceinfo.keys()): + # NB: look at sources in reverse order, overwriting, so 1st ones take priority + src_id = sourceinfo[src] + if src == "ncbi": + has_ncbi = True + elif not has_ncbi and src == "ncbi_silva": + # only use the ncbi_via_silva id if no 'normal' ncbi already set + src = "ncbi" + source_ptrs.setdefault(src, {})[src_id] = {"id": src_id} + ott_data["sources"][src] = source_ptrs[src][src_id] + + def main(): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument( @@ -397,6 +442,18 @@ def main(): default="data/filtered/OneZoom_provider_ids.csv", help=("EOL identifiers file, from " "https://opendata.eol.org/dataset/identifiers-csv-gz"), ) + parser.add_argument( + "--extra_source_file", + default=None, + type=str, + help=( + "An optional additional file to supplement the taxonomy.tsv file, " + "providing additional mappings from OTTs to source ids (useful for overriding). " + 'The first line should be a header contining "uid" and "sourceinfo" column ' + "headers, similar to those in the taxonomy.tsv file. NB the OTT can be a " + 'number, or an ID of the form "mrcaott409215ott616649").' + ), + ) parser.add_argument( "-o", type=argparse.FileType("w"), @@ -416,24 +473,15 @@ def main(): OTT_ptrs = {} source_ptrs = {} for r in read_ot_taxonomy(args.OpenTreeTaxonomy): - OTTid = r["uid"] - OTT_ptrs[OTTid] = {"ott": OTTid, "sources": {}} - - has_ncbi = False - for src in reversed( - r["sourceinfo"].keys() - ): # NB: look at sources in reverse order, overwriting, so 1st ones take priority - src_id = r["sourceinfo"][src] - if src == "ncbi": - has_ncbi = True - elif not has_ncbi and src == "ncbi_silva": - # only use the ncbi_via_silva id if no 'normal' ncbi already set - src = "ncbi" - if src not in source_ptrs: - source_ptrs[src] = {} - source_ptrs[src][src_id] = {"id": src_id} - OTT_ptrs[OTTid]["sources"][src] = source_ptrs[src][src_id] - OTT_ptrs[OTTid]["rank"] = r["rank"] + add_taxon_sources(OTT_ptrs, source_ptrs, r["uid"], r["sourceinfo"], r["rank"]) + + if args.extra_source_file is not None: + logger.info(f"Supplementing source IDs from {args.extra_source_file}") + extra_otts = 0 + for r in read_extra_source_file(args.extra_source_file): + add_taxon_sources(OTT_ptrs, source_ptrs, r["uid"], r["sourceinfo"], r.get("rank")) + extra_otts += 1 + logger.info(f"✔ {extra_otts} OTTs supplemented from {args.extra_source_file}") eol_sources = { "ncbi": 676, diff --git a/tests/test_taxon_map.py b/tests/test_taxon_map.py new file mode 100644 index 00000000..e07f56f9 --- /dev/null +++ b/tests/test_taxon_map.py @@ -0,0 +1,209 @@ +""" +Unit tests for the taxonomy-reading parts of taxon_map +""" + +from oz_tree_build.taxon_mapping_and_popularity.taxon_map import ( + add_taxon_sources, + parse_sourceinfo, + read_extra_source_file, + read_ot_taxonomy, +) + + +def write_ot_taxonomy(path, rows): + """Write rows (dicts) out in OpenTree taxonomy.tsv format, i.e. "\t|\t"-separated""" + header = ["uid", "parent_uid", "name", "rank", "sourceinfo", "uniqname", "flags"] + with open(path, "w", encoding="utf-8") as f: + for r in [{k: k for k in header}, *rows]: + f.write("".join(f"{r.get(k, '')}\t|\t" for k in header) + "\n") + + +def write_extra_source_file(path, rows, header=("uid", "name", "sourceinfo", "notes")): + with open(path, "w", encoding="utf-8") as f: + f.write("\t".join(header) + "\n") + for r in rows: + f.write("\t".join(str(r.get(k, "")) for k in header) + "\n") + + +class TestParseSourceinfo: + def test_numeric_ids_become_ints(self): + assert parse_sourceinfo("ncbi:1274384,gbif:8094325") == { + "ncbi": 1274384, + "gbif": 8094325, + } + + def test_non_numeric_ids_stay_strings(self): + # e.g. SILVA accessions, and GBIF ids of the form "D11377/#1" + assert parse_sourceinfo("silva:JX948102,gbif:D11377/#1") == { + "silva": "JX948102", + "gbif": "D11377/#1", + } + + def test_only_first_colon_separates(self): + assert parse_sourceinfo("silva:AB:CD") == {"silva": "AB:CD"} + + def test_zero_ids_are_kept(self): + # "life" is silva:0,ncbi:1,gbif:0,irmng:0, so 0 must not be treated as absent + assert parse_sourceinfo("silva:0,ncbi:1,gbif:0") == {"silva": 0, "ncbi": 1, "gbif": 0} + + def test_empty_sourceinfo(self): + assert parse_sourceinfo("") == {} + + def test_order_is_preserved(self): + # add_taxon_sources relies on the order for source priority + assert list(parse_sourceinfo("silva:0,ncbi:1,gbif:0,irmng:0")) == [ + "silva", + "ncbi", + "gbif", + "irmng", + ] + + +class TestReadOtTaxonomy: + def test_fields_are_split_and_converted(self, tmp_path): + path = tmp_path / "taxonomy.tsv" + write_ot_taxonomy( + path, + [ + {"uid": 805080, "name": "life", "rank": "no rank", "sourceinfo": "ncbi:1"}, + { + "uid": 93302, + "parent_uid": 805080, + "name": "cellular organisms", + "rank": "no rank", + "sourceinfo": "ncbi:131567", + }, + ], + ) + rows = list(read_ot_taxonomy(path)) + + assert [r["uid"] for r in rows] == [805080, 93302] + assert [r["parent_uid"] for r in rows] == [None, 805080] + assert [r["name"] for r in rows] == ["life", "cellular organisms"] + assert [r["rank"] for r in rows] == ["no rank", "no rank"] + assert [r["sourceinfo"] for r in rows] == [{"ncbi": 1}, {"ncbi": 131567}] + + def test_trailing_separator_is_not_a_field(self, tmp_path): + # Each line ends with "\t|\t", which must not yield an extra empty column + path = tmp_path / "taxonomy.tsv" + write_ot_taxonomy(path, [{"uid": 1, "sourceinfo": "ncbi:1", "flags": "sibling_higher"}]) + (row,) = list(read_ot_taxonomy(path)) + + assert set(row) == {"uid", "parent_uid", "name", "rank", "sourceinfo", "uniqname", "flags"} + assert row["flags"] == "sibling_higher" + + +class TestReadExtraSourceFile: + def test_row_is_parsed_like_a_taxonomy_row(self, tmp_path): + path = tmp_path / "SupplementaryTaxonomy.tsv" + write_extra_source_file( + path, + [ + { + "uid": 809432, + "name": "Strigops habroptilus", + "sourceinfo": "ncbi:2489341,irmng:11435975", + "notes": "Add in missing kakapo", + } + ], + ) + (row,) = list(read_extra_source_file(path)) + + assert row["uid"] == 809432 + assert row["sourceinfo"] == {"ncbi": 2489341, "irmng": 11435975} + assert row["name"] == "Strigops habroptilus" + assert row["notes"] == "Add in missing kakapo" + + def test_non_numeric_uid_stays_a_string(self, tmp_path): + path = tmp_path / "extra.tsv" + write_extra_source_file(path, [{"uid": "mrcaott409215ott616649", "sourceinfo": "ncbi:1"}]) + (row,) = list(read_extra_source_file(path)) + + assert row["uid"] == "mrcaott409215ott616649" + + def test_only_uid_and_sourceinfo_are_required(self, tmp_path): + path = tmp_path / "extra.tsv" + write_extra_source_file(path, [{"uid": 1, "sourceinfo": "gbif:2"}], header=("uid", "sourceinfo")) + (row,) = list(read_extra_source_file(path)) + + assert row == {"uid": 1, "sourceinfo": {"gbif": 2}} + + def test_missing_file_is_ignored_with_a_warning(self, tmp_path, caplog): + path = tmp_path / "nonexistent.tsv" + + assert list(read_extra_source_file(path)) == [] + assert "not found" in caplog.text + + +class TestAddTaxonSources: + def test_adds_a_new_ott_with_its_sources(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 770315, {"ncbi": 9999, "gbif": 1234}, "species") + + assert OTT_ptrs == { + 770315: { + "ott": 770315, + "rank": "species", + "sources": {"ncbi": {"id": 9999}, "gbif": {"id": 1234}}, + } + } + assert source_ptrs == {"ncbi": {9999: {"id": 9999}}, "gbif": {1234: {"id": 1234}}} + + def test_ott_and_source_entries_are_the_same_object(self): + # Wikidata data is added via source_ptrs, and read back out via OTT_ptrs, + # so the two must point at one shared dict + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5}) + + assert OTT_ptrs[1]["sources"]["ncbi"] is source_ptrs["ncbi"][5] + + def test_non_numeric_source_ids_are_usable(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": "JX948102"}) + + assert source_ptrs["silva"]["JX948102"] == {"id": "JX948102"} + + def test_a_second_call_overrides_only_the_sources_given(self): + # i.e. how an extra_source_file supplements the OpenTree taxonomy + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 9999, "gbif": 1234}, "species") + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 2489341, "irmng": 11435975}) + + assert OTT_ptrs[1]["sources"] == { + "ncbi": {"id": 2489341}, + "gbif": {"id": 1234}, + "irmng": {"id": 11435975}, + } + + def test_rank_is_kept_when_not_given(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 1}, "species") + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"gbif": 2}) + + assert OTT_ptrs[1]["rank"] == "species" + + def test_rank_is_absent_if_never_given(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 1}) + + assert "rank" not in OTT_ptrs[1] + + def test_ncbi_silva_is_used_as_ncbi_if_no_ncbi(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi_silva": 777, "irmng": 3}) + + assert OTT_ptrs[1]["sources"] == {"ncbi": {"id": 777}, "irmng": {"id": 3}} + + def test_a_normal_ncbi_id_takes_priority(self): + # Sources are read in reverse, so the first ncbi in the row wins + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5, "ncbi_silva": 777}) + + assert OTT_ptrs[1]["sources"]["ncbi"] == {"id": 5} + + def test_empty_sourceinfo_still_adds_the_ott(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {}, "species") + + assert OTT_ptrs == {1: {"ott": 1, "rank": "species", "sources": {}}} + assert source_ptrs == {} From 9ebf4c25337b2730babf9a2b11e9a1954472c753 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 25 Aug 2026 09:01:51 +0000 Subject: [PATCH 42/62] taxon_map: Preserve structures when updating src Without, previously set data would get lost. --- .../taxon_mapping_and_popularity/taxon_map.py | 4 ++-- tests/test_taxon_map.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py index b5f82930..f0f56dad 100644 --- a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py +++ b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py @@ -390,8 +390,8 @@ def add_taxon_sources(OTT_ptrs, source_ptrs, ott, sourceinfo, rank=None): elif not has_ncbi and src == "ncbi_silva": # only use the ncbi_via_silva id if no 'normal' ncbi already set src = "ncbi" - source_ptrs.setdefault(src, {})[src_id] = {"id": src_id} - ott_data["sources"][src] = source_ptrs[src][src_id] + # NB: reuse any existing entry, so OTTs sharing a source id share its (wikidata) data + ott_data["sources"][src] = source_ptrs.setdefault(src, {}).setdefault(src_id, {"id": src_id}) def main(): diff --git a/tests/test_taxon_map.py b/tests/test_taxon_map.py index e07f56f9..a556585f 100644 --- a/tests/test_taxon_map.py +++ b/tests/test_taxon_map.py @@ -157,6 +157,26 @@ def test_ott_and_source_entries_are_the_same_object(self): assert OTT_ptrs[1]["sources"]["ncbi"] is source_ptrs["ncbi"][5] + def test_otts_sharing_a_source_id_share_its_entry(self): + # Otherwise the first OTT is left pointing at an orphaned dict, which never + # gets the wikidata item that is added via source_ptrs + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5}) + add_taxon_sources(OTT_ptrs, source_ptrs, 2, {"ncbi": 5}) + + source_ptrs["ncbi"][5]["wd"] = "Q123" + assert OTT_ptrs[1]["sources"]["ncbi"] is OTT_ptrs[2]["sources"]["ncbi"] + assert OTT_ptrs[1]["sources"]["ncbi"]["wd"] == "Q123" + + def test_reading_a_source_keeps_data_added_to_its_entry(self): + # An extra_source_file row re-stating an id must not wipe out its wikidata item + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5}) + source_ptrs["ncbi"][5]["wd"] = "Q123" + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5}) + + assert OTT_ptrs[1]["sources"]["ncbi"] == {"id": 5, "wd": "Q123"} + def test_non_numeric_source_ids_are_usable(self): OTT_ptrs, source_ptrs = {}, {} add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": "JX948102"}) From a2c7d360180245d59157f4c178e77369225ba0f3 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 25 Aug 2026 09:21:44 +0000 Subject: [PATCH 43/62] taxon_mapping_and_popularity: Remove NCBI_via_silva hack The issue is seemingly solved in latest OT versions, and only using silva NCBI IDs when desperate is losing us a significant number of matches. In addition, the regex only matches a particular ordering of arguments, which generally not matching anyway. --- .../OTT_popularity_mapping.py | 20 +++++++------------ .../taxon_mapping_and_popularity/taxon_map.py | 6 ------ tests/test_taxon_map.py | 14 ++++--------- 3 files changed, 11 insertions(+), 29 deletions(-) diff --git a/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py b/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py index 89f1ce49..e1724d04 100755 --- a/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py +++ b/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py @@ -365,11 +365,12 @@ def create_from_taxonomy(OTTtax_filename, sources, OTT_ptrs, extra_taxonomy_file unused_sources = set() source_ptrs = {s: {} for s in sources} - # hack for NCBI_via_silva - # (see https://groups.google.com/d/msg/opentreeoflife/L2x3Ond16c4/CVp6msiiCgAJ) - silva_regexp = re.compile(r"ncbi:(\d+),silva:([^,$]+)") - # keep ncbi_id as ncbi_silva, but chop off the silva ID as it's not used in wikidata/EoL - silva_sub = r"ncbi_silva:\1" + # NB: NCBI ids that OpenTree got via SILVA used to be marked as "ncbi_silva" and + # treated with suspicion (see + # https://groups.google.com/d/msg/opentreeoflife/L2x3Ond16c4/CVp6msiiCgAJ). + # Against taxonomy v16.1 they behave just like any other NCBI id: they agree with + # the wikidata item found via gbif/irmng/worms 95.5% of the time, vs 96.2% for + # non-SILVA ids, so they are now used as-is. data_files = [OTTtax_filename] if extra_taxonomy_file is not None: @@ -395,16 +396,9 @@ def create_from_taxonomy(OTTtax_filename, sources, OTT_ptrs, extra_taxonomy_file OTTid = OTTrow["uid"] logging.warning(f" Found an ott value which is not an integer: {OTTid}") - sourceinfo = silva_regexp.sub(silva_sub, OTTrow["sourceinfo"]) - ncbi = False - for srcs in reversed(sourceinfo.split(",")): + for srcs in reversed(OTTrow["sourceinfo"].split(",")): # look at sources in reverse order, overwriting, so 1st ones take priority src, src_id = srcs.split(":", 1) - if src == "ncbi": - ncbi = True - elif (src == "ncbi_silva") and (not ncbi): - # only use the ncbi_via_silva id if no 'normal' ncbi already set - src = "ncbi" if src not in source_ptrs: if src not in unused_sources: logging.info(f" New and unused source: {src} (in '{srcs}')") diff --git a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py index f0f56dad..afc01636 100644 --- a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py +++ b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py @@ -381,15 +381,9 @@ def add_taxon_sources(OTT_ptrs, source_ptrs, ott, sourceinfo, rank=None): if rank is not None: ott_data["rank"] = rank - has_ncbi = False for src in reversed(sourceinfo.keys()): # NB: look at sources in reverse order, overwriting, so 1st ones take priority src_id = sourceinfo[src] - if src == "ncbi": - has_ncbi = True - elif not has_ncbi and src == "ncbi_silva": - # only use the ncbi_via_silva id if no 'normal' ncbi already set - src = "ncbi" # NB: reuse any existing entry, so OTTs sharing a source id share its (wikidata) data ott_data["sources"][src] = source_ptrs.setdefault(src, {}).setdefault(src_id, {"id": src_id}) diff --git a/tests/test_taxon_map.py b/tests/test_taxon_map.py index a556585f..fc2991dc 100644 --- a/tests/test_taxon_map.py +++ b/tests/test_taxon_map.py @@ -208,18 +208,12 @@ def test_rank_is_absent_if_never_given(self): assert "rank" not in OTT_ptrs[1] - def test_ncbi_silva_is_used_as_ncbi_if_no_ncbi(self): + def test_a_silva_derived_ncbi_id_is_used_like_any_other(self): + # NCBI ids from SILVA-sourced rows used to be singled out as "ncbi_silva" OTT_ptrs, source_ptrs = {}, {} - add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi_silva": 777, "irmng": 3}) + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": "JX948102", "ncbi": 1274384}) - assert OTT_ptrs[1]["sources"] == {"ncbi": {"id": 777}, "irmng": {"id": 3}} - - def test_a_normal_ncbi_id_takes_priority(self): - # Sources are read in reverse, so the first ncbi in the row wins - OTT_ptrs, source_ptrs = {}, {} - add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5, "ncbi_silva": 777}) - - assert OTT_ptrs[1]["sources"]["ncbi"] == {"id": 5} + assert OTT_ptrs[1]["sources"]["ncbi"] == {"id": 1274384} def test_empty_sourceinfo_still_adds_the_ott(self): OTT_ptrs, source_ptrs = {}, {} From d3d2d2299b1b4e5e578fa98c6e2d6a26e9ea70ae Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 25 Aug 2026 09:34:58 +0000 Subject: [PATCH 44/62] taxon_map: Add source whitelist Claude noticed that along with the entries we're expecting, there's a lot of junk in the sources we find, likely upstream parsing problems bubbling up. Whilst removing them doesn't make a dramatic difference practically, it does make the output easier to read. --- .../taxon_mapping_and_popularity/taxon_map.py | 28 +++++++++++++++---- tests/test_taxon_map.py | 23 +++++++++++++-- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py index afc01636..92ee345a 100644 --- a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py +++ b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py @@ -16,6 +16,11 @@ logger = logging.getLogger(__name__) +# The taxonomy sources we can map to wikidata, most trusted first (the order is used to +# choose between wikidata items when they disagree). Any other source in the taxonomy +# (silva, h2007, "additions-6520052-6520144", ...) is ignored. +SOURCES = ("ncbi", "if", "worms", "irmng", "gbif") + def map_wiki_info( source_ptrs, @@ -371,11 +376,12 @@ def read_extra_source_file(path): logger.warning(f" Extra source file '{path}' not found, so ignored") -def add_taxon_sources(OTT_ptrs, source_ptrs, ott, sourceinfo, rank=None): +def add_taxon_sources(OTT_ptrs, source_ptrs, ott, sourceinfo, rank=None, unused_sources=None): """ Point OTT_ptrs[ott] at an entry in source_ptrs for each source in sourceinfo, adding the OTT if not already there, and overwriting any existing source of - the same name. + the same name. Sources not in SOURCES are ignored, their names being added to + the unused_sources set, if given. """ ott_data = OTT_ptrs.setdefault(ott, {"ott": ott, "sources": {}}) if rank is not None: @@ -384,6 +390,10 @@ def add_taxon_sources(OTT_ptrs, source_ptrs, ott, sourceinfo, rank=None): for src in reversed(sourceinfo.keys()): # NB: look at sources in reverse order, overwriting, so 1st ones take priority src_id = sourceinfo[src] + if src not in SOURCES: + if unused_sources is not None: + unused_sources.add(src) + continue # NB: reuse any existing entry, so OTTs sharing a source id share its (wikidata) data ott_data["sources"][src] = source_ptrs.setdefault(src, {}).setdefault(src_id, {"id": src_id}) @@ -465,18 +475,24 @@ def main(): # Replaces get_OTT_list & OTT_popularity_mapping.create_from_taxonomy respectively logger.info("Generating OTT_ptrs / source_ptrs from taxonomy") OTT_ptrs = {} - source_ptrs = {} + source_ptrs = {s: {} for s in SOURCES} # NB: all sources must exist, even if empty + unused_sources = set() for r in read_ot_taxonomy(args.OpenTreeTaxonomy): - add_taxon_sources(OTT_ptrs, source_ptrs, r["uid"], r["sourceinfo"], r["rank"]) + add_taxon_sources(OTT_ptrs, source_ptrs, r["uid"], r["sourceinfo"], r["rank"], unused_sources) if args.extra_source_file is not None: logger.info(f"Supplementing source IDs from {args.extra_source_file}") extra_otts = 0 for r in read_extra_source_file(args.extra_source_file): - add_taxon_sources(OTT_ptrs, source_ptrs, r["uid"], r["sourceinfo"], r.get("rank")) + add_taxon_sources(OTT_ptrs, source_ptrs, r["uid"], r["sourceinfo"], r.get("rank"), unused_sources) extra_otts += 1 logger.info(f"✔ {extra_otts} OTTs supplemented from {args.extra_source_file}") + logger.info( + f"✔ {len(OTT_ptrs)} OTTs with sources {[f'{s}: {len(source_ptrs[s])}' for s in SOURCES]}. " + f"Ignored {len(unused_sources)} unused sources, e.g. {sorted(unused_sources)[:5]}" + ) + eol_sources = { "ncbi": 676, "worms": 459, @@ -487,7 +503,7 @@ def main(): map_wiki_info( source_ptrs=source_ptrs, - source_order=["ncbi", "if", "worms", "irmng", "gbif"], + source_order=SOURCES, OTT_ptrs=OTT_ptrs, WD_filename=args.wikidataDumpFile, lang=args.wikilang, diff --git a/tests/test_taxon_map.py b/tests/test_taxon_map.py index fc2991dc..812be536 100644 --- a/tests/test_taxon_map.py +++ b/tests/test_taxon_map.py @@ -179,9 +179,26 @@ def test_reading_a_source_keeps_data_added_to_its_entry(self): def test_non_numeric_source_ids_are_usable(self): OTT_ptrs, source_ptrs = {}, {} - add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": "JX948102"}) + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"gbif": "D11377/#1"}) - assert source_ptrs["silva"]["JX948102"] == {"id": "JX948102"} + assert source_ptrs["gbif"]["D11377/#1"] == {"id": "D11377/#1"} + + def test_unknown_sources_are_ignored(self): + # The taxonomy carries ~150 source names we can't map to wikidata + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": 0, "h2007": 1, "additions-6520052-6520144": 2}) + + assert OTT_ptrs == {1: {"ott": 1, "sources": {}}} + assert source_ptrs == {} + + def test_unknown_sources_are_collected_when_asked(self): + OTT_ptrs, source_ptrs = {}, {} + unused = set() + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": 0, "ncbi": 1}, unused_sources=unused) + add_taxon_sources(OTT_ptrs, source_ptrs, 2, {"silva": 3, "h2007": 4}, unused_sources=unused) + + assert unused == {"silva", "h2007"} + assert set(source_ptrs) == {"ncbi"} def test_a_second_call_overrides_only_the_sources_given(self): # i.e. how an extra_source_file supplements the OpenTree taxonomy @@ -213,7 +230,7 @@ def test_a_silva_derived_ncbi_id_is_used_like_any_other(self): OTT_ptrs, source_ptrs = {}, {} add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": "JX948102", "ncbi": 1274384}) - assert OTT_ptrs[1]["sources"]["ncbi"] == {"id": 1274384} + assert OTT_ptrs[1]["sources"] == {"ncbi": {"id": 1274384}} def test_empty_sourceinfo_still_adds_the_ott(self): OTT_ptrs, source_ptrs = {}, {} From f35053059ae9babf30d3bb67d5d62d8a4b399e79 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 25 Aug 2026 10:23:19 +0000 Subject: [PATCH 45/62] taxon_map: ifung -> if as far as we can With at least the current OpenTree, ifung is actually if: https://github.com/OpenTreeOfLife/reference-taxonomy/wiki/Interim-taxonomy-file-format https://tree.opentreeoflife.org/taxonomy/browse?id=75257 https://tree.opentreeoflife.org/taxonomy/browse?id=199764 Use the correct column name up until the DB, where it's not worth the faff of renaming. --- .../CSV_base_table_creator.py | 4 +- .../taxon_mapping_and_popularity/taxon_map.py | 42 +++++++++---------- oz_tree_build/tree_build/step_output.py | 8 ++-- tests/test_taxon_map.py | 27 ++++++++++++ tests/test_tree_build_step_output.py | 6 ++- tests/test_tree_build_step_popularity.py | 2 +- 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py b/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py index 21305361..20dd7106 100755 --- a/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py +++ b/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py @@ -676,7 +676,7 @@ def output_simplified_tree(tree, taxonomy_file, outdir, version, seed, save_sql= leaf_extras["popularity_rank"] = ["popularity_rank"] leaf_extras["price"] = None leaf_extras["ncbi"] = ["sources", "ncbi", "id"] - leaf_extras["ifung"] = ["sources", "ifung", "id"] + leaf_extras["ifung"] = ["sources", "if", "id"] # NB: index fungorum is "if" in the taxonomy leaf_extras["worms"] = ["sources", "worms", "id"] leaf_extras["irmng"] = ["sources", "irmng", "id"] leaf_extras["gbif"] = ["sources", "gbif", "id"] @@ -693,7 +693,7 @@ def output_simplified_tree(tree, taxonomy_file, outdir, version, seed, save_sql= node_extras["raw_popularity"] = ["wd", "raw_popularity"] node_extras["popularity"] = ["popularity"] node_extras["ncbi"] = ["sources", "ncbi", "id"] - node_extras["ifung"] = ["sources", "ifung", "id"] + node_extras["ifung"] = ["sources", "if", "id"] node_extras["worms"] = ["sources", "worms", "id"] node_extras["irmng"] = ["sources", "irmng", "id"] node_extras["gbif"] = ["sources", "gbif", "id"] diff --git a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py index 92ee345a..f209e9b7 100644 --- a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py +++ b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py @@ -514,10 +514,20 @@ def main(): ) populate_iucn(OTT_ptrs, args.EOLidentifiers) - # Write collated data out into CSV format - writer = csv.writer(args.o, dialect="excel") + write_taxon_map(args.o, OTT_ptrs) + args.o.close() + + +def write_taxon_map(out_file, OTT_ptrs): + """Write the collated OTT data out as CSV, as read back by read_taxon_map() + + Source columns are named as the taxonomy names them, i.e. index fungorum is "if". + They are only renamed ("ifung") when written to the ordered_leaves/ordered_nodes + files, whose header is used as the column list when importing into the database. + """ + writer = csv.writer(out_file, dialect="excel") writer.writerow( - ( + [ "ott", "wikidata", "wikipedia_lang_flag", @@ -525,17 +535,13 @@ def main(): "eol", "rank", "raw_popularity", - "ncbi", - "ifung", - "worms", - "irmng", - "gbif", + *SOURCES, "ipni", - ) + ] ) for o in OTT_ptrs.values(): writer.writerow( - ( + [ o["ott"], o.get("wd", {}).get("Q"), o.get("wd", {}).get("wikipedia_lang_flag"), @@ -543,15 +549,10 @@ def main(): o.get("eol"), o.get("rank"), o.get("wd", {}).get("raw_popularity"), - o["sources"].get("ncbi", {}).get("id"), - o["sources"].get("ifung", {}).get("id"), - o["sources"].get("worms", {}).get("id"), - o["sources"].get("irmng", {}).get("id"), - o["sources"].get("gbif", {}).get("id"), + *(o["sources"].get(src, {}).get("id") for src in SOURCES), o.get("ipni"), - ) + ] ) - args.o.close() def read_taxon_map(path): @@ -559,13 +560,12 @@ def read_taxon_map(path): Empty fields become ``None``. Numeric fields are converted to ``int`` or ``float``; ``iucn`` is left as a string because multiple values may be - joined with ``|``. Source IDs (``ncbi``, ``ifung``, ``worms``, ``irmng``, - ``gbif``) are converted to ``int`` where possible and otherwise left as - strings. + joined with ``|``. Source IDs (see ``SOURCES``) are converted to ``int`` + where possible and otherwise left as strings. """ int_fields = ("ott", "wikidata", "wikipedia_lang_flag", "eol", "ipni") float_fields = ("raw_popularity",) - source_fields = ("ncbi", "ifung", "worms", "irmng", "gbif") + source_fields = SOURCES out = {} with open(path, encoding="utf-8", newline="") as f: reader = csv.DictReader(f, dialect="excel") diff --git a/oz_tree_build/tree_build/step_output.py b/oz_tree_build/tree_build/step_output.py index 9316ec19..7bcab034 100644 --- a/oz_tree_build/tree_build/step_output.py +++ b/oz_tree_build/tree_build/step_output.py @@ -119,7 +119,7 @@ def output_mysqlexport(tree, out_dir): "popularity_rank", "price", "ncbi", - "ifung", + "ifung", # NB: the DB's name for the taxonomy's "if" source "worms", "irmng", "gbif", @@ -143,7 +143,7 @@ def output_mysqlexport(tree, out_dir): "raw_popularity", "popularity", "ncbi", - "ifung", + "ifung", # NB: the DB's name for the taxonomy's "if" source "worms", "irmng", "gbif", @@ -186,7 +186,7 @@ def output_mysqlexport(tree, out_dir): node.props.get("popularity_rank", "\\N"), None, # "price" node.props["taxon"].get("ncbi", "\\N"), - node.props["taxon"].get("ifung", "\\N"), + node.props["taxon"].get("if", "\\N"), # NB: "ifung" in the DB node.props["taxon"].get("worms", "\\N"), node.props["taxon"].get("irmng", "\\N"), node.props["taxon"].get("gbif", "\\N"), @@ -211,7 +211,7 @@ def output_mysqlexport(tree, out_dir): node.props["taxon"].get("raw_popularity", "\\N"), node.props.get("popularity", "\\N"), node.props["taxon"].get("ncbi", "\\N"), - node.props["taxon"].get("ifung", "\\N"), + node.props["taxon"].get("if", "\\N"), # NB: "ifung" in the DB node.props["taxon"].get("worms", "\\N"), node.props["taxon"].get("irmng", "\\N"), node.props["taxon"].get("gbif", "\\N"), diff --git a/tests/test_taxon_map.py b/tests/test_taxon_map.py index 812be536..3dfd3f9d 100644 --- a/tests/test_taxon_map.py +++ b/tests/test_taxon_map.py @@ -7,6 +7,8 @@ parse_sourceinfo, read_extra_source_file, read_ot_taxonomy, + read_taxon_map, + write_taxon_map, ) @@ -238,3 +240,28 @@ def test_empty_sourceinfo_still_adds_the_ott(self): assert OTT_ptrs == {1: {"ott": 1, "rank": "species", "sources": {}}} assert source_ptrs == {} + + +class TestWriteTaxonMap: + def build(self, tmp_path, sourceinfo): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, parse_sourceinfo(sourceinfo), "species") + path = tmp_path / "taxon_map.csv" + with open(path, "w", encoding="utf-8", newline="") as f: + write_taxon_map(f, OTT_ptrs) + return path + + def test_every_source_reaches_its_column(self, tmp_path): + # NB: sources are named as the taxonomy names them, i.e. index fungorum is "if" + path = self.build(tmp_path, "ncbi:1,if:2,worms:3,irmng:4,gbif:5") + row = read_taxon_map(path)[1] + + assert (row["ncbi"], row["if"], row["worms"], row["irmng"], row["gbif"]) == (1, 2, 3, 4, 5) + + def test_absent_sources_are_empty(self, tmp_path): + path = self.build(tmp_path, "ncbi:1") + row = read_taxon_map(path)[1] + + assert row["ncbi"] == 1 + assert (row["if"], row["worms"], row["irmng"], row["gbif"]) == (None, None, None, None) + assert (row["ott"], row["rank"]) == (1, "species") diff --git a/tests/test_tree_build_step_output.py b/tests/test_tree_build_step_output.py index 87a3410d..e73bcae4 100644 --- a/tests/test_tree_build_step_output.py +++ b/tests/test_tree_build_step_output.py @@ -377,6 +377,7 @@ def test_taxon_props_are_written_to_leaf_row(self, tmp_path): "iucn": "LC", "eol": "42", "ncbi": "999", + "if": "531546", }, }, ) @@ -389,6 +390,8 @@ def test_taxon_props_are_written_to_leaf_row(self, tmp_path): assert a[LEAF_HEADER.index("iucn")] == "LC" assert a[LEAF_HEADER.index("eol")] == "42" assert a[LEAF_HEADER.index("ncbi")] == "999" + # The taxonomy's "if" source is the DB's "ifung" column + assert a[LEAF_HEADER.index("ifung")] == "531546" # B had no overrides → \N everywhere taxon-derived. b = rows["B"] assert b[LEAF_HEADER.index("ott")] == "\\N" @@ -398,12 +401,13 @@ def test_taxon_props_are_written_to_node_row(self, tmp_path): # Internal nodes get the same taxon projection — but with `rnk` # in place of the leaf-only `iucn`/`extinction_date` columns. t = ete4.Tree("(A,B)R;", parser=1) - _prep(t, taxon_overrides={"R": {"ott": "777", "rnk": "family"}}) + _prep(t, taxon_overrides={"R": {"ott": "777", "rnk": "family", "if": "9257"}}) output_mysqlexport(t, str(tmp_path)) nodes = _read_csv(tmp_path, "ordered_nodes.csv") root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") assert root[NODE_HEADER.index("ott")] == "777" assert root[NODE_HEADER.index("rnk")] == "family" + assert root[NODE_HEADER.index("ifung")] == "9257" def test_missing_extinction_date_and_popularity_are_backslash_N(self, tmp_path): # Leaf-only props (extinction_date, popularity, popularity_rank) diff --git a/tests/test_tree_build_step_popularity.py b/tests/test_tree_build_step_popularity.py index e263571f..4637a037 100644 --- a/tests/test_tree_build_step_popularity.py +++ b/tests/test_tree_build_step_popularity.py @@ -22,7 +22,7 @@ "rank", "raw_popularity", "ncbi", - "ifung", + "if", "worms", "irmng", "gbif", From e7819ab3584d5102f3918d7245bee9e530cb7657 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Tue, 25 Aug 2026 10:48:22 +0000 Subject: [PATCH 46/62] CSV_base_table_creator: Remove It's now been replaced with taxon_map, tidy up old code. --- dvc.yaml | 33 - oz_tree_build/_OZglobals.py | 1 - .../CSV_base_table_creator.py | 1141 ----------------- .../OTT_popularity_mapping.py | 187 --- .../dendropy_extras.py | 526 -------- .../taxon_mapping_and_popularity/taxon_map.py | 5 +- oz_tree_build/tree_build/step_jsnewick.py | 3 +- oz_tree_build/utilities/filter_eol.py | 2 +- pyproject.toml | 1 - .../ordered_leaves_0.csv | 41 - .../ordered_leaves_0.csv.mySQL | 41 - .../ordered_nodes_0.csv | 40 - .../ordered_nodes_0.csv.mySQL | 40 - .../ordered_tree_0.nwk | 1 - .../ordered_tree_0.poly | 1 - tests/test_full_generation.py | 49 - 16 files changed, 6 insertions(+), 2106 deletions(-) delete mode 100755 oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py delete mode 100755 oz_tree_build/taxon_mapping_and_popularity/dendropy_extras.py delete mode 100644 tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv delete mode 100644 tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv.mySQL delete mode 100644 tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv delete mode 100644 tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv.mySQL delete mode 100644 tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.nwk delete mode 100644 tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.poly delete mode 100644 tests/test_full_generation.py diff --git a/dvc.yaml b/dvc.yaml index f224eb31..7f8d9325 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -207,36 +207,3 @@ stages: - ot_version outs: - data/taxon_map.csv - - # ~10 mins - CSV_base_table_creator: - cmd: - - mkdir -p data/output_files - - >- - .venv/bin/CSV_base_table_creator - data/OZTreeBuild/${oz_tree}/${oz_tree}_full_tree.phy - data/OpenTree/${ot_version}/taxonomy.tsv - data/filtered/OneZoom_provider_ids.csv - data/filtered/OneZoom_latest-all.json - data/filtered/OneZoom_enwiki-latest-page.sql - data/filtered/pageviews/OneZoom_pageviews* - -o data/output_files -v - --version ${build_version} - --exclude ${exclude_from_popularity} - --extra_source_file data/OZTreeBuild/${oz_tree}/BespokeTree/SupplementaryTaxonomy.tsv - 2> data/CSV_base_table_creator.log - deps: - - data/OZTreeBuild/${oz_tree}/${oz_tree}_full_tree.phy - - data/OpenTree/${ot_version}/taxonomy.tsv - - data/filtered/OneZoom_provider_ids.csv - - data/filtered/OneZoom_latest-all.json - - data/filtered/OneZoom_enwiki-latest-page.sql - - data/filtered/pageviews/ - - data/OZTreeBuild/${oz_tree}/BespokeTree/SupplementaryTaxonomy.tsv - params: - - oz_tree - - ot_version - - build_version - - exclude_from_popularity - outs: - - data/output_files/ diff --git a/oz_tree_build/_OZglobals.py b/oz_tree_build/_OZglobals.py index 71056637..714f21ff 100644 --- a/oz_tree_build/_OZglobals.py +++ b/oz_tree_build/_OZglobals.py @@ -14,7 +14,6 @@ current = type("", (), {})() # allow us to set e.g. current.OZglobals # bitwise flags for existence of different language wikipedia articles -# this variable is also used in construct_wiki_info in CSV_base_table_creator.py wikiflags = cache.ram( "wikiflags", lambda: { diff --git a/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py b/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py deleted file mode 100755 index 20dd7106..00000000 --- a/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py +++ /dev/null @@ -1,1141 +0,0 @@ -""" -Creates the base files for the dynamically-loaded tree, on the basis of a -single newick tree with OTT numbers on the leaves and nodes. - -Final output is 2 csv files, two long newick string with braces and commas -(one with curly braces for polytomies) and a dates file. - -First we remove polytomies, subspecies, and leaves and nodes in two large python structures - -Using the Open Tree of Life taxonomy file (taxonomy.tsv), point leaves and nodes to -source ids (e.g. ncbi:1234, etc etc), adding any missing leaves/nodes/subspecies etc -from the OpenTree of life (this helps calculate phylogenetic popularity later). - -Then use these source ids to map against Encyclopedia of Life and WikiData ids, -using the EOL identifiers.csv file and the WikiData JSON dump. Using wikidata, -also flag whether or not a wikipedia english language page for that taxon exists -(this helps decide whether or not to show the wikipedia tab). Additionally, to get extra -EoL ids, we supplement the EOL identifiers list by EOL numbers gleaned from wikidata - -We also use wikidata and EoL lists to populate the IUCN id field (potentially problematic & -out of date, but at least means we don't need to do any taxonomic name matching) - -==Popularity== - -Wikidata ids are used to get pagesize and pageview count files, to calculate raw popularities. - -Finally, use the full OpenTree to calculate phylogenetic popularities from base popularities - -==Nested set structure, to get leaves from nodes=== - -At the end of the script, nodes & leaves from the original tree are placed into 2 CSV files. -Each node provides a lft and rgt bracket delimiting all the leaves descended from it. -This allows us to quickly find all terminal children of a given node. - -CSVs can be imported into a mysql database by removing the existing data via 'TRUNCATE TABLE' -then using the mysql command 'LOAD DATA INFILE', which is orders of magnitude faster than -using the builtin web2py import (https://groups.google.com/forum/#!topic/web2py/1bGR8ojrEfs). - - -The OTT id is matched using a reg expr for names such as Aptenodytes_forsteri_ott494370. -This script also allows for temporary node names such as _1234, which are taken as (arbitrary) -*negative* OTT IDs (i.e. -1234 in this case) in the database. This -allows us to find children of unnamed nodes too. - -Download: - * the OpenTree taxonomy from https://tree.opentreeoflife.org/about/taxonomy-version/ott2.9 - * an eol mapping file from http://beta.eol.org/uploads/data_search_files/identifiers.csv.gz - * the wikidata JSON dump from http://dumps.wikimedia.org/wikidatawiki/entities/ - -To test, try e.g. - -Usage: -OT_VERSION=9.1 -ServerScripts/TaxonMappingAndPopularity/CSV_base_table_creator.py \ - ../static/FinalOutputs/Life_full_tree.phy data/OpenTree/ott/taxonomy.tsv \ - data/EOL/identifiers.csv data/Wiki/wd_JSON/* data/Wiki/wp_SQL/* \ - data/Wiki/wp_pagecounts/pagecounts* \ - --OpenTreeFile data/OpenTree/draftversion${OT_VERSION}.tre\ - -o data/output_files/ordered -v --exclude Archosauria_ott335588 Dinosauria_ott90215 > \ - data/output_files/ordered_output.log - -ServerScripts/TaxonMappingAndPopularity/CSV_base_table_creator.py \ - ../static/FinalOutputs/Life_full_tree.phy data/OpenTree/ott/taxonomy.tsv \ - data/EOL/identifiers.csv data/Wiki/wd_JSON/* data/Wiki/wp_SQL/* \ - data/Wiki/wp_pagecounts/* - --OpenTreeFile data/OpenTree/draftversion${OT_VERSION}.tre -o data/output_files/ordered -n - -currently results in: - Out of 3526952 OTT taxa, 2394052 (67.88%) have EOL ids from EOL. \ - Supplementing these with 371071 EOL ids from wikidata gives a coverage of 78.4 %. - Populating IUCN IDs using EOL csv file (or if absent, wikidata) -""" - -import argparse -import csv -import json -import logging -import os.path -import random -import re -import sys -import time -from collections import OrderedDict, defaultdict -from math import log - -from dendropy import Node, Tree - -from ..utilities.debug_util import parse_args_and_add_logging_switch -from ..utilities.file_utils import open_file_based_on_extension -from ..utilities.wikidata_utils import get_qid_from_taxa_data -from . import OTT_popularity_mapping - -# local packages -from .dendropy_extras import write_pop_newick - -__author__ = "Yan Wong" -__license__ = """This is free and unencumbered software released into the public domain by the author, Yan Wong, for OneZoom CIO. - -Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. - -In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to """ # noqa E501 - -sql_subs_string = "" # ? for sqlite, %s for mysql - -# DendroPy performs lots of recursion when reading large trees, this is expected -# https://github.com/jeetsukumaran/DendroPy/issues/52 -sys.setrecursionlimit(3000) - - -def is_unnamed_OTT(OTTid): - """ - TO DO: I'm not sure when we use unnamed nodes with an OTT, so is this needed? - """ - try: - return OTTid < 0 - except TypeError: - return False - - -def get_OTT_species(taxonomy_filename): - with open(taxonomy_filename) as taxonomy_file: - species_list = set() - taxonomy_file.seek(0) - reader = csv.DictReader(taxonomy_file, delimiter="\t") - for row in reader: - if row["rank"] == "species": - species_list.add(int(row["uid"])) - return species_list - - -def parse_tree(tree_filename): - """ - Parses (tree_filename) and returns the DendroPy tree object - """ - try: - tree = Tree.get_from_path( - tree_filename, - schema="newick", - preserve_underscores=True, - suppress_leaf_node_taxa=True, - ) - return tree - except Exception as e: - sys.exit("Problem reading tree from " + tree_filename + ": " + str(e)) - logging.info(" > read tree from " + tree_filename) - - -def get_OTT_list(tree, sources): - """ - Takes a base tree and creates objects for each node and leaf, attaching them as 'data' - dictionaries to each node in the DendroPy tree. Nodes and leaves with an OTT id also - have pointers to their data dicts stored in an OTT-keyed dict, so that mappings to other - databases (ncbi id, etc etc) can be created. - - We want to allow duplicate leaf names, so for the entire procedure we ignore the Dendropy - concept of a taxon list and simply use labels. Returns the Dendropy tree and the OTT dict. - """ - indexed_by_ott = {} - - ott_node = re.compile(r"(.*) ott(\d+)(@\d*)?$") # matches the OTT number - mrca_ott_node = re.compile( - r"(.*) (mrcaott\d+ott\d+)(@\d*)?$" - ) # matches a node with an "mrca" node number (no unique OTT) - tot = 0 - for node in tree.preorder_node_iter(): - tot += 1 - node.data = {} - if node.label: - node.label = node.label.replace("_", " ") - m = ott_node.search(node.label) - if m is not None: - if m.group(3): - logging.warning( - "Node has an @ sign at the end ({node.label}), meaning it has " - "probably not been substituted by an OpenTree equivalent. You " - "may want to provide an alternative subtree from this node " - "downwards, as otherwise it will probably be deleted from the " - "main tree." - ) - node.label = m.group(1) - node.data["ott"] = int(m.group(2)) - indexed_by_ott[node.data["ott"]] = node.data - node.data["sources"] = {} - else: - m = mrca_ott_node.search(node.label) - if m is not None: - if m.group(3): - logging.warning( - f"Node has an @ sign at the end ({node.label}), meaning it " - "has probably not been substituted by an OpenTree " - "equivalent. You may want to provide an alternative subtree " - "from this node downwards, as otherwise it will probably be " - "deleted from the main tree." - ) - node.label = m.group(1) - # this is an 'mrca' node, so we want to save sources but *not* save - # the ott number in node.data - indexed_by_ott[m.group(2)] = node.data - node.data["sources"] = {} - elif node.is_leaf(): - logging.warning( - f"Leaf without an OTT id: '{node.label}'. " "This will not be associated with any other data" - ) - # Finally, put underscores at the start or the end of the new label back as these - # denote "fake" names that are hidden and only used for mapping. We could keep - # them as spaces, but leading/trailing underscores are easier to see by eye. - if node.label[0] == " ": - node.label = "_" + node.label[1:] - if node.label[-1] == " ": - node.label = node.label[:-1] + "_" - logging.info( - f"✔ extracted {len(indexed_by_ott)} otts from {tot} leaves & nodes. " - f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - return indexed_by_ott - - -def add_eol_IDs_from_EOL_table_dump(source_ptrs, identifiers_filename, source_mapping): - used = 0 - EOL2OTT = {v: k for k, v in source_mapping.items()} - with open_file_based_on_extension(identifiers_filename, "rt") as identifiers_file: - reader = csv.DictReader(identifiers_file) - for EOLrow in reader: - if reader.line_num % 1000000 == 0: - logging.info( - f"... {reader.line_num} rows read, {used} used, " - f"mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - provider = int(EOLrow["resource_id"]) - if provider in EOL2OTT: - src = source_ptrs[EOL2OTT[provider]] - if EOL2OTT[provider] == "gbif" and not EOLrow["resource_pk"].isdigit(): - # The EoL file has duplicate (non numeric) IDs for GBIF: ignore these - continue - providerid = EOLrow["resource_pk"] - EOLid = int(EOLrow["page_id"]) - try: - if int(providerid) in src: - used += 1 - src[int(providerid)]["EoL"] = EOLid - except ValueError: - if providerid in src: - used += 1 - src[providerid]["EoL"] = EOLid - logging.info( - f"✔ Matched {used} EoL entries in the EoL identifiers file. " - f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - - -def identify_best_EoLdata(OTT_ptrs, sources): - """ - Each OTT number may point to several EoL entries, one for the NCBI number, - another for the WORMS number, etc etc. Hopefully these will be the same entry, - but they may not be. If they are different we need to choose the best one - to use. We take the one with the most sources supporting this entry: - if there is a tie, we take the lowest, as recommended by JRice from EoL - """ - validOTTs = OTTs_with_EOLmatch = dups = 0 - for OTTid, data in OTT_ptrs.items(): - if is_unnamed_OTT(OTTid): - continue - validOTTs += 1 - choose = {} - for src in sources: - if src in data["sources"] and data["sources"][src] is not None: - if "EoL" in data["sources"][src]: - EOLid = int(data["sources"][src]["EoL"]) - if EOLid not in choose: - choose[EOLid] = [] - choose[EOLid] += [src] - if len(choose) == 0: - data["eol"] = None - else: - OTTs_with_EOLmatch += 1 - errstr = None - if len(choose) > 1: - # weed out those EOLids with the least support. - errstr = f"More than one EoL ID {choose} for taxon OTT: {OTTid}" - dups += 1 - max_refs = max([len(choose[i]) for i in choose]) - choose = [EOLid for EOLid in choose if len(choose[EOLid]) == max_refs] - best = min(choose) - data["eol"] = best - if errstr: - logging.debug(f" {errstr}, chosen {best}") - logging.info( - f" ✔ Of {validOTTs} OpenTree taxa, {OTTs_with_EOLmatch} " - f"({OTTs_with_EOLmatch / validOTTs * 100:.2f}%) have EoL entries in the EoL " - f"identifiers file, and {dups} have multiple possible EOL ids. " - f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - - -def set_wikidata(bz2_filename, source_ptrs, lang): - """ - Will alter the source_ptrs. - Returns WDitems (Q->WD), WPnames (name-WD), common_name_Qs (Q->Q) - """ - WDitems = {} - WPnames = {} - common_name_Qs = {} - sum_info = defaultdict(int) - - ( - Q_to_WD, - WPname_to_WD, - src_to_WD, - replace_Q, - info, - ) = OTT_popularity_mapping.wikidata_info(bz2_filename, source_ptrs, lang) - - WDitems.update(Q_to_WD) - WPnames.update(WPname_to_WD) - common_name_Qs.update(replace_Q) - # Add 'wd' item to source_ptrs - for src, ids in src_to_WD.items(): - for src_id, WD in ids.items(): - source_ptrs[src][src_id]["wd"] = WD - for k, v in info.items(): - sum_info[k] += v - - logging.info( - f"✔ {len(WDitems)} wikidata matches, of which " - f"{sum_info['n_eol']} have EOL ids, {sum_info['n_iucn']} have IUCN ids, " - f"{sum_info['n_ipni']} have IPNI, and {len(WPnames)} " - f"({(len(WPnames)/len(WDitems)*100):.2f}%) have titles that exist on " - f"{lang}.wikipedia. Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - return WDitems, WPnames, common_name_Qs - - -def set_wikipedia_pageviews(filenames, WPnames, lang): - names_found = 0 - for fn in filenames: - WPnames_views = OTT_popularity_mapping.pageviews_for_titles(fn, set(WPnames.keys()), lang) - for name, n_views in WPnames_views.items(): - if not hasattr(WPnames[name], "pageviews"): - names_found += 1 - WPnames[name].pageviews = [] - WPnames[name].pageviews.append(n_views) - logging.info( - f" ✔ Of {len(WPnames)} WikiData taxon entries, {names_found} " - f"({(names_found/len(WPnames) * 100):.2f}%) have pageview data for '{lang}' in " - f"{len(filenames)} files. Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - - -def supplement_from_wikidata(OTT_ptrs): - """ - If no OTT_ptrs[OTTid]['eol'] exists, but there is an - OTT_ptrs[OTTid]['wd']['initial_wiki_item']['EoL'] then put this into - OTT_ptrs[OTTid]['eol'] - Similarly for IPNI (although this is currently unpopulated) - """ - EOLalready = n_eol = n_ipni = n = 0 - for OTTid, data in OTT_ptrs.items(): - if is_unnamed_OTT(OTTid): - logging.info(f" unlabelled node (OTT: {OTTid}) when iterating through OTT_ptrs") - continue - n += 1 - if data.get("eol") is None: - try: - data["eol"] = int(data["wd"].EoL) - n_eol += 1 - except (AttributeError, KeyError, TypeError, ValueError): - pass - else: - EOLalready += 1 - if data.get("ipni") is None: - try: - data["ipni"] = int(data["wd"].ipni) - n_ipni += 1 - except (AttributeError, KeyError, TypeError, ValueError): - pass - logging.info( - f"✔ Out of {n} OTT taxa, {EOLalready} ({(EOLalready/n * 100):.2f}%) already " - f"have EOL ids from the EOL file. Supplementing these with {n_eol} EOL ids from " - f"wikidata gives a coverage of {((EOLalready + n_eol)/n * 100):.1f} %." - + (f" An addition {n_ipni} IPNI identifiers added via wikidata" if n_ipni else "") - ) - - -iucn_num = 5 - - -def populate_iucn(OTT_ptrs, identifiers_filename, verbosity=0): - """ - Port the IUCN number from both EoL and Wikidata, and keep both if there is a conflict - """ - used = 0 - - eol_mapping = {} # to store eol=>iucn - for OTTid, data in OTT_ptrs.items(): - if "eol" in data: - if data["eol"] in eol_mapping: - eol_mapping[data["eol"]].append(OTTid) - else: - eol_mapping[data["eol"]] = [OTTid] - - with open_file_based_on_extension(identifiers_filename, "rt") as identifiers_file: - reader = csv.DictReader(identifiers_file) - for EOLrow in reader: - if reader.line_num % 1000000 == 0: - logging.info( - f" - {reader.line_num} rows read, {used} used. " f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - if int(EOLrow["resource_id"]) == iucn_num and EOLrow["resource_pk"].isdigit(): - # there are lots of non-species IUCN rows with pk == str (e.g. Animalia) - try: - for ott in eol_mapping[int(EOLrow["page_id"])]: - OTT_ptrs[ott]["iucn"] = EOLrow["resource_pk"] - used += 1 - except LookupError: - pass # no equivalent eol id in eol_mapping - logging.info( - f" > matched {used} IUCN entries in the EoL identifiers file. " - f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - - # now go through and double-check against IUCN stored on wikidata - for OTTid, data in OTT_ptrs.items(): - try: - wd_iucn = str(int(data["wd"].iucn)) - if "iucn" not in data: - data["iucn"] = wd_iucn - used += 1 - else: - if wd_iucn not in data["iucn"].split("|"): - data["iucn"] += "|" + wd_iucn - logging.debug( - f' conflicting IUCN IDs for OTT {OTTid}: EoL = {data["iucn"]} ' - f'(via http://eol.org/pages/{data["eol"]}), wikidata = ' - f'{wd_iucn} (via http://http://wikidata.org/wiki/Q{data["wd"].Q}).' - ) - except ValueError: - logging.warning(f" Cannot convert wikidata IUCN ID {data['wd'].iucn} to integer.") - except (KeyError, AttributeError): - pass # can't find a wd instance or an iucn within the wd instance. Oh well. - - logging.info(f" > Increased IUCN coverage to {used} taxa using wikidata") - - -def popularity_function( - sum_of_all_ancestor_popularities, - sum_of_all_descendant_popularities, - number_of_ancestors, - number_of_descendants, -): - """ - a) Dividing by number_of_ancestors+number_of_descendants would mean averaging - popularity over all nodes, which would bias against taxa which have many - unvisited/unpopular children - b) Alternatively, dividing by a constant is equivalent to summing popularity over - all nodes, which biases towards taxa with many fine taxonomic divisions - We do something between the two by dividing by the log of the number of nodes. - """ - if ( - (sum_of_all_ancestor_popularities is None) - or (sum_of_all_descendant_popularities is None) - or (number_of_ancestors is None) - or (number_of_descendants is None) - ): - return None - elif number_of_ancestors + number_of_descendants == 1: - # Avoid a divide by zero error if this adds up to 1 - # Though the need for this makes me think that the log calculation - # may not be mathematically sound - return sum_of_all_ancestor_popularities + sum_of_all_descendant_popularities - else: - return (sum_of_all_ancestor_popularities + sum_of_all_descendant_popularities) / log( - number_of_ancestors + number_of_descendants - ) - - -def resolve_polytomies_add_popularity(tree, seed): - """ - If there are polytomies in the tree, resolve them, but make sure that the newly - created nodes get popularity values too. These can be recalculated from the - descendants and ancestors of the children - - """ - prev_num_nodes = sum(1 for i in tree.postorder_node_iter()) - random.seed(seed) # so we get the same bifurcations each time - - # We implement a slightly non-random resolution to group nodes with the same genus together - # See https://github.com/OneZoom/OZtree/issues/958 - tree.group_genera_in_polytomies() - tree.resolve_polytomies(rng=random) - num_new_nodes = sum(1 for i in tree.postorder_node_iter()) - prev_num_nodes - for node in tree.postorder_node_iter(): - if not hasattr(node, "data"): - # this is a new node - it should always have 2 children - try: - n = ancestor_pop_sum = descendant_pop_sum = n_ancestors_sum = n_descendants_sum = 0 - for c in node.child_node_iter(): - n += 1 - ancestor_pop_sum += c.ancestors_popsum - descendant_pop_sum += c.descendants_popsum - n_ancestors_sum += c.n_ancestors - n_descendants_sum += c.n_descendants - - node.data = { - "popularity": popularity_function( - ancestor_pop_sum / n, - descendant_pop_sum, - n_ancestors_sum / n, - n_descendants_sum, - ) - } - except AttributeError: - # probably popularity values undefined for one of the children - pass - return num_new_nodes - - -def create_leaf_popularity_rankings(tree): - """ - Make a rank of all existing leaves by phylogenetic popularity - Must be run once all invalid tips etc have been removed. - If there are no popularities, set all ranks to None - """ - leaf_popularities = defaultdict(int) - for node in tree.leaf_node_iter(): - leaf_popularities[node.data.get("popularity")] += 1 - cumsum = 1 - if None in leaf_popularities: - return - for k in sorted(leaf_popularities.keys(), reverse=True): - add_next = leaf_popularities[k] - leaf_popularities[k] = cumsum - cumsum += add_next - for leaf in tree.leaf_node_iter(): - leaf.data["popularity_rank"] = leaf_popularities[leaf.data.get("popularity")] - - -def write_popularity_tree(tree, outdir, filename, version, verbosity=0): - Node.write_pop_newick = write_pop_newick - with open(os.path.join(outdir, f"{filename}_{version}.nwk"), "w+") as popularity_newick: - tree.seed_node.write_pop_newick(popularity_newick) - - -def output_simplified_tree(tree, taxonomy_file, outdir, version, seed, save_sql=True, extinct_tree_mode=False): - """ - We should now have leaf entries attached to each node in the tree like - data = { - 'ott':, - 'wd': WikidataItem(Q=15478814, EoL=1100788, l={'en','fr'}), - 'pop_dscdt': 0, - 'pop_ancst': 220183.23395609166, - 'sources': {'ncbi': None, - 'worms': None, - 'gbif': {'wd': WikidataItem(Q=15478814, EoL=1100788), 'id': '2840414'}, - 'if': None, - 'irmng': None}, - 'popularity': 220183.23395609166, - 'eol': 1100788 - 'iucn':XXXXXXX} - - ... or, if we have managed to calculate popularity ... - - data = { - 'wd': WikidataItem( - Q=15478814, EoL=1100788, l={'en','fr'}, pageviews=[64, 47], pagesize=1465, raw_pop=285.1 - ), - 'pop_dscdt': 0, - 'pop_ancst': 392245.76075749274, - 'sources': { - 'ncbi': { - 'wd': WikidataItem( - pageviews=[64, 47], raw_popularity=285.1, Q=4672161, EoL=281897, pagesize=1465), - 'EoL': 281897, - 'id': '691616' - }, - 'worms': None, - 'gbif': { - 'wd': WikidataItem( - pageviews=[64, 47], raw_popularity=285.1, Q=4672161, EoL=281897, pagesize=1465), - 'id': '1968205' - } - 'if': None, - 'irmng': {'EoL': 281897, 'id': '10290975'} - }, - 'eol': 281897 - } - - Removes non-species from tips, outputs simplified versions. - """ - from .dendropy_extras import ( - group_genera_in_polytomies, - prune_children_of_otts, - prune_non_species, - remove_unifurcations_keeping_higher_taxa, - set_node_ages, - set_real_parent_nodes, - write_brief_newick, - write_preorder_to_csv, - ) - - # monkey patch the existing dendropy objects - Tree.prune_children_of_otts = prune_children_of_otts - Tree.prune_non_species = prune_non_species - Tree.set_node_ages = set_node_ages - Tree.set_real_parent_nodes = set_real_parent_nodes - Tree.remove_unifurcations_keeping_higher_taxa = remove_unifurcations_keeping_higher_taxa - Tree.write_preorder_to_csv = write_preorder_to_csv - Tree.group_genera_in_polytomies = group_genera_in_polytomies - - Tree.create_leaf_popularity_rankings = ( - create_leaf_popularity_rankings # not defined in dendropy_extras, but in this file - ) - Tree.resolve_polytomies_add_popularity = resolve_polytomies_add_popularity - Node.write_brief_newick = write_brief_newick - - # For the extinct tree, we don't want to remove any species - if not extinct_tree_mode: - logging.info(f" > removing children labeled species in '{taxonomy_file}'") - n = len(tree.prune_children_of_otts(get_OTT_species(taxonomy_file))) - logging.info(f" ✔ removed all children of {n} nodes") - - logging.info(" > removing tips that appear not to be species") - # species names containing these (even initially) are discarded - bad_sp = ["cf.", "aff.", "subsp.", "environmental sample"] - # species names containing these within the name are discarded: - bad_sp += [" cv.", " sp."] - n = {k: len(v) for k, v in tree.prune_non_species(bad_matches=bad_sp, extinct_tree_mode=extinct_tree_mode).items()} - logging.info( - f" ✔ removed {n['unlabelled']} blank leaves, {n['no_space']} lacking a space, " - f"& {n['bad_match']} containing {bad_sp} (assumed bad tips)" - ) - - logging.info(" > setting node ages & removing extinction props") - a, n = tree.set_node_ages() - logging.info(f" ✔ set ages on {a} nodes and leaves & removed {n} extinction props") - - logging.info(" > removing unary nodes, keeping monotypic species or highest taxon") - n_deleted_nodes = tree.remove_unifurcations_keeping_higher_taxa() - # see https://github.com/jeetsukumaran/DendroPy/issues/75 - logging.info(f" ✔ removed {n_deleted_nodes} unifurcations") - - logging.info(" > splitting polytomies by genera and assigning popularities to new nodes") - n_new = tree.resolve_polytomies_add_popularity(seed) - tree.create_leaf_popularity_rankings() - logging.info(f" ✔ polytomies split with seed={seed}: {n_new} extra nodes created") - - # NB: we shouldn't need to (re)set popularity or ages, since deleting nodes - # does not affect these, and both have been calculated *after* new - # nodes were created by resolve_polytomies. - logging.info(" > setting real parents and ranking leaf popularity") - tree.set_real_parent_nodes() - logging.info(" ✔ real parents and popularity ranks set") - - logging.info(" > ladderizing tree (groups with fewer leaves first)") - tree.ladderize(ascending=True) # warning: ladderize ascending is needed for the short OZ newick-like form - logging.info(" ✔ ladderized") - - logging.info(" > writing tree, dates, and csv to files") - with open(os.path.join(outdir, f"ordered_tree_{version}.nwk"), "w+") as condensed_newick: - tree.seed_node.write_brief_newick(condensed_newick) - with open(os.path.join(outdir, f"ordered_tree_{version}.poly"), "w+") as condensed_poly: - tree.seed_node.write_brief_newick(condensed_poly, "{}") - - # these are the extra columns output to the leaf csv file - leaf_extras = OrderedDict() - leaf_extras["ott"] = ["ott"] - leaf_extras["wikidata"] = ["wd", "Q"] - leaf_extras["wikipedia_lang_flag"] = ["wd", "wikipedia_lang_flag"] - leaf_extras["iucn"] = ["iucn"] - leaf_extras["eol"] = ["eol"] - leaf_extras["raw_popularity"] = ["wd", "raw_popularity"] - leaf_extras["popularity"] = ["popularity"] - leaf_extras["popularity_rank"] = ["popularity_rank"] - leaf_extras["price"] = None - leaf_extras["ncbi"] = ["sources", "ncbi", "id"] - leaf_extras["ifung"] = ["sources", "if", "id"] # NB: index fungorum is "if" in the taxonomy - leaf_extras["worms"] = ["sources", "worms", "id"] - leaf_extras["irmng"] = ["sources", "irmng", "id"] - leaf_extras["gbif"] = ["sources", "gbif", "id"] - leaf_extras["ipni"] = ["ipni"] - - # these are the extra columns output to the node csv file - node_extras = OrderedDict() - node_extras["ott"] = ["ott"] - node_extras["wikidata"] = ["wd", "Q"] - node_extras["wikipedia_lang_flag"] = ["wd", "wikipedia_lang_flag"] - node_extras["eol"] = ["eol"] - # We avoid using 'rank' as it is a reserved word in mysql - node_extras["rnk"] = ["rank"] - node_extras["raw_popularity"] = ["wd", "raw_popularity"] - node_extras["popularity"] = ["popularity"] - node_extras["ncbi"] = ["sources", "ncbi", "id"] - node_extras["ifung"] = ["sources", "if", "id"] - node_extras["worms"] = ["sources", "worms", "id"] - node_extras["irmng"] = ["sources", "irmng", "id"] - node_extras["gbif"] = ["sources", "gbif", "id"] - node_extras["ipni"] = ["ipni"] - node_extras["vern_synth"] = None - for representative_image_type in ["rep", "rtr", "rpd"]: - for i in [str(x + 1) for x in range(8)]: - node_extras[representative_image_type + i] = None - for iucn_type in ["NE", "DD", "LC", "NT", "VU", "EN", "CR", "EW", "EX"]: - node_extras["iucn" + iucn_type] = None - - with ( - open(os.path.join(outdir, f"ordered_leaves_{version}.csv"), "w+", encoding="utf-8") as leaves, - open(os.path.join(outdir, f"ordered_nodes_{version}.csv"), "w+", encoding="utf-8") as nodes, - ): - tree.write_preorder_to_csv(leaves, leaf_extras, nodes, node_extras, -version) - logging.info(f" ✔ written into {outdir}/ordered_..._{version}...") - - # make a copy of the csv file that can be imported into mySQL (has \\N for null values) - if save_sql: - from shutil import copyfile - from subprocess import call - - with open(os.path.join(outdir, f"import_{version}.sql"), "w", encoding="utf-8") as sql_f: - # make CSV files that can be imported into mySQL (subs \\N for null values) - logging.info(" > saving extra file copies in mySQL format: import them using:") - for tab in ["_leaves", "_nodes"]: - fn = os.path.join(outdir, "ordered" + tab + f"_{version}" + ".csv") - sqlfile = fn + ".mySQL" - copyfile(fn, sqlfile) - call(["perl", "-pi", "-e", r"s/,(?=(,|\n))/,\\N/g", sqlfile]) - sql_f.writelines( - [ - f"TRUNCATE TABLE ordered{tab};\n" - f"LOAD DATA LOCAL INFILE '{sqlfile}' REPLACE INTO TABLE `ordered{tab}` \n" - f" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' \n" - f" IGNORE 1 LINES ({open(fn).readline().rstrip()}) SET id = NULL;\n" - ] - ) - - -def display_WD_ott_stats(OTT_ptrs): - """ - Display some stats about OTTs coming from Wikidata - """ - matching_otts = 0 - mismatching_otts = 0 - no_wd_otts = 0 - for ott in OTT_ptrs: - try: - if OTT_ptrs[ott]["rank"] == "species": - if OTT_ptrs[ott]["wd"].get("wd_ott") is not None: - if ott == OTT_ptrs[ott]["wd"].wd_ott: - matching_otts += 1 - else: - logging.debug( - f"Q{OTT_ptrs[ott]['wd'].Q}: OTT {ott} does not match {OTT_ptrs[ott]['wd'].wd_ott}" - ) - mismatching_otts += 1 - else: - no_wd_otts += 1 - except (KeyError, AttributeError): - pass - - logging.info("✔ Stats on Wikidata OTT matching:") - logging.info(f" Leaves where the WD ott matches the ott: {matching_otts}") - logging.info(f" Leaves where the WD ott does not match the wd_ott: {mismatching_otts}") - logging.info(f" Leaves where WD does not have an ott: {no_wd_otts}") - - -def map_wiki_info( - source_ptrs, - source_order, - OTT_ptrs, - WD_filename, - lang, - WP_SQL_filename, - WP_pageviews_filenames, -): - """ - 1) use the wikidata JSON dump to map identifiers from the source_ptrs structure to - wikidata Qids, and then on to wikipedia pages - 2) if sql and pagevisits filenames are given - - Return True if popularity is mapped - """ - logging.debug(f"Processing wikidata json dump in parallel for {lang}") - popularity_steps = 0 - WDitems, WPnames, swap_Qs = set_wikidata(WD_filename, source_ptrs, lang) - - if WP_SQL_filename is not None: - logging.info(f" > Adding wikipedia page sizes from {WP_SQL_filename}") - # Can't easily parallelize this as it is gzip compressed (not a block format) - OTT_popularity_mapping.add_pagesize_for_titles(WPnames, WP_SQL_filename) - popularity_steps += 1 - - if len(WP_pageviews_filenames) > 0: - logging.info(f" > Adding wikipedia visit counts from {len(WP_pageviews_filenames)} files") - set_wikipedia_pageviews(WP_pageviews_filenames, WPnames, lang) - popularity_steps += 1 - - if popularity_steps == 2: - logging.info(" > Calculating raw popularity measures") - tot = 0 - for WDinstance in WDitems.values(): - if WDinstance.set_raw_popularity(): - tot += 1 - logging.info(f" ✔ Raw popularity measures set on {tot} wikidata items") - else: - logging.info(" x Skipping popularity calculations") - - # Here we might want to multiply up some taxa, e.g. plants, - # see https://github.com/OneZoom/OZtree/issues/130 - logging.info(" > Finding best wiki matches") - OTT_popularity_mapping.identify_best_wikidata(OTT_ptrs, lang, source_order) - - logging.info(" > Swapping vernacular wikidata items into taxon items") - OTT_popularity_mapping.overwrite_wd(WDitems, swap_Qs, only_if_more_popular=(popularity_steps == 2), check_lang=lang) - - logging.info(" > Supplementing ids (EOL/IPNI) with ones from wikidata") - supplement_from_wikidata(OTT_ptrs) - - logging.info("✔ Wikidata/wikipedia data mapped") - - display_WD_ott_stats(OTT_ptrs) - - return popularity_steps == 2 - - -def percolate_popularity( - tree, - exclude_taxa, - output_location, - popularity_file, - version, - info_on_focal_labels=None, -): - """ - NB: we must percolate popularities through the tree before deleting monotomies, - since these often contain useful popularity information. - This should also allocate popularities even for nodes that have been - created by polytomy resolving. - - We should also check that there are not multiple uses of the same Qid - (https://github.com/OneZoom/OZtree/issues/132) - """ - if info_on_focal_labels is None: - info_on_focal_labels = [] - OTT_popularity_mapping.sum_popularity_over_tree(tree, exclude=exclude_taxa) - # now apply the popularity function - Qids = set() - for node in tree.preorder_node_iter(): - try: - Q = node.data["wd"]["Q"] - if Q in Qids: - logging.warning( - f"duplicate wikidata Qids used (Q{Q}) - this will cause " - f"popularity double-counting for OTT {node.data['ott']}" - ) - else: - Qids.add(Q) - except KeyError: - pass - pop = popularity_function( - node.ancestors_popsum, - node.descendants_popsum, - node.n_ancestors, - node.n_descendants, - ) - - # Round to 2 decimal places - node.data["popularity"] = round(pop, 2) - - if popularity_file: - write_popularity_tree(tree, output_location, popularity_file, version) - # NB to examine a taxon for popularity contributions here, you could try - for focal_label in info_on_focal_labels: - focal_taxon = focal_label.replace("_", " ") - node = tree.find_node_with_label(focal_taxon) - try: - print( - "{}: own pop = {} (Q{}) descendant pop sum = {}".format( - focal_taxon, - node.pop_store, - node.data["wd"].get("Q", " absent"), - node.descendants_popsum, - ) - ) - try: - leaf_iter = node.leaf_node_iter() - except AttributeError: - leaf_iter = node.leaf_iter() - for t, tip in enumerate(leaf_iter): - print( - "Tip {} = {}: own_pop = {}, Qid = Q{}".format( - t, - tip.label, - getattr(tip, "pop_store", None), - tip.data["wd"].get("Q", " absent"), - ) - ) - if t > 100: - print("More tips exist, but have been omitted") - break - while node.parent_node: - node = node.parent_node - if node.pop_store: - print(f"Ancestors: {node.label} = {node.pop_store:.2f}") - except (IndexError, AttributeError) as e: - logging.warning(f"Problem reporting on focal taxon '{focal_taxon}': {e}") - - -def switch_otts_to_qids(taxa_data_file, tree): - """ - For the extinct tree, OTTs don't work well, as many species don't have one, or have the wrong one. - However, we trust the QIDs that we got from the previous extinct tree building phase. - So we switch all the OTTs to QIDs, and essentially pretend that they are OTTs. This works - because OneZoom doesn't actually rely on the ID being an OTT, just that it is unique. - """ - taxa_data = {} - with open(taxa_data_file) as f: - taxa_data = json.load(f) - - for node in tree.preorder_node_iter(): - try: - # Get the QID for this node's taxon - qid = get_qid_from_taxa_data(taxa_data, node.label) - if qid: - # Replace the node's OTT with the QID - node.data["ott"] = qid - # Also, update the QID in the wikidata item, as it may not have - # one at all, or the one it has may be wrong - if "wd" in node.data: - if isinstance(node.data["wd"], dict): - node.data["wd"]["Q"] = qid - else: - node.data["wd"].Q = qid - else: - node.data["wd"] = OTT_popularity_mapping.WikidataItem({"id": f"Q{qid}"}) - except Exception as e: - logging.warning(f"switch_otts_to_qids error: {node.label} qid={qid} Error: {e}") - - -def process_all(args): - random_seed_addition = 42 - start = time.time() - logging.info(f"OneZoom data generation started on {time.asctime(time.localtime(time.time()))}") - skip_popularity = ( - args.popularity_file is None - ) # Default is "": None is when popularity_file explictly specified with no name - - # From http://eol.org/api/docs/provider_hierarchies - # These need to be an ordered dict with the first being the preferred id used when getting - # a corresponding wikidata ID. - # All the ids for these are integers >= 0 - sources = ["ncbi", "if", "worms", "irmng", "gbif"] - eol_sources = { - "ncbi": 676, - "worms": 459, - "gbif": 767, - } # update when EoL has harvested index fungorum & IRMNG - # the ids for these sources may not be numbers (e.g. Silva has things like D11377/#1 - - logging.info("> Creating tree structure") - tree = parse_tree(args.Tree) - OTT_ptrs = get_OTT_list(tree, sources) - - logging.info("> Adding source IDs") - source_ptrs = OTT_popularity_mapping.create_from_taxonomy( - args.OpenTreeTaxonomy, sources, OTT_ptrs, args.extra_source_file - ) - - logging.info("> Adding EOL IDs from EOL csv file") - add_eol_IDs_from_EOL_table_dump(source_ptrs, args.EOLidentifiers, eol_sources) - logging.info("> Finding best EoL matches") - identify_best_EoLdata(OTT_ptrs, eol_sources) - - if args.wikidataDumpFile: - logging.info("> Adding wikidata info") - has_popularity = map_wiki_info( - source_ptrs, - sources, - OTT_ptrs, - args.wikidataDumpFile, - args.wikilang, - None if skip_popularity else args.wikipediaSQLDumpFile, - None if skip_popularity else args.wikipedia_totals_bz2_pageviews, - ) - if has_popularity: - logging.info("> Percolating popularity through the tree") - percolate_popularity( - tree, - args.exclude, - args.output_location, - args.popularity_file, - args.version, - args.info_on_focal_labels, - ) - else: - logging.info("No wikidataDumpFile given: skipping wiki mapping and popularity calc") - - logging.info("> Populating IUCN IDs using EOL csv file (or if absent, wikidata)") - populate_iucn(OTT_ptrs, args.EOLidentifiers) - - # If a taxa_data_file is passed in (typically for the extinct tree), we use it to - # switch the OTTs to QIDs, which work more reliably for the extinct tree - extinct_tree_mode = False - if args.taxa_data_file: - extinct_tree_mode = True - logging.info("> Switching OTTs to QIDs, which works better for the extinct tree") - switch_otts_to_qids(args.taxa_data_file, tree) - - logging.info(f"> Writing out results to {args.output_location}/xxx") - output_simplified_tree( - tree, - args.OpenTreeTaxonomy, - args.output_location, - args.version, - random_seed_addition, - extinct_tree_mode=extinct_tree_mode, - ) - t_fmt = "%H hrs %M min %S sec" - logging.info(f"✔ ALL DONE IN {time.strftime(t_fmt, time.gmtime(time.time()-start))}") - - -def main(): - parser = argparse.ArgumentParser( - description=( - "Convert a newick file with OpenTree labels into refined trees and CSV tables, " - "while mapping OpenTree Taxonomy IDs to other ids (including EoL & Wikidata)" - ) - ) - parser.add_argument("Tree", help="The newick format tree to use") - parser.add_argument( - "OpenTreeTaxonomy", - help="The OpenTree taxonomy.tsv file, from http://files.opentreeoflife.org/ott/", - ) - parser.add_argument( - "EOLidentifiers", - help=("The gzipped EOL identifiers file, from " "https://opendata.eol.org/dataset/identifiers-csv-gz"), - ) - parser.add_argument( - "wikidataDumpFile", - nargs="?", - help=( - "The very large wikidata JSON dump, " - "from https://dumps.wikimedia.org/wikidatawiki/entities/ (latest-all.json.bz2)." - "A filtered version can be used for faster processing." - ), - ) - parser.add_argument( - "wikipediaSQLDumpFile", - nargs="?", - help=( - "The gzipped >1GB wikipedia -latest-page.sql.gz dump, " - "from https://dumps.wikimedia.org/enwiki/latest/ (enwiki-page.sql.gz) " - ), - ) - parser.add_argument( - "wikipedia_totals_bz2_pageviews", - nargs="*", - help=( - 'One or more b2zipped "totals" pageview count files, ' - "from https://dumps.wikimedia.org/other/pagecounts-ez/merged/ " - "(e.g. pagecounts-2016-01-views-ge-5-totals.bz2, or pagecounts*totals.bz2)" - ), - ) - parser.add_argument( - "--popularity_file", - "-p", - nargs="?", - const=None, - default="", - help=( - "Save popularity as branch lengths in a tree under this filename. If no " - "filename given, skip the tedious process of calculating popularity altogether", - ), - ) - parser.add_argument( - "--exclude", - "-x", - nargs="*", - default=[], - help=( - "(Optional) taxa to exclude from calculation of phylogenetic popularities, " - "such as Dinosauria_ott90215, Archosauria_ott335588" - ), - ) - parser.add_argument( - "--output_location", - "-o", - default="output", - help="The directory to store the csv, newick, and date files", - ) - parser.add_argument( - "--wikilang", - "-l", - default="en", - help=( - 'The language wikipedia to check for popularity, e.g. "en". ' - "Where there are multiple Wikidata items for a taxon " - "(e.g. one under the common name, one under the scientific name), " - "then we also default to using the WD item with the sitelink in this language." - ), - ) - parser.add_argument( - "--version", - default=int(time.time() / 60.0), - type=int, - help=( - "A unique version number for the tree, to be saved in the DB tables & output " - "files. Defaults to minutes since epoch (time()/60)" - ), - ) - parser.add_argument( - "--extra_source_file", - default=None, - type=str, - help=( - "An optional additional file to supplement the taxonomy.tsv file, " - "providing additional mappings from OTTs to source ids (useful for overriding). " - 'The first line should be a header contining "uid" and "sourceinfo" column ' - "headers, similar to those in the taxonomy.tsv file. NB the OTT can be a " - 'number, or an ID of the form "mrcaott409215ott616649").' - ), - ) - parser.add_argument( - "--info_on_focal_labels", - nargs="*", - default=[], - help=('Output some extra information for these named taxa (e.g. "Canis_lupus"), ' "for debugging purposes"), - ) - parser.add_argument( - "--taxa-data-file", - default=None, - type=str, - help="JSON file with persisted data about taxa, typically used for the extinct tree", - ) - - args = parse_args_and_add_logging_switch(parser) - process_all(args) - - -if __name__ == "__main__": - main() diff --git a/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py b/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py index e1724d04..cf4baf70 100755 --- a/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py +++ b/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py @@ -350,77 +350,6 @@ def mem(): return mem -def create_from_taxonomy(OTTtax_filename, sources, OTT_ptrs, extra_taxonomy_file=None): - """ - Creates object data and a source_ptrs array pointing to elements within it. - Also fills out the OTT_ptrs array to point to the right place. OTT_ptrs can be - partially filled: new OTT numbers are simply appended OTT id in the taxonomy. - - src ids are ints where possible, although can be strings if they contain characters - - "extra_taxonomy_map" allows us to inject mappings that are missing from the OpenTree - e.g. - """ - - unused_sources = set() - source_ptrs = {s: {} for s in sources} - - # NB: NCBI ids that OpenTree got via SILVA used to be marked as "ncbi_silva" and - # treated with suspicion (see - # https://groups.google.com/d/msg/opentreeoflife/L2x3Ond16c4/CVp6msiiCgAJ). - # Against taxonomy v16.1 they behave just like any other NCBI id: they agree with - # the wikidata item found via gbif/irmng/worms 95.5% of the time, vs 96.2% for - # non-SILVA ids, so they are now used as-is. - - data_files = [OTTtax_filename] - if extra_taxonomy_file is not None: - try: - data_files.append(extra_taxonomy_file) - except FileNotFoundError: - logging.warning(f" Extra taxonomy file '{extra_taxonomy_file}' not found, so ignored") - - used = 0 - for fn in data_files: - with open(fn, encoding="utf-8") as f: - reader = csv.DictReader(f, delimiter="\t") - for OTTrow in reader: - # first 2 lines are header & blank in taxonomy.tsv - if ((reader.line_num - 2) % 1000000 == 0) and reader.line_num > 2: - logging.info( - f"Reading taxonomy file {fn}: {reader.line_num-2} rows read, " - f"{used} identifiers used, mem usage {mem():.1f} Mb" - ) - try: - OTTid = int(OTTrow["uid"]) - except ValueError: - OTTid = OTTrow["uid"] - logging.warning(f" Found an ott value which is not an integer: {OTTid}") - - for srcs in reversed(OTTrow["sourceinfo"].split(",")): - # look at sources in reverse order, overwriting, so 1st ones take priority - src, src_id = srcs.split(":", 1) - if src not in source_ptrs: - if src not in unused_sources: - logging.info(f" New and unused source: {src} (in '{srcs}')") - unused_sources.update([src]) - continue - used += 1 - if src_id.isdigit(): - src_id = int(src_id) - source_ptrs[src][src_id] = {"id": src_id} - try: - OTT_ptrs[OTTid]["sources"][src] = source_ptrs[src][src_id] - OTT_ptrs[OTTid]["rank"] = OTTrow["rank"] - except LookupError: - pass - - logging.info( - f"✔ created {used} source pointers for {len(source_ptrs)} sources " - f"{list(source_ptrs.keys())}. Mem usage {mem():.1f} Mb" - ) - return source_ptrs - - # P31 (instance of) values to search for # See https://en.wikipedia.org/wiki/Template:Taxonbar/whitelist for the full list match_taxa = { @@ -865,119 +794,3 @@ def pageviews_for_titles( pageviews[title] = int(views) return pageviews - - -def sum_popularity_over_tree(tree, OTT_ptrs=None, exclude=None, pop_store="pop", verbosity=0): - """ - Add popularity indices for branch lengths based on a phylogenetic tree (and return the - tree, or the number of root descendants). - We might want to exclude some names from the popularity metric (e.g. exclude archosaurs, - to ensure birds don't gather popularity intended for dinosaurs). This is done by passing - an array such as ['Dinosauria_ott90215', 'Archosauria_ott335588'] as the exclude argument. - - 'tree' can be the name of a tree file or a dendropy tree object - - 'pop_store' is the name of the attribute in which to store the popularity. If you wish to - create a tree with popularity on the branches, you can pass in pop_store='edge_length' - - NB: if OTT_ptrs is given, then the raw popularity is stored in the object pointed to by - OTT_ptrs[OTTid]['wd'], where OTTid can be extracted from the node label in the tree. - If OTT_ptrs is None, then the popularity is stored in the node object itself, in - Node.data['wd']['pop']. - - Popularity summed up and down the tree depends on the OpenTree structure, and is stored in - OTT_ptrs[OTTid]['pop_ancst'] (popularity summed upwards for all ancestors of this node) - and OTT_ptrs[OTTid]['pop_dscdt'] (popularity summed over all descendants). To get a - measure of the sum of both ancestor and descendant popularity, just add these together - - We also count up the *number* of edges above each node to the root and the number of those - that have a popularity measure. These are stored in - - OTT_ptrs[OTTid]['n_ancst'] and OTT_ptrs[OTTid]['n_pop_ancst'] - - we also flag up the poor seed plants (Spermatophyta_ott1007992) - we could add a little - to their pop value later - """ - from dendropy import Tree - - if exclude is None: - exclude = [] - if not isinstance(tree, Tree): - tree = Tree.get( - file=tree, - schema="newick", - suppress_edge_lengths=True, - preserve_underscores=True, - suppress_leaf_node_taxa=True, - ) - - logging.info(f" Tree read for phylogenetic popularity calc: mem usage {mem():.1f} Mb") - - # put popularity into the pop_store attribute - for node in tree.preorder_node_iter(): - if node.label in exclude: - node.pop_store = 0 - node.has_pop = False - else: - try: - node.pop_store = node.data["wd"].raw_popularity - node.has_pop = True - except (LookupError, AttributeError): - node.pop_store = 0 - node.has_pop = False - - # go up the tree from the tips, summing up the popularity indices beneath and - # adding the number of descendants - for node in tree.postorder_node_iter(): - if node.is_leaf(): - node.descendants_popsum = 0 - node.n_descendants = 0 - try: - node._parent_node.n_descendants += 1 + node.n_descendants - node._parent_node.descendants_popsum += node.pop_store + node.descendants_popsum - except AttributeError: # could be the first time we have checked the parent - try: - node._parent_node.n_descendants = 1 + node.n_descendants - node._parent_node.descendants_popsum = node.pop_store + node.descendants_popsum - except AttributeError: # this could be the root, with node._parent_node = None - pass - # root_descendants = node.n_descendants - - # go down the tree from the root, summing up the popularity indices above, - # and summing up numbers of nodes - for node in tree.preorder_node_iter(): - if node.parent_node is None: - # this is the root. - node.seedplant = False - node.n_ancestors = 0 - node.n_pop_ancestors = 0 - node.ancestors_popsum = 0.0 - else: - node.n_ancestors = node._parent_node.n_ancestors + 1 - node.ancestors_popsum = node._parent_node.ancestors_popsum + node.pop_store - if getattr(node, "has_pop", None): - node.n_pop_ancestors = node._parent_node.n_pop_ancestors + 1 - else: - node.n_pop_ancestors = node._parent_node.n_pop_ancestors - if node.label and node.label == "Spermatophyta": - node.seedplant = True - logging.info("Found plant root") - else: - node.seedplant = node._parent_node.seedplant - - # place these values into the OTT_ptrs structure - if OTT_ptrs: - for node in tree.preorder_node_iter(): - try: - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["pop_self"] = node.pop_store - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["pop_ancst"] = ( - node.ancestors_popsum - ) # nb, this includes popularity of self - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["pop_dscdt"] = node.descendants_popsum - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["n_ancst"] = node.n_ancestors - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["n_dscdt"] = node.n_descendants - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["n_pop_ancst"] = node.n_pop_ancestors - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["is_seed_plant"] = node.seedplant - except (LookupError, AttributeError): - pass - return tree diff --git a/oz_tree_build/taxon_mapping_and_popularity/dendropy_extras.py b/oz_tree_build/taxon_mapping_and_popularity/dendropy_extras.py deleted file mode 100755 index cacdf5f0..00000000 --- a/oz_tree_build/taxon_mapping_and_popularity/dendropy_extras.py +++ /dev/null @@ -1,526 +0,0 @@ -#!/usr/bin/env -S python3 -u -""" -A set of functions for monkey patching into dendropy objects. -These all assume that the tree has been loaded with suppress_leaf_node_taxa=True -""" - -# -# To be patched into the Tree object -# -import collections -import itertools -import logging - -import dendropy - - -def prune_children_of_otts(self, ott_species_list): - to_trim = set() - for nd in self.postorder_internal_node_iter(): - try: - if nd.data.get("ott") in ott_species_list: - trim_me = True - # check for extinction props - if ( - (nd.num_child_nodes() == 1) - and (next(iter(nd.child_nodes())).num_child_nodes() == 0) - and (next(iter(nd.child_nodes())).edge.length) - ): - trim_me = False # this is an extinction prop - # check for species within this group - for sub_nd in nd.postorder_internal_node_iter(): - if sub_nd in to_trim: - logging.warning( - f"Species {sub_nd.label} is contained within another species {nd.label}: not trimming it" - ) - trim_me = False - if trim_me: - to_trim.add(nd) - except AttributeError: - pass # nodes created by breaking polytomies will have no data attribute - for nd in to_trim: - nd.clear_child_nodes() - return to_trim - - -def prune_non_species( - self, - recursive=True, - bad_matches=None, # any strings in here indicate non-species (e.g. ' cf.') - update_bipartitions=False, - extinct_tree_mode=False, -): - """ - Removes all terminal nodes whose name is '' or does not contain a space. - Recursive=true means remove tips (which may create more tips) & keep going until - none left to prune. Extinction props should be unlabelled nodes with a length, e.g. - ((:65)Tyrannosaurus_rex,Birds) - """ - if bad_matches is None: - bad_matches = [] - nodes_removed = {"no_space": [], "unlabelled": [], "bad_match": []} - done = False - while not done: - nodes_to_remove = {k: [] for k in nodes_removed.keys()} - for nd in self.leaf_node_iter(): - if nd.label is None: - if ( - nd.edge.length - and nd.parent_node - and nd.parent_node.label - and (nd.parent_node.num_child_nodes() == 1) - ): - # only an extinction prop, if it has a length AND the parent - # node is a named unifurcation - pass - else: - nodes_to_remove["unlabelled"].append(nd) - elif " " not in nd.label and not extinct_tree_mode: - # For the extinct tree, we allow tips with no spaces, as we often end up with genera as tips - - # num_spaces is 0: a leaf, but prob not a species. Also catches label=='' - logging.info( - f"Removing '{nd.label}' since it does not seem to be a species " "(it does not contain a space)" - ) - nodes_to_remove["no_space"].append(nd) - elif any(match in nd.label for match in bad_matches): - logging.info(f"Removing '{nd.label}' since it contains one of {bad_matches}") - nodes_to_remove["bad_match"].append(nd) - for k, nodes in nodes_to_remove.items(): - for nd in nodes: - nd.edge.tail_node.remove_child(nd) - nodes_removed[k] += nodes - if not recursive: - done = True - if all([len(v) == 0 for v in nodes_to_remove.values()]): - done = True - - if update_bipartitions: - self.update_bipartitions() - return nodes_removed - - -def set_node_ages(self): - """ - Adds an attribute called 'age' to each node, with the value equal to - the sum of edge lengths from the node to the tips. Also adds the attribute - extinction_date to terminal nodes that have been propped to earlier in time than 0Ma - - By convention, null branch lengths are of unspecified length, whereas zero-length - branches (e.g. injected by resolving polytomies) are of a fixed length = 0. - Fossil species are denoted by a terminal unnamed monotomy (an 'extinction prop') - which allows us to set the extinction date of any taxon. That means the - entire tree is expected to be ultrametric. - - Returns number of nodes with age set, and number of deleted extinction props - """ - # Percolate age up the tree (go from tips upwards, assuming all tips at 0Ma) - # Where children disagree on the age of their parent, take the larger number - tot_ages = 0 - for node in self.postorder_node_iter(): - if node.is_leaf(): - node.age = 0 - if node.parent_node is not None: - l = node.edge.length - if getattr(node, "age", None) is not None and l is not None: - if l < 0: - logging.warning(f"length <0 for {node.label}") - l = 0 if l < 0 else l - if getattr(node.parent_node, "age", None) is None: - node.parent_node.age = node.age + l - tot_ages += 1 - else: - if node.parent_node.age < (node.age + l): - node.parent_node.age = node.age + l - if abs(node.parent_node.age - (node.age + l)) >= 1: - parent = [n.label for n in node.ancestor_iter() if n.label][:3] - logging.warning( - f"Age of node '{node.parent_node.label}' (child of {parent}) " - f"is {node.parent_node.age} via one route, but has a child " - f"node '{node.label}' of age {node.age}, attached by a " - f"branch of length {l}, which sums to {node.age+l}." - ) - # Round to 6 decimal places to prevent floating point errors - node.parent_node.age = round(node.parent_node.age, 6) - - # For newly fixed ages, percolate them down the tree if we know the age of a deeper node - for node in self.preorder_node_iter(): - if getattr(node, "age", None) is not None: - for ch in node.child_node_iter(): - if getattr(ch, "age", None) is None and ch.edge.length is not None: - ch.age = node.age - (ch.edge.length if ch.edge.length > 0 else 0) - tot_ages += 1 - - # Now we have calculated dates, remove 'extinction props', and flag up extinct species - removed = 0 - for leaf in self.leaf_node_iter(): - if (leaf.label is None) and leaf.edge.length > 0: - assert leaf.parent_node.num_child_nodes() == 1 - leaf.parent_node.extinction_date = getattr(leaf.parent_node, "age", None) - removed += 1 - # this is an extinction prop - remove the prop - leaf.parent_node.clear_child_nodes() - return tot_ages, removed - - -def set_real_parent_nodes(self): - """ - Adds an attribute called 'real_parent' to each leaf and node, which - represents the parent node ignoring randomly resolved polytomies. We can then find - all 'true' children by looking for leaves and nodes which have this as a 'real_parent'. - edge.length==0 indicates a node which is a polytomy. - """ - for node in self.preorder_node_iter(): - for ch in node.child_node_iter(): - if node.edge.length == 0 and node.real_parent_node: - ch.real_parent_node = node.real_parent_node - else: - ch.real_parent_node = node - - -def is_on_unifurcation_path(node): - """ - this is a node which is either a unifurcation or the first node in a path of - successive unifurcations - """ - return node.num_child_nodes() == 1 or (node.parent_node and node.parent_node.num_child_nodes() == 1) - - -def remove_unifurcations_keeping_higher_taxa(self): - """ - Does a more sophisticated pass than the remove_unifurcations flag in Dendropy4: - * If this is a unifurcation ending in a leaf, the lowest level (i.e. species) is - retained as the name of the leaf - * If this is a series of unifurcations within the tree, the node with the highest - raw popularity score should be kept. If there is a tie, any *named* nodes are - given priority, then the nodes highest in the tree. - """ - nd_iter = self.postorder_node_iter() - n_deleted = 0 - for k, g in itertools.groupby(nd_iter, is_on_unifurcation_path): - # k should alternate between 0 (not on unifurcation path) and 1 - if k: - # the group could consist of multiple unifurcation paths - # we need to separate them into groups themselves - node_lists = [[next(g)]] - for next_node in g: - if next_node == node_lists[-1][-1].parent_node: - node_lists[-1].append(next_node) - else: - node_lists.append([next_node]) - for sequential_unary_nodes in node_lists: - if sequential_unary_nodes[0].num_child_nodes() == 0: - # this ends in a tip, we can rely on the normal suppress_unifurcation - # behaviour (by default Dendropy keeps the lowest level taxa) - logging.debug( - "Unary nodes ending in tip left so that first is used: " - + ", ".join([(x.label or "None") for x in sequential_unary_nodes]) - ) - else: - # sort so that best is last - by popularity then presence of label, - # finally by existing position - sorted_unary_nodes = sorted( - sequential_unary_nodes, - key=lambda n: ( - n.data.get("raw_popularity"), - bool(getattr(n, "label", "")), - ), - ) - keep_node = sorted_unary_nodes[-1] - logging.debug( - "Unary nodes collapsed to last in this list: " - + ", ".join([(x.label or "None") for x in sorted_unary_nodes]) - ) - for nd in sequential_unary_nodes: - # these should still be in postorder - if nd != keep_node: - n_deleted += 1 - nd.edge.collapse(adjust_collapsed_head_children_edge_lengths=True) - n_deleted += len(self.suppress_unifurcations()) - return n_deleted - - -def write_preorder_to_csv( - self, - leaf_file, - extra_leaf_data_properties, - node_file, - extra_node_data_properties, - root_parent_id, -): - """ - Write the leaf and node info for this tree to csv files. - * for leaves, always write the parent, name, and extinction_data - * for nodes, always write the parent,node_rgt,leaf_lft,leaf_rgt, name, and age - - In addition to these, also write out the extra_leaf_data_properties and - extra_node_data_properties contained in the data property dictionary of each node - (or blank if the property does not exist), e.g. for leaves this might be - extinction_date, - ott, - wikidata, - wikipedia_lang_flag, - eol, - iucn, - popularity, - popularity_rank, - price, - ncbi, - ifung, - worms, - irmng, - gbif - - for nodes: - age, - ott, - wikidata, - wikipedia_lang_flag, - eol, - popularity, - ncbi, - ifung, - worms, - irmng, - gbif, - vern_synth, - rep1,..., - rtr1,..., - iucnNE,... - """ - import csv - from collections import OrderedDict - - leaf_csv = csv.writer(leaf_file, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") - leaf_csv.writerow(["parent", "real_parent", "name", "extinction_date", *extra_leaf_data_properties]) - node_csv = csv.writer(node_file, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") - node_csv.writerow( - [ - "parent", - "real_parent", - "node_rgt", - "leaf_lft", - "leaf_rgt", - "name", - "age", - *extra_node_data_properties, - ] - ) - - # allocate node numbers - internal_node_number = 0 - for node in self.preorder_internal_node_iter(): - # NB: increment first, since we use a 1-base numbering system, for mySQL row numbering - internal_node_number += 1 - node.id = internal_node_number - - # postorder traversal to allocate rgt side of ranges - internal_leaf_count = 0 - prev_node = None - for node in self.postorder_node_iter(): - # find rightmost leaf by postorder iteration. - # For rightmost node, if previously visited node is a leaf, then (because we ladderize - # ascending) the rightmost node must be self (i.e. this is a terminal internal node). - # Otherwise it is the previously visted node - if node.is_leaf(): - internal_leaf_count += 1 - else: - node.last_leaf = internal_leaf_count # should have counted all the internal leaves by now - if prev_node.is_leaf(): - node.node_rgt = node.id # node_rgt == self - else: - # the node_rgt should be the same as the node_rgt of the previous node - node.node_rgt = prev_node.node_rgt - prev_node = node - - leaf_count = 1 - extra_leaf_output = OrderedDict() - extra_node_output = OrderedDict() - for node in self.preorder_node_iter(): - if node.is_leaf(): - base_output = [ - node.parent_node.id, - # negative real_parent ids if this is a polytomy - (-node.real_parent_node.id if node.edge.length == 0 else node.real_parent_node.id), - node.label, - getattr(node, "extinction_date", None), - ] - for colname, keys in extra_leaf_data_properties.items(): - try: - extra_leaf_output[colname] = node.data - for k in keys: - extra_leaf_output[colname] = extra_leaf_output[colname][k] - except ( - KeyError, - TypeError, - AttributeError, - ): - # catch non-existent key name, None, or no data attribute - # (e.g. for polytomies) - extra_leaf_output[colname] = None - leaf_csv.writerow(base_output + list(extra_leaf_output.values())) - leaf_count += 1 - else: - base_output = [ - node.parent_node.id if node.parent_node else root_parent_id, - ( - (-node.real_parent_node.id if node.edge.length == 0 else node.real_parent_node.id) - if hasattr(node, "real_parent_node") - else 0 - ), - node.node_rgt, - leaf_count, - node.last_leaf, - node.label, - getattr(node, "age", None), - ] - for colname, keys in extra_node_data_properties.items(): - try: - extra_node_output[colname] = node.data - for k in keys: - extra_node_output[colname] = extra_node_output[colname][k] - except (KeyError, TypeError, AttributeError): - # catch none existent key name, None, or no data attribute - # (e.g. for polytomies) - extra_node_output[colname] = None - node_csv.writerow(base_output + list(extra_node_output.values())) - - -# -# To be patched into the Node object -# - - -def write_brief_newick(self, out, polytomy_braces="()", write_otts=False): - """ - Copied from the default dendropy 4 function Node._write_newick - The function requires a binary tree, and the tree should have been ladderized beforehand - It outputs a string consisting of braces only (no commas), such that the number of tips - (& therefore the number of nodes-1, since this is a binary tree) is equal to the number of - characters in the string. Edges lengths are omitted, but internal nodes that have an edge - length of 0 are represented by 'polytomy_braces' which can be specified, e.g. '{}' or '<>' - """ - child_nodes = self.child_nodes() - if child_nodes: - if self.edge and self.edge.length == 0: # if 0, this is a polytomy - out.write(polytomy_braces[0]) # added - else: - out.write("(") - assert len(child_nodes) == 2 # added - f_child = child_nodes[0] - for child in child_nodes: - if child is not f_child: - out.write(",") - child.write_brief_newick(out, polytomy_braces) - if self.edge and self.edge.length == 0: - out.write(polytomy_braces[1]) # added - if write_otts and "ott" in self.data: - out.write(str(self.data["ott"])) - else: - out.write(")") - if write_otts and "ott" in self.data: - out.write(str(self.data["ott"])) - - -def write_pop_newick(self, out): - """ - Copied from the default dendropy 4 function Node._write_newick - This returns the Node as a NEWICK statement but uses node.pop_store - instead of node.edge.length for the edge length values. - """ - child_nodes = self.child_nodes() - if child_nodes: - out.write("(") - f_child = child_nodes[0] - for child in child_nodes: - if child is not f_child: - out.write(",") - child.write_pop_newick(out) - out.write(")") - - label = self._get_node_token( - suppress_leaf_node_labels=False, - suppress_rooting=True, - unquoted_underscores=True, - ) - try: - ott = self.data["ott"] - if label.endswith("'"): - out.write(label[:-1] + f"_ott{ott}'") - else: - out.write(label + f"_ott{ott}") - except (AttributeError, KeyError): - out.write(label) - e = self.edge - if e: - sel = e.length - if sel is not None: - s = "" - try: - s = float(sel) - s = str(s) - except ValueError: - s = str(sel) - if s: - out.write(f":{s}") - - -def group_genera_in_polytomies(self): - """ - Group together nodes in a polytomy whose - tips have the same genus name. - """ - polytomies = [] - for node in self.postorder_node_iter(): - if len(node._child_nodes) > 2: - polytomies.append(node) - for node in polytomies: - # New code here to group children by genus - child_genera = collections.defaultdict(list) - for child in node._child_nodes: - genus = None - for leaf in child.leaf_iter(): - g = (leaf.label or "").split(" ")[0] - if genus is None: - genus = g - elif g != genus: - genus = "" - break - if genus: # ignore if None or empty string - child_genera[genus].append(child) - # first group children by genus - for _, children in child_genera.items(): - if len(children) > 1 and len(children) < len(node._child_nodes): - new_node = dendropy.Node() - node.add_child(new_node) - new_node.edge.length = 0.0 - for child in children: - node.remove_child(child) - new_node.add_child(child) - - -if __name__ == "__main__": - # test - import sys - from collections import OrderedDict - - from dendropy import Tree - - Tree.write_preorder_to_csv = write_preorder_to_csv - Tree.set_node_ages = set_node_ages - Tree.group_genera_in_polytomies = group_genera_in_polytomies - t = Tree.get_from_path( - sys.argv[1], - schema="newick", - suppress_internal_node_taxa=True, - suppress_leaf_node_taxa=True, - ) - for i, nd in enumerate(t.preorder_node_iter()): - nd.data = {"preorder_index": i} - t.set_node_ages() - # print(t.find_node_with_label("Primates").data) - t.ladderize(ascending=True) - with open("test_leaves.csv", "w+") as l, open("test_nodes.csv", "w+") as n: - node_extras = OrderedDict() - node_extras["preorder index"] = ["preorder_index"] - t.write_preorder_to_csv(l, {}, n, node_extras, -1) diff --git a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py index f209e9b7..64185321 100644 --- a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py +++ b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py @@ -21,6 +21,9 @@ # (silva, h2007, "additions-6520052-6520144", ...) is ignored. SOURCES = ("ncbi", "if", "worms", "irmng", "gbif") +# The EoL "resource_id" identifying IUCN rows in the EoL identifiers file +iucn_num = 5 + def map_wiki_info( source_ptrs, @@ -272,7 +275,7 @@ def identify_best_EoLdata(OTT_ptrs, sources): ) -def populate_iucn(OTT_ptrs, identifiers_filename, verbosity=0, iucn_num=5): +def populate_iucn(OTT_ptrs, identifiers_filename, verbosity=0, iucn_num=iucn_num): """ Port the IUCN number from both EoL and Wikidata, and keep both if there is a conflict """ diff --git a/oz_tree_build/tree_build/step_jsnewick.py b/oz_tree_build/tree_build/step_jsnewick.py index 585dc5af..516874ed 100644 --- a/oz_tree_build/tree_build/step_jsnewick.py +++ b/oz_tree_build/tree_build/step_jsnewick.py @@ -28,8 +28,7 @@ def jsnewick_brief_newick(tree, polytomy_braces="()"): ``polytomy_braces`` is a two-character string overriding the brackets used for any *non-root* internal whose ``dist == 0`` — the marker ``resolve_polytomy`` leaves on an artificial split. - Pass e.g. ``"{}"`` to flag those nodes for the frontend (matches - ``dendropy_extras.write_brief_newick``). + Pass e.g. ``"{}"`` to flag those nodes for the frontend. """ parts = [] for node, action in _walk_internal(tree): diff --git a/oz_tree_build/utilities/filter_eol.py b/oz_tree_build/utilities/filter_eol.py index 99a92095..b756e404 100644 --- a/oz_tree_build/utilities/filter_eol.py +++ b/oz_tree_build/utilities/filter_eol.py @@ -4,7 +4,7 @@ import logging import sys -from ..taxon_mapping_and_popularity.CSV_base_table_creator import iucn_num +from ..taxon_mapping_and_popularity.taxon_map import iucn_num from .file_utils import open_file_based_on_extension from .filter_common import read_taxonomy_source_ids diff --git a/pyproject.toml b/pyproject.toml index cbdc8f97..e0701055 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ download_and_filter_wikidata = "oz_tree_build.utilities.download_and_filter_wiki discover_latest_wikidata_dump_url = "oz_tree_build.utilities.download_and_filter_wikidata:discover_main" discover_latest_enwiki_sql_url = "oz_tree_build.utilities.filter_wikipedia_sql:discover_main" download_opentree = "oz_tree_build.utilities.download_opentree:main" -CSV_base_table_creator = "oz_tree_build.taxon_mapping_and_popularity.CSV_base_table_creator:main" get_wiki_images = "oz_tree_build.images.get_wiki_images:main" get_wiki_vernaculars = "oz_tree_build.vernaculars.get_wiki_vernaculars:main" process_image_bits = "oz_tree_build.images.process_image_bits:main" diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv b/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv deleted file mode 100644 index bf2216c3..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv +++ /dev/null @@ -1,41 +0,0 @@ -parent,real_parent,name,extinction_date,ott,wikidata,wikipedia_lang_flag,iucn,eol,raw_popularity,popularity,popularity_rank,price,ncbi,ifung,worms,irmng,gbif,ipni -1,1,Acinonyx jubatus,,752759,23907,1048575,219,328680,128090.45,128090.45,2,,32536,,,10856341,2435270, -3,3,Puma yagouaroundi,,86162,182304,1048575,,1053885,30183.81,42905.99,9,,1608482,,,10910448,2435146, -3,3,Puma concolor,,42307,35255,1048575,18868,311910,108261.25,113975.14,3,,9696,,,10212875,2435099, -5,5,Prionailurus planiceps,,86166,274177,1040383,18148,311659,7675.98,9545.35,29,,61403,,,11429819,2434917, -6,6,Prionailurus rubiginosus,,507541,309274,1048575,18149,312856,17500.57,14326.27,25,,61387,,,11039646,2434895, -7,7,Prionailurus viverrinus,,862641,190674,1048575,18150,1037335,27649.05,18532.47,22,,61388,,,10594861,2434899, -8,8,Prionailurus bengalensis,,280108,42627,1048575,18146|223138747,1041047,27344.18,16907.7,23,,37029,,,10210660,2434903, -8,8,Prionailurus iriomotensis,,418475,19829419,0,18151,1053884,0,2855.57,40,,37030,,,,2434901, -10,10,Leptailurus serval,,86170,42699,1048575,11638,1041048,42607.28,26473.39,17,,61405,,,11216434,2435172, -11,11,Caracal caracal,,1033549,30847,1048575,3847,312855,63025.4,35175.15,11,,61394,,,11060971,2435010, -12,12,Felis manul,,86183,166794,1048575,15640,328665,52482.91,34320.86,13,,61408,,,10859216,2435023, -14,14,Felis bieti,,54743,204322,1048575,8539,328664,7804.55,10061.31,27,,458418,,,11269571,2435040, -14,14,Felis chaus,,983181,42623,1048575,8540,328671,26429.95,18538.09,21,,61376,,,10591049,2435066, -16,16,Felis margarita,,983177,175329,1048575,8541,328670,34349.37,21129.2,19,,61378,,,11270420,2435028, -16,16,Felis nigripes,,983179,204814,1048575,8542,328666,36443.03,22038.46,18,,61379,,,10785265,2435037, -17,17,Felis silvestris,,563163,43576,1048575,60354712|181049859,328605,33214.97,20636.53,20,,9683,,,10201332,7964291, -17,17,Felis catus,,563166,146,1048575,,1037781,307566.59,139785.93,1,,9685,,,,8625722, -20,20,Leopardus pardalis,,752746,33261,1048575,11509,313991,52797.12,31945.02,14,,32538,,,10212871,2434982, -20,20,Leopardus wiedii,,507553,192421,1048575,11511,311954,14868.56,12453.59,26,,61382,,,10592818,2434950, -22,22,Leopardus geoffroyi,,774303,42682,1048575,15310,925988,6828.5,7787.43,34,,46844,,,10765089,2434942, -23,22,Leopardus tigrinus,,774309,205948,1048575,54012637,311661,9685.65,8670.33,31,,46842,,,11116645,2434930, -23,22,Leopardus guigna,,507542,211042,1048575,15311,1053887,12068.12,9754.64,28,,61386,,,11170429,2434923, -24,24,Leopardus jacobitus,,904397,213047,1048575,15452,1053886,7787.0,8248.37,33,,713925,,,10909680,2434979, -25,25,Leopardus colocolo,,86175,210314,1039871,15309,47054070,8894.45,8310.24,32,,61406,,,10534600,2434919, -26,25,Leopardus braccatus,,3613208,133763,1048559,,1053889,1345.42,4651.48,37,,,,,,2434927, -26,25,Leopardus pajeros,,3613206,311417,1044463,,925987,0,4067.17,39,,,,,,2434935, -28,28,Profelis aurata,,660447,192231,1048575,,311555,9315.95,4787.45,36,,61412,,,11097445,9546725, -29,29,Catopuma badia,,763032,213044,1048575,4037,311552,7483.18,4898.87,35,,61454,,,10224214,5787235, -29,29,Catopuma temminckii,,763025,192233,1048575,4038,311553,15930.33,8961.09,30,,61455,,,10999935,2435094, -31,31,Lynx rufus,,507545,131907,1048575,12521,328602,80838.36,57974.43,7,,61384,,,10199263,2435246, -32,32,Lynx pardinus,,442049,129727,1048575,12520,347432,25562.25,29709.45,16,,191816,,,10229763,2435261, -33,33,Lynx canadensis,,507549,146457,1048575,12518,328604,39433.81,34374.36,12,,61383,,,10201330,2435263, -33,33,Lynx lynx,,886829,43375,1048575,12519,328603,42295.46,35617.16,10,,13125,,,10199264,2435240, -34,34,Pardofelis marmorata,,660452,80191,1048575,16218,311554,8700.5,4184.06,38,,61410,,,10733346,2435089, -35,35,Neofelis nebulosa,,763016,36135,1048575,14519,328675,32620.44,14846.2,24,,61452,,,10200770,2435079, -36,36,Uncia uncia,,532117,30197,1048575,22732,328676,70362.32,30557.97,15,,29064,,,11222977,2435238, -37,37,Panthera tigris,,42314,19939,1048575,15955,328674,167475.09,88532.08,4,,9694,,,10762914,5219416, -38,38,Panthera onca,,42322,35694,1048575,15953,328606,110310.18,62427.19,6,,9690,,,10201333,5219426, -39,39,Panthera leo,,563151,140,1048575,15951,328672,164462.75,81591.6,5,,9689,,,10196306,5219404, -39,39,Panthera pardus,,42324,34706,1048575,15954,328673,103768.13,57928.51,8,,9691,,,10200769,5219436, diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv.mySQL b/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv.mySQL deleted file mode 100644 index 06151b02..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv.mySQL +++ /dev/null @@ -1,41 +0,0 @@ -parent,real_parent,name,extinction_date,ott,wikidata,wikipedia_lang_flag,iucn,eol,raw_popularity,popularity,popularity_rank,price,ncbi,ifung,worms,irmng,gbif,ipni -1,1,Acinonyx jubatus,\N,752759,23907,1048575,219,328680,128090.45,128090.45,2,\N,32536,\N,\N,10856341,2435270,\N -3,3,Puma yagouaroundi,\N,86162,182304,1048575,\N,1053885,30183.81,42905.99,9,\N,1608482,\N,\N,10910448,2435146,\N -3,3,Puma concolor,\N,42307,35255,1048575,18868,311910,108261.25,113975.14,3,\N,9696,\N,\N,10212875,2435099,\N -5,5,Prionailurus planiceps,\N,86166,274177,1040383,18148,311659,7675.98,9545.35,29,\N,61403,\N,\N,11429819,2434917,\N -6,6,Prionailurus rubiginosus,\N,507541,309274,1048575,18149,312856,17500.57,14326.27,25,\N,61387,\N,\N,11039646,2434895,\N -7,7,Prionailurus viverrinus,\N,862641,190674,1048575,18150,1037335,27649.05,18532.47,22,\N,61388,\N,\N,10594861,2434899,\N -8,8,Prionailurus bengalensis,\N,280108,42627,1048575,18146|223138747,1041047,27344.18,16907.7,23,\N,37029,\N,\N,10210660,2434903,\N -8,8,Prionailurus iriomotensis,\N,418475,19829419,0,18151,1053884,0,2855.57,40,\N,37030,\N,\N,\N,2434901,\N -10,10,Leptailurus serval,\N,86170,42699,1048575,11638,1041048,42607.28,26473.39,17,\N,61405,\N,\N,11216434,2435172,\N -11,11,Caracal caracal,\N,1033549,30847,1048575,3847,312855,63025.4,35175.15,11,\N,61394,\N,\N,11060971,2435010,\N -12,12,Felis manul,\N,86183,166794,1048575,15640,328665,52482.91,34320.86,13,\N,61408,\N,\N,10859216,2435023,\N -14,14,Felis bieti,\N,54743,204322,1048575,8539,328664,7804.55,10061.31,27,\N,458418,\N,\N,11269571,2435040,\N -14,14,Felis chaus,\N,983181,42623,1048575,8540,328671,26429.95,18538.09,21,\N,61376,\N,\N,10591049,2435066,\N -16,16,Felis margarita,\N,983177,175329,1048575,8541,328670,34349.37,21129.2,19,\N,61378,\N,\N,11270420,2435028,\N -16,16,Felis nigripes,\N,983179,204814,1048575,8542,328666,36443.03,22038.46,18,\N,61379,\N,\N,10785265,2435037,\N -17,17,Felis silvestris,\N,563163,43576,1048575,60354712|181049859,328605,33214.97,20636.53,20,\N,9683,\N,\N,10201332,7964291,\N -17,17,Felis catus,\N,563166,146,1048575,\N,1037781,307566.59,139785.93,1,\N,9685,\N,\N,\N,8625722,\N -20,20,Leopardus pardalis,\N,752746,33261,1048575,11509,313991,52797.12,31945.02,14,\N,32538,\N,\N,10212871,2434982,\N -20,20,Leopardus wiedii,\N,507553,192421,1048575,11511,311954,14868.56,12453.59,26,\N,61382,\N,\N,10592818,2434950,\N -22,22,Leopardus geoffroyi,\N,774303,42682,1048575,15310,925988,6828.5,7787.43,34,\N,46844,\N,\N,10765089,2434942,\N -23,22,Leopardus tigrinus,\N,774309,205948,1048575,54012637,311661,9685.65,8670.33,31,\N,46842,\N,\N,11116645,2434930,\N -23,22,Leopardus guigna,\N,507542,211042,1048575,15311,1053887,12068.12,9754.64,28,\N,61386,\N,\N,11170429,2434923,\N -24,24,Leopardus jacobitus,\N,904397,213047,1048575,15452,1053886,7787.0,8248.37,33,\N,713925,\N,\N,10909680,2434979,\N -25,25,Leopardus colocolo,\N,86175,210314,1039871,15309,47054070,8894.45,8310.24,32,\N,61406,\N,\N,10534600,2434919,\N -26,25,Leopardus braccatus,\N,3613208,133763,1048559,\N,1053889,1345.42,4651.48,37,\N,\N,\N,\N,\N,2434927,\N -26,25,Leopardus pajeros,\N,3613206,311417,1044463,\N,925987,0,4067.17,39,\N,\N,\N,\N,\N,2434935,\N -28,28,Profelis aurata,\N,660447,192231,1048575,\N,311555,9315.95,4787.45,36,\N,61412,\N,\N,11097445,9546725,\N -29,29,Catopuma badia,\N,763032,213044,1048575,4037,311552,7483.18,4898.87,35,\N,61454,\N,\N,10224214,5787235,\N -29,29,Catopuma temminckii,\N,763025,192233,1048575,4038,311553,15930.33,8961.09,30,\N,61455,\N,\N,10999935,2435094,\N -31,31,Lynx rufus,\N,507545,131907,1048575,12521,328602,80838.36,57974.43,7,\N,61384,\N,\N,10199263,2435246,\N -32,32,Lynx pardinus,\N,442049,129727,1048575,12520,347432,25562.25,29709.45,16,\N,191816,\N,\N,10229763,2435261,\N -33,33,Lynx canadensis,\N,507549,146457,1048575,12518,328604,39433.81,34374.36,12,\N,61383,\N,\N,10201330,2435263,\N -33,33,Lynx lynx,\N,886829,43375,1048575,12519,328603,42295.46,35617.16,10,\N,13125,\N,\N,10199264,2435240,\N -34,34,Pardofelis marmorata,\N,660452,80191,1048575,16218,311554,8700.5,4184.06,38,\N,61410,\N,\N,10733346,2435089,\N -35,35,Neofelis nebulosa,\N,763016,36135,1048575,14519,328675,32620.44,14846.2,24,\N,61452,\N,\N,10200770,2435079,\N -36,36,Uncia uncia,\N,532117,30197,1048575,22732,328676,70362.32,30557.97,15,\N,29064,\N,\N,11222977,2435238,\N -37,37,Panthera tigris,\N,42314,19939,1048575,15955,328674,167475.09,88532.08,4,\N,9694,\N,\N,10762914,5219416,\N -38,38,Panthera onca,\N,42322,35694,1048575,15953,328606,110310.18,62427.19,6,\N,9690,\N,\N,10201333,5219426,\N -39,39,Panthera leo,\N,563151,140,1048575,15951,328672,164462.75,81591.6,5,\N,9689,\N,\N,10196306,5219404,\N -39,39,Panthera pardus,\N,42324,34706,1048575,15954,328673,103768.13,57928.51,8,\N,9691,\N,\N,10200769,5219436,\N diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv b/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv deleted file mode 100644 index fd633893..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv +++ /dev/null @@ -1,40 +0,0 @@ -parent,real_parent,node_rgt,leaf_lft,leaf_rgt,name,age,ott,wikidata,wikipedia_lang_flag,eol,rnk,raw_popularity,popularity,ncbi,ifung,worms,irmng,gbif,ipni,vern_synth,rep1,rep2,rep3,rep4,rep5,rep6,rep7,rep8,rtr1,rtr2,rtr3,rtr4,rtr5,rtr6,rtr7,rtr8,rpd1,rpd2,rpd3,rpd4,rpd5,rpd6,rpd7,rpd8,iucnNE,iucnDD,iucnLC,iucnNT,iucnVU,iucnEN,iucnCR,iucnEW,iucnEX -0,0,39,1,40,Felidae,13.882716,563159,25265,1048575,7674,family,57879.95,476707.47,9681,,,104889,9703,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -1,1,39,2,40,,12.358025,,,,,,,448635.47,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -2,2,3,2,3,Puma,10.191358,86161,270748,1046527,34428,genus,16953.24,112096.18,146712,,,1405111,2435098,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -2,2,39,4,40,,12.197531,,,,,,,416672.81,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -4,4,8,4,8,Prionailurus,8.265432,570215,42592,1048575,27870,genus,5556.68,35750.71,37028,,,1040948,2434894,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -5,5,8,5,8,,6.5,,,,,,,33896.89,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -6,6,8,6,8,,6.179012,,,,,,,27557.45,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -7,7,8,7,8,,3.37037,,,,,,,15821.97,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -4,4,39,9,40,,9.067901,,,,,,,409080.5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -9,9,17,9,17,,7.54321,,,,,,,206369.06,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -10,10,17,10,17,,7.382716,,,,,,,195493.67,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -11,11,17,11,17,Felis,5.938272,563165,228283,1048575,20189,genus,14302.4,177345.27,9682,,,1179423,2435022,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -12,12,17,12,17,,4.092593,,,,,,,162398.95,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -13,13,14,12,13,,2.006173,,,,,,,21079.31,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -13,13,17,14,17,,4.012346,,,,,,,161374.43,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -15,15,16,14,15,,2.888889,,,,,,,35487.29,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -15,15,17,16,17,,1.925926,,,,,,,148081.51,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -9,9,39,18,40,,8.987654,,,,,,,281420.15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -18,18,26,18,26,Leopardus,7.54321,774314,318414,1048575,14320,genus,9365.01,40610.58,46841,,,1384113,2434918,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -19,19,20,18,19,,4.975309,,,,,,,37043.93,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -19,19,26,20,26,,6.259259,,,,,,,19365.73,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -21,21,23,20,22,,6.098765,,,,,,,15825.24,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -22,-22,23,21,22,,6.098765,,,,,,,13514.71,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -21,21,26,23,26,,4.333333,,,,,,,10679.31,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -24,24,26,24,26,,3.450617,,,,,,,7889.58,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -25,-25,26,25,26,,3.450617,,,,,,,4466.6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -18,18,39,27,40,,8.907407,,,,,,,281245.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -27,27,29,27,29,,6.259259,,,,,,,15388.44,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -28,28,29,28,29,Catopuma,5.537037,763015,1419858,1048567,35821,genus,2703.74,11886.47,61453,,,1294543,2435092,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -27,27,39,30,40,,8.746913,,,,,,,285553.52,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -30,30,33,30,33,Lynx,7.141975,886828,677014,1048575,18767,genus,39716.08,88830.59,13124,,,1026710,2435239,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -31,31,33,31,33,,4.333333,,,,,,,59160.21,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -32,32,33,32,33,,2.888889,,,,,,,50646.64,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -30,30,39,34,40,,6.660493,,,,,,,238590.44,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -34,34,39,35,40,Clade7728_,6.580246,,,,,,,240043.34,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -35,35,39,36,40,,4.895061,,,,,,,233372.49,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -36,36,39,37,40,Panthera,4.814814,563154,127960,1048575,14134,genus,44815.57,213097.5,9688,,,1330102,2435194,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -37,37,39,38,40,,4.734567,,,,,,,156332.64,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -38,38,39,39,40,,4.65432,,,,,,,118620.56,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv.mySQL b/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv.mySQL deleted file mode 100644 index c54df5ac..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv.mySQL +++ /dev/null @@ -1,40 +0,0 @@ -parent,real_parent,node_rgt,leaf_lft,leaf_rgt,name,age,ott,wikidata,wikipedia_lang_flag,eol,rnk,raw_popularity,popularity,ncbi,ifung,worms,irmng,gbif,ipni,vern_synth,rep1,rep2,rep3,rep4,rep5,rep6,rep7,rep8,rtr1,rtr2,rtr3,rtr4,rtr5,rtr6,rtr7,rtr8,rpd1,rpd2,rpd3,rpd4,rpd5,rpd6,rpd7,rpd8,iucnNE,iucnDD,iucnLC,iucnNT,iucnVU,iucnEN,iucnCR,iucnEW,iucnEX -0,0,39,1,40,Felidae,13.882716,563159,25265,1048575,7674,family,57879.95,476707.47,9681,\N,\N,104889,9703,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -1,1,39,2,40,\N,12.358025,\N,\N,\N,\N,\N,\N,448635.47,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -2,2,3,2,3,Puma,10.191358,86161,270748,1046527,34428,genus,16953.24,112096.18,146712,\N,\N,1405111,2435098,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -2,2,39,4,40,\N,12.197531,\N,\N,\N,\N,\N,\N,416672.81,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -4,4,8,4,8,Prionailurus,8.265432,570215,42592,1048575,27870,genus,5556.68,35750.71,37028,\N,\N,1040948,2434894,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -5,5,8,5,8,\N,6.5,\N,\N,\N,\N,\N,\N,33896.89,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -6,6,8,6,8,\N,6.179012,\N,\N,\N,\N,\N,\N,27557.45,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -7,7,8,7,8,\N,3.37037,\N,\N,\N,\N,\N,\N,15821.97,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -4,4,39,9,40,\N,9.067901,\N,\N,\N,\N,\N,\N,409080.5,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -9,9,17,9,17,\N,7.54321,\N,\N,\N,\N,\N,\N,206369.06,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -10,10,17,10,17,\N,7.382716,\N,\N,\N,\N,\N,\N,195493.67,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -11,11,17,11,17,Felis,5.938272,563165,228283,1048575,20189,genus,14302.4,177345.27,9682,\N,\N,1179423,2435022,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -12,12,17,12,17,\N,4.092593,\N,\N,\N,\N,\N,\N,162398.95,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -13,13,14,12,13,\N,2.006173,\N,\N,\N,\N,\N,\N,21079.31,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -13,13,17,14,17,\N,4.012346,\N,\N,\N,\N,\N,\N,161374.43,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -15,15,16,14,15,\N,2.888889,\N,\N,\N,\N,\N,\N,35487.29,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -15,15,17,16,17,\N,1.925926,\N,\N,\N,\N,\N,\N,148081.51,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -9,9,39,18,40,\N,8.987654,\N,\N,\N,\N,\N,\N,281420.15,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -18,18,26,18,26,Leopardus,7.54321,774314,318414,1048575,14320,genus,9365.01,40610.58,46841,\N,\N,1384113,2434918,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -19,19,20,18,19,\N,4.975309,\N,\N,\N,\N,\N,\N,37043.93,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -19,19,26,20,26,\N,6.259259,\N,\N,\N,\N,\N,\N,19365.73,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -21,21,23,20,22,\N,6.098765,\N,\N,\N,\N,\N,\N,15825.24,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -22,-22,23,21,22,\N,6.098765,\N,\N,\N,\N,\N,\N,13514.71,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -21,21,26,23,26,\N,4.333333,\N,\N,\N,\N,\N,\N,10679.31,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -24,24,26,24,26,\N,3.450617,\N,\N,\N,\N,\N,\N,7889.58,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -25,-25,26,25,26,\N,3.450617,\N,\N,\N,\N,\N,\N,4466.6,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -18,18,39,27,40,\N,8.907407,\N,\N,\N,\N,\N,\N,281245.7,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -27,27,29,27,29,\N,6.259259,\N,\N,\N,\N,\N,\N,15388.44,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -28,28,29,28,29,Catopuma,5.537037,763015,1419858,1048567,35821,genus,2703.74,11886.47,61453,\N,\N,1294543,2435092,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -27,27,39,30,40,\N,8.746913,\N,\N,\N,\N,\N,\N,285553.52,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -30,30,33,30,33,Lynx,7.141975,886828,677014,1048575,18767,genus,39716.08,88830.59,13124,\N,\N,1026710,2435239,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -31,31,33,31,33,\N,4.333333,\N,\N,\N,\N,\N,\N,59160.21,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -32,32,33,32,33,\N,2.888889,\N,\N,\N,\N,\N,\N,50646.64,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -30,30,39,34,40,\N,6.660493,\N,\N,\N,\N,\N,\N,238590.44,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -34,34,39,35,40,Clade7728_,6.580246,\N,\N,\N,\N,\N,\N,240043.34,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -35,35,39,36,40,\N,4.895061,\N,\N,\N,\N,\N,\N,233372.49,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -36,36,39,37,40,Panthera,4.814814,563154,127960,1048575,14134,genus,44815.57,213097.5,9688,\N,\N,1330102,2435194,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -37,37,39,38,40,\N,4.734567,\N,\N,\N,\N,\N,\N,156332.64,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -38,38,39,39,40,\N,4.65432,\N,\N,\N,\N,\N,\N,118620.56,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.nwk b/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.nwk deleted file mode 100644 index 2c296378..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.nwk +++ /dev/null @@ -1 +0,0 @@ -(,((,),((,(,(,(,)))),((,(,(,((,),((,),(,)))))),(((,),((,(,)),(,(,(,))))),((,(,)),((,(,(,))),(,(,(,(,(,(,))))))))))))) \ No newline at end of file diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.poly b/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.poly deleted file mode 100644 index de137a6a..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.poly +++ /dev/null @@ -1 +0,0 @@ -(,((,),((,(,(,(,)))),((,(,(,((,),((,),(,)))))),(((,),((,{,}),(,(,{,})))),((,(,)),((,(,(,))),(,(,(,(,(,(,))))))))))))) \ No newline at end of file diff --git a/tests/test_full_generation.py b/tests/test_full_generation.py deleted file mode 100644 index b61e819b..00000000 --- a/tests/test_full_generation.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import types - -from oz_tree_build.taxon_mapping_and_popularity import CSV_base_table_creator -from oz_tree_build.utilities.file_utils import check_identical_files - -from .felidae_helpers import get_felidae_test_folders - - -def test_full_felidae_generation(): - """ - This is more of a functional test than a unit test. It runs the full pipeline - on a small clade. It then compares the output to the expected output. - """ - - args = types.SimpleNamespace() - - ( - input_path, - expected_output_path, - args.output_location, - ) = get_felidae_test_folders("generation") - - # Set all the arguments, to mimic the command line - args.Tree = os.path.join(input_path, "Felidae_AllLife_full_tree.phy") - args.OpenTreeTaxonomy = os.path.join(input_path, "Felidae_taxonomy.tsv") - args.EOLidentifiers = os.path.join(input_path, "Felidae_provider_ids.csv") - args.wikidataDumpFile = os.path.join(input_path, "Felidae_latest-all.json") - args.wikipediaSQLDumpFile = os.path.join(input_path, "Felidae_enwiki-latest-page.sql") - args.wikipedia_totals_bz2_pageviews = [ - os.path.join(input_path, f) for f in os.listdir(input_path) if f.startswith("Felidae_pageviews") - ] - - # Sort the list of pagecount files so that the order is consistent - args.wikipedia_totals_bz2_pageviews.sort() - - args.verbosity = 0 - args.version = 0 - args.wikilang = "en" - args.popularity_file = "" - args.extra_source_file = None - args.taxa_data_file = None - args.exclude = [] - args.info_on_focal_labels = [] - - CSV_base_table_creator.process_all(args) - - # Check that the output files are the same as the expected files - check_identical_files(args.output_location, expected_output_path) From c600c24ddbe8909c9bd963ba284a44b1bb4073ac Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 27 Aug 2026 09:41:49 +0000 Subject: [PATCH 47/62] date_tree: Don't clobber tree_build's log settings --- oz_tree_build/date_tree/date_tree.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/oz_tree_build/date_tree/date_tree.py b/oz_tree_build/date_tree/date_tree.py index 8ebe4a69..8a5e3101 100644 --- a/oz_tree_build/date_tree/date_tree.py +++ b/oz_tree_build/date_tree/date_tree.py @@ -46,7 +46,6 @@ import argparse import logging logger = logging.getLogger(__name__) -logging.basicConfig(filename="main.log", filemode="w", force=True, level=logging.ERROR) def nwk_write(tree, outfile): @@ -271,6 +270,10 @@ def generate_trees(args): def main(): + # NB: Only configure logging when run as a script, not on import, otherwise + # we clobber the logging config of anything importing us + logging.basicConfig(filename="main.log", filemode="w", force=True, level=logging.ERROR) + parser = argparse.ArgumentParser( description=( "Generate a set of dated trees of all life, based on the Open Tree of Life and Chronosynth. " From f383150066e994ef606eb582c66315c4780b1dee Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Thu, 27 Aug 2026 10:08:13 +0000 Subject: [PATCH 48/62] debug_util: Force logging settings As well as date_tree, ete4 will also clobber logging settings. Force logging options, and restore the error_handler so scripts still stop properly. --- oz_tree_build/utilities/debug_util.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/oz_tree_build/utilities/debug_util.py b/oz_tree_build/utilities/debug_util.py index 86b7735c..29c07df4 100644 --- a/oz_tree_build/utilities/debug_util.py +++ b/oz_tree_build/utilities/debug_util.py @@ -51,15 +51,26 @@ def parse_args_and_add_logging_switch(parser): args = parser.parse_args() if args.verbosity == 0: - logging.basicConfig(stream=sys.stderr, level=logging.WARNING) + level = logging.WARNING elif args.verbosity == 1: - logging.basicConfig(stream=sys.stderr, level=logging.INFO) + level = logging.INFO else: - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + level = logging.DEBUG + + # NB: force=True, since importing ete4 configures the root logger for us + # (see ete4/smartview/explorer.py), which would make this call a no-op. + # For the same reason we spell out the format rather than inheriting one. + logging.basicConfig( + stream=sys.stderr, + level=level, + format="%(asctime)s %(levelname)s %(module)s: %(message)s", + force=True, + ) if _error_handler is None: _error_handler = _ErrorCountingHandler() - logging.getLogger().addHandler(_error_handler) atexit.register(_exit_if_errors_logged) + # NB: Added after basicConfig(), which removes any pre-existing handlers + logging.getLogger().addHandler(_error_handler) return args From b78fa2b1292ee9080b6aba466fabdd4cec79b4ae Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 10:47:55 +0000 Subject: [PATCH 49/62] dvc.lock: Manually fix .venv/bin/ additions Avoid re-runs by splicing the cmd changes into the lockfile. --- dvc.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dvc.lock b/dvc.lock index 3f087215..9652f0d4 100644 --- a/dvc.lock +++ b/dvc.lock @@ -1,7 +1,7 @@ schema: '2.0' stages: download_and_filter_wikidata: - cmd: download_and_filter_wikidata --url "$(cat + cmd: .venv/bin/download_and_filter_wikidata --url "$(cat data/Wiki/wd_JSON/latest-all-json-bz2-url.txt)" -o data/filtered/OneZoom_latest-all.json deps: @@ -15,8 +15,8 @@ stages: md5: e6f69def9d6fa2bb4b90060509c079bb size: 1567956473 extract_wikidata_titles: - cmd: extract_wikidata_titles data/filtered/OneZoom_latest-all.json -o - data/filtered/wikidata_titles.txt + cmd: .venv/bin/extract_wikidata_titles data/filtered/OneZoom_latest-all.json + -o data/filtered/wikidata_titles.txt deps: - path: data/filtered/OneZoom_latest-all.json hash: md5 @@ -41,7 +41,7 @@ stages: md5: 541324eaa9f3f1a14bb6ddcf7ea95de6 size: 2397370114 download_and_filter_pageviews: - cmd: download_and_filter_pageviews --titles-file + cmd: .venv/bin/download_and_filter_pageviews --titles-file data/filtered/wikidata_titles.txt --months 12 -o data/filtered/pageviews deps: - path: data/filtered/wikidata_titles.txt @@ -55,7 +55,7 @@ stages: size: 128216689 nfiles: 13 filter_wikipedia_sql: - cmd: filter_wikipedia_sql data/Wiki/wp_SQL/enwiki-page.sql.gz + cmd: .venv/bin/filter_wikipedia_sql data/Wiki/wp_SQL/enwiki-page.sql.gz data/filtered/wikidata_titles.txt -o data/filtered/OneZoom_enwiki-latest-page.sql deps: @@ -87,7 +87,7 @@ stages: cmd: - rm -rf data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1 - mkdir -p data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1 - - add_ott_numbers_to_trees --savein + - .venv/bin/add_ott_numbers_to_trees --savein data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1 --output_info data/add_ott_numbers_to_trees.log data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/*.[pP][hH][yY] @@ -153,7 +153,7 @@ stages: md5: cb59f793fb021063b7c35e821ee8086a size: 31 filter_eol: - cmd: filter_eol data/EOL/provider_ids.csv.gz + cmd: .venv/bin/filter_eol data/EOL/provider_ids.csv.gz data/OpenTree/v16.1/taxonomy.tsv -o data/filtered/OneZoom_provider_ids.csv deps: - path: data/EOL/provider_ids.csv.gz @@ -264,7 +264,7 @@ stages: size: 1158320679 nfiles: 7 discover_latest_wikidata_dump_url: - cmd: discover_latest_wikidata_dump_url > + cmd: .venv/bin/discover_latest_wikidata_dump_url > data/Wiki/wd_JSON/latest-all-json-bz2-url.txt outs: - path: data/Wiki/wd_JSON/latest-all-json-bz2-url.txt @@ -272,7 +272,7 @@ stages: md5: aa1692624401f4a1c759bc9526787fd7 size: 90 discover_latest_enwiki_sql_url: - cmd: discover_latest_enwiki_sql_url > + cmd: .venv/bin/discover_latest_enwiki_sql_url > data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt outs: - path: data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt From a0a1359f231d16d9e17e59005bc0b65fb02832fb Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 13:09:14 +0000 Subject: [PATCH 50/62] step_treeprop: Read the "date" prop, not the unset "age" treeprop_geological read node.props["age"], but nothing in the pipeline ever sets that prop, so every node logged "has no age property" and was assigned geological=0. The function was ported from a DendroPy implementation whose docstring correctly said "date" while its code read node.age -- an attribute the old pipeline really did populate, via dendropy_extras.set_node_ages(). The ete4 port kept the "age" spelling and reconciled the docstring the wrong way; set_node_ages() then left with CSV_base_table_creator. "date" is the name the rest of the pipeline uses, so read that. The tests passed only because their helper synthesised the missing prop. Co-Authored-By: Claude Opus 5 --- oz_tree_build/tree_build/step_treeprop.py | 8 +++---- tests/test_tree_build_step_treeprop.py | 28 +++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/oz_tree_build/tree_build/step_treeprop.py b/oz_tree_build/tree_build/step_treeprop.py index 5ef5d764..a95e35d1 100644 --- a/oz_tree_build/tree_build/step_treeprop.py +++ b/oz_tree_build/tree_build/step_treeprop.py @@ -72,7 +72,7 @@ def treeprop_geological(tree): Given an ete4 tree object, add a "geological" prop to each node, representing a 1-based period index. - Assumes the tree already has an "age" prop representing an absolute age in Mya. + Assumes the tree already has a "date" prop representing an absolute age in Mya. Return name of prop just added. """ @@ -80,9 +80,9 @@ def treeprop_geological(tree): lookup = [(p["mya_start"], idx) for idx, p in enumerate(GEOLOGICAL_PERIODS)] for node in tree.traverse("preorder"): - n_age = node.props.get("age") + n_age = node.props.get("date") if n_age is None: - logger.warning(f"Node {node.name} has no age property") + logger.warning(f"Node {node.name} has no date property") node.props["geological"] = 0 else: for mya_start, idx in lookup: # noqa: B007 # idx is used outside the lookup, not inside @@ -90,7 +90,7 @@ def treeprop_geological(tree): break else: # Fell off end - idx = None + idx = 0 node.props["geological"] = idx prop_format = tree.root.props.setdefault("prop_format", {}) diff --git a/tests/test_tree_build_step_treeprop.py b/tests/test_tree_build_step_treeprop.py index b492083a..eb9226bf 100644 --- a/tests/test_tree_build_step_treeprop.py +++ b/tests/test_tree_build_step_treeprop.py @@ -16,35 +16,35 @@ ######################################## -def set_ages_from_dist(tree): +def set_dates_from_dist(tree): """ - Postorder pass: leaves get age 0, interior nodes get max(child.age + child.dist). - If any child has an unknown age or dist, the parent's age becomes None. + Postorder pass: leaves get date 0, interior nodes get max(child.date + child.dist). + If any child has an unknown date or dist, the parent's date becomes None. """ for node in tree.traverse("postorder"): if node.is_leaf: - node.props["age"] = 0 + node.props["date"] = 0 continue - parent_age = 0 + parent_date = 0 for c in node.children: - if c.props.get("age") is None or c.dist is None: - parent_age = None + if c.props.get("date") is None or c.dist is None: + parent_date = None break - new_age = c.props["age"] + c.dist - if new_age > parent_age: - parent_age = new_age - node.props["age"] = parent_age + new_date = c.props["date"] + c.dist + if new_date > parent_date: + parent_date = new_date + node.props["date"] = parent_date def do_treeprop_geological(nwk, date_tree=True): t = ete4.Tree(nwk, parser=1) - # Our tree needs to have the age prop set for this to work + # Our tree needs to have the date prop set for this to work if date_tree: - set_ages_from_dist(t) + set_dates_from_dist(t) assert treeprop_geological(t) == "geological" # Traverse tree, returning all periods - return [(n.name, n.props.get("age"), n.props["geological"]) for n in t.traverse("preorder")] + return [(n.name, n.props.get("date"), n.props["geological"]) for n in t.traverse("preorder")] class TestTreepropGeological: From 88e4bce69d24d97489ddc1615c802ca694f9cb3b Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 13:10:13 +0000 Subject: [PATCH 51/62] tree_build: Anchor the root date at LUCA, without clobbering impute_missing_dates documents that it "Assumes root node is dated", and no bespoke tree dates biota itself -- tidy_infill_dates_bottomup cannot reach the root, as it has no dist and its Eubacteria child contributes date 0. So the root needs an explicit date or imputation fails outright. The value was 4567, the age of the solar system, which is ~1400My older than the tree's own oldest date (3200) and drags every undated node near the root older with it. 4000 is a Last Universal Common Ancestor estimate, which is what the root actually represents. Use setdefault rather than add_prop so a date supplied by the data is no longer silently overwritten. Co-Authored-By: Claude Opus 5 --- oz_tree_build/tree_build/tree_build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 519f62ce..8ac43d38 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -31,6 +31,8 @@ logger = logging.getLogger(__name__) +ROOT_DATE_MYA = 4000 # Last Universal Common Ancestor estimate + def main(): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) @@ -115,6 +117,7 @@ def main(): tree_fixing.delete_one_child_nodes(base_t) logger.info("Re-interpoltate missing dates") + base_t.root.props.setdefault("date", ROOT_DATE_MYA) for n in base_t.traverse(): # First do some tidying to force tree_dating to work if n.is_leaf and n.name == "mrcaimp": # Bin imputed mrca nodes made by fix_polyphyly left dangling by grafting process From bf839d35ebcaa313fe65df71e293ec79a622be03 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 13:13:12 +0000 Subject: [PATCH 52/62] tree_build: Identify polytomies by prop, not zero branch length step_output and step_jsnewick treated dist == 0 as the marker for an artificially resolved polytomy. That was never reliable and is about to become wrong: branch lengths are regenerated from dates in a following commit, which overwrites the zeros. It was already missing the OT trees entirely. Their polytomies are resolved upstream by tree_fixing.fix_polytomy, and date_tree writes the tree with no branch lengths at all, so those nodes arrive with dist None -- 965,498 of them in the current dated_tree_pre.tre, every one of them written out as a genuine parent. Record the resolution as a "polytomy" prop instead. Its value names how the topology was chosen, as the two stages differ: ete4's resolve_polytomy is deterministic and pairs children off in order, so every polytomy becomes the same comb, while fix_polytomy draws uniformly at random from the possible topologies. OT nodes are marked by the "mrcapoly" name fix_polytomy gives them, that being all that survives into the newick. Consumers only test the prop for truth, so both kinds work unchanged. Co-Authored-By: Claude Opus 5 --- oz_tree_build/tree_build/step_jsnewick.py | 10 +-- oz_tree_build/tree_build/step_output.py | 17 ++--- oz_tree_build/tree_build/step_tidy.py | 64 +++++++++++++++++ oz_tree_build/tree_build/tree_build.py | 19 +++-- tests/test_tree_build_step_jsnewick.py | 39 +++++++--- tests/test_tree_build_step_output.py | 35 ++++++--- tests/test_tree_build_step_tidy.py | 86 +++++++++++++++++++++++ 7 files changed, 234 insertions(+), 36 deletions(-) diff --git a/oz_tree_build/tree_build/step_jsnewick.py b/oz_tree_build/tree_build/step_jsnewick.py index 516874ed..afffc74c 100644 --- a/oz_tree_build/tree_build/step_jsnewick.py +++ b/oz_tree_build/tree_build/step_jsnewick.py @@ -14,6 +14,8 @@ output directly from an ete4 tree, with no on-disk round-trip. """ +from .step_tidy import POLYTOMY_PROP + def jsnewick_brief_newick(tree, polytomy_braces="()"): """ @@ -26,13 +28,13 @@ def jsnewick_brief_newick(tree, polytomy_braces="()"): ``"(())"``. ``polytomy_braces`` is a two-character string overriding the - brackets used for any *non-root* internal whose ``dist == 0`` — - the marker ``resolve_polytomy`` leaves on an artificial split. - Pass e.g. ``"{}"`` to flag those nodes for the frontend. + brackets used for any *non-root* internal carrying the ``polytomy`` + prop — the marker ``tidy_resolve_polytomies`` leaves on an artificial + split. Pass e.g. ``"{}"`` to flag those nodes for the frontend. """ parts = [] for node, action in _walk_internal(tree): - braces = polytomy_braces if (node.up is not None and node.dist == 0) else "()" + braces = polytomy_braces if (node.up is not None and node.props.get(POLYTOMY_PROP)) else "()" parts.append(braces[0] if action == "open" else braces[1]) return "".join(parts) diff --git a/oz_tree_build/tree_build/step_output.py b/oz_tree_build/tree_build/step_output.py index 7bcab034..97734239 100644 --- a/oz_tree_build/tree_build/step_output.py +++ b/oz_tree_build/tree_build/step_output.py @@ -4,6 +4,7 @@ import struct from ..utilities.ete import node_name_without_ott +from .step_tidy import POLYTOMY_PROP def output_add_prop_ids(tree): @@ -84,12 +85,12 @@ def output_mysqlexport(tree, out_dir): - ``\\N`` is the marker for missing values (MySQL ``LOAD DATA`` treats it as NULL). - ``real_parent`` walks past randomly-resolved polytomies: any - ancestor with ``dist == 0`` is skipped so the column points at the - nearest biologically meaningful parent. The raw ``parent`` column - still references the immediate parent. - - A node that is itself a polytomy resolution (``dist == 0``) records - its ``real_parent`` as the *negative* of the resolved parent's id, - flagging the relationship as artificial. + ancestor carrying the ``polytomy`` prop is skipped so the column + points at the nearest biologically meaningful parent. The raw + ``parent`` column still references the immediate parent. + - A node that is itself a polytomy resolution (has the ``polytomy`` + prop) records its ``real_parent`` as the *negative* of the resolved + parent's id, flagging the relationship as artificial. - The leaf ``name`` column has any trailing ``_ottNNN`` suffix stripped (the OTT is carried separately in its own column). - An internal node's ``date`` prop is exposed via the ``age`` column. @@ -157,12 +158,12 @@ def output_mysqlexport(tree, out_dir): for node in tree.traverse("preorder"): # Find our real parent, ignoring randomly resolved polytomies real_parent = node.parent - while real_parent and real_parent.dist == 0: # TODO: Is this still how we identify polytomies? + while real_parent and real_parent.props.get(POLYTOMY_PROP): real_parent = real_parent.parent if not real_parent: real_parent_id = 0 - elif node.dist == 0: + elif node.props.get(POLYTOMY_PROP): # real_parent is negative iff we're a polytomy real_parent_id = -real_parent.props["id"] else: diff --git a/oz_tree_build/tree_build/step_tidy.py b/oz_tree_build/tree_build/step_tidy.py index 696e566a..e3876b32 100644 --- a/oz_tree_build/tree_build/step_tidy.py +++ b/oz_tree_build/tree_build/step_tidy.py @@ -1,3 +1,67 @@ +# Prop marking a node as an artificial split inserted to break up a polytomy. +# Its value says *how* the topology was chosen, since the two stages that +# resolve polytomies do so very differently. Any node carrying the prop is +# artificial, so consumers that only care about that can test it for truth. +POLYTOMY_PROP = "polytomy" + +# ete4's resolve_polytomy: deterministic, and not a sample of anything -- it +# pairs children off in whatever order they happen to be in, so a polytomy of +# n children always becomes the same left-nested comb. +POLYTOMY_COMB = "comb" + +# dated_complete_tree.tree_fixing.fix_polytomy: draws uniformly at random from +# the possible topologies, using a seeded rng. +POLYTOMY_RANDOM = "random" + +# Name fix_polytomy gives the nodes it inserts, and the only trace of them that +# survives into the newick date_tree writes out +OT_POLYTOMY_NAME = "mrcapoly" + + +def tidy_resolve_polytomies(tree, kind=POLYTOMY_COMB): + """ + Resolve any polytomies in ``tree`` via ete4's ``resolve_polytomy``, marking + each artificially inserted node with ``POLYTOMY_PROP`` set to ``kind``. + + ``resolve_polytomy`` leaves inserted nodes with ``dist == 0``, but branch + lengths are regenerated from dates later in the pipeline, so the resolution + has to be recorded as a prop to survive that. + + Return number of nodes inserted. + """ + pre_existing = {id(n) for n in tree.traverse()} + tree.resolve_polytomy() + + count = 0 + for node in tree.traverse(): + if id(node) not in pre_existing: + node.props[POLYTOMY_PROP] = kind + count += 1 + return count + + +def tidy_mark_resolved_polytomies(tree, kind=POLYTOMY_RANDOM, name=OT_POLYTOMY_NAME): + """ + Mark nodes in an already-resolved ``tree`` with ``POLYTOMY_PROP`` set to + ``kind``, so they match those ``tidy_resolve_polytomies`` marks itself. + + OpenTree subtrees arrive polytomy-resolved by + ``dated_complete_tree.tree_fixing.fix_polytomy``, which identifies the nodes + it inserts by giving them the name ``mrcapoly``. That name is all we have to + go on: ``date_tree.nwk_write`` emits only the ``date`` prop, and the tree is + written before ``compute_branch_lengths`` runs, so the nodes arrive with no + branch length either. + + Return number of nodes marked. + """ + count = 0 + for node in tree.traverse(): + if node.name == name: + node.props[POLYTOMY_PROP] = kind + count += 1 + return count + + def tidy_infill_dates_bottomup(tree): """ Working bottom-upwards, fill in missing date properties based on branch lengths. diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 8ac43d38..b3758a38 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -22,7 +22,12 @@ from .step_parse import parse_bespoke_trees, parse_ot_orphans from .step_popularity import popularity_add_prop, popularity_add_rank from .step_taxon import taxon_add_prop -from .step_tidy import tidy_clear_conflicting_dates_topdown, tidy_infill_dates_bottomup +from .step_tidy import ( + tidy_clear_conflicting_dates_topdown, + tidy_infill_dates_bottomup, + tidy_mark_resolved_polytomies, + tidy_resolve_polytomies, +) from .step_treeprop import ( treeprop_geological, treeprop_sliding_window, @@ -74,10 +79,12 @@ def main(): base_t, bespoke_ts = parse_bespoke_trees(args.bespoke_dir) missing_inclusions = graft_tree(base_t, additional_trees=bespoke_ts, prefer_subtree_name=True) - logger.info("Random resoution of polytomies for bespoke trees") + logger.info("Resolve polytomies in bespoke trees") # https://etetoolkit.org/docs/latest/reference/reference_tree.html#ete3.TreeNode.resolve_polytomy - # NB: Doesn't shuffle children like the DendroPy equivalent, but given a fixed seed do we care? - base_t.resolve_polytomy() + # NB: Despite the DendroPy equivalent this replaces, this isn't a random draw: it + # doesn't shuffle children, so every polytomy becomes the same comb. Marked as + # POLYTOMY_COMB to distinguish it from the OT trees' random resolution below. + logger.info(f"Resolved bespoke polytomies with {tidy_resolve_polytomies(base_t)} new nodes") logger.info("Resolve branch lengths to dates bottom-up. Remove (or not care about) branch lengths") tidy_infill_dates_bottomup(base_t) @@ -86,7 +93,9 @@ def main(): tidy_clear_conflicting_dates_topdown(base_t) logger.info("Graft OT subtrees onto our trees. Already polytomy-resolved & date pins from chronosynth applied") - opentree_ts = graft_extract_ot_subtrees(date_tree.nwk_read(args.opentree), missing_inclusions) + opentree_t = date_tree.nwk_read(args.opentree) + logger.info(f"Marked {tidy_mark_resolved_polytomies(opentree_t)} pre-resolved OT polytomy nodes") + opentree_ts = graft_extract_ot_subtrees(opentree_t, missing_inclusions) opentree_ts.update(parse_ot_orphans(args.orphan_dir, missing_inclusions)) missing_inclusions = graft_tree( base_t, additional_trees=opentree_ts, prefer_subtree_name=False, disable_recursion=True diff --git a/tests/test_tree_build_step_jsnewick.py b/tests/test_tree_build_step_jsnewick.py index 84706769..6ae47957 100644 --- a/tests/test_tree_build_step_jsnewick.py +++ b/tests/test_tree_build_step_jsnewick.py @@ -5,6 +5,15 @@ jsnewick_cutpositionmap_binary, jsnewick_cutpositionmap_polytomy, ) +from oz_tree_build.tree_build.step_tidy import POLYTOMY_COMB, POLYTOMY_PROP + + +def _mark_polytomies(tree, *names): + """Flag the named internal nodes as polytomy resolutions.""" + for node in tree.traverse(): + if node.name in names: + node.props[POLYTOMY_PROP] = POLYTOMY_COMB + ######################################## # jsnewick_brief_newick @@ -36,29 +45,39 @@ def test_brief_newick_deep_caterpillar(): def test_brief_newick_polytomy_braces_default(): - """With the default polytomy_braces the dist=0 marker is invisible.""" - t = ete4.Tree("((A:1,B:1):0,C:2);", parser=1) + """With the default polytomy_braces the polytomy prop is invisible.""" + t = ete4.Tree("((A:1,B:1)P:1,C:2);", parser=1) + _mark_polytomies(t, "P") assert jsnewick_brief_newick(t) == "(())" def test_brief_newick_polytomy_braces_overridden(): - """An internal with dist=0 gets the override braces; non-zero dist does not.""" - t = ete4.Tree("((A:1,B:1):0,C:2);", parser=1) + """An internal with the polytomy prop gets the override braces; one without does not.""" + t = ete4.Tree("((A:1,B:1)P:1,C:2);", parser=1) + _mark_polytomies(t, "P") assert jsnewick_brief_newick(t, polytomy_braces="{}") == "({})" - t_nonzero = ete4.Tree("((A:1,B:1):3,C:2);", parser=1) - assert jsnewick_brief_newick(t_nonzero, polytomy_braces="{}") == "(())" + t_unmarked = ete4.Tree("((A:1,B:1)P:1,C:2);", parser=1) + assert jsnewick_brief_newick(t_unmarked, polytomy_braces="{}") == "(())" + + +def test_brief_newick_polytomy_ignores_zero_dist(): + """A zero-length branch is no longer a polytomy marker on its own.""" + t = ete4.Tree("((A:1,B:1)P:0,C:2);", parser=1) + assert jsnewick_brief_newick(t, polytomy_braces="{}") == "(())" def test_brief_newick_polytomy_root_excluded(): - """The root's own dist is ignored even when set to 0.""" - t = ete4.Tree("(A:1,B:1):0;", parser=1) + """The root is never treated as a polytomy resolution, even if marked.""" + t = ete4.Tree("(A:1,B:1)R:1;", parser=1) + _mark_polytomies(t, "R") assert jsnewick_brief_newick(t, polytomy_braces="{}") == "()" def test_brief_newick_polytomy_braces_nested(): - """Multiple dist=0 ancestors each get the polytomy braces.""" - t = ete4.Tree("(((A:1,B:1):0,C:2):0,D:1);", parser=1) + """Multiple marked ancestors each get the polytomy braces.""" + t = ete4.Tree("(((A:1,B:1)P:1,C:2)Q:1,D:1);", parser=1) + _mark_polytomies(t, "P", "Q") assert jsnewick_brief_newick(t, polytomy_braces="{}") == "({{}})" diff --git a/tests/test_tree_build_step_output.py b/tests/test_tree_build_step_output.py index e73bcae4..66ba8a94 100644 --- a/tests/test_tree_build_step_output.py +++ b/tests/test_tree_build_step_output.py @@ -12,6 +12,7 @@ output_mysqlexport, output_proparray, ) +from oz_tree_build.tree_build.step_tidy import POLYTOMY_COMB, POLYTOMY_PROP def _by_name(tree): @@ -447,13 +448,14 @@ def test_internal_node_date_written_as_age(self, tmp_path): assert root[NODE_HEADER.index("age")] == "12.5" def test_polytomy_parent_is_skipped_for_real_parent(self, tmp_path): - # A non-polytomy node whose immediate parent has dist==0 (a - # randomly-resolved polytomy node) should attribute its - # real_parent to the next ancestor with dist!=0. - # Tree shape: G -> P (dist=0 polytomy) -> X (dist=1 leaf). + # A non-polytomy node whose immediate parent carries the polytomy + # prop (a randomly-resolved polytomy node) should attribute its + # real_parent to the next unmarked ancestor. + # Tree shape: G -> P (polytomy) -> X (leaf). # X's real_parent must be G, not P. - t = ete4.Tree("((X:1,Y:1)P:0,Z:1)G:1;", parser=1) + t = ete4.Tree("((X:1,Y:1)P:1,Z:1)G:1;", parser=1) nodes = _by_name(t) + nodes["P"].props[POLYTOMY_PROP] = POLYTOMY_COMB _prep(t) output_mysqlexport(t, str(tmp_path)) leaves = _read_csv(tmp_path, "ordered_leaves.csv") @@ -463,17 +465,32 @@ def test_polytomy_parent_is_skipped_for_real_parent(self, tmp_path): assert x[LEAF_HEADER.index("real_parent")] == str(nodes["G"].props["id"]) def test_polytomy_self_emits_negative_real_parent(self, tmp_path): - # A node that is itself a polytomy resolution (dist=0) writes a - # negative real_parent_id, flagging the relationship as artificial. - t = ete4.Tree("((X:1,Y:1)P:0,Z:1)G:1;", parser=1) + # A node that is itself a polytomy resolution writes a negative + # real_parent_id, flagging the relationship as artificial. + t = ete4.Tree("((X:1,Y:1)P:1,Z:1)G:1;", parser=1) nodes = _by_name(t) + nodes["P"].props[POLYTOMY_PROP] = POLYTOMY_COMB _prep(t) output_mysqlexport(t, str(tmp_path)) node_rows = _read_csv(tmp_path, "ordered_nodes.csv") p = next(r for r in node_rows[1:] if r[NODE_HEADER.index("name")] == "P") - # P's dist is 0; G is its (non-polytomy) parent. + # P is marked as a polytomy; G is its (unmarked) parent. assert p[NODE_HEADER.index("real_parent")] == str(-nodes["G"].props["id"]) + def test_zero_dist_is_not_a_polytomy_marker(self, tmp_path): + # Branch lengths are regenerated from dates, so a zero-length branch + # no longer implies an artificial split. + t = ete4.Tree("((X:1,Y:1)P:0,Z:1)G:1;", parser=1) + nodes = _by_name(t) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + node_rows = _read_csv(tmp_path, "ordered_nodes.csv") + p = next(r for r in node_rows[1:] if r[NODE_HEADER.index("name")] == "P") + assert p[NODE_HEADER.index("real_parent")] == str(nodes["G"].props["id"]) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + x = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "X") + assert x[LEAF_HEADER.index("real_parent")] == str(nodes["P"].props["id"]) + def test_import_sql_contains_load_data_for_both_tables(self, tmp_path): # The SQL script should truncate-and-load both CSV files. The # column list inside `LOAD DATA INFILE` is read from the first diff --git a/tests/test_tree_build_step_tidy.py b/tests/test_tree_build_step_tidy.py index f0d06ddf..cae422ec 100644 --- a/tests/test_tree_build_step_tidy.py +++ b/tests/test_tree_build_step_tidy.py @@ -1,8 +1,13 @@ import ete4 from oz_tree_build.tree_build.step_tidy import ( + POLYTOMY_COMB, + POLYTOMY_PROP, + POLYTOMY_RANDOM, tidy_clear_conflicting_dates_topdown, tidy_infill_dates_bottomup, + tidy_mark_resolved_polytomies, + tidy_resolve_polytomies, ) @@ -10,6 +15,87 @@ def _by_name(tree): return {n.name: n for n in tree.traverse()} +class TestTidyResolvePolytomies: + def test_binary_tree_is_untouched(self): + t = ete4.Tree("((A:1,B:1)X:1,C:2)Root;", parser=1) + assert tidy_resolve_polytomies(t) == 0 + assert not any(n.props.get(POLYTOMY_PROP) for n in t.traverse()) + + def test_inserted_nodes_are_marked(self): + # A 4-way polytomy needs 2 extra nodes to become binary. + t = ete4.Tree("(A:1,B:1,C:1,D:1)Root;", parser=1) + assert tidy_resolve_polytomies(t) == 2 + marked = [n for n in t.traverse() if n.props.get(POLYTOMY_PROP)] + assert len(marked) == 2 + assert all(len(n.children) == 2 for n in t.traverse() if not n.is_leaf) + + def test_marked_as_comb_not_random(self): + # ete4's resolve_polytomy pairs children off in order rather than + # sampling a topology, so these are combs, not random draws. + t = ete4.Tree("(A:1,B:1,C:1)Root;", parser=1) + tidy_resolve_polytomies(t) + marked = [n for n in t.traverse() if n.props.get(POLYTOMY_PROP)] + assert [n.props[POLYTOMY_PROP] for n in marked] == [POLYTOMY_COMB] + + def test_resolution_is_deterministic_comb(self): + # Documents *why* the value is "comb": repeated runs give the same + # left-nested shape, so this is a systematic artefact, not a sample. + shapes = set() + for _ in range(5): + t = ete4.Tree("(A,B,C,D,E)Root;", parser=1) + tidy_resolve_polytomies(t) + shapes.add(t.write(parser=1, props=[])) + assert shapes == {"((((A,B):0,C):0,D):0,E);"} + + def test_pre_existing_nodes_are_not_marked(self): + t = ete4.Tree("(A:1,B:1,C:1)Root;", parser=1) + tidy_resolve_polytomies(t) + named = _by_name(t) + for name in ("Root", "A", "B", "C"): + assert not named[name].props.get(POLYTOMY_PROP) + + def test_marks_survive_zeroed_branch_lengths(self): + # The prop, not dist, is what identifies a resolution — clobbering + # branch lengths (as compute_branch_lengths later does) must not + # lose the marking. + t = ete4.Tree("(A:1,B:1,C:1)Root;", parser=1) + tidy_resolve_polytomies(t) + for n in t.traverse(): + n.dist = 5 + assert len([n for n in t.traverse() if n.props.get(POLYTOMY_PROP)]) == 1 + + +class TestTidyMarkResolvedPolytomies: + def test_marks_nodes_by_name_as_random(self): + t = ete4.Tree("((A:1,B:1)mrcapoly:1,C:2)Root;", parser=1) + assert tidy_mark_resolved_polytomies(t) == 1 + assert _by_name(t)["mrcapoly"].props[POLYTOMY_PROP] == POLYTOMY_RANDOM + + def test_both_kinds_are_truthy_for_consumers(self): + # step_output and step_jsnewick only ask "is this artificial?", so + # every kind has to survive a plain truth test. + assert POLYTOMY_COMB + assert POLYTOMY_RANDOM + assert POLYTOMY_COMB != POLYTOMY_RANDOM + + def test_leaves_other_nodes_alone(self): + t = ete4.Tree("((A:1,B:1)mrcapoly:1,C:2)Root;", parser=1) + tidy_mark_resolved_polytomies(t) + named = _by_name(t) + for name in ("Root", "A", "B", "C"): + assert not named[name].props.get(POLYTOMY_PROP) + + def test_no_matching_names_marks_nothing(self): + t = ete4.Tree("((A:1,B:1)X:0,C:2)Root;", parser=1) + assert tidy_mark_resolved_polytomies(t) == 0 + assert not any(n.props.get(POLYTOMY_PROP) for n in t.traverse()) + + def test_custom_name(self): + t = ete4.Tree("((A:1,B:1)poly:1,C:2)Root;", parser=1) + assert tidy_mark_resolved_polytomies(t, name="poly") == 1 + assert _by_name(t)["poly"].props[POLYTOMY_PROP] == POLYTOMY_RANDOM + + class TestTidyInfillDatesBottomup: def test_single_leaf_gets_date_zero(self): t = ete4.Tree("A;", parser=1) From a0643f842d571de17cba4e96685ae02b4adc269f Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 13:14:15 +0000 Subject: [PATCH 53/62] tree_build: Drop dangling mrcaimp leaves before removing unary nodes The mrcaimp detach ran inside the loop after delete_one_child_nodes, so every leaf it removed left its parent with a single child and nothing came along afterwards to collapse it: after delete_one_child_nodes: ((A:1,mrcaimp:1)X:1,B:1); after mrcaimp detach : ((A:1)X:1,B:1); <- X now unary mrcaimp appears 1,084,269 times in the current dated tree, so this was not a corner case, and it defeated the point of the preceding step. Do the detach first, in its own pass. That makes it possible for the root itself to end up unary, in which case delete_one_child_nodes replaces it -- so assign its return value, which was being discarded. Co-Authored-By: Claude Opus 5 --- oz_tree_build/tree_build/tree_build.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index b3758a38..ca1a4259 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -122,17 +122,21 @@ def main(): ) tidy_clear_conflicting_dates_topdown(base_t) + logger.info("Bin imputed mrca nodes made by fix_polyphyly left dangling by grafting process") + # NB: Has to happen before delete_one_child_nodes. Detaching a leaf leaves its parent + # with one child, and there is no second unary-node pass to tidy those up afterwards. + dangling = [n for n in base_t.traverse() if n.is_leaf and n.name == "mrcaimp"] + for n in dangling: + n.detach() + logger.info(f"Detached {len(dangling)} dangling mrcaimp leaves") + logger.info("Remove unary nodes (they are likely uninteresting, and make a mess of the tree rendering)") - tree_fixing.delete_one_child_nodes(base_t) + # NB: Returns the tree, which is a *new* root if the old one was itself unary + base_t = tree_fixing.delete_one_child_nodes(base_t) logger.info("Re-interpoltate missing dates") base_t.root.props.setdefault("date", ROOT_DATE_MYA) for n in base_t.traverse(): # First do some tidying to force tree_dating to work - if n.is_leaf and n.name == "mrcaimp": - # Bin imputed mrca nodes made by fix_polyphyly left dangling by grafting process - n.detach() - continue - if not n.name: # dated-complete-tree will assume all nodes have a name n.name = "" From e9f9fda7f074429f2db49b08ac84cd720edcc43e Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 13:14:39 +0000 Subject: [PATCH 54/62] tree_build: Regenerate branch lengths from the final dates tidy_infill_dates_bottomup converts branch lengths into dates and the pipeline treats 'date' as canonical from then on, but nothing ever recomputed 'dist' from the result. So branch lengths were whatever the input newicks happened to carry, before dates were cleared, infilled and imputed -- and simply absent for nodes whose newick had no length, since ete4 leaves those as None rather than defaulting to 1. On a tree run through the real steps, dist disagreed with the dates it was supposed to describe by more than an order of magnitude: node date dist parent.date - date x 10.0 100.0 4557.0 c 0 None 2283.5 treeprop_sliding_window and treeprop_weighted_mean consume dist, so both were computed from lengths unrelated to the tree's own dates. compute_branch_lengths exists for exactly this and is called by date_tree and every dated-complete-tree main, just never here. Co-Authored-By: Claude Opus 5 --- oz_tree_build/tree_build/tree_build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index ca1a4259..0e0df0ee 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -150,6 +150,12 @@ def main(): tree_dating.date_labelling(base_t) tree_dating.impute_missing_dates(base_t, l=0.25) + logger.info("Regenerate branch lengths from the final dates") + # Up to here 'dist' is whatever the input newicks happened to carry, which is + # stale (or missing) now dates have been infilled/cleared/imputed. Everything + # downstream that uses branch lengths wants them to agree with 'date'. + tree_dating.compute_branch_lengths(base_t) + logger.info("Rank popularities, post-node removal") popularity_add_rank(base_t) From f30f37d347ba08f66bc77a79d0363f6e5a3b80cd Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 13:15:31 +0000 Subject: [PATCH 55/62] step_popularity: Share popularity between OTTs duplicating a Qid OTT frequently holds one taxon twice, split across source taxonomies -- Q26285047 is OTT 988455 (ncbi + gbif ids) and OTT 4018224 (worms + irmng ids), same genus, same rank. Both rows carry the same raw_popularity, so summing them unaltered counted that popularity once per copy. The code noticed and warned about this but did nothing about it. In the assembled tree 2,473 Qids sit on more than one node, 2,721 nodes in total, and every group shares an identical raw_popularity. They are not duplicates in the sense of being adjacent: only 96 of 2,473 groups are siblings, the median leaf/leaf pair is 73 leaves apart and 292 pairs are separated by over 10,000 leaves. That scatter rules out crediting one copy and zeroing the others -- it would hand a genus's entire popularity to one of two distant clades on an arbitrary basis. Divide it between them instead, which keeps the tree's total right without choosing. Only nodes that actually contribute are counted, so an excluded or unpopulated duplicate does not dilute its twin. Rebuilding, 4.59% of leaves change popularity. popularity_rank changes for 99% of leaves, but 94.56% of those have an unchanged popularity and zero order inversions between them -- competition ranking renumbers everything below any changed value. The 15 most popular leaves are unchanged. The per-node warning is now inaccurate, as the double-counting no longer happens; it moves to debug with a single summary warning in its place. Co-Authored-By: Claude Opus 5 --- oz_tree_build/tree_build/step_popularity.py | 48 ++++++- tests/test_tree_build_step_popularity.py | 143 ++++++++++++++++++++ 2 files changed, 184 insertions(+), 7 deletions(-) diff --git a/oz_tree_build/tree_build/step_popularity.py b/oz_tree_build/tree_build/step_popularity.py index bfb43c38..c4fc35ea 100644 --- a/oz_tree_build/tree_build/step_popularity.py +++ b/oz_tree_build/tree_build/step_popularity.py @@ -25,8 +25,10 @@ def popularity_add_prop( caller zero out the raw popularity of named nodes before summation (e.g. excluding Dinosauria so its popularity is not credited to birds). - A warning is logged for any Wikidata Qid that appears on more than one node, - since that causes the same popularity to be counted twice. + Wikidata Qids appearing on more than one node are counted and summarised in a + single warning; ``sum_popularity_over_tree`` splits their raw popularity + between the nodes sharing them, so this is a data-quality note rather than a + miscalculation. Set debug logging to list the individual nodes. Must be run before monotomies / unary nodes are removed: those nodes often carry useful popularity that needs to percolate to their relatives first. @@ -37,13 +39,15 @@ def popularity_add_prop( # now apply the popularity function Qids = set() + duplicate_qid_nodes = 0 for node in tree.traverse(strategy="preorder"): Q = node.props["taxon"].get("wikidata") if Q is not None: if Q in Qids: - logger.warning( - f"duplicate wikidata Qids used (Q{Q}) - this will cause " - f"popularity double-counting for OTT {node_get_ott(node)}" + duplicate_qid_nodes += 1 + logger.debug( + f"duplicate wikidata Qid (Q{Q}) on OTT {node_get_ott(node)} - " + f"its raw popularity is shared with the other nodes using this Qid" ) else: Qids.add(Q) @@ -57,6 +61,13 @@ def popularity_add_prop( # Round to 2 decimal places node.props["popularity"] = round(pop, 2) + if duplicate_qid_nodes: + logger.warning( + f"{duplicate_qid_nodes} nodes share a wikidata Qid with an earlier node, usually " + f"because OTT holds the same taxon more than once. Their raw popularity has been " + f"divided between the nodes sharing each Qid; enable debug logging to list them." + ) + def popularity_add_rank(tree): """ @@ -174,6 +185,10 @@ def sum_popularity_over_tree(tree, exclude_taxa=None): It is copied onto ``node.props["pop"]`` and then summed across ancestors and descendants. + Where several nodes share a Wikidata Qid they each hold the same raw + popularity, so it is divided between them before summing -- see the comment + on ``qid_counts`` below. + We might want to exclude some names from the popularity metric (e.g. exclude archosaurs, to ensure birds don't gather popularity intended for dinosaurs). This is done by passing an array such as @@ -193,13 +208,32 @@ def sum_popularity_over_tree(tree, exclude_taxa=None): logger.info("Tree read for phylogenetic popularity calc") + def has_own_pop(node): + return node.name not in exclude_taxa and node.props["taxon"].get("raw_popularity") is not None + + # A Qid on several nodes is nearly always one taxon held twice by OTT (usually + # split across source taxonomies), and every copy carries the *same* full + # raw_popularity, so summing them unaltered counts that popularity once per + # copy. Share it out evenly instead: that keeps the tree's total popularity + # right without having to pick which copy is the "real" one -- which we can't + # do sensibly anyway, as the copies are frequently in quite distant clades. + # Only nodes that actually contribute popularity are counted, so an excluded + # or unpopulated duplicate doesn't dilute its twin. + qid_counts = collections.Counter( + node.props["taxon"]["wikidata"] + for node in tree.traverse(strategy="preorder") + if has_own_pop(node) and node.props["taxon"].get("wikidata") is not None + ) + # put popularity into the "pop" attribute for node in tree.traverse(strategy="preorder"): - if node.name in exclude_taxa or node.props["taxon"].get("raw_popularity") is None: + if not has_own_pop(node): node.props["pop"] = 0 node.props["has_pop"] = False else: - node.props["pop"] = node.props["taxon"]["raw_popularity"] + Q = node.props["taxon"].get("wikidata") + raw_popularity = node.props["taxon"]["raw_popularity"] + node.props["pop"] = raw_popularity if Q is None else raw_popularity / qid_counts[Q] node.props["has_pop"] = True # go up the tree from the tips, summing up the popularity indices beneath and diff --git a/tests/test_tree_build_step_popularity.py b/tests/test_tree_build_step_popularity.py index 4637a037..88020f63 100644 --- a/tests/test_tree_build_step_popularity.py +++ b/tests/test_tree_build_step_popularity.py @@ -1,4 +1,5 @@ import csv +import logging from math import log import ete4 @@ -212,6 +213,148 @@ def test_exclude_taxa_zeroes_pop_for_named_node(self, tmp_path): assert root.props["descendants_popsum"] == 30.0 +class TestSharedWikidataQids: + """ + OTT frequently holds one taxon twice (split across source taxonomies), and + every copy carries the same raw_popularity. It is shared between them rather + than counted once per copy. + """ + + def test_shared_qid_splits_raw_popularity(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["A_ott1"].props["pop"] == 50.0 + assert by_name["B_ott2"].props["pop"] == 50.0 + # The clade's total is what one copy of the taxon is worth, not two. + assert by_name["Root_ott3"].props["descendants_popsum"] == 100.0 + + def test_distinct_qids_are_untouched(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 43, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["A_ott1"].props["pop"] == 100.0 + assert by_name["B_ott2"].props["pop"] == 100.0 + + def test_node_without_a_qid_is_untouched(self, tmp_path): + # No Qid means nothing to share with, so no division. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 100.0}, + {"ott": 2, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["A_ott1"].props["pop"] == 100.0 + assert by_name["B_ott2"].props["pop"] == 100.0 + + def test_qid_shared_by_three_nodes_splits_three_ways(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2,C_ott4)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 90.0}, + {"ott": 2, "wikidata": 42, "raw_popularity": 90.0}, + {"ott": 4, "wikidata": 42, "raw_popularity": 90.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["A_ott1"].props["pop"] == 30.0 + assert by_name["Root_ott3"].props["descendants_popsum"] == 90.0 + + def test_excluded_duplicate_does_not_dilute_its_twin(self, tmp_path): + # An excluded node contributes no popularity, so the remaining node + # should keep the full score rather than be halved against a zero. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t, exclude_taxa=["B_ott2"]) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["B_ott2"].props["pop"] == 0 + assert by_name["A_ott1"].props["pop"] == 100.0 + + def test_duplicate_without_raw_popularity_does_not_dilute_its_twin(self, tmp_path): + # Likewise for a duplicate that has a Qid but no popularity to give. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 42}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["B_ott2"].props["has_pop"] is False + assert by_name["A_ott1"].props["pop"] == 100.0 + + def test_duplicates_are_summarised_in_one_warning(self, tmp_path, caplog): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + with caplog.at_level(logging.WARNING): + popularity_add_prop(t) + + warnings = [r for r in caplog.records if r.levelno == logging.WARNING] + assert len(warnings) == 1 + assert "1 nodes share a wikidata Qid" in warnings[0].message + + class TestPopularityAddProp: def test_popularity_set_on_every_node_and_rounded(self, tmp_path): t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) From 045a9f9db8529e944691248bbe6520909c665e29 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 14:02:44 +0000 Subject: [PATCH 56/62] step_graft: Remove ancestors left empty by subtree extraction graft_extract_ot_subtrees detaches each requested subtree from the OpenTree tree in place, but left behind any ancestor that detaching emptied. An emptied node has no children, so everything downstream reads it as a leaf: it was written to ordered_leaves.csv as though it were a species, with no OTT but a real popularity and leaf rank, and date_labelling warned that a leaf had a non-zero date. BonyFishOpenTree.PHY requests every family under Beryciformes individually and arranges them itself, so the whole OT clade is relocated and its scaffolding hollowed out: Beryciformes_ott587933 (128 leaves in OT) |- mrcaott90320ott98190 | |- Cetomimidae_ott118790 <- requested | `- mrcaimp -> Melamphaidae, Berycidae, Gibberichthyidae <- requested `- mrcaott118778ott325745 |- mrcaott118778ott781203 -> Barbourisiidae, Stephanoberycidae <- req. `- Rondeletiidae_ott190706 <- requested Prune at the point of detachment, walking up while nodes are left empty. Only nodes emptied by us are removed -- a node that was already a tip is a real taxon and is left alone -- and the root is never detached. Doing this by name after the fact does not work: nothing in the final tree distinguishes an emptied taxon from a genuine leaf, and pruning just the synthetic mrca* nodes merely moves the problem up to Beryciformes itself. tidy_prune_synthetic_leaves still exists for mrcaimp debris that arrives in the source data independently of extraction, and replaces the narrower mrcaimp pass in tree_build. Rebuilding: both warnings gone with no replacement, and four fewer leaves (2287173 -> 2287169) -- the three mrca* nodes plus Beryciformes. The no-OTT leaf count drops to exactly the 2051 genuine unplaced taxa, so nothing real was pruned. Co-Authored-By: Claude Opus 5 --- oz_tree_build/tree_build/step_graft.py | 14 ++++++- oz_tree_build/tree_build/step_tidy.py | 40 +++++++++++++++++++ oz_tree_build/tree_build/tree_build.py | 8 ++-- tests/test_tree_build_step_graft.py | 53 +++++++++++++++++++++++++- tests/test_tree_build_step_tidy.py | 50 ++++++++++++++++++++++++ 5 files changed, 157 insertions(+), 8 deletions(-) diff --git a/oz_tree_build/tree_build/step_graft.py b/oz_tree_build/tree_build/step_graft.py index 5bb5358e..c840fb16 100644 --- a/oz_tree_build/tree_build/step_graft.py +++ b/oz_tree_build/tree_build/step_graft.py @@ -77,7 +77,8 @@ def graft_extract_ot_subtrees(opentree_t, inclusions): (see ``decypher_inclusion_syntax``). ``opentree_t`` is walked and any node whose OTT matches one of those base OTTs is detached and returned as a standalone subtree. ``opentree_t`` is mutated in place — every extracted - subtree is removed from it. + subtree is removed from it, along with any ancestor left with no + descendants as a result (see the loop below). The extraction recurses into each detached subtree, so a requested OTT that lies inside another requested subtree is still extracted (and its @@ -110,9 +111,20 @@ def is_leaf_fn(n): del start_otts[node_ott] # Prune this tree, extract any required subtrees from this subtree + parent = n.up sub_t = n.detach() out_trees.update(prune_ot_subtrees(sub_t)) # NB: node_ott now removed from start_otts, so won't loop out_trees[r["orig_name"]] = sub_t + + # Detaching can leave (parent) with nothing below it -- every one of its + # descendants was requested separately, so the whole clade now lives in + # the bespoke tree. An emptied node reads as a leaf from here on and + # would be written out as though it were a species, so drop it, and any + # ancestor it empties in turn. Only nodes emptied by *us* are removed: + # a node that was already a tip is a real taxon and is left alone. + while parent is not None and parent.up is not None and not parent.children: + emptied, parent = parent, parent.up + emptied.detach() return True for _ in ot_t.traverse(strategy="preorder", is_leaf_fn=is_leaf_fn): diff --git a/oz_tree_build/tree_build/step_tidy.py b/oz_tree_build/tree_build/step_tidy.py index e3876b32..50174685 100644 --- a/oz_tree_build/tree_build/step_tidy.py +++ b/oz_tree_build/tree_build/step_tidy.py @@ -1,3 +1,5 @@ +import re + # Prop marking a node as an artificial split inserted to break up a polytomy. # Its value says *how* the topology was chosen, since the two stages that # resolve polytomies do so very differently. Any node carrying the prop is @@ -17,6 +19,12 @@ # survives into the newick date_tree writes out OT_POLYTOMY_NAME = "mrcapoly" +# Names belonging to synthetic nodes that stand in for an ancestor rather than +# naming a taxon: OpenTree's own MRCA labels, plus those dated_complete_tree +# gives the nodes it inserts resolving polyphyly ("mrcaimp") and polytomies +# ("mrcapoly"). None of them is ever a taxon in its own right. +SYNTHETIC_NAME_RE = re.compile(r"^mrca(ott\d+ott\d+|imp|poly)$") + def tidy_resolve_polytomies(tree, kind=POLYTOMY_COMB): """ @@ -62,6 +70,38 @@ def tidy_mark_resolved_polytomies(tree, kind=POLYTOMY_RANDOM, name=OT_POLYTOMY_N return count +def tidy_prune_synthetic_leaves(tree, name_re=SYNTHETIC_NAME_RE): + """ + Drop childless synthetic nodes, repeating until none are left. + + ``graft_extract_ot_subtrees`` detaches each requested subtree from the + OpenTree tree in place and does not tidy up the ancestors that empties. An + emptied node has no children, so it reads as a leaf from then on and is + written out as though it were a species -- keeping the date it had as an + internal node, which is what makes ``date_labelling`` complain that a leaf + has a non-zero date. + + Removing one can empty its parent in turn (an OT MRCA node above a + ``mrcaimp`` node above two extracted subtrees, say), hence the fixed point. + Nodes that were *already* childless in the input are debris for the same + reason and go the same way. + + Only synthetic names are pruned. A named taxon left childless is left alone + deliberately: it would be a real taxon losing its whole subtree, which is + worth noticing rather than quietly deleting. + + Return number of nodes removed. + """ + removed = 0 + while True: + emptied = [n for n in tree.traverse() if n.is_leaf and n.up is not None and name_re.match(n.name or "")] + if not emptied: + return removed + for node in emptied: + node.detach() + removed += len(emptied) + + def tidy_infill_dates_bottomup(tree): """ Working bottom-upwards, fill in missing date properties based on branch lengths. diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 0e0df0ee..9909254d 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -26,6 +26,7 @@ tidy_clear_conflicting_dates_topdown, tidy_infill_dates_bottomup, tidy_mark_resolved_polytomies, + tidy_prune_synthetic_leaves, tidy_resolve_polytomies, ) from .step_treeprop import ( @@ -122,13 +123,10 @@ def main(): ) tidy_clear_conflicting_dates_topdown(base_t) - logger.info("Bin imputed mrca nodes made by fix_polyphyly left dangling by grafting process") + logger.info("Bin synthetic mrca nodes left childless by the grafting process") # NB: Has to happen before delete_one_child_nodes. Detaching a leaf leaves its parent # with one child, and there is no second unary-node pass to tidy those up afterwards. - dangling = [n for n in base_t.traverse() if n.is_leaf and n.name == "mrcaimp"] - for n in dangling: - n.detach() - logger.info(f"Detached {len(dangling)} dangling mrcaimp leaves") + logger.info(f"Pruned {tidy_prune_synthetic_leaves(base_t)} childless synthetic nodes") logger.info("Remove unary nodes (they are likely uninteresting, and make a mess of the tree rendering)") # NB: Returns the tree, which is a *new* root if the old one was itself unary diff --git a/tests/test_tree_build_step_graft.py b/tests/test_tree_build_step_graft.py index bd5aa009..8ba73ad9 100644 --- a/tests/test_tree_build_step_graft.py +++ b/tests/test_tree_build_step_graft.py @@ -133,8 +133,57 @@ def test_recurses_into_extracted_subtrees(self, tmp_path): result = graft_extract_ot_subtrees(ete4.Tree(str(ot_file), parser=1), ["Outer_ott2@", "Nested_ott1@"]) assert set(result.keys()) == {"Outer_ott2@", "Nested_ott1@"} assert result["Nested_ott1@"].write() == "(I1_ott11,I2_ott12);" - # NB: Outer tree no longer contains inner tree - assert result["Outer_ott2@"].write() == "(Filler_ott99);" + # NB: Outer tree no longer contains inner tree. Filler_ott99 held nothing + # but the nested subtree, so extracting it emptied Filler too and Filler + # goes with it -- leaving the outer subtree with nothing in it at all. + assert result["Outer_ott2@"].write() == ";" + + def test_ancestor_emptied_by_extraction_is_removed(self, tmp_path): + # Every child of Inner_ott3 is requested separately, so Inner is left + # with nothing below it. It would otherwise survive as a childless node, + # reading as a leaf and being written out as though it were a species. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((X_ott11)Sub1_ott1,(Y_ott21)Sub2_ott2)Inner_ott3,Z_ott4)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub1_ott1@", "Sub2_ott2@"]) + assert [n.name for n in ot_t.traverse()] == ["Root_ott99", "Z_ott4"] + + def test_emptying_cascades_up_the_ancestry(self, tmp_path): + # Removing Inner empties Middle, which empties Outer. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("((((((X_ott11)Sub_ott1)Inner_ott5)Middle_ott6)Outer_ott7),Z_ott4)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub_ott1@"]) + assert not any(n.name.startswith(("Inner", "Middle", "Outer")) for n in ot_t.traverse()) + assert "Z_ott4" in [n.name for n in ot_t.traverse()] + + def test_ancestor_keeping_a_child_is_left_alone(self, tmp_path): + # Inner still holds Keep_ott9, so it is a genuine ancestor, not debris. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((X_ott11)Sub_ott1,Keep_ott9)Inner_ott3,Z_ott4)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub_ott1@"]) + names = [n.name for n in ot_t.traverse()] + assert "Inner_ott3" in names + assert "Keep_ott9" in names + + def test_pre_existing_tips_are_not_pruned(self, tmp_path): + # A node that was already a tip is a real taxon, not something we + # emptied, so it must survive even though it has no children. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((X_ott11)Sub_ott1,Tip_ott8)Inner_ott3,Z_ott4)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub_ott1@"]) + assert "Tip_ott8" in [n.name for n in ot_t.traverse()] + + def test_root_is_never_detached(self, tmp_path): + # Extracting everything leaves the root childless; it must stay put. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("((X_ott11)Sub_ott1)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub_ott1@"]) + assert ot_t.up is None + assert ot_t.name == "Root_ott99" class TestPresentInTree: diff --git a/tests/test_tree_build_step_tidy.py b/tests/test_tree_build_step_tidy.py index cae422ec..4eaa473b 100644 --- a/tests/test_tree_build_step_tidy.py +++ b/tests/test_tree_build_step_tidy.py @@ -7,10 +7,60 @@ tidy_clear_conflicting_dates_topdown, tidy_infill_dates_bottomup, tidy_mark_resolved_polytomies, + tidy_prune_synthetic_leaves, tidy_resolve_polytomies, ) +class TestTidyPruneSyntheticLeaves: + def test_childless_ot_mrca_node_is_pruned(self): + # An OT MRCA label left childless because both its children were + # extracted as separate subtrees. + t = ete4.Tree("(mrcaott118778ott781203,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 1 + assert [n.name for n in t.leaves()] == ["B"] + + def test_mrcaimp_and_mrcapoly_are_pruned(self): + t = ete4.Tree("(mrcaimp,mrcapoly,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 2 + assert [n.name for n in t.leaves()] == ["B"] + + def test_cascades_upwards(self): + # Root -> M (OT mrca) -> mrcaimp -> nothing. Pruning the mrcaimp + # empties M, which must then go too. + t = ete4.Tree("((mrcaimp)mrcaott90320ott145150,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 2 + assert [n.name for n in t.leaves()] == ["B"] + + def test_named_taxa_are_left_alone(self): + # A real taxon left childless is not ours to delete silently. + t = ete4.Tree("(Berycidae_ott118776,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 0 + assert sorted(n.name for n in t.leaves()) == ["B", "Berycidae_ott118776"] + + def test_synthetic_node_with_children_is_kept(self): + # Only *childless* synthetic nodes are debris; one still holding a + # subtree is doing its job as an ancestor. + t = ete4.Tree("((X:1,Y:1)mrcaimp,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 0 + assert sorted(n.name for n in t.leaves()) == ["B", "X", "Y"] + + def test_similar_names_are_not_matched(self): + # Guard the regex against eating real taxa that merely start "mrca". + t = ete4.Tree("(mrcaimposter,Mrcaimp,mrcaott12,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 0 + + def test_returns_zero_on_a_clean_tree(self): + t = ete4.Tree("(A:1,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 0 + + def test_root_is_never_detached(self): + # A tree that collapses entirely must not try to detach its own root. + t = ete4.Tree("(mrcaimp)mrcapoly;", parser=1) + tidy_prune_synthetic_leaves(t) + assert t.up is None + + def _by_name(tree): return {n.name: n for n in tree.traverse()} From 0261b4efef74f5131a84e989cac5eee7face4ba0 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 14:05:45 +0000 Subject: [PATCH 57/62] step_treeprop: Remove sliding-window / weighted-mean They ended up being redundant, only geological is used by the renderer in #1013. --- oz_tree_build/tree_build/step_treeprop.py | 88 ---------- oz_tree_build/tree_build/tree_build.py | 4 - tests/test_tree_build_step_treeprop.py | 195 ---------------------- 3 files changed, 287 deletions(-) diff --git a/oz_tree_build/tree_build/step_treeprop.py b/oz_tree_build/tree_build/step_treeprop.py index a95e35d1..699b7a0e 100644 --- a/oz_tree_build/tree_build/step_treeprop.py +++ b/oz_tree_build/tree_build/step_treeprop.py @@ -1,5 +1,4 @@ import logging -import math logger = logging.getLogger(__name__) @@ -97,90 +96,3 @@ def treeprop_geological(tree): prop_format["geological"] = "u8" return "geological" - - -def treeprop_sliding_window(tree, local_mean_width=5): - """ - Given an ete4 tree object, add a "sliding_window" prop to each node, - representing the log-ratio between the node's edge length and the mean - edge length of nearby ancestors (up to local_mean_width upwards) and - descendants (up to local_mean_width deep). - - Return name of prop just added. - """ - for node in tree.traverse("preorder"): - edge_length = node.dist - if edge_length == 0: - node.props["sliding_window"] = 0 - continue - if edge_length is None or edge_length < 0: - logger.warning(f"Node {node.name} has no / negative branch length {edge_length}") - node.props["sliding_window"] = 0.0 - continue - window_count = 0 - window_sum = 0 - - # Work upwards, adding parents to window - nn = node - for _ in range(local_mean_width): - if nn.up is None: - break - nn_length = nn.dist - if nn_length is None or nn_length < 0: - logger.warning(f"Node {nn.name} has no / negative branch length {nn_length}") - break - window_sum += nn_length - window_count += 1 - nn = nn.up - - # Work downwards, adding descendents to window - stack = [(c, 1) for c in node.children] - while stack: - dn, depth = stack.pop() - dn_length = dn.dist - if dn_length is None or dn_length < 0: - logger.warning(f"Node {dn.name} has no / negative branch length {dn_length}") - continue - window_sum += dn_length - window_count += 1 - if depth < local_mean_width: - stack.extend((c, depth + 1) for c in dn.children) - - if window_count == 0: - node.props["sliding_window"] = 0.0 - else: - node.props["sliding_window"] = math.log(edge_length / (window_sum / window_count)) - - prop_format = tree.root.props.setdefault("prop_format", {}) - prop_format["sliding_window"] = "f16" - - return "sliding_window" - - -def treeprop_weighted_mean(tree, weighting=0.8): - """ - Given an ete4 tree object, add a "weighted_mean_ratio" prop. - - Return name of prop just added. - """ - # Calculate weighted mean vs. ancestors - for node in tree.traverse("preorder"): - edge_length = node.dist - parent = node.up - if parent is None: - node.props["weighted_mean"] = 0.0 - node.props["weighted_mean_ratio"] = 0.0 - elif edge_length is None or parent.props.get("weighted_mean") is None: - if edge_length is None: - logger.warning(f"Node {node.name} has no branch length") - node.props["weighted_mean"] = 0.0 - node.props["weighted_mean_ratio"] = 0.0 - else: - node.props["weighted_mean"] = (edge_length + (parent.props["weighted_mean"] * weighting)) / (1 + weighting) - node.props["weighted_mean_ratio"] = edge_length / max(node.props["weighted_mean"], 1e-6) - - prop_format = tree.root.props.setdefault("prop_format", {}) - prop_format["weighted_mean"] = "f16" - prop_format["weighted_mean_ratio"] = "f16" - - return "weighted_mean_ratio" diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py index 9909254d..0caa2b81 100644 --- a/oz_tree_build/tree_build/tree_build.py +++ b/oz_tree_build/tree_build/tree_build.py @@ -31,8 +31,6 @@ ) from .step_treeprop import ( treeprop_geological, - treeprop_sliding_window, - treeprop_weighted_mean, ) logger = logging.getLogger(__name__) @@ -191,8 +189,6 @@ def main(): logger.info("Generate tree properties and output arrays") output_proparray(base_t, args.out_dir, treeprop_geological(base_t)) - output_proparray(base_t, args.out_dir, treeprop_sliding_window(base_t)) - output_proparray(base_t, args.out_dir, treeprop_weighted_mean(base_t)) if __name__ == "__main__": diff --git a/tests/test_tree_build_step_treeprop.py b/tests/test_tree_build_step_treeprop.py index eb9226bf..ec2aecdc 100644 --- a/tests/test_tree_build_step_treeprop.py +++ b/tests/test_tree_build_step_treeprop.py @@ -1,14 +1,8 @@ -import logging -import math -import random - import ete4 from oz_tree_build.tree_build.step_treeprop import ( GEOLOGICAL_PERIODS, treeprop_geological, - treeprop_sliding_window, - treeprop_weighted_mean, ) ######################################## @@ -83,192 +77,3 @@ def get_period(x): assert get_period(520.99) == ("Cambrian", "Series 2", 521) assert get_period(521) == ("Cambrian", "Series 2", 521) assert get_period(521.01) == ("Cambrian", "Terreneuvian", 538.8) - - -######################################## -# treeprop_weighted_mean -######################################## - - -def do_treeprop_weighted_mean(nwk, weighting=0.8): - t = ete4.Tree(nwk, parser=1) - assert treeprop_weighted_mean(t, weighting=weighting) == "weighted_mean_ratio" - - # Traverse tree, returning all periods - return [(n.name, n.props.get("date"), n.props["weighted_mean_ratio"]) for n in t.traverse("preorder")] - - -def generate_tree(dists): - tree_str = "" - for i, d in enumerate(dists): - if tree_str != "": - tree_str = f"({tree_str})" - tree_str += f"n{i}" - if d is not None: - tree_str += f":{d}" - tree_str += ";" - return tree_str - - -def expected_results_wm(dists, weighting): - """ - Compute expected weighted_mean_ratio values for a linear caterpillar tree - built from dists, where dists[k] is the branch length of node nk, nk's - parent is n(k+1), and n(len-1) is the root. - - Traversal is preorder, so results start at the root (n_last) and walk - down to the leaf (n0). The root and any node with a missing branch - length get weighted_mean and weighted_mean_ratio of 0.0; that 0.0 then - feeds back into the recurrence for descendants like any other value, - so a single missing dist no longer poisons the whole subtree below it. - """ - n = len(dists) - weighted_mean = [None] * n - - # Walk from root (index n-1) down to leaf (index 0) - for i in range(n - 1, -1, -1): - if i == n - 1 or dists[i] is None: - weighted_mean[i] = 0.0 - else: - weighted_mean[i] = (dists[i] + weighted_mean[i + 1] * weighting) / (1 + weighting) - - results = [] - for i in range(n - 1, -1, -1): - if i == n - 1 or dists[i] is None: - ratio = 0.0 - else: - ratio = dists[i] / weighted_mean[i] - results.append((f"n{i}", None, ratio)) - return results - - -class TestTreepropWeightedMean: - def test_weighting(self): - """weighting param honoured""" - dists = [random.randrange(10, 100) for _ in range(20)] - tree_str = generate_tree(dists) - - assert do_treeprop_weighted_mean(tree_str) == expected_results_wm(dists, weighting=0.8) - assert do_treeprop_weighted_mean(tree_str, weighting=3) == expected_results_wm(dists, weighting=3) - assert expected_results_wm(dists, 0.8) != expected_results_wm(dists, 3) - - def test_missing_branch_length(self, caplog): - """Nodes with missing branch length get a 0.0 weighted_mean and emit a warning. - - The missing node's 0.0 feeds back into the recurrence for its descendants - like any other value, so only the missing node itself (and the root, which - is always 0.0) shows a zero ratio. - """ - dists = [random.randrange(10, 100) for _ in range(20)] - dists[10] = None - tree_str = generate_tree(dists) - - with caplog.at_level( - logging.WARNING, - logger="oz_tree_build.taxon_mapping_and_popularity.tree_props.weighted_mean", - ): - result = do_treeprop_weighted_mean(tree_str, weighting=3) - - assert result == expected_results_wm(dists, weighting=3) - # Preorder visits n19..n0. Only the root (index 0 → n19) and the missing - # node (index 9 → n10) have ratio 0.0; descendants of n10 compute normally. - assert [i for i, x in enumerate(result) if x[2] == 0.0] == [0, 9] - assert any("n10" in r.message and r.levelno == logging.WARNING for r in caplog.records) - - -######################################## -# treeprop_sliding_window -######################################## - - -def do_treeprop_sliding_window(nwk, local_mean_width=5): - t = ete4.Tree(nwk, parser=1) - assert treeprop_sliding_window(t, local_mean_width=local_mean_width) == "sliding_window" - - # Traverse tree, returning all periods - return [(n.name, n.props.get("date"), n.props["sliding_window"]) for n in t.traverse("preorder")] - - -def expected_results_sw(dists, local_mean_width): - """ - Compute expected sliding_window values from a linear caterpillar tree built - from dists, where dists[k] is the branch length of node nk, nk's parent is - n(k+1), and n(len-1) is the root. - - Traversal order is preorder: root (n_last) down to leaf (n0). The window - walks up to local_mean_width ancestors (stopping at the root, whose own - edge length is never counted, or at any None / negative edge length) and - up to local_mean_width descendants downwards (in this linear tree a single - chain; again stopping at a None / negative edge length). A node whose own - edge length is exactly 0 short-circuits to 0; a None / negative own edge - length short-circuits to 0.0; a node that contributes nothing to the - window returns 0.0. - """ - n = len(dists) - - def node_sw(k): - if dists[k] == 0: - return 0 - if dists[k] is None or dists[k] < 0: - return 0.0 - window = [] - kk = k - for _ in range(local_mean_width): - if kk >= n - 1: - break - if dists[kk] is None or dists[kk] < 0: - break - window.append(dists[kk]) - kk += 1 - kk = k - 1 - for _ in range(local_mean_width): - if kk < 0: - break - if dists[kk] is None or dists[kk] < 0: - break - window.append(dists[kk]) - kk -= 1 - if not window: - return 0.0 - return math.log(dists[k] / (sum(window) / len(window))) - - return [(f"n{i}", None, node_sw(i)) for i in range(n - 1, -1, -1)] - - -class TestSlidingWindow: - def test_local_mean_width(self): - """local_mean_width param honoured""" - dists = [random.randrange(10, 100) for _ in range(20)] - tree_str = generate_tree(dists) - - assert do_treeprop_sliding_window(tree_str) == expected_results_sw(dists, local_mean_width=5) - assert do_treeprop_sliding_window(tree_str, local_mean_width=3) == expected_results_sw( - dists, local_mean_width=3 - ) - assert expected_results_sw(dists, 5) != expected_results_sw(dists, 3) - - def test_missing_branch_length(self, caplog): - """Nodes with missing branch length get sliding_window 0.0 and emit a warning""" - dists = [random.randrange(10, 100) for _ in range(20)] - dists[10] = None - tree_str = generate_tree(dists) - - with caplog.at_level( - logging.WARNING, logger="oz_tree_build.taxon_mapping_and_popularity.tree_props.sliding_window" - ): - result = do_treeprop_sliding_window(tree_str, local_mean_width=3) - - assert result == expected_results_sw(dists, local_mean_width=3) - assert any("n10" in r.message and r.levelno == logging.WARNING for r in caplog.records) - - def test_zero_branch_length(self): - """Nodes with edge length == 0 short-circuit to sliding_window 0 without affecting siblings""" - dists = [random.randrange(10, 100) for _ in range(20)] - dists[10] = 0 - tree_str = generate_tree(dists) - - result = do_treeprop_sliding_window(tree_str, local_mean_width=3) - - assert result == expected_results_sw(dists, local_mean_width=3) - # n10 is at preorder index 9 and was given dist 0, so it should be exactly 0. - assert result[9] == ("n10", None, 0) From 4af34c52af5554a3d0f8668ccc276132be639578 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 14:28:23 +0000 Subject: [PATCH 58/62] step_output: Write \N for missing values, not empty strings read_taxon_map() gives every taxon a full set of keys, using None for the columns that were empty, so .get(key, "\N") never fired and csv.writer wrote the None as an empty string. Add mysql_null() and use it for every taxon-derived and node column. Whilst here: * Read the taxon map's "rank" column, rather than the DB's name for it ("rnk"), which meant the rank was never exported at all. * Write the unpopulated "price" / "vern_synth" columns as \N too. --- oz_tree_build/tree_build/step_output.py | 77 +++++++++++++---------- tests/test_tree_build_step_output.py | 84 ++++++++++++++++++++++++- 2 files changed, 126 insertions(+), 35 deletions(-) diff --git a/oz_tree_build/tree_build/step_output.py b/oz_tree_build/tree_build/step_output.py index 97734239..c62d34b3 100644 --- a/oz_tree_build/tree_build/step_output.py +++ b/oz_tree_build/tree_build/step_output.py @@ -62,6 +62,17 @@ def output_add_prop_ids(tree): prev_node = node +def mysql_null(value): + """ + Map a missing value to the ``\\N`` marker MySQL's ``LOAD DATA`` reads as NULL + + Values are missing if either absent or None: read_taxon_map() produces a full + set of keys for every taxon, with None for the columns that were empty, so + ``dict.get(key, "\\N")`` is not enough on its own. + """ + return "\\N" if value is None else value + + def output_mysqlexport(tree, out_dir): """ Write the three files needed to load the tree into the OneZoom MySQL @@ -75,8 +86,8 @@ def output_mysqlexport(tree, out_dir): Preconditions ------------- Every node must carry ``props["taxon"]``, a dict of taxon-derived - columns (``ott``, ``wikidata``, ``ncbi``, ...); missing keys are - written as ``\\N``. Every internal node must additionally carry + columns (``ott``, ``wikidata``, ``ncbi``, ...); keys that are absent + or None are written as ``\\N``. Every internal node must additionally carry ``id`` / ``node_rgt`` / ``leaf_lft`` / ``leaf_rgt`` as produced by `output_add_prop_ids`. @@ -176,22 +187,22 @@ def output_mysqlexport(tree, out_dir): # TODO: negative real_parent ids if this is a polytomy real_parent_id, node_name_without_ott(node), - node.props.get("extinction_date", "\\N"), - node.props["taxon"].get("ott", "\\N"), - node.props["taxon"].get("wikidata", "\\N"), - node.props["taxon"].get("wikipedia_lang_flag", "\\N"), - node.props["taxon"].get("iucn", "\\N"), - node.props["taxon"].get("eol", "\\N"), - node.props["taxon"].get("raw_popularity", "\\N"), - node.props.get("popularity", "\\N"), - node.props.get("popularity_rank", "\\N"), - None, # "price" - node.props["taxon"].get("ncbi", "\\N"), - node.props["taxon"].get("if", "\\N"), # NB: "ifung" in the DB - node.props["taxon"].get("worms", "\\N"), - node.props["taxon"].get("irmng", "\\N"), - node.props["taxon"].get("gbif", "\\N"), - node.props["taxon"].get("ipni", "\\N"), + mysql_null(node.props.get("extinction_date")), + mysql_null(node.props["taxon"].get("ott")), + mysql_null(node.props["taxon"].get("wikidata")), + mysql_null(node.props["taxon"].get("wikipedia_lang_flag")), + mysql_null(node.props["taxon"].get("iucn")), + mysql_null(node.props["taxon"].get("eol")), + mysql_null(node.props["taxon"].get("raw_popularity")), + mysql_null(node.props.get("popularity")), + mysql_null(node.props.get("popularity_rank")), + "\\N", # "price" + mysql_null(node.props["taxon"].get("ncbi")), + mysql_null(node.props["taxon"].get("if")), # NB: "ifung" in the DB + mysql_null(node.props["taxon"].get("worms")), + mysql_null(node.props["taxon"].get("irmng")), + mysql_null(node.props["taxon"].get("gbif")), + mysql_null(node.props["taxon"].get("ipni")), ] ) else: @@ -203,21 +214,21 @@ def output_mysqlexport(tree, out_dir): node.props["leaf_lft"], node.props["leaf_rgt"], node_name_without_ott(node), - node.props.get("date", "\\N"), # TODO: But only if it's not imputed - node.props["taxon"].get("ott", "\\N"), - node.props["taxon"].get("wikidata", "\\N"), - node.props["taxon"].get("wikipedia_lang_flag", "\\N"), - node.props["taxon"].get("eol", "\\N"), - node.props["taxon"].get("rnk", "\\N"), - node.props["taxon"].get("raw_popularity", "\\N"), - node.props.get("popularity", "\\N"), - node.props["taxon"].get("ncbi", "\\N"), - node.props["taxon"].get("if", "\\N"), # NB: "ifung" in the DB - node.props["taxon"].get("worms", "\\N"), - node.props["taxon"].get("irmng", "\\N"), - node.props["taxon"].get("gbif", "\\N"), - node.props["taxon"].get("ipni", "\\N"), - None, # "vern_synth" + mysql_null(node.props.get("date")), # TODO: But only if it's not imputed + mysql_null(node.props["taxon"].get("ott")), + mysql_null(node.props["taxon"].get("wikidata")), + mysql_null(node.props["taxon"].get("wikipedia_lang_flag")), + mysql_null(node.props["taxon"].get("eol")), + mysql_null(node.props["taxon"].get("rank")), # NB: "rnk" in the DB + mysql_null(node.props["taxon"].get("raw_popularity")), + mysql_null(node.props.get("popularity")), + mysql_null(node.props["taxon"].get("ncbi")), + mysql_null(node.props["taxon"].get("if")), # NB: "ifung" in the DB + mysql_null(node.props["taxon"].get("worms")), + mysql_null(node.props["taxon"].get("irmng")), + mysql_null(node.props["taxon"].get("gbif")), + mysql_null(node.props["taxon"].get("ipni")), + "\\N", # "vern_synth" ] + ["\\N" for _ in ("rep", "rtr", "rpd") for _ in range(8)] + ["\\N" for _ in ("NE", "DD", "LC", "NT", "VU", "EN", "CR", "EW", "EX")] diff --git a/tests/test_tree_build_step_output.py b/tests/test_tree_build_step_output.py index 66ba8a94..b6e38cee 100644 --- a/tests/test_tree_build_step_output.py +++ b/tests/test_tree_build_step_output.py @@ -399,17 +399,97 @@ def test_taxon_props_are_written_to_leaf_row(self, tmp_path): assert b[LEAF_HEADER.index("ncbi")] == "\\N" def test_taxon_props_are_written_to_node_row(self, tmp_path): - # Internal nodes get the same taxon projection — but with `rnk` + # Internal nodes get the same taxon projection — but with the rank # in place of the leaf-only `iucn`/`extinction_date` columns. t = ete4.Tree("(A,B)R;", parser=1) - _prep(t, taxon_overrides={"R": {"ott": "777", "rnk": "family", "if": "9257"}}) + _prep(t, taxon_overrides={"R": {"ott": "777", "rank": "family", "if": "9257"}}) output_mysqlexport(t, str(tmp_path)) nodes = _read_csv(tmp_path, "ordered_nodes.csv") root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") assert root[NODE_HEADER.index("ott")] == "777" + # The taxon map's "rank" is the DB's "rnk" column assert root[NODE_HEADER.index("rnk")] == "family" assert root[NODE_HEADER.index("ifung")] == "9257" + def test_none_taxon_values_are_backslash_N(self, tmp_path): + # read_taxon_map() gives every taxon a full set of keys, using None for + # the columns that were empty, so a present-but-None value has to be + # written as \N just like an absent key would be. + t = ete4.Tree("(A,B)R;", parser=1) + none_taxon = dict.fromkeys( + ( + "ott", + "wikidata", + "wikipedia_lang_flag", + "iucn", + "eol", + "rank", + "raw_popularity", + "ncbi", + "if", + "worms", + "irmng", + "gbif", + "ipni", + ) + ) + _prep(t, taxon_overrides={"A": none_taxon, "R": none_taxon}) + output_mysqlexport(t, str(tmp_path)) + + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + a = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "A") + for col in ("ott", "wikidata", "wikipedia_lang_flag", "iucn", "eol", "raw_popularity"): + assert a[LEAF_HEADER.index(col)] == "\\N", col + for col in ("ncbi", "ifung", "worms", "irmng", "gbif", "ipni"): + assert a[LEAF_HEADER.index(col)] == "\\N", col + + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + for col in ("ott", "wikidata", "wikipedia_lang_flag", "eol", "rnk", "raw_popularity"): + assert root[NODE_HEADER.index(col)] == "\\N", col + for col in ("ncbi", "ifung", "worms", "irmng", "gbif", "ipni"): + assert root[NODE_HEADER.index(col)] == "\\N", col + + def test_none_node_props_are_backslash_N(self, tmp_path): + # Ditto for props set on the node itself rather than its taxon. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + nodes_by_name = _by_name(t) + nodes_by_name["A"].props["extinction_date"] = None + nodes_by_name["A"].props["popularity"] = None + nodes_by_name["A"].props["popularity_rank"] = None + nodes_by_name["R"].props["date"] = None + nodes_by_name["R"].props["popularity"] = None + output_mysqlexport(t, str(tmp_path)) + + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + a = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "A") + for col in ("extinction_date", "popularity", "popularity_rank"): + assert a[LEAF_HEADER.index(col)] == "\\N", col + + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + for col in ("age", "popularity"): + assert root[NODE_HEADER.index(col)] == "\\N", col + + def test_unpopulated_columns_are_backslash_N(self, tmp_path): + # Columns tree_build doesn't (yet) generate are still NULL, not empty + # strings — "price" for leaves, "vern_synth" and the rep/rtr/rpd/iucn* + # summary columns for internal nodes. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + for row in leaves[1:]: + assert row[LEAF_HEADER.index("price")] == "\\N" + + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + for row in nodes[1:]: + assert row[NODE_HEADER.index("vern_synth")] == "\\N" + assert row[NODE_HEADER.index("rep1")] == "\\N" + assert row[NODE_HEADER.index("iucnEX")] == "\\N" + def test_missing_extinction_date_and_popularity_are_backslash_N(self, tmp_path): # Leaf-only props (extinction_date, popularity, popularity_rank) # default to \N when not set. From 2d32f96768865f3cfbaf45f95ef004fdff31c5bd Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 14:49:48 +0000 Subject: [PATCH 59/62] step_output: Temporary parent value, not NULL We don't know the tree serial here, so can't fill it. Using \\N doesn't work, since the column is NOT NULL. Put a placeholder value to get replaced later by the import script. --- oz_tree_build/tree_build/step_output.py | 14 ++++++++++---- tests/test_tree_build_step_output.py | 15 ++++++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/oz_tree_build/tree_build/step_output.py b/oz_tree_build/tree_build/step_output.py index c62d34b3..1959113a 100644 --- a/oz_tree_build/tree_build/step_output.py +++ b/oz_tree_build/tree_build/step_output.py @@ -105,8 +105,11 @@ def output_mysqlexport(tree, out_dir): - The leaf ``name`` column has any trailing ``_ottNNN`` suffix stripped (the OTT is carried separately in its own column). - An internal node's ``date`` prop is exposed via the ``age`` column. - - The root's ``parent`` is ``\\N`` but its ``real_parent`` is the - sentinel ``0``. + - The root has no parent, but the ``parent`` column is NOT NULL, so it + gets the placeholder ``-999`` for the import script to replace. Its + ``real_parent`` is the sentinel ``0``. + - Every leaf is required to have a parent; a parentless leaf raises + ``ValueError``. """ with ( @@ -181,9 +184,11 @@ def output_mysqlexport(tree, out_dir): real_parent_id = real_parent.props["id"] if node.is_leaf: + if not node.parent: + raise ValueError(f"Leaf {node} has no parent") leaf_csv.writerow( [ - node.parent.props["id"] if node.parent else "\\N", # "parent" + node.parent.props["id"], # "parent" # TODO: negative real_parent ids if this is a polytomy real_parent_id, node_name_without_ott(node), @@ -208,7 +213,8 @@ def output_mysqlexport(tree, out_dir): else: node_csv.writerow( [ - node.parent.props["id"] if node.parent else "\\N", # "parent" + # NB: This has to be NOT NULL, but root doesn't have a parent. Bodge temporary value + node.parent.props["id"] if node.parent else "-999", # "parent" real_parent_id, node.props["node_rgt"], node.props["leaf_lft"], diff --git a/tests/test_tree_build_step_output.py b/tests/test_tree_build_step_output.py index b6e38cee..4d37dbeb 100644 --- a/tests/test_tree_build_step_output.py +++ b/tests/test_tree_build_step_output.py @@ -316,14 +316,23 @@ def test_leaf_name_strips_ott_suffix(self, tmp_path): names = [r[LEAF_HEADER.index("name")] for r in leaves[1:]] assert names == ["A", "B"] - def test_root_parent_field_is_backslash_N(self, tmp_path): - # Root has no parent → "parent" column is \N (MySQL NULL marker). + def test_root_parent_field_is_placeholder(self, tmp_path): + # Root has no parent, but "parent" is NOT NULL in the DB, so it gets a + # placeholder for the import script to replace rather than \N. t = ete4.Tree("(A,B)R;", parser=1) _prep(t) output_mysqlexport(t, str(tmp_path)) nodes = _read_csv(tmp_path, "ordered_nodes.csv") root_row = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") - assert root_row[NODE_HEADER.index("parent")] == "\\N" + assert root_row[NODE_HEADER.index("parent")] == "-999" + + def test_parentless_leaf_is_rejected(self, tmp_path): + # A single-leaf tree has a leaf at the root. There's no placeholder for + # this case, so it's an error rather than a bad "parent" value. + t = ete4.Tree("A;", parser=1) + _prep(t) + with pytest.raises(ValueError, match="no parent"): + output_mysqlexport(t, str(tmp_path)) def test_root_real_parent_is_zero(self, tmp_path): # Root has no parent, so real_parent is the sentinel 0. From 754077b3434f3159eddff3795b2a4148432c44ef Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 15:51:23 +0000 Subject: [PATCH 60/62] step_treeprop: Sync GEOLOGICAL_PERIODS colours --- oz_tree_build/tree_build/step_treeprop.py | 78 +++++++++++------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/oz_tree_build/tree_build/step_treeprop.py b/oz_tree_build/tree_build/step_treeprop.py index 699b7a0e..70962ed6 100644 --- a/oz_tree_build/tree_build/step_treeprop.py +++ b/oz_tree_build/tree_build/step_treeprop.py @@ -6,61 +6,61 @@ # Sourced from https://stratigraphy.org/supplementary#data # fmt: off GEOLOGICAL_PERIODS = [ - {"eon": "Unknown","era": "Unknown","period": "Unknown","epoch": "Unknown","short_text": "Unknown","long_text": "Unknown","color": "#1A1A1A","mya_start": -1e9,"number": 1}, # noqa: E501 + {"eon": "Unknown","era": "Unknown","period": "Unknown","epoch": "Unknown","short_text": "Unknown","long_text": "Unknown","color": "#999999","mya_start": -1e9,"number": 0}, # noqa: E501 {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Anthropocene","short_text": "Anthropocene","long_text": "Anthropocene mass extinction event","color": "#1A1A1A","mya_start": 0.000246,"number": 1}, # noqa: E501 - {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Holocene","short_text": "Holocene Epoch","long_text": "Holocene Epoch, Quaternary Period","color": "#7A7A72","mya_start": 0.0117,"number": 2}, # noqa: E501 - {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Pleistocene","short_text": "Pleistocene Epoch","long_text": "Pleistocene Epoch, Quaternary Period","color": "#7A7A72","mya_start": 2.58,"number": 3}, # noqa: E501 - {"eon": "Phanerozoic","era": "Cenozoic","period": "Neogene","epoch": "Pliocene","short_text": "Neogene Period","long_text": "Pliocene Epoch, Neogene Period","color": "#A08050","mya_start": 5.333,"number": 4}, # noqa: E501 - {"eon": "Phanerozoic","era": "Cenozoic","period": "Neogene","epoch": "Miocene","short_text": "Neogene Period","long_text": "Miocene Epoch, Neogene Period","color": "#A08050","mya_start": 23.04,"number": 5}, # noqa: E501 - {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Oligocene","short_text": "Paleogene Period","long_text": "Oligocene Epoch, Paleogene Period","color": "#8A6A3A","mya_start": 33.9,"number": 6}, # noqa: E501 - {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Eocene","short_text": "Paleogene Period","long_text": "Eocene Epoch, Paleogene Period","color": "#8A6A3A","mya_start": 56,"number": 7}, # noqa: E501 - {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Paleocene","short_text": "Paleogene Period","long_text": "Paleocene Epoch, Paleogene Period","color": "#8A6A3A","mya_start": 66,"number": 8}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Holocene","short_text": "Holocene Epoch","long_text": "Holocene Epoch, Quaternary Period","color": "#74746C","mya_start": 0.0117,"number": 2}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Pleistocene","short_text": "Pleistocene Epoch","long_text": "Pleistocene Epoch, Quaternary Period","color": "#818179","mya_start": 2.58,"number": 3}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Neogene","epoch": "Pliocene","short_text": "Neogene Period","long_text": "Pliocene Epoch, Neogene Period","color": "#9B7B4D","mya_start": 5.333,"number": 4}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Neogene","epoch": "Miocene","short_text": "Neogene Period","long_text": "Miocene Epoch, Neogene Period","color": "#A98555","mya_start": 23.04,"number": 5}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Oligocene","short_text": "Paleogene Period","long_text": "Oligocene Epoch, Paleogene Period","color": "#826333","mya_start": 33.9,"number": 6}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Eocene","short_text": "Paleogene Period","long_text": "Eocene Epoch, Paleogene Period","color": "#8C6C3D","mya_start": 56,"number": 7}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Paleocene","short_text": "Paleogene Period","long_text": "Paleocene Epoch, Paleogene Period","color": "#967447","mya_start": 66,"number": 8}, # noqa: E501 {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Upper","short_text": "Cretaceous–Paleogene extinction","long_text": "Cretaceous–Paleogene extinction","color": "#1A1A1A","mya_start": 65.9999,"number": 9}, # noqa: E501 RUF001 - {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Upper","short_text": "Cretaceous Period","long_text": "(Upper) Cretaceous Period","color": "#6C7A4D","mya_start": 100.5,"number": 10}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Upper","short_text": "Cretaceous Period","long_text": "(Upper) Cretaceous Period","color": "#657347","mya_start": 100.5,"number": 10}, # noqa: E501 {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Lower","short_text": "Cretaceous Period","long_text": "(Lower) Cretaceous Period","color": "#6C7A4D","mya_start": 143.1,"number": 11}, # noqa: E501 - {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Upper","short_text": "Jurassic Period","long_text": "(Upper) Jurassic Period","color": "#3E5B3A","mya_start": 161.5,"number": 12}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Upper","short_text": "Jurassic Period","long_text": "(Upper) Jurassic Period","color": "#385536","mya_start": 161.5,"number": 12}, # noqa: E501 {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Middle","short_text": "Jurassic Period","long_text": "(Middle) Jurassic Period","color": "#3E5B3A","mya_start": 174.7,"number": 13}, # noqa: E501 - {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Lower","short_text": "Jurassic Period","long_text": "(Lower) Jurassic Period","color": "#3E5B3A","mya_start": 201.4,"number": 14}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Lower","short_text": "Jurassic Period","long_text": "(Lower) Jurassic Period","color": "#45613F","mya_start": 201.4,"number": 14}, # noqa: E501 {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Upper","short_text": "Triassic–Jurassic extinction event","long_text": "Triassic–Jurassic extinction event","color": "#1A1A1A","mya_start": 201.3,"number": 15}, # noqa: E501 RUF001 - {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Upper","short_text": "Triassic Period","long_text": "(Upper) Triassic Period","color": "#7A4A2B","mya_start": 237,"number": 16}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Upper","short_text": "Triassic Period","long_text": "(Upper) Triassic Period","color": "#704329","mya_start": 237,"number": 16}, # noqa: E501 {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Middle","short_text": "Triassic Period","long_text": "(Middle) Triassic Period","color": "#7A4A2B","mya_start": 246.7,"number": 17}, # noqa: E501 {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Lower","short_text": "Triassic Period","long_text": "(Lower) Triassic Period","color": "#7A4A2B","mya_start": 251.902,"number": 18}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Lopingian","short_text": "Permian–Triassic extinction event","long_text": "Permian–Triassic extinction event ""Great dying""","color": "#1A1A1A","mya_start": 252,"number": 19}, # noqa: E501 RUF001 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Lopingian","short_text": "Permian Period","long_text": "Lopingian Epoch, Permian Period","color": "#6A5D35","mya_start": 259.51,"number": 20}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Lopingian","short_text": "Permian–Triassic extinction event","long_text": "Permian–Triassic extinction event \"Great dying\"","color": "#1A1A1A","mya_start": 252,"number": 19}, # noqa: E501 RUF001 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Lopingian","short_text": "Permian Period","long_text": "Lopingian Epoch, Permian Period","color": "#62572F","mya_start": 259.51,"number": 20}, # noqa: E501 {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Guadalupian","short_text": "Permian Period","long_text": "Guadalupian Epoch, Permian Period","color": "#6A5D35","mya_start": 274.4,"number": 21}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Cisuralian","short_text": "Permian Period","long_text": "Cisuralian Epoch, Permian Period","color": "#6A5D35","mya_start": 298.9,"number": 22}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Carboniferous","epoch": "Pennsylvanian","short_text": "Carboniferous Period","long_text": "Pennsylvanian Epoch, Carboniferous Period","color": "#2F4F2F","mya_start": 323.4,"number": 23}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Cisuralian","short_text": "Permian Period","long_text": "Cisuralian Epoch, Permian Period","color": "#71643A","mya_start": 298.9,"number": 22}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Carboniferous","epoch": "Pennsylvanian","short_text": "Carboniferous Period","long_text": "Pennsylvanian Epoch, Carboniferous Period","color": "#294A2B","mya_start": 323.4,"number": 23}, # noqa: E501 {"eon": "Phanerozoic","era": "Paleozoic","period": "Carboniferous","epoch": "Mississippian","short_text": "Carboniferous Period","long_text": "Mississippian Epoch, Carboniferous Period","color": "#2F4F2F","mya_start": 358.86,"number": 24}, # noqa: E501 {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Upper","short_text": "Late Devonian mass extinction","long_text": "Late Devonian mass extinction","color": "#1A1A1A","mya_start": 372,"number": 25}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Upper","short_text": "Devonian Period","long_text": "(Upper Epoch, Devonian Period","color": "#6B6B2F","mya_start": 382.31,"number": 26}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Upper","short_text": "Devonian Period","long_text": "(Upper Epoch, Devonian Period","color": "#62632B","mya_start": 382.31,"number": 26}, # noqa: E501 {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Middle","short_text": "Devonian Period","long_text": "(Middle) Devonian Period","color": "#6B6B2F","mya_start": 393.47,"number": 27}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Lower","short_text": "Devonian Period","long_text": "(Lower) Devonian Period","color": "#6B6B2F","mya_start": 419.62,"number": 28}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Pridoli","short_text": "Silurian Period","long_text": "Pridoli Epoch, Silurian Period","color": "#5A6A3A","mya_start": 422.7,"number": 29}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Lower","short_text": "Devonian Period","long_text": "(Lower) Devonian Period","color": "#727333","mya_start": 419.62,"number": 28}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Pridoli","short_text": "Silurian Period","long_text": "Pridoli Epoch, Silurian Period","color": "#536534","mya_start": 422.7,"number": 29}, # noqa: E501 {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Ludlow","short_text": "Silurian Period","long_text": "Ludlow Epoch, Silurian Period","color": "#5A6A3A","mya_start": 426.7,"number": 30}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Wenlock","short_text": "Silurian Period","long_text": "Wenlock Epoch, Silurian Period","color": "#5A6A3A","mya_start": 432.9,"number": 31}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Llandovery","short_text": "Silurian Period","long_text": "Llandovery Epoch, Silurian Period","color": "#5A6A3A","mya_start": 443.1,"number": 32}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Wenlock","short_text": "Silurian Period","long_text": "Wenlock Epoch, Silurian Period","color": "#60713D","mya_start": 432.9,"number": 31}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Llandovery","short_text": "Silurian Period","long_text": "Llandovery Epoch, Silurian Period","color": "#566F3A","mya_start": 443.1,"number": 32}, # noqa: E501 {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Upper","short_text": "Late Ordovician mass extinction","long_text": "Late Ordovician mass extinction","color": "#1A1A1A","mya_start": 445,"number": 33}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Upper","short_text": "Ordovician Period","long_text": "(Upper) Ordovician Period","color": "#486B4A","mya_start": 458.2,"number": 34}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Upper","short_text": "Ordovician Period","long_text": "(Upper) Ordovician Period","color": "#426846","mya_start": 458.2,"number": 34}, # noqa: E501 {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Middle","short_text": "Ordovician Period","long_text": "(Middle) Ordovician Period","color": "#486B4A","mya_start": 471.3,"number": 35}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Lower","short_text": "Ordovician Period","long_text": "(Lower) Ordovician Period","color": "#486B4A","mya_start": 486.85,"number": 36}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Furongian","short_text": "Cambrian Period","long_text": "Furongian Epoch, Cambrian Period","color": "#2E5D50","mya_start": 497,"number": 37}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Lower","short_text": "Ordovician Period","long_text": "(Lower) Ordovician Period","color": "#4E704C","mya_start": 486.85,"number": 36}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Furongian","short_text": "Cambrian Period","long_text": "Furongian Epoch, Cambrian Period","color": "#28594D","mya_start": 497,"number": 37}, # noqa: E501 {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Miaolingian","short_text": "Cambrian Period","long_text": "Miaolingian Epoch, Cambrian Period","color": "#2E5D50","mya_start": 506.5,"number": 38}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Series 2","short_text": "Cambrian Period","long_text": "Series 2 Epoch, Cambrian Period","color": "#2E5D50","mya_start": 521,"number": 39}, # noqa: E501 - {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Terreneuvian","short_text": "Cambrian Period","long_text": "Terreneuvian Epoch, Cambrian Period","color": "#2E5D50","mya_start": 538.8,"number": 40}, # noqa: E501 - {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Ediacaran","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Ediacaran Period, Neo-proterozoic Era","color": "#3B4A52","mya_start": 635,"number": 41}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Series 2","short_text": "Cambrian Period","long_text": "Series 2 Epoch, Cambrian Period","color": "#336353","mya_start": 521,"number": 39}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Terreneuvian","short_text": "Cambrian Period","long_text": "Terreneuvian Epoch, Cambrian Period","color": "#2A6255","mya_start": 538.8,"number": 40}, # noqa: E501 + {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Ediacaran","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Ediacaran Period, Neo-proterozoic Era","color": "#35464F","mya_start": 635,"number": 41}, # noqa: E501 {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Cryogenian","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Cryogenian Period, Neo-proterozoic Era","color": "#3B4A52","mya_start": 720,"number": 42}, # noqa: E501 - {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Tonian","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Tonian Period, Neo-proterozoic Era","color": "#3B4A52","mya_start": 1000,"number": 43}, # noqa: E501 - {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Stenian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Stenian Period, Meso-proterozoic Era","color": "#3B4A52","mya_start": 1200,"number": 44}, # noqa: E501 - {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Ectasian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Ectasian Period, Meso-proterozoic Era","color": "#3B4A52","mya_start": 1400,"number": 45}, # noqa: E501 - {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Calymmian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Calymmian Period, Meso-proterozoic Era","color": "#3B4A52","mya_start": 1600,"number": 46}, # noqa: E501 - {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Statherian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Statherian Period, Paleo-proterozoic Era","color": "#3B4A52","mya_start": 1800,"number": 47}, # noqa: E501 - {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Orosirian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Orosirian Period, Paleo-proterozoic Era","color": "#3B4A52","mya_start": 2050,"number": 48}, # noqa: E501 - {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Rhyacian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Rhyacian Period, Paleo-proterozoic Era","color": "#3B4A52","mya_start": 2300,"number": 49}, # noqa: E501 - {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Siderian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Siderian Period, Paleo-proterozoic Era","color": "#3B4A52","mya_start": 2500,"number": 50}, # noqa: E501 - {"eon": "Archean","era": "Neo-Archean","period": "-","epoch": "-","short_text": "Neo-Archean Era","long_text": "Neo-Archean Era","color": "#2C2A28","mya_start": 2800,"number": 51}, # noqa: E501 - {"eon": "Archean","era": "Meso-Archean","period": "-","epoch": "-","short_text": "Meso-Archean Era","long_text": "Meso-Archean Era","color": "#2C2A28","mya_start": 3200,"number": 52}, # noqa: E501 - {"eon": "Archean","era": "Paleo-Archean","period": "-","epoch": "-","short_text": "Paleo-Archean Era","long_text": "Paleo-Archean Era","color": "#2C2A28","mya_start": 3600,"number": 53}, # noqa: E501 - {"eon": "Archean","era": "Eo-Archean","period": "-","epoch": "-","short_text": "Eo-Archean Era","long_text": "Eo-Archean Era","color": "#2C2A28","mya_start": 4031,"number": 54}, # noqa: E501 + {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Tonian","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Tonian Period, Neo-proterozoic Era","color": "#40505A","mya_start": 1000,"number": 43}, # noqa: E501 + {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Stenian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Stenian Period, Meso-proterozoic Era","color": "#3A4D58","mya_start": 1200,"number": 44}, # noqa: E501 + {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Ectasian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Ectasian Period, Meso-proterozoic Era","color": "#45525A","mya_start": 1400,"number": 45}, # noqa: E501 + {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Calymmian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Calymmian Period, Meso-proterozoic Era","color": "#394953","mya_start": 1600,"number": 46}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Statherian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Statherian Period, Paleo-proterozoic Era","color": "#46535A","mya_start": 1800,"number": 47}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Orosirian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Orosirian Period, Paleo-proterozoic Era","color": "#33444D","mya_start": 2050,"number": 48}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Rhyacian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Rhyacian Period, Paleo-proterozoic Era","color": "#3E4B55","mya_start": 2300,"number": 49}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Siderian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Siderian Period, Paleo-proterozoic Era","color": "#2F414A","mya_start": 2500,"number": 50}, # noqa: E501 + {"eon": "Archean","era": "Neo-Archean","period": "-","epoch": "-","short_text": "Neo-Archean Era","long_text": "Neo-Archean Era","color": "#302D2A","mya_start": 2800,"number": 51}, # noqa: E501 + {"eon": "Archean","era": "Meso-Archean","period": "-","epoch": "-","short_text": "Meso-Archean Era","long_text": "Meso-Archean Era","color": "#35312D","mya_start": 3200,"number": 52}, # noqa: E501 + {"eon": "Archean","era": "Paleo-Archean","period": "-","epoch": "-","short_text": "Paleo-Archean Era","long_text": "Paleo-Archean Era","color": "#2B2927","mya_start": 3600,"number": 53}, # noqa: E501 + {"eon": "Archean","era": "Eo-Archean","period": "-","epoch": "-","short_text": "Eo-Archean Era","long_text": "Eo-Archean Era","color": "#272624","mya_start": 4031,"number": 54}, # noqa: E501 {"eon": "Hadean","era": "-","period": "-","epoch": "-","short_text": "Hadean Eon","long_text": "Hadean Eon","color": "#1A1A1A","mya_start": 4567,"number": 55}, # noqa: E501 ] # fmt: on From d0463f43f59aea65cc480d1fd1586ba19ae51b61 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 15:53:04 +0000 Subject: [PATCH 61/62] dvc.lock: Update with replacement pipeline --- dvc.lock | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 128 insertions(+), 12 deletions(-) diff --git a/dvc.lock b/dvc.lock index 9652f0d4..8a1e68ae 100644 --- a/dvc.lock +++ b/dvc.lock @@ -73,15 +73,18 @@ stages: md5: f7437ba9e2a89ff658749e434a38b39a size: 21507159 download_opentree: - cmd: download_opentree --version v16.1 --output-dir data/OpenTree + cmd: + - rm -rf data/OpenTree/v16.1/ + - mkdir -p data/OpenTree/v16.1/ + - .venv/bin/download_opentree --version v16.1 --output-dir data/OpenTree params: params.yaml: ot_version: v16.1 outs: - path: data/OpenTree/v16.1/ hash: md5 - md5: 87ff995e9d5028efc185857f34448746.dir - size: 587064765 + md5: 8c9c0fadda5ac919e5573462208a5c8f.dir + size: 564222021 nfiles: 3 add_ott_numbers_to_trees: cmd: @@ -94,9 +97,9 @@ stages: deps: - path: data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/ hash: md5 - md5: c3c1ebf2453c636e3ffdfcef58722d9c.dir - size: 1231291 - nfiles: 56 + md5: 4097a3d5af5e05a587882129df7882df.dir + size: 1271857 + nfiles: 45 params: params.yaml: ot_version: v16.1 @@ -104,9 +107,9 @@ stages: outs: - path: data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1/ hash: md5 - md5: c12514e92740949250ecdb4375d6c360.dir - size: 1534814 - nfiles: 55 + md5: 4d9484418e30d363e0c33ffb2e12e5fd.dir + size: 1586233 + nfiles: 44 get_open_trees_from_one_zoom: cmd: - cd data/OZTreeBuild/AllLife && get_open_trees_from_one_zoom @@ -280,9 +283,122 @@ stages: md5: 1018a5c664d01747fdd7e218190cb4ac size: 72 download_node_ages: - cmd: download_node_ages data/node_ages.json + cmd: .venv/bin/download_node_ages data/node_ages.json outs: - path: data/node_ages.json hash: md5 - md5: 2440f9bb2139301bf352797a9802c2d9 - size: 14044769 + md5: 02b43f39966ac4c461bed8f1fa3247a9 + size: 13157027 + date_tree: + cmd: + - rm -rf data/dated_tree + - mkdir -p data/dated_tree + - .venv/bin/date_tree --output_folder=data/dated_tree --date_cache + data/node_ages.json --annotations data/OpenTree/v16.1/annotations.json + --taxonomy data/OpenTree/v16.1/taxonomy.tsv --supertree + data/OpenTree/v16.1/labelled_supertree_ottnames.tre + deps: + - path: data/node_ages.json + hash: md5 + md5: 02b43f39966ac4c461bed8f1fa3247a9 + size: 13157027 + params: + params.yaml: + ot_version: v16.1 + outs: + - path: data/dated_tree/dated_tree_pre.tre + hash: md5 + md5: 531c294f7e80f3d588369af73047ddfe + size: 134355939 + taxon_map: + cmd: .venv/bin/taxon_map --OpenTreeTaxonomy data/OpenTree/v16.1/taxonomy.tsv + --wikidataDumpFile data/filtered/OneZoom_latest-all.json + --wikipediaSQLDumpFile data/filtered/OneZoom_enwiki-latest-page.sql + --wikipedia_totals_bz2_pageviews data/filtered/pageviews/ --EOLidentifiers + data/filtered/OneZoom_provider_ids.csv --extra_source_file + data/OZTreeBuild/AllLife/BespokeTree/SupplementaryTaxonomy.tsv -o + data/taxon_map.csv + deps: + - path: data/OZTreeBuild/AllLife/BespokeTree/SupplementaryTaxonomy.tsv + hash: md5 + md5: 8e861649388bf88595b93c0199f2cc3a + size: 312 + isexec: true + - path: data/OpenTree/v16.1/taxonomy.tsv + hash: md5 + md5: d7a58eaaf132522b89a506e96ca5098f + size: 417054016 + - path: data/filtered/OneZoom_enwiki-latest-page.sql + hash: md5 + md5: f7437ba9e2a89ff658749e434a38b39a + size: 21507159 + - path: data/filtered/OneZoom_latest-all.json + hash: md5 + md5: e6f69def9d6fa2bb4b90060509c079bb + size: 1567956473 + - path: data/filtered/OneZoom_provider_ids.csv + hash: md5 + md5: f7c9bb8374957c07168bec36d6591347 + size: 221682224 + - path: data/filtered/pageviews/ + hash: md5 + md5: 701c03fc874aa8a4d5578a6a74267bee.dir + size: 128216689 + nfiles: 13 + params: + params.yaml: + ot_version: v16.1 + oz_tree: AllLife + outs: + - path: data/taxon_map.csv + hash: md5 + md5: 08e16ef58049999ecb1608be819976d4 + size: 230041544 + tree_build: + cmd: + - rm -rf data/out + - mkdir -p data/out + - .venv/bin/tree_build --bespoke_dir + data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1/ --orphan_dir + data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/ --opentree + data/dated_tree/dated_tree_pre.tre --taxon_map data/taxon_map.csv + --exclude Archosauria_ott335588 Dinosauria_ott90215 --out_dir data/out + deps: + - path: data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1/ + hash: md5 + md5: 4d9484418e30d363e0c33ffb2e12e5fd.dir + size: 1586233 + nfiles: 44 + - path: data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/ + hash: md5 + md5: ba7be58a908cdd00297088d0227ffc54.dir + size: 103205 + nfiles: 4 + - path: data/dated_tree/dated_tree_pre.tre + hash: md5 + md5: 531c294f7e80f3d588369af73047ddfe + size: 134355939 + - path: data/taxon_map.csv + hash: md5 + md5: 08e16ef58049999ecb1608be819976d4 + size: 230041544 + params: + params.yaml: + ot_version: v16.1 + oz_tree: AllLife + outs: + - path: data/out + hash: md5 + md5: 8681e904a6f6cc834f85b981bcdcb78c.dir + size: 774309948 + nfiles: 7 + versioned_outputs: + cmd: + - mkdir -p data/out_versioned + - .venv/bin/versioned_outputs --out_dir data/out_versioned data/out/* -vv + deps: + - path: data/out/ + hash: md5 + md5: 8681e904a6f6cc834f85b981bcdcb78c.dir + size: 774309948 + nfiles: 7 From a0553b8b482163b5f370b75d03d7e0ff6c7fbab2 Mon Sep 17 00:00:00 2001 From: Jamie Lentin Date: Fri, 28 Aug 2026 16:05:08 +0000 Subject: [PATCH 62/62] .github/workflows/dvc.yml: make_js_treefiles -> versioned_outputs --- .github/workflows/dvc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dvc.yml b/.github/workflows/dvc.yml index 37d98b78..17c8fadc 100644 --- a/.github/workflows/dvc.yml +++ b/.github/workflows/dvc.yml @@ -45,6 +45,6 @@ jobs: [ -n "${{ secrets.DVC_READONLY_SECRET_ACCESS_KEY }}" ] || { echo "DVC_READONLY_SECRET_ACCESS_KEY secret is not set"; exit 1; } dvc remote modify onezoom-r2 access_key_id ${{ secrets.DVC_READONLY_ACCESS_KEY_ID }} dvc remote modify onezoom-r2 secret_access_key ${{ secrets.DVC_READONLY_SECRET_ACCESS_KEY }} - dvc freeze make_js_treefiles + dvc freeze versioned_outputs dvc repro --allow-missing --dry | tee /dev/stderr | grep -q "Data and pipelines are up to date." if dvc data status --not-in-remote | grep -q "Not in remote"; then exit 1; fi