diff --git a/.github/workflows/dvc.yml b/.github/workflows/dvc.yml index 9c6b8bde..17c8fadc 100644 --- a/.github/workflows/dvc.yml +++ b/.github/workflows/dvc.yml @@ -26,6 +26,7 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install '.[dev]' + python3 -m pip install -r requirements.txt - name: Checkout merge ref if: github.event_name == 'pull_request_target' uses: actions/checkout@v7 @@ -44,6 +45,6 @@ jobs: [ -n "${{ secrets.DVC_READONLY_SECRET_ACCESS_KEY }}" ] || { echo "DVC_READONLY_SECRET_ACCESS_KEY secret is not set"; exit 1; } dvc remote modify onezoom-r2 access_key_id ${{ secrets.DVC_READONLY_ACCESS_KEY_ID }} dvc remote modify onezoom-r2 secret_access_key ${{ secrets.DVC_READONLY_SECRET_ACCESS_KEY }} - dvc freeze make_js_treefiles + dvc freeze versioned_outputs dvc repro --allow-missing --dry | tee /dev/stderr | grep -q "Data and pipelines are up to date." if dvc data status --not-in-remote | grep -q "Not in remote"; then exit 1; fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index df9233ac..d203e83a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,7 +40,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - python3 -m pip install '.[dev]' + python3 -m pip install -r requirements.txt - name: Test with pytest run: | python3 -m pytest tests --conf-file tests/appconfig.ini diff --git a/README.markdown b/README.markdown index 08109ef1..016532f9 100644 --- a/README.markdown +++ b/README.markdown @@ -13,7 +13,7 @@ The first step to using this repo is to create a Python virtual environment and source .venv/bin/activate # Install it - pip install -e '.[dev]' + pip install -r requirements.txt # Set up git hooks including linting and DVC pre-commit install --hook-type pre-push --hook-type post-checkout --hook-type pre-commit diff --git a/data/.gitignore b/data/.gitignore index 0992b36e..700db5ee 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1,2 +1,7 @@ /js_output /output_files +/node_ages.json +/dated_tree/ +/taxon_map.csv +/out +/out_versioned diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Batoids_Aschliman2012.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Batoids_Aschliman2012.PHY deleted file mode 100755 index e48d543b..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Batoids_Aschliman2012.PHY +++ /dev/null @@ -1,14 +0,0 @@ -[Batoids & dates from: -Body plan convergence in the evolution of skates and rays (Chondrichthyes: Batoidea) -N.C. Aschliman et al. / Molecular Phylogenetics and Evolution 63 (2012) 28–42 https://doi.org/10.1016/j.ympev.2011.12.012 with basal divergence set to 300Ma, and ray/other divergence at 200Ma, i.e. - -(Squalus:300.0,((Rajidae_ott978560@:78.3688,(Anacanthobatidae_ott802681@:69.5035,Arhynchobatidae_ott406376@:69.5035):8.8652)Rajiformes:121.6312,(((Platyrhinodis_ott1032962@:63.8298,Platyrhina_ott456578@:63.8298)Platyrhinidae:100.3546,((Torpedinidae_ott553102,Hypnidae_ott356637@):72.695,(Narcinidae_ott818997@:63.8298,Narkidae_ott932203@:63.8298):8.8652)Torpediniformes:91.4894):13.4752,( - -From Aschliman fig 1 I split Rhiniformes into 2 groups, and move Zanobatidae (which in OpenTree v5 is in this Pristiformes/Rhiniformes group) outside both -(Zapteryx_ott356651@:79.078,Trygonorrhina_ott1041304@:79.078)Rhiniformes1_:79.7872,(Rhiniformes2__ott356644~-456585-356651-1041304@:152.4823,(Zanobatidae_ott456585@:142.1986, - - -(Hexatrygonidae_ott456584@:92.9078,Myliobatiformes_minus_Hexatrygon_ott~706576-456584@:92.9078)Myliobatiformes:49.2908):10.2837):6.383):18.7943):22.3404):100.0); - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((Rajidae_ott978560@:78.3688,(Anacanthobatidae_ott802681@:69.5035,Arhynchobatidae_ott406376@:69.5035):8.8652)Rajiformes:121.6312,(((Platyrhinodis_ott1032962@:63.8298,Platyrhina_ott456578@:63.8298)Platyrhinidae:100.3546,((Torpedinidae_ott553102@,Hypnidae_ott356637@):72.695,(Narcinidae_ott818997@:63.8298,Narkidae_ott932203@:63.8298):8.8652)Torpediniformes:91.4894):13.4752,((Zapteryx_ott356651@:79.078,Trygonorrhina_ott1041304@:79.078)Rhiniformes1_:79.7872,(Rhiniformes2__ott356644~-456585-356651-1041304@:152.4823,(Zanobatidae_ott456585@:142.1986,(Hexatrygonidae_ott456584@:92.9078,Myliobatiformes_minus_Hexatrygon__ott~706576-456584@:92.9078)Myliobatiformes:49.2908):10.2837):6.383):18.7943):22.3404)Batoidea; diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Renz2013.phy b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Renz2013.phy deleted file mode 100755 index b3acaa90..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Renz2013.phy +++ /dev/null @@ -1,7 +0,0 @@ -[Chondrichthyes tree: - -Basal split from Revealing Less Derived Nature of Cartilaginous Fish Genomes with Their Evolutionary Time Scale Inferred with Nuclear Genes ( https://doi.org/10.1371/journal.pone.0066400 ), dates need setting so that -(HOLOCEPHALI@:420,(BATOIDEA@:300,SELACHIMORPHA@:300):120); - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(HOLOCEPHALI@,(BATOIDEA@,SELACHII@:300):120)CHONDRICHTHYES; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Stein2018.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Stein2018.PHY new file mode 100644 index 00000000..e515ec3a --- /dev/null +++ b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Chondrichthyes_Stein2018.PHY @@ -0,0 +1,49 @@ +[Chondrichthyes (sharks, rays and chimaeras), 1189 spp., crown at 375.51Mya. + +Sourced from https://vertlife.org/data/sharks/ , i.e. Stein et al. (2018) "Global priorities for +conserving the evolutionary history of sharks, rays and chimaeras", +https://www.nature.com/articles/s41559-017-0448-4 + +My method (for records) - read in the distribution of 10,000 trees, calculate phylogenetic +diversity of each, find out which tree yields the median value, interpret this as a median tree +to move forward with, output as newick. NB: R's double-square-bracket list indexing is written +with curly braces below, since a closing square bracket would terminate this newick comment: + +rm(list=ls()) + +# install useful packages +require('ape') +require('caper') + +# Read all trees +trees <- ape::read.tree(file='Chond.10Cal.10kTreeSet.tre') + +# Calculate PD for each tree +pd <- sapply(trees, function(tr) sum(tr$edge.length)) + +# Median PD value +median_pd <- median(pd) + +# Tree closest to the median +median_idx <- which.min(abs(pd - median_pd)) +median_tree <- trees{{median_idx}} + +# Newick string +newick <- write.tree(median_tree) +writeLines(newick, "median_PD_shark_tree.txt") +Three tips of the published 1192 have been dropped, as WoRMS and GBIF agree each is a junior +synonym of a species that is *also* already a tip in this tree, i.e. the source tree scores one +taxon twice. In each case the two tips are not sisters, so this is not a resolution question: + Raja atriventralis = Okamejei kenojei (ott167107, other tip 68.2Ma away) + Torpedo zugmayeri = Torpedo sinuspersici (ott254411, other tip 12.0Ma away) + Rhinoptera sewelli = Rhinoptera jayakari (ott801133, other tip 53.0Ma away) +Each was removed by splicing its sibling onto its grandparent with the two branch lengths summed, +so every surviving tip keeps its exact original depth. + +Five further tips have no OTT in OpenTree v16.1, so will carry no metadata. Narcine nigra +(gbif:9209986) and Glaucostegus spinosus (worms:1577337, gbif:11555527) are valid species merely +absent from OpenTree; Rajella alia, Narcine bicolor and Rhinoptera hainanensis have no +species-level record in either WoRMS or the GBIF backbone. + +#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] +(((Callorhinchus_callorynchus:8.358674601,(Callorhinchus_milii:6.062028924,Callorhinchus_capensis:6.062028924):2.296645677):183.545478,(((Neoharriotta_pinnata:23.47814296,(Neoharriotta_pumila:5.362641919,Neoharriotta_carri:5.362641919):18.11550104):98.09750975,((Harriotta_raleighana:2.167434783,Harriotta_haeckeli:2.167434783):45.49684829,(Rhinochimaera_pacifica:28.17523151,(Rhinochimaera_africana:10.67730884,Rhinochimaera_atlantica:10.67730884):17.49792266):19.48905057):73.91136963):29.19241684,((((Hydrolagus_mitsukurii:7.770233588,Hydrolagus_africanus:7.770233588):63.10423394,((Hydrolagus_matallanasi:8.086410705,Hydrolagus_pallidus:8.086410705):41.79791888,(((Hydrolagus_alberti:37.29381117,Hydrolagus_alphus:37.29381117):1.001440697,Chimaera_bahamaensis:38.29525187):4.716001781,(Chimaera_jordani:31.28687314,Chimaera_cubana:31.28687314):11.7243805):6.873075936):20.99013895):22.57191234,((Chimaera_phantasma:56.11744559,((Hydrolagus_lemures:0.7134329691,Hydrolagus_ogilbyi:0.7134329691):3.531579029,Chimaera_owstoni:4.245011998):51.87243359):36.52182889,((Hydrolagus_melanophasma:24.11616128,((Hydrolagus_macrophthalmus:0.6714718955,Hydrolagus_purpurescens:0.6714718955):13.77257212,Hydrolagus_marmoratus:14.44404401):9.672118271):33.60542615,((((Hydrolagus_trolli:4.335658096,(Hydrolagus_affinis:2.263558323,Hydrolagus_mirabilis:2.263558323):2.072099773):25.16991186,Hydrolagus_novaezealandiae:29.50556995):10.64297717,(Chimaera_macrospina:0.2239522907,Chimaera_monstrosa:0.2239512907):39.92459584):10.37280832,((Chimaera_fulva:7.366992129,Hydrolagus_lusitanicus:7.366992129):35.56949083,((Chimaera_opalescens:21.28026693,(Chimaera_lignaria:0.3028240304,Chimaera_notafricana:0.3028240304):20.9774429):13.49399672,(Hydrolagus_mccoskeri:28.45812427,(Chimaera_panthera:24.23156051,Hydrolagus_bemisi:24.23156051):4.226563753):6.316139386):8.162219305):7.584873489):7.200231985):34.91768605):0.8071063885):22.08228939,((Chimaera_obscura:33.90186107,(Hydrolagus_deani:31.1475967,(Hydrolagus_barbouri:1.228210918,Chimaera_argiloba:1.228210918):29.91938578):2.754264375):20.18437018,(Hydrolagus_colliei:10.37477941,(Hydrolagus_homonycteris:5.829567273,Hydrolagus_eidolon:5.829567273):4.545212136):43.71145184):61.442439):35.23939929):41.13608306):183.6071783,((((((Sinobatis_borneensis:39.17614414,(Sinobatis_filicauda:5.646536957,Sinobatis_bulbicauda:5.646536957):33.52960718):26.84262147,(Sinobatis_melanosoma:53.38554254,Sinobatis_caerulea:53.38554254):12.63322307):65.92837501,(((((Irolita_waitii:7.040727833,Irolita_westraliensis:7.040727833):51.81997166,(((((Brochiraja_vittacauda:10.22961535,Brochiraja_heuresa:10.22961535):3.055060773,Brochiraja_aenigma:13.28467612):14.02245351,Brochiraja_microspinifera:27.30712963):0.6018088931,((Brochiraja_asperula:4.976380405,Brochiraja_spinifera:4.976380405):6.968529938,(Brochiraja_leviveneta:0.6397389676,Brochiraja_albilabiata:0.6397389676):11.30517037):15.96402918):11.49240089,((((Pavoraja_alleni:3.473419762,Pavoraja_umbrosa:3.473419762):10.75805432,(Pavoraja_mosaica:5.08545657,Pavoraja_pseudonitida:5.08545657):9.14601751):2.857309725,(Pavoraja_nitida:7.871088436,Pavoraja_arenaria:7.871088436):9.217695368):20.66914121,(((((Notoraja_sticta:9.925666549,(Notoraja_longiventralis:4.738130075,Notoraja_inusitata:4.738130075):5.187536474):1.648937547,(Notoraja_tobitukai:1.266431615,Notoraja_lira:1.266431615):10.30817248):0.8839082218,Notoraja_alisae:12.45851232):1.011039725,((Notoraja_azurea:8.755770931,(Notoraja_ochroderma:4.410926525,(Notoraja_sapphira:3.841511327,Notoraja_hirticauda:3.841511327):0.5694151984):4.344844406):1.542934181,Notoraja_fijiensis:10.29870511):3.170846932):0.7775969335,(Insentiraja_subtilispinosa:10.96454473,Insentiraja_laxipella:10.96454473):3.282604249):23.51077503):1.643415396):19.45935909):15.8676792,(((Rioraja_agassizii:33.3705862,(Atlantoraja_castelnaui:28.19579882,(Atlantoraja_platana:18.13784829,Atlantoraja_cyclophora:18.13784829):10.05795052):5.174787384):10.87431488,((Bathyraja_andriashevi:19.11773684,((Bathyraja_simoterus:6.560069018,Bathyraja_tzinovskii:6.560069018):6.17497096,Bathyraja_notoroensis:12.73503998):6.382696862):17.45591983,((((Bathyraja_scaphiops:10.47632346,((Bathyraja_brachyurops:0.6969068085,Bathyraja_diplotaenia:0.6969048085):4.88199018,(Bathyraja_panthera:1.92466328,Rhinoraja_magellanica:1.92466528):3.654231709):4.897428468):11.14091704,(Bathyraja_griseocauda:18.89341279,(((Bathyraja_kincaidii:0.1790584979,Bathyraja_violacea:0.1790584979):1.148156186,(Bathyraja_mariposa:0.6875087035,Bathyraja_aguja:0.6875067035):0.6397069803):4.581032185,(Bathyraja_matsubarai:1.845381476,Rhinoraja_taranetzi:1.845382476):4.062864392):12.98516592):2.723828705):6.643969351,(((((Bathyraja_tunae:3.32733383,Bathyraja_peruana:3.32733383):1.577556814,Bathyraja_minispinosa:4.904891643):5.841272448,(Bathyraja_longicauda:5.967620675,(Bathyraja_fedorovi:2.416912331,Bathyraja_cousseauae:2.416913331):3.550708344):4.778542416):5.43735687,((Bathyraja_pallida:10.5806181,((Rhinoraja_murrayi:5.435463142,(Bathyraja_irrasa:2.87594671,Bathyraja_papilionifera:2.87594571):2.559516432):3.756099099,Bathyraja_meridionalis:9.19156224):1.389056861):3.237185484,((Bathyraja_parmifera:3.825072969,(Rhinoraja_odai:1.881583418,Bathyraja_smirnovi:1.881585418):1.943487551):7.712375304,Bathyraja_shuntovi:11.53744727):2.280357312):2.365716377):1.035349203,(Bathyraja_schroederi:8.146900523,(Rhinoraja_albomaculata:2.330425709,Rhinoraja_macloviana:2.330425709):5.816476814):9.071968642):11.04234068):1.07702797,(((Bathyraja_ishiharai:14.84805665,Bathyraja_spinosissima:14.84805665):11.19580746,(((Rhinoraja_longicauda:0.7550978383,Rhinoraja_multispinis:0.7550988383):14.28914483,(Bathyraja_maculata:1.327631923,Bathyraja_lindbergi:1.327631923):13.71661174):5.722606066,((Bathyraja_abyssicola:12.01095504,Bathyraja_trachura:12.01095504):8.381503389,((((Bathyraja_isotrachys:12.72829711,Bathyraja_aleutica:12.72829811):1.213623875,((Bathyraja_eatonii:4.392058529,Bathyraja_spinicauda:4.392059529):5.595300283,Bathyraja_microtrachys:9.987358811):3.954562172):2.950771985,((Bathyraja_maccaini:0.9494173912,Rhinoraja_kujiensis:0.9494173912):15.24487839,Bathyraja_bergi:16.19429578):0.6983971896):3.474385439,Bathyraja_hesperafricana:20.36707841):0.02537902003):0.3743913068):5.277015375):2.078710296,(Bathyraja_richardsoni:25.49240586,(Bathyraja_smithii:10.0701836,(Bathyraja_leucomelanos:2.716500375,Bathyraja_trachouros:2.716500375):7.353683224):15.42222126):2.630169545):1.215663409):7.235418858):7.671244406):4.483878586,((((Sympterygia_acuta:8.263860235,Sympterygia_lima:8.263860235):17.50978683,(Sympterygia_bonapartii:20.53289355,Sympterygia_brevicaudata:20.53289355):5.240753521):13.80557435,((((Psammobatis_rutrum:8.333778171,Psammobatis_parvacauda:8.333778171):10.02832979,Psammobatis_extenta:18.36210796):3.192798292,(Psammobatis_scobina:0.1046593915,Psammobatis_bergi:0.1046593915):21.45024687):14.19881628,(Psammobatis_rudis:14.6054161,(Psammobatis_normani:6.748291324,Psammobatis_lentiginosa:6.748291324):7.857124776):21.14830643):3.825498887):4.097396449,(Arhynchobatis_asperrimus:7.393882799,Pseudoraja_fischeri:7.393882799):36.28273507):5.052161795):25.99959803):21.46362374,(Cruriraja_parcomaculata:32.96965624,(Cruriraja_poeyi:16.56034743,(Cruriraja_rugosa:14.07928554,(Cruriraja_atlantis:7.735276647,Cruriraja_durbanensis:7.735276647):6.344008892):2.481061888):16.40930982):63.22234519):4.976529961,(Cruriraja_hulleyi:77.48091226,((Anacanthobatis_longirostris:32.40952188,(Anacanthobatis_folirostris:18.83687194,(((Anacanthobatis_nanhaiensis:5.3918596,Anacanthobatis_marmoratus:5.3918596):8.05536903,Anacanthobatis_donghaiensis:13.44722863):5.102418714,((Anacanthobatis_ori:9.385791171,Anacanthobatis_americanus:9.385791171):6.188292923,Anacanthobatis_stenosoma:15.57408409):2.97556325):0.2872245957):13.57264994):7.526547653,(Cruriraja_cadenati:7.519803767,Cruriraja_andamanica:7.519803767):32.41626677):37.54484173):23.68762013):30.77860923):0.4428895613,(((((Raja_texana:3.166585749,Raja_cervigoni:3.166585749):7.316143459,((Raja_rouxi:6.971347569,((Raja_herwigi:0.5569348506,Raja_undulata:0.5569348506):1.77734269,Raja_equatorialis:2.33427754):4.637070028):1.786027024,Raja_miraletus:8.757374593):1.725354615):34.71744076,((Raja_asterias:25.7065491,((Raja_africana:11.191193391,(((Raja_straeleni:0.512673198,Raja_cortezensis:0.512673198):6.896253357,Raja_polystigma:7.408926555):3.745552218,(Raja_maderensis:8.453013722,(Raja_radula:4.772818913,Raja_clavata:4.772818913):3.680194808):2.701465051):0.03671461803):2.023333395,Raja_bahamensis:13.21452679):12.49202232):6.320659222,(Raja_montagui:22.4338096,(((Raja_brachyura:3.855638977,Raja_chinensis:3.855638977):7.809305279,(Raja_rondeleti:5.867856493,(Raja_ackleyi:4.325066784,Raja_macrorynchus:4.325066784):1.542789709):5.797087763):3.853175257,Raja_microocellata:15.51811951):6.915690085):9.593398727):13.17296164):22.96236808,((((Okamejei_leptoura:16.85192558,(Okamejei_meerdervoortii:3.16553417,Okamejei_mengae:3.16553417):13.68639141):20.70399803,Okamejei_powelli:37.55592361):11.93197211,(((Okamejei_acutispina:23.65243425,(Okamejei_porosa:23.39886067,Okamejei_heemstrai:23.39886067):0.2535725731):11.08388534,(Okamejei_pita:25.26613029,((Okamejei_philipi:6.045728956,Okamejei_kenojei:6.045729956):4.230855597,((Okamejei_boesemani:6.896892747,Okamejei_cairae:6.896892747):0.2855586432,Okamejei_schmidti:7.18245039):3.094134162):14.98954574):9.470188301):5.877338636,(Okamejei_arafurensis:8.123219002,Okamejei_hollandi:8.123220002):32.49043822):8.874238497):9.906376556,((Hongeo_koreana:49.4451325,((Beringraja_binoculata:17.25206725,Beringraja_pulchra:17.25206725):25.80350595,((Raja_stellulata:5.678675253,Raja_rhina:5.678675253):25.44142192,Raja_inornata:31.12009618):11.93547703):6.389559302):2.649090585,((Dipturus_oculus:39.20229066,(((Dipturus_melanospilus:28.65631037,((Dipturus_pullopunctata:5.511556864,(Dipturus_olseni:0.736989903,Dipturus_flavirostris:0.736989903):4.774566961):4.488935698,Dipturus_oregoni:10.00049256):18.65581781):9.534985746,((Dipturus_bullisi:7.3559195,Dipturus_garricki:7.3559195):28.76926653,(((Dipturus_springeri:5.391234476,Dipturus_queenslandicus:5.391234476):28.13086063,((((Dipturus_gigas:10.02264915,(Dipturus_grahami:4.066597938,Dipturus_stenorhynchus:4.066597938):5.956051212):6.277449585,Dipturus_nidarosiensis:16.30009874):3.91787814,((Spiniraja_whitleyi:0.6994862958,Dipturus_polyommata:0.6994862958):5.301824778,Dipturus_endeavouri:6.001311074):14.2166658):8.651483567,(((Zearaja_chilensis:3.574525543,Zearaja_nasuta:3.574525543):11.37020723,(Zearaja_argentinensis:13.01696807,Zearaja_maugeana:13.01696807):1.927764699):9.22426834,(Dipturus_trachydermus:0.7414619029,Dipturus_leptocaudus:0.7414619029):23.42753921):4.70046033):4.652634664):0.1296576574,(((Dipturus_gudgeri:12.3875517,Dipturus_innominatus:12.3875517):8.73735124,((Dipturus_confusus:5.265413949,(Dipturus_australis:2.606517517,Dipturus_cerva:2.606517517):2.658896432):5.00625359,((Dipturus_canutus:4.687016366,(Dentiraja_lemprieri:3.023365297,Dentiraja_flindersi:3.023365297):1.663653068):3.385802388,Dipturus_johannisdavesi:8.072818753):2.198846786):10.8532354):5.575985752,(((Dipturus_lanceorostratus:12.16586385,Dipturus_mennii:12.16586385):6.014323239,(Dipturus_crosnieri:10.0690654,(Dipturus_healdi:8.148318605,(Dipturus_teevani:6.013058991,(Dipturus_wengi:3.800047011,(Dipturus_falloargus:2.122104733,Dipturus_wuhanlingi:2.122104733):1.677942278):2.21301198):2.135258615):1.920747793):8.111121686):0.01180091955,(Dipturus_macrocauda:0.2515679286,Dipturus_tengu:0.2515679286):17.94042108):8.508898692):6.950866067):2.47343327):2.066110081):0.2104667985,((((Dipturus_apricus:2.487523728,Dipturus_batis:2.487522728):3.750537835,Dipturus_oxyrinchus:6.238060563):10.5374782,(Dipturus_acrobelus:11.20098378,(Dipturus_ecuadoriensis:9.118548994,Dipturus_intermedia:9.118548994):2.082434789):5.57455598):7.425229406,((Dipturus_laevis:2.63344629,Dipturus_campbelli:2.63344629):11.85892953,Dipturus_doutrei:14.49237582):9.708393344):14.20099374):0.8005277506):7.808813209,Dipturus_kwangtungensis:47.01110387):5.083120216):7.300049191):8.768264774):23.95119521,((((((Gurgesiella_dorsalifera:9.193446746,Gurgesiella_atlantica:9.193446746):15.31929839,Gurgesiella_furvescens:24.51274514):35.09986193,(Okamejei_jensenae:58.97528969,((Raja_velezi:0.6990403687,Raja_eglanteria:0.6990403687):34.74950308,Rostroraja_alba:35.44854344):23.52674625):0.637316378):21.68274158,(((((Leucoraja_ocellata:10.44357804,((Leucoraja_yucatanensis:0.5084598457,Leucoraja_leucosticta:0.5084598457):7.193407559,Leucoraja_virginica:7.701867405):2.74171164):8.128054054,Leucoraja_erinacea:18.5716321):7.085630183,((Leucoraja_garmani:19.55854395,Leucoraja_lentiginosa:19.55854495):5.887003073,Leucoraja_compagnoi:25.44554802):0.2117152589):27.72626899,(((Rajella_bathyphila:31.76419226,(((Rajella_leopardus:2.962789352,Rajella_purpuriventralis:2.962789352):6.211535839,Rajella_dissimilis:9.174325191):20.95394229,((Rajella_ravidula:23.24397328,(Rajella_lintea:19.47028017,Rajella_nigerrima:19.47028017):3.773693108):0.4004479477,(((Rajella_kukujevi:8.897621508,Rajella_fuliginea:8.897622508):2.175991339,((Rajella_alia:0.2268500174,Rajella_sadowskii:0.2268500174):7.787764885,Rajella_annandalei:8.014614903):3.058998944):11.05590689,Rajella_barnardi:22.12951974):1.514900492):6.483846251):1.63592578):3.670537043,((Rajella_challengeri:15.7985789,Rajella_bigelowi:15.7985779):5.589226473,((Rajella_eisenhardti:14.10028167,Rajella_caudaspinosa:14.10028067):3.527269429,Rajella_fyllae:17.6275501):3.760254282):14.04692493):4.547874484,((Leucoraja_caribbaea:20.43016075,Leucoraja_pristispina:20.43016075):7.768825148,((Amblyraja_robertsi:12.8370791,((Amblyraja_taaf:1.957320159,Amblyraja_georgiana:1.957320159):6.173320086,((Amblyraja_frerichsi:3.537299873,Amblyraja_doellojuradoi:3.537299873):2.131030062,((Amblyraja_reversa:1.499503147,Amblyraja_hyperborea:1.499503147):1.039350746,(Amblyraja_jenseni:0.04960359504,Amblyraja_badia:0.04960359504):2.489250298):3.129476042):2.462310311):4.706438859):2.665534523,Amblyraja_radiata:15.50261363):12.69637227):11.78361889):13.40092749):7.438985845,(Leucoraja_circularis:25.02143978,(Leucoraja_naevus:10.90476428,(Leucoraja_wallacei:8.875025509,(Leucoraja_fullonica:4.48038987,Leucoraja_melitensis:4.48039087):4.39463464):2.029738766):14.1166755):35.80107934):20.47282953):0.6501199846,((((Malacoraja_kreffti:4.589430327,Malacoraja_spinacidermis:4.589430327):21.07207038,(Malacoraja_obscura:6.873534683,Malacoraja_senta:6.873534683):18.78796602):16.41535361,(Dactylobatus_armatus:38.55805552,Dactylobatus_clarkii:38.55805552):3.518798788):11.5473781,(Neoraja_stehmanni:19.47148225,(Neoraja_carolinensis:14.78936932,((Neoraja_africana:0.6125252306,Neoraja_iberica:0.6125252306):10.20121094,Neoraja_caerulea:10.81373617):3.975633148):4.682112933):34.15275016):28.32123623):6.695045356,(((((Fenestraja_cubensis:7.542189809,Fenestraja_sinusmexicanus:7.542189809):4.40795778,Fenestraja_maceachrani:11.95014759):8.841224208,Fenestraja_mamillidens:20.7913718):7.088910034,(Fenestraja_ishiyamai:26.94523382,((Fenestraja_plutonia:3.718224306,Fenestraja_atripinna:3.718224306):1.512678504,Fenestraja_sibogae:5.230902811):21.71433101):0.9350480135):33.88379945,(((Breviraja_colesi:6.999599827,Breviraja_claramaculata:6.999599827):1.497909702,Breviraja_spinosa:8.497509529):6.387544282,Breviraja_nigriventralis:14.88505381):46.87902747):26.87643271):3.473220277):40.27629691):98.13618487,(((Platyrhinoidis_triseriata:119.9002818,((Platyrhina_sinensis:30.06173956,Platyrhina_tangi:30.06173956):7.895263815,Platyrhina_hyugaensis:37.95700338):81.9432784):96.36647121,(Hypnos_monopterygius:141.3830394,((((Narcine_nelsoni:25.18935277,(((Narcine_insolita:8.234021176,Narcine_brunnea:8.234021176):0.398777927,Narcine_nigra:8.632799103):7.461296303,(Narcine_lingula:7.798639659,(Narcine_timlei:5.192366724,Narcine_bancroftii:5.192366724):2.606272935):8.295455747):9.095257362):20.32197661,((Narcine_ornata:36.57948628,((Narcine_firma:3.643040183,Narcine_maculata:3.643040183):19.50524876,(Narcine_rierai:3.966175777,Narcine_brevilabiata:3.966175777):19.18211317):13.43119734):5.613996735,((Narcine_oculifera:14.4769021,Narcine_atzi:14.4769021):7.183710371,(Narcine_brasiliensis:3.021940785,Narcine_vermiculatus:3.021940785):18.63867169):20.53287055):3.317846364):44.10934401,((Narcine_prodorsalis:19.68586978,(Narcine_bicolor:13.57147845,Narcine_entemedor:13.57147845):6.114391324):13.09420591,Narcine_leoparda:32.78007568):56.84059771):29.92629281,(((((((Diplobatis_ommata:10.42239302,Diplobatis_pictus:10.42239302):5.575263305,Diplobatis_colombiensis:15.99765632):28.75059464,Diplobatis_guamachensis:44.74825096):34.72188584,(Discopyge_castelloi:1.721389097,Discopyge_tschudii:1.721389097):77.7487477):8.938328581,(((Torpedo_mackayana:38.05239618,Torpedo_tokionis:38.05239618):1.706995777,((Torpedo_torpedo:9.019999346,Torpedo_fuscomaculata:9.019999345999999):3.016684923,(Torpedo_marmorata:11.2143906,(Torpedo_andersoni:8.748488211,(Torpedo_sinuspersici:2.781079598,Torpedo_tremens:2.781079598):5.967408613):2.46590239):0.8222926684):27.72270869):24.49537913,(((((((Torpedo_fairchildi:8.620533445,Torpedo_alexandrinsis:8.620533445):7.639430498,Torpedo_microdiscus:16.25996394):12.13761001,(Torpedo_californica:3.314106777,Torpedo_puelcha:3.314106777):25.08346718):4.683021765,((Torpedo_formosa:1.447130747,Torpedo_adenensis:1.447130747):12.32644516,(Torpedo_macneilli:5.634587003,(Torpedo_nobiliana:1.561207266,Torpedo_polleni:1.561206266):4.073379737):8.138989904):19.30701982):4.415574925,Torpedo_semipelagica:37.49617065):12.64928991,(Torpedo_peruana:7.671210488,Torpedo_panthera:7.671210488):42.47425007):13.70452567,(Torpedo_suessii:19.01594773,Torpedo_bauchotae:19.01594773):44.8340385):0.404784859):24.15369429):5.550481244,((((Heteronarce_bentuviai:2.643170898,Heteronarce_mollis:2.643170898):5.014099166,Heteronarce_prabhui:7.657270064):7.247006764,Heteronarce_garmani:14.90427683):71.65860573,((Crassinarke_dormitor:13.99829617,((Electrolux_addisoni:6.689196302,((Narke_dipterygia:0.3921102731,Narke_capensis:0.3921102731):1.918801914,Narke_japonica:2.310912187):4.378284115):4.772910039,Temera_hardwickii:11.46210634):2.536189826):7.325293331,(Typhlonarke_aysoni:4.830802912,Typhlonarke_tarakea:4.830802912):16.49278658):65.23929306):7.396064065):9.601013473,((Benthobatis_marcida:35.84723151,(Benthobatis_moresbyi:14.01411778,(Benthobatis_kreffti:2.854311606,Benthobatis_yangi:2.854311606):11.15980617):21.83311373):40.7505562,(Narcine_westraliensis:47.94212957,(Narcine_lasti:20.82209261,Narcine_tasmaniensis:20.82209261):27.12003696):28.65565713):26.96217239):15.9870061):21.83607325):74.88371354):9.155085965,((((Trygonorrhina_dumerilii:25.68937262,Trygonorrhina_fasciata:25.68937262):37.73137024,Trygonorrhina_melaleuca:63.42074286):87.0534649,((Zapteryx_brevirostris:78.54367083,(Zapteryx_xyster:4.180896346,Zapteryx_exasperata:4.180896346):74.36277448):42.11287348,((Aptychotrema_timorensis:5.250606906,Aptychotrema_rostrata:5.250606906):28.32078516,Aptychotrema_vincentiana:33.57139207):87.08515124):29.81766445):59.55858144,((((Rhina_ancylostoma:98.28393074,((Rhynchobatus_palpebratus:9.034476135,(Rhynchobatus_laevis:1.20464812,Rhynchobatus_springeri:1.20464812):7.829828015):14.08607483,((Rhynchobatus_djiddensis:5.309800286,Rhynchobatus_australiae:5.309800286):7.526694087,Rhynchobatus_luebberti:12.83649437):10.2840566):75.16337978):42.91154007,((((Glaucostegus_cemiculus:3.596731869,Glaucostegus_granulatus:3.596731869):17.50155713,Glaucostegus_halavi:21.098289):33.52481676,(((Glaucostegus_microphthalmus:2.838521318,Glaucostegus_thouin:2.838521318):3.841786822,(Glaucostegus_obtusus:2.266205897,Glaucostegus_spinosus:2.266205897):4.414102243):3.476136604,Glaucostegus_typus:10.15644474):44.46666102):73.96671497,(Anoxypristis_cuspidata:121.4912742,(Pristis_pristis:86.46471243,(Pristis_clavata:67.40996125,(Pristis_zijsron:55.50754067,Pristis_pectinata:55.50754067):11.90242159):19.05475118):35.02656073):7.098546561):12.60565009):16.14049037,((((Rhinobatos_irvinei:8.515294888,Rhinobatos_lentiginosus:8.515294888):23.84950284,Acroteriobatus_leucospilus:32.36479773):62.32153268,((((Rhinobatos_horkelii:14.57186817,Acroteriobatus_zanzibarensis:14.57186817):2.041070533,Acroteriobatus_ocellatus:16.6129387):45.22996714,(((Acroteriobatus_salalah:27.90728152,Rhinobatos_glaucostigma:27.90728152):3.523539446,Rhinobatos_albomaculatus:31.43082097):5.441804393,(Rhinobatos_productus:23.75569542,Rhinobatos_leucorhynchus:23.75569542):13.11692994):24.97028048):11.19236706,(Rhinobatos_planiceps:18.56568436,(Rhinobatos_sainsburyi:7.834219269,Rhinobatos_percellens:7.834219269):10.73146509):54.46958854):21.65105752):55.061003,(((Acroteriobatus_annulatus:29.09716844,(Rhinobatos_nudidorsalis:4.942349945,(Rhinobatos_annandalei:0.4155657118,Acroteriobatus_variegatus:0.4155657118):4.526785233):24.1548175):30.05673669,((((Rhinobatos_schlegelii:3.59019968,Rhinobatos_formosensis:3.59019968):3.711698758,Rhinobatos_hynnicephalus:7.301898438):11.25426297,Rhinobatos_punctifer:18.55616141):23.00296064,(((Rhinobatos_rhinobatos:6.97377461,Rhinobatos_prahli:6.97377361):13.47524679,Acroteriobatus_blochii:20.4490204):16.19413788,Rhinobatos_lionotus:36.64315828):4.915963766):17.59478208):54.96684368,(Rhinobatos_penggali:63.18266917,(Rhinobatos_jimbaranensis:5.319046449,Rhinobatos_holcorhynchus:5.319046449):57.86362273):50.93807864):35.6265856):7.588627765):48.31474389,(Zanobatus_schoenleinii:192.2080066,(Hexatrygon_bickelli:138.3849967,((((Aetobatus_flagellum:46.6446165,Aetobatus_narinari:46.6446165):69.95446405,(((Aetomylaeus_vespertilio:78.73843232,(Aetomylaeus_maculatus:47.54500358,(Aetomylaeus_nichofii:31.43173836,(Pteromylaeus_bovinus:10.89042125,Pteromylaeus_asperrimus:10.89042125):20.54131711):16.11326523):31.19342873):17.54098204,((Myliobatis_californicus:40.34839692,((Myliobatis_freminvillei:18.00077731,Myliobatis_longirostris:18.00077731):11.77966248,(Myliobatis_peruvianus:10.96873197,Myliobatis_tenuicaudatus:10.96873197):18.81170882):10.56795613):27.75548727,((Myliobatis_chilensis:31.05809892,(((Myliobatis_rhombus:1.978671904,Myliobatis_hamlyni:1.978671904):11.58887439,(Myliobatis_ridens:5.681763206,Myliobatis_tobijei:5.681763206):7.885783084):14.81855788,Myliobatis_aquila:28.38610417):2.671994751):17.53018513,(Myliobatis_australis:0.9710848798,Myliobatis_goodei:0.9710848798):47.61719916):19.51560015):28.17553017):14.59774079,((((Rhinoptera_jayakari:26.97373417,Rhinoptera_steindachneri:26.97373417):2.250287622,Rhinoptera_bonasus:29.22402279):23.76489512,(Rhinoptera_hainanensis:22.08866197,(Rhinoptera_marginata:19.325558787000002,(((Rhinoptera_javanica:3.63613518,Rhinoptera_adspersa:3.63613418):2.332327264,Rhinoptera_neglecta:5.968462444):7.747452472,(Rhinoptera_brasiliensis:12.18654202,Rhinoptera_peli:12.18654202):1.529371895):5.609644867):2.763103183):30.90025494):12.02837268,(((Manta_birostris:11.91938335,Manta_alfredi:11.91938335):35.11961696,((Mobula_tarapacana:5.255516264,Mobula_japanica:5.255516264):35.8487036,((Mobula_thurstoni:10.32470245,Mobula_kuhlii:10.32470245):24.3401836,Mobula_rochebrunei:34.66488605):6.439333805):5.93478044):13.85046125,((Mobula_mobular:16.00237161,Mobula_diabolus:16.00237161):28.52895421,(Mobula_eregoodootenkee:31.48935902,(Mobula_munkiana:10.11769162,Mobula_hypostoma:10.11769162):21.3716674):13.0419668):16.35813573):4.127828043):45.85986456):5.721925401):10.98139957,(Plesiobatis_daviesi:118.3929008,(((Trygonoptera_ovalis:43.25067798,Trygonoptera_galba:43.25067798):14.2669157,((Trygonoptera_testacea:33.41323343,(Trygonoptera_mucosa:10.47827509,Trygonoptera_imitata:10.47827509):22.93495834):12.13157807,Trygonoptera_personata:45.5448115):11.97278118):52.33068576,(((((((Urolophus_javanicus:4.676652958,(Urolophus_piperatus:2.692627086,Urolophus_deforgesi:2.692627086):1.984025873):7.058608284,Urolophus_papilio:11.73526124):2.241394263,(Urolophus_armatus:13.44642159,Urolophus_westraliensis:13.44642259):0.5302339137):12.64550216,Urolophus_aurantiacus:26.62215866):6.792264437,((((Urolophus_sufflavus:1.161252086,(Urolophus_cruciatus:0.9809657173,Urolophus_paucimaculatus:0.9809657173):0.1802863682):13.57730048,Urolophus_orarius:14.73855256):15.48198989,(Urolophus_kapalensis:0.1629809544,Urolophus_lobatus:0.1629799544):30.0575625):0.3003893461,(Urolophus_kaianus:23.9749954,((Urolophus_viridis:6.404147975,Urolophus_neocaledoniensis:6.404146975):1.259339021,Urolophus_expansus:7.663486996):16.31150941):6.5459364):2.893490297):20.26640125,Urolophus_gigas:53.68082335):24.31071648,(Urolophus_mitosis:70.05391057,((Urolophus_flavomosaicus:2.793087349,Urolophus_bucculentus:2.793087349):47.09514786,Urolophus_circularis:49.88823521):20.16567536):7.937629262):31.85673961):8.54462132):9.187578364):3.0838548,((((((Gymnura_altavela:1.003578991,Gymnura_tentaculata:1.003578991):12.64017567,Gymnura_natalensis:13.64375466):17.55706166,(Gymnura_marmorata:9.113746944,Gymnura_crebripunctata:9.113746944):22.08706938):53.78547763,(Gymnura_afuerae:24.3219565,Gymnura_micrura:24.3219565):60.66433746):14.48528783,(((Gymnura_poecilura:26.70871372,Gymnura_hirundo:26.70871372):37.65412001,((Gymnura_zonura:14.9934195,Gymnura_bimaculata:14.9934195):13.24167512,Gymnura_crooki:28.23509463):36.1277391):17.5127712,(Gymnura_japonica:8.642964755,Gymnura_australis:8.642964755):73.23264017):17.59597686):15.49008132,((((Urobatis_halleri:23.01344967,(Urobatis_maculatus:3.784106032,Urobatis_concentricus:3.784106032):19.22934364):49.78050761,((Urobatis_marmoratus:37.66096818,(Urobatis_tumbesensis:16.19197771,Urobatis_jamaicensis:16.19197771):21.46899047):23.33160834,((((Urotrygon_serrula:4.569639857,Urotrygon_microphthalmum:4.569639857):19.43510139,Urotrygon_reticulata:24.00474125):4.496135171,(Urotrygon_rogersi:27.42416404,(((Urotrygon_munda:5.185434779,(Urotrygon_simulatrix:1.434351486,Urotrygon_caudispinosus:1.434351486):3.751083293):3.321145402,Urotrygon_venezuelae:8.50658018):9.717290073,(Urotrygon_cimar:2.031257341,Urotrygon_nana:2.031257341):16.19261291):9.200293791):1.076712372):11.350753,(Urotrygon_aspidura:16.25541205,(Urotrygon_chilensis:5.490024193,Urotrygon_peruanus:5.490024193):10.76538786):23.59621737):21.14094711):11.80138175):26.97451415,((Himantura_schmardae:21.00392628,Himantura_pacifica:21.00392628):48.67470127,(((Potamotrygon_constellata:34.15146208,((((Potamotrygon_marinae:9.791352925,Potamotrygon_falkneri:9.791352925):15.85315491,Potamotrygon_schuemacheri:25.64450784):1.24617883,(Potamotrygon_brachyura:15.56881797,((Potamotrygon_henlei:12.93304235,Potamotrygon_humerosa:12.93304235):0.2347456696,((Potamotrygon_schroederi:5.778233393,Potamotrygon_tigrina:5.778233393):2.88440591,((Potamotrygon_orbignyi:5.290202191,Potamotrygon_castexi:5.290202191):0.8720133288,(Potamotrygon_dumerilii:1.949356052,(Potamotrygon_scobina:0.8323102727,Potamotrygon_motoro:0.8323102727):1.117045779):4.212859467):2.500423783):4.505148713):2.401029957):11.32186869):6.199393309,(Potamotrygon_tatianae:26.42922045,(((Plesiotrygon_iwamae:8.140454789,Plesiotrygon_nana:8.140454789):11.89682724,((Potamotrygon_boesemani:7.566258501,(Potamotrygon_ocellata:4.4909062,Potamotrygon_yepezi:4.4909062):3.075352302):8.26559798,(Potamotrygon_signata:1.602703281,Potamotrygon_leopoldi:1.602703281):14.2291532):4.205425543):5.083948972,(Potamotrygon_magdalenae:23.20066416,Potamotrygon_hystrix:23.20066416):1.920566837):1.30798945):6.660859528):1.061382102):5.626480181,Paratrygon_aiereba:39.77794226):4.294282415,(Heliotrygon_gomesi:6.194699588,Heliotrygon_rosai:6.194699588):37.87752508):25.60640288):30.08984487):11.17388531,(((Taeniura_lymma:62.52887828,((Neotrygon_kuhlii:34.15370367,(Neotrygon_leylandi:8.287011947,(Neotrygon_picta:5.585087182,Neotrygon_annotata:5.585088182):2.701924765):25.86669073):14.60678344,Neotrygon_ningalooensis:48.76048711):13.76839116):33.81315457,((((Dasyatis_sabina:22.08146439,Dasyatis_sinensis:22.08146439):42.77626045,(Dasyatis_gigantea:11.83878879,(Dasyatis_americana:10.94243584,(Dasyatis_marianae:6.211223363,Dasyatis_longa:6.211222363):4.731213481):0.8963519514):53.01893604):6.74349893,((Dasyatis_dipterura:25.64525288,(Dasyatis_say:12.57079926,Dasyatis_acutirostra:12.57079926):13.07445362):31.01472413,(((Dasyatis_guttata:0.4518722463,Dasyatis_hypostigma:0.4518722463):11.05350182,Dasyatis_geijskesi:11.50537407):13.40840928,Dasyatis_colarensis:24.91378335):31.74619366):14.94124676):15.80078832,((((((Dasyatis_garouaensis:2.206612624,(Dasyatis_fluviorum:0.6754233275,Dasyatis_tortonesei:0.6754243275):1.531188297):19.76099128,((Dasyatis_akajei:2.657677184,Dasyatis_hastata:2.657678184):4.251614106,Dasyatis_laosensis:6.909291291):15.05831162):6.208149952,(Dasyatis_parvonigra:14.70960625,(Dasyatis_bennetti:0.4322282121,Dasyatis_navarrae:0.4322282121):14.27737804):13.46614661):21.64236123,(Dasyatis_izuensis:9.2521103,Dasyatis_laevigata:9.2521103):40.56600279):11.51894307,Dasyatis_zugei:61.33705716):10.94806241,((((Dasyatis_ushiei:10.19611082,(Dasyatis_thetidis:3.112784547,Dasyatis_centroura:3.112784547):7.083326269):38.69947863,(Pteroplatytrygon_violacea:40.80396813,((Dasyatis_matsubarai:5.65689369,Dasyatis_brevicaudata:5.65689369):5.614220823,Dasyatis_multispinosa:11.27111551):29.53285361):8.09162032):1.516733331,Dasyatis_marmorata:50.41232278):6.526621418,((Taeniurops_grabata:41.10593149,Taeniurops_meyeni:41.10593149):10.23939925,((Dasyatis_rudis:23.25591861,Dasyatis_pastinaca:23.25591761):2.642200187,Dasyatis_chrysonota:25.8981178):25.44721394):5.59361145):15.34617638):15.11689151):8.940020764):12.08153846,((Dasyatis_microps:85.80139935,((Dasyatis_margaritella:16.1479049,Dasyatis_margarita:16.1479049):49.93428254,((((((Himantura_kittipongi:1.445502181,Himantura_fai:1.445502181):20.98134335,Himantura_hortlei:22.42684553):10.44385934,((Himantura_uarnak:13.74454599,(Himantura_marginata:10.98247985,Himantura_undulata:10.98248085):2.762065139):10.21940705,Himantura_leoparda:23.96395204):8.906752824):5.451015932,(((Himantura_toshi:0.9928705457,Himantura_astra:0.9928705457):2.750259474,Himantura_pastinacoides:3.74312902):33.0828758,((Himantura_signifer:24.89887026,Himantura_oxyrhyncha:24.89887026):9.683152292,(((Himantura_imbricata:0.2560154644,Dasyatis_lata:0.2560154644):12.59589248,(Himantura_walga:3.646202386,Himantura_dalyensis:3.646202386):9.20570556):12.39337829,Himantura_gerrardi:25.24528624):9.336735319):2.243983262):1.495715982):7.17636378,(Himantura_uarnacoides:23.97461919,(Himantura_jenkinsii:12.64499635,Himantura_randalli:12.64499535):11.32962284):21.52346639):11.48986326,(((Urogymnus_asperrimus:21.52653296,Urogymnus_ukpam:21.52653296):20.21474893,(Himantura_polylepis:1.66908045,Himantura_granulata:1.66908045):40.07220144):6.412166046,Himantura_lobistoma:48.15344794):8.834499902):9.094238607):19.7192129):9.643849013,(Makararaja_chindwinensis:45.06544892,((Pastinachus_stellurostris:22.17693541,Pastinachus_solocirostris:22.17693541):10.53528892,((Pastinachus_sephen:9.801828233,Pastinachus_gracilicaudus:9.801828233):1.899023909,Pastinachus_atrus:11.70085114):21.01137319):12.35322359):50.37979944):12.97832195):2.518787433):4.019305366):15.70267182):7.720662796):53.82300883):13.44269852):4.382084126):15.38904876):5.104377093):46.83220985,((((Chlamydoselachus_anguineus:0.007482449505,Chlamydoselachus_africana:0.007482449505):145.2157711,(Notorynchus_cepedianus:81.34156367,((Hexanchus_vitulus:31.74940647,(Hexanchus_griseus:2.130500002,Hexanchus_nakamurai:2.130500002):29.61890647):13.14305906,Heptranchias_perlo:44.89246553):36.44909814):63.88168992):96.54902134,(((((Deania_quadrispinosa:12.68890734,Deania_profundorum:12.68890734):26.17505355,(Deania_hystricosa:4.370110851,Deania_calcea:4.370110851):34.49385005):76.48501793,(((Centrophorus_squamosus:45.63838927,(Centrophorus_granulosus:8.094764847,Centrophorus_tessellatus:8.094763847):37.54362542):3.806351082,((((Centrophorus_harrissoni:10.5961307,Centrophorus_seychellorum:10.5961297):2.607291109,Centrophorus_isodon:13.20342181):11.9716952,(Centrophorus_lusitanicus:3.527989816,Centrophorus_moluccensis:3.527989816):21.64712619):7.546269764,Centrophorus_westraliensis:32.72138577):16.72335458):47.02953903,(Centrophorus_atromarginatus:38.29852257,Centrophorus_zeehaani:38.29852257):58.1757568):18.87469945):52.50371713,((((Aculeola_nigra:29.83898388,(((Centroscyllium_granulatum:3.977778692,Centroscyllium_excelsum:3.977778692):1.420277553,(Centroscyllium_kamoharai:0.407860159,Centroscyllium_nigrum:0.407860159):4.990196086):9.239732255,((Centroscyllium_fabricii:8.583871699,Centroscyllium_ritteri:8.583871699):2.561024524,Centroscyllium_ornatum:11.14489622):3.492892278):15.20119538):71.25157141,(Trigonognathus_kabeyai:100.7445743,(((((Etmopterus_dianthus:16.4730376,((Etmopterus_unicolor:10.67136892,Etmopterus_litvinovi:10.67136892):0.590435874,(((Etmopterus_bullisi:0.3514589177,Etmopterus_villosus:0.3514589177):0.08591578253,Etmopterus_spinax:0.4373747002):4.698804343,(Etmopterus_princeps:1.118831664,Etmopterus_granulosus:1.118832664):4.017347379):6.125625754):5.211231806):5.865991676,Etmopterus_compagnoi:22.33902828):28.02093677,((Etmopterus_gracilispinis:35.15946206,(Etmopterus_schultzi:32.76960336,(Etmopterus_polli:20.51165759,Etmopterus_virens:20.51165759):12.25794577):2.389858699):12.34156966,Etmopterus_schmidti:47.50103072):2.85893433):6.030278684,(((Etmopterus_robinsi:41.97012785,((Etmopterus_caudistigmus:7.571816072,Etmopterus_decacuspidatus:7.571816072):21.72816884,(Etmopterus_sculptus:3.182478294,Etmopterus_sentosus:3.182478294):26.11750662):12.67014294):5.413346116,(Etmopterus_pusillus:28.60879292,Etmopterus_bigelowi:28.60879292):18.77468105):1.878429206,(((Etmopterus_splendidus:18.33271446,(Etmopterus_burgessi:15.19831161,Etmopterus_fusus:15.19831161):3.134402847):22.96376579,((Etmopterus_pycnolepis:2.019557697,Etmopterus_carteri:2.019557697):10.71401361,(Etmopterus_pseudosqualiolus:1.69604974,Etmopterus_perryi:1.69604974):11.03752157):28.56290894):7.919852134,(Etmopterus_evansi:16.84862558,Etmopterus_hillianus:16.84862558):32.3677068):0.04557078673):7.12834056):27.6488323,((((Etmopterus_dislineatus:13.18228509,Etmopterus_joungi:13.18228409):4.614555323,Etmopterus_molleri:17.79684041):10.65117952,Etmopterus_lucifer:28.44801993):11.51949607,((Etmopterus_brachyurus:10.01544602,Etmopterus_viator:10.01544502):5.944409694,Etmopterus_sheikoi:15.95985571):24.00766029):44.07156103):16.70549827):0.3459809901):45.8250891,(((Squaliolus_laticaudus:62.82489696,(Mollisquama_parini:61.30760484,((Heteroscymnoides_marleyi:12.24488571,Squaliolus_aliae:12.24488571):17.20399701,(Euprotomicroides_zantedeschia:3.433907158,Euprotomicrus_bispinatus:3.433907158):26.01497557):31.85872211):1.517292123):64.20837255,(((Isistius_labialis:4.453458847,Isistius_brasiliensis:4.453458847):0.2585786812,Isistius_plutodus:4.712037528):80.74217458,Dalatias_licha:85.45421211):41.5790574):16.87340225,((Cirrhigaleus_asper:56.62274364,(Cirrhigaleus_barbifer:16.26028245,Cirrhigaleus_australis:16.26028245):40.36246119):28.02037768,(((((Squalus_notocaudatus:11.54517606,Squalus_suckleyi:11.54517606):3.348574088,Squalus_acanthias:14.89375014):0.982797746,Squalus_altipinnis:15.87654789):3.620252502,Squalus_griffini:19.49680039):60.46530723,((Squalus_megalops:31.0995569,((Squalus_melanurus:5.549231904,Squalus_albifrons:5.549231904):6.711899161,Squalus_brevirostris:12.26113206):18.83842484):20.44138766,((Squalus_grahami:15.16814855,((Squalus_mitsukurii:10.89410676,(Squalus_rancureli:9.256679907,(Squalus_montalbani:0.2121823748,Squalus_chloroculus:0.2121823748):9.044497533):1.637426852):2.670714492,Squalus_cubensis:13.56482125):1.603327303):19.74048663,((((Squalus_japonicus:7.840485518,(Squalus_blainville:5.925567146,(Squalus_nasutus:2.123959344,Squalus_raoulensis:2.123959344):3.801607802):1.914918372):5.16888256,(Squalus_edmundsi:9.427592629,Squalus_hemipinnis:9.427592629):3.581775449):16.76605042,((Squalus_lalannei:3.523880279,Squalus_formosus:3.523880279):10.83367267,Squalus_crassispinus:14.35755295):15.41786555):4.889925312,Squalus_bucephalus:34.66534381):0.2432913743):16.63230837):28.42116407):4.681012695):59.26355144):3.008971639):11.13974098,(((Somniosus_rostratus:18.77438309,Somniosus_longus:18.77438309):5.47100368,((Somniosus_antarcticus:2.957963686,Somniosus_pacificus:2.957963686):4.865521765,Somniosus_microcephalus:7.823484452):16.42190232):133.6791905,(((Centroscymnus_coelolepis:46.44653144,((Scymnodalatias_garricki:36.91845704,Scymnodalatias_oligodon:36.91845704):6.036486787,(Scymnodalatias_sherwoodi:13.57875359,Scymnodalatias_albicauda:13.57875359):29.37619023):3.491587614):7.876093264,(((Proscymnodon_plunketi:2.219131166,Proscymnodon_macracanthus:2.219131166):16.8221658,Centroscymnus_owstonii:19.04129697):26.26245382,((Oxynotus_caribbaeus:37.8724312,((Oxynotus_japonicus:0.6878710176,Oxynotus_centrina:0.6878710176):29.69117686,(Oxynotus_bruniensis:22.53812185,Oxynotus_paradoxus:22.53812185):7.840926034):7.493383322):0.07144831636,Scymnodon_ringens:37.94387952):7.35987227):9.018872916):18.23334479,((Zameus_squamulosus:17.00885569,Zameus_ichiharai:17.00885569):37.83023217,Centroselachus_crepidater:54.83908785):17.71688164):85.36860775):0.1308081315):9.797311583):39.66946926,(((Echinorhinus_brucus:27.72473684,Echinorhinus_cookei:27.72473684):77.95912971,(Pliotrema_warreni:92.13477035,(Pristiophorus_japonicus:65.30120692,(((Pristiophorus_delicatus:6.547860922,Pristiophorus_nancyae:6.547860922):12.963068,Pristiophorus_cirratus:19.51092892):29.43889705,(Pristiophorus_nudipinnis:3.312010713,Pristiophorus_schroederi:3.312010713):45.63781526):16.35138094):26.83356243):13.5490972):8.323688048,(((((Squatina_legnota:9.604536318,Squatina_tergocellatoides:9.604536318):17.49546032,(Squatina_formosa:9.314628034,Squatina_nebulosa:9.314628034):17.7853686):22.45677473,Squatina_japonica:49.55677137):4.913014869,((((Squatina_californica:6.859721993,Squatina_dumeril:6.859721993):17.31768277,(Squatina_occulta:13.49984386,Squatina_guggenheim:13.49984386):10.6775609):11.51363075,Squatina_armata:35.69103452):1.745392124,Squatina_africana:37.43642664):17.0333596):24.60483021,(((Squatina_albipunctata:5.899470433,(Squatina_tergocellata:4.249635109,Squatina_pseudocellata:4.249635109):1.649836324):24.4988268,(Squatina_oculata:11.54457034,Squatina_australis:11.54457034):18.85372689):16.13258772,((Squatina_argentina:26.24570098,Squatina_aculeata:26.24570098):13.89943507,(Squatina_squatina:38.99982089,Squatina_caillieti:38.99982089):1.145315161):6.385748895):32.54373049):34.93293915):93.51461062):34.25010871):12.72360209,((Heterodontus_francisci:105.6242033,(((((Heterodontus_zebra:1.753230771,Heterodontus_quoyi:1.753229771):40.13547186,(Heterodontus_portusjacksoni:15.55596067,Heterodontus_galeatus:15.55596067):26.33274196):14.2828199,Heterodontus_ramalheira:56.17152153):2.99347869,(Heterodontus_japonicus:2.763072658,Heterodontus_omanensis:2.763072658):56.40192756):26.50798964,Heterodontus_mexicanus:85.67299086):19.95121344):132.9841361,((((((Parascyllium_variolatum:8.749520313,(Parascyllium_ferrugineum:4.631975505,Parascyllium_elongatum:4.631975505):4.117544808):3.397765747,Parascyllium_collare:12.14728606):13.17740164,Parascyllium_sparsimaculatum:25.3246877):10.23383615,((Cirrhoscyllium_expolitum:7.100178173,Cirrhoscyllium_formosanum:7.100178173):8.724796254,Cirrhoscyllium_japonicum:15.82497443):19.73354942):177.53247,(((Brachaelurus_colcloughi:58.96101233,Brachaelurus_waddi:58.96101233):89.43427313,(Eucrossorhinus_dasypogon:87.6432052,(((Orectolobus_ornatus:48.7988366,((Orectolobus_floridus:13.20647288,Orectolobus_reticulatus:13.20647288):14.24983141,Sutorectus_tentaculatus:27.45630428):21.34253232):7.869474173,((Orectolobus_leptolineatus:2.79886054,Orectolobus_wardi:2.79886054):2.491146835,Orectolobus_halei:5.290007376):51.3783034):7.058662484,((Orectolobus_japonicus:3.465035802,Orectolobus_maculatus:3.465035802):39.59176907,(Orectolobus_hutchinsi:34.28860258,Orectolobus_parvimaculatus:34.28860258):8.768202295):20.67016938):23.91623194):60.75208026):22.1035266,((Pseudoginglymostoma_brevicaudatum:89.85151626,((Rhincodon_typus:1.715041029,Nebrius_ferrugineus:1.715042029):43.30663305,(Ginglymostoma_cirratum:0.9474585788,Stegostoma_fasciatum:0.9474575788):44.0742165):44.82984218):46.3422819,((Hemiscyllium_ocellatum:63.53364463,((Hemiscyllium_trispeculare:6.991417458,Hemiscyllium_henryi:6.991417458):19.93514048,(Hemiscyllium_galei:17.96942478,((Hemiscyllium_freycineti:3.950928032,Hemiscyllium_hallstromi:3.950928032):0.6781280339,(Hemiscyllium_strahani:2.208443563,Hemiscyllium_michaeli:2.208443563):2.420612503):13.34036871):8.957133162):36.60708669):1.310389773,(((Chiloscyllium_arabicum:7.932681842,(Chiloscyllium_hasseltii:0.524748455,Chiloscyllium_griseum:0.524748455):7.407933387):4.875320699,Chiloscyllium_burmensis:12.80800254):47.28612329,((Chiloscyllium_indicum:34.10675713,Chiloscyllium_plagiosum:34.10675713):13.7129619,Chiloscyllium_punctatum:47.81971904):12.2744078):4.749908564):71.34976376):34.3050139):42.59218175):10.60480141,((Mitsukurina_owstoni:134.2111241,((Alopias_superciliosus:99.96974986,((Alopias_vulpinus:72.96925002,Alopias_pelagicus:72.96925002):17.07828172,((Megachasma_pelagios:32.12557447,(Odontaspis_noronhai:14.41254216,Odontaspis_ferox:14.41254216):17.7130323):11.22494932,Pseudocarcharias_kamoharai:43.35052379):46.69700795):9.922218118):18.32518103,(Carcharias_taurus:102.5805044,(Cetorhinus_maximus:91.2720053,((Lamna_ditropis:42.00458329,Lamna_nasus:42.00458329):19.00560201,(Carcharodon_carcharias:53.86403447,(Isurus_oxyrinchus:40.40124976,Isurus_paucus:40.40124976):13.46278471):7.14615083):30.26182):11.30849913):15.71442646):15.91619222):61.34502001,(((Cephalurus_cephalus:9.955041462,(Poroderma_pantherinum:3.364614017,Poroderma_africanum:3.364614017):6.590427444):75.65488141,(((((Scyliorhinus_torrei:3.314098104,Scyliorhinus_haeckelii:3.314098104):19.39441154,(Scyliorhinus_canicula:16.16749142,Scyliorhinus_stellaris:16.16749142):6.541017221):5.79368327,(Scyliorhinus_cervigoni:2.083816541,Scyliorhinus_retifer:2.083816541):26.41837637):6.802751424,Scyliorhinus_capensis:35.30494434):36.59685206,((((Cephaloscyllium_fasciatum:7.189015599,Cephaloscyllium_pictum:7.189015599):22.36660692,(Cephaloscyllium_ventriosum:4.504281183,(Cephaloscyllium_signourum:0.8699138181,Cephaloscyllium_stevensi:0.8699138181):3.634367365):25.05134134):18.08144583,(Cephaloscyllium_maculatum:41.63865528,(((Cephaloscyllium_silasi:28.7542332,(Cephaloscyllium_pardelotum:20.97856321,Cephaloscyllium_speccum:20.97856321):7.775669992):2.408632527,((Cephaloscyllium_variegatum:14.94241244,(Cephaloscyllium_laticeps:13.81944885,((Cephaloscyllium_isabellum:11.17429585,(Cephaloscyllium_albipinnum:10.49348232,((Cephaloscyllium_zebrum:8.841682101,Cephaloscyllium_hiscosellum:8.841681101):1.118112451,Cephaloscyllium_umbratile:9.959794552):0.5336887676):0.680812532):0.3117720506,Cephaloscyllium_sarawakensis:11.4860679):2.333381953):1.122963588):8.946962885,Cephaloscyllium_cooki:23.88937633):7.273489403):5.506260095,Cephaloscyllium_sufflans:36.66912583):4.969529456):5.998413067):4.723532436,(((Scyliorhinus_garmani:12.32715371,Scyliorhinus_tokubee:12.32715371):7.726367517,(Scyliorhinus_comoroensis:18.01809592,(Scyliorhinus_meadi:7.433352724,Scyliorhinus_boa:7.433352724):10.58474319):2.035425309):20.11875643,(Scyliorhinus_hesperius:36.2008802,(Scyliorhinus_besnardi:13.17757833,Scyliorhinus_torazame:13.17757833):23.02330187):3.971397461):12.18832313):19.54119561):13.70812548):92.43974859,((((Schroederichthys_bivius:8.1387007,Schroederichthys_tenuis:8.1387007):101.5515934,(Schroederichthys_maculatus:6.766576398,(Schroederichthys_saurisqualus:5.890957235,Schroederichthys_chilensis:5.890957235):0.8756191631):102.9237177):42.81891313,((Aulohalaelurus_labiosus:2.891202693,Aulohalaelurus_kanakorum:2.891202693):64.37069769,((Atelomycterus_baliensis:18.22823845,Atelomycterus_marmoratus:18.22823845):39.00973161,((Atelomycterus_fasciatus:5.591448261,Atelomycterus_macleayi:5.591448261):1.697598664,Atelomycterus_marnkalha:7.289046925):49.94892313):10.02393033):85.24730688):15.00625565,(((((Eridacnis_barbouri:8.710690946,(Eridacnis_sinuans:2.845802643,Eridacnis_radcliffei:2.845802643):5.864888303):7.914124045,Ctenacis_fehlmanni:16.62481499):64.86875846,((Proscyllium_magnificum:4.064034456,Proscyllium_venustum:4.064034456):14.76182168,Proscyllium_habereri:18.82585614):62.66771731):36.50912235,(Pseudotriakis_microdon:66.95646575,(Planonasus_parini:58.23666935,(Gollum_attenuatus:15.26942161,Gollum_suluensis:15.26942161):42.96724774):8.719796402):51.04623005):31.27796832,(((Leptocharias_smithii:107.0922393,((Hemipristis_elongata:94.46893312,(((Paragaleus_tengi:17.6427788,((Paragaleus_pectoralis:3.909892049,Paragaleus_randalli:3.909892049):1.508668647,Paragaleus_leucolomatus:5.418560696):12.2242181):27.98934651,Chaenogaleus_macrostoma:45.63212531):11.63883623,(Hemigaleus_microstoma:27.07340816,Hemigaleus_australiensis:27.07340816):30.19755338):37.19797258):0.5613666624,(Galeocerdo_cuvier:86.9459618,((Eusphyra_blochii:53.1393355,(Sphyrna_mokarran:47.42667078,(Sphyrna_zygaena:43.41342972,(Sphyrna_lewini:34.24620219,(Sphyrna_corona:22.19814277,(Sphyrna_tiburo:16.53687161,(Sphyrna_tudes:12.59117963,Sphyrna_media:12.59117963):3.94569198):5.661270164):12.04805941):9.167227532):4.013242061):5.712664722):26.62345531,(((Loxodon_macrorhinus:48.90488323,(Scoliodon_macrorhynchos:8.820852934,Scoliodon_laticaudus:8.820852934):40.0840303):16.57297315,((Rhizoprionodon_oligolinx:18.88932096,(Rhizoprionodon_acutus:16.18622104,Rhizoprionodon_taylori:16.18622104):2.703099917):28.935825,((Rhizoprionodon_longurio:14.32369121,Rhizoprionodon_lalandii:14.32369121):6.111505624,(Rhizoprionodon_terraenovae:5.234978623,Rhizoprionodon_porosus:5.234978623):15.20021821):27.38994912):17.65270943):7.022891076,(((((((Triaenodon_obesus:2.54764861,Carcharhinus_sorrah:2.54764961):19.78250327,Carcharhinus_amboinensis:22.33015288):14.81174703,(((Carcharhinus_borneensis:24.30103648,Carcharhinus_macloti:24.30103648):1.015984226,(Carcharhinus_sealei:16.7032399,Carcharhinus_dussumieri:16.7032399):8.613779806):6.798182979,Carcharhinus_porosus:32.11520369):5.026696227):0.9944017699,((Carcharhinus_perezii:29.38734982,((Carcharhinus_tjutjot:10.65879678,Carcharhinus_cerdale:10.65879678):8.05936036,((Isogomphodon_oxyrhynchus:1.600992739,Carcharhinus_longimanus:1.600992739):15.24528018,(Carcharhinus_coatesi:11.45412096,(Carcharhinus_obscurus:1.812165757,Carcharhinus_galapagensis:1.812165757):9.641956198):5.392151969):1.87188422):10.66919268):6.524379538,(Carcharhinus_brachyurus:18.48480739,Carcharhinus_brevipinna:18.48480739):17.42692097):2.224572324):6.76243056,(((((Prionace_glauca:10.05156668,Carcharhinus_albimarginatus:10.05156668):14.00699862,Carcharhinus_hemiodon:24.0585663):6.323652981,(Carcharhinus_falciformis:6.55314985,Carcharhinus_wheeleri:6.55315085):23.82906843):7.400774404,Carcharhinus_amblyrhynchos:37.78299268):3.009925421,((Carcharhinus_leucas:8.410255553,(Carcharhinus_altimus:6.337232296,Carcharhinus_plumbeus:6.337232296):2.073024257):13.73914499,(Carcharhinus_isodon:19.94134125,(Nasolamia_velox:8.570558778,Carcharhinus_acronotus:8.570558778):11.37078247):2.208059295):18.64351756):4.105813144):11.7376862,(Carcharhinus_signatus:40.09984274,((Carcharhinus_fitzroyensis:33.31310476,(Carcharhinus_melanopterus:9.226559455,Carcharhinus_cautus:9.226559455):24.08654531):5.711069596,((Carcharhinus_limbatus:6.667418612,(Carcharhinus_tilstoni:5.79104573,Carcharhinus_amblyrhynchoides:5.79104573):0.876371882):2.130438024,Carcharhinus_leiodon:8.797855636):30.22631872):1.075669385):16.53657471):7.707605265,((Negaprion_acutidens:24.74747595,Negaprion_brevirostris:24.74747595):25.05448423,((Lamiopsis_temminckii:25.12177474,Lamiopsis_tephrodes:25.12177474):10.17607834,((Glyphis_glyphis:17.66547909,(Glyphis_garricki:8.820389447,Glyphis_siamensis:8.820389447):8.845089648):10.2966767,((Glyphis_fowlerae:1.984076128,Glyphis_gangeticus:1.984076128):23.14678293,Glyphis_sp.1:25.13085906):2.83129674):7.33569729):14.5041061):14.54206353):8.156723744):7.262044354):7.183169985):8.084337986):12.06193951):2.229628852,(((Triakis_semifasciata:41.32357614,((Triakis_maculata:1.483507267,Triakis_acutipinna:1.483507267):1.959037845,Triakis_scyllium:3.442545112):37.88103103):45.14173476,(Furgaleus_macki:54.91111657,(Hemitriakis_indroyonoi:41.69888773,((Hemitriakis_japanica:3.14389151,(Hemitriakis_abdita:2.849363983,Hemitriakis_leucoperiptera:2.849363983):0.2945275274):23.37255922,(Hemitriakis_falcata:4.388073915,Hemitriakis_complicofasciata:4.388073915):22.12837682):15.182437):13.21222884):31.55419434):22.01328173,(((Gogolia_filewoodi:45.87822513,(Iago_garricki:45.18324133,Iago_omanensis:45.18324133):0.694983798):19.24910388,(Galeorhinus_galeus:63.06271669,Hypogaleus_hyugaensis:63.06271669):2.064612314):39.85048517,((((((Mustelus_mento:1.007166999,Mustelus_albipinnis:1.007166999):10.47476897,((Mustelus_palumbes:3.80104806,Mustelus_asterias:3.80104806):7.457029063,(Mustelus_stevensi:6.330832452,(Mustelus_antarcticus:5.339255298,(Mustelus_dorsalis:2.337101569,Mustelus_lenticulatus:2.337101569):3.002153728):0.9915781542):4.927243671):0.223858845):1.892883402,Mustelus_manazo:13.37481837):6.81737378,Mustelus_schmitti:20.19219315):35.78000078,((Mustelus_griseus:36.5700188,((Scylliogaleus_quecketti:34.12247321,Triakis_megalopterus:34.12247321):0.9692223434,Mustelus_mangalorensis:35.09169555):1.478323253):3.709686355,Mustelus_whitneyi:40.27970516):15.69248877):13.30129623,((Mustelus_sinusmexicanus:41.1552099,((Mustelus_norrisi:23.98723683,((Mustelus_canis:18.8959876,(Mustelus_higmani:10.19952419,Mustelus_henlei:10.19952419):8.696463411):0.3851234186,Mustelus_walkeri:19.28111102):4.70612581):4.643084131,(((Mustelus_lunulatus:18.79953762,(Mustelus_mosis:10.90941679,(Mustelus_ravidus:2.616055768,Mustelus_widodoi:2.616055768):8.293361022):7.890120833):2.292606408,(Mustelus_mustelus:2.088403143,Mustelus_fasciatus:2.088403143):19.00374189):0.9500885653,Mustelus_californicus:22.0422346):6.588087366):12.52488894):15.07172521,(Mustelus_minicanis:5.372763639,Mustelus_punctulatus:5.372763639):50.85417147):13.04655505):35.70432302):3.500779463):0.8432745072):18.67871258,((((((Bythaelurus_hispidus:9.483733485,(Bythaelurus_immaculatus:2.314472081,Bythaelurus_clevai:2.314472081):7.169261404):28.69656724,Bythaelurus_canescens:38.18030072):28.10877385,((Bythaelurus_alcockii:5.691939533,Bythaelurus_giddingsi:5.691939533):25.68566054,(Bythaelurus_lutarius:6.89256777,(Bythaelurus_incanus:1.061122649,Bythaelurus_dawsoni:1.061122649):5.831445122):24.48503231):34.91147449):17.95220284,((Figaro_boardmani:30.75405074,Figaro_striatus:30.75405074):17.55586784,(Asymbolus_funebris:35.54050268,(Asymbolus_pallidus:29.40412131,(Asymbolus_galacticus:26.88628887,((Asymbolus_occiduus:2.775123752,(Asymbolus_vincenti:2.339680136,Asymbolus_parvus:2.339680136):0.435443616):15.51525631,((Asymbolus_submaculatus:1.173350135,Asymbolus_rubiginosus:1.173350135):2.022175151,Asymbolus_analis:3.195525285):15.09485478):8.595908808):2.517832444):6.136381367):12.76941589):35.93135783):24.35045098,(((Galeus_antillensis:7.343732315,Galeus_murinus:7.343732315):60.42023634,((((Galeus_piperatus:1.780975267,Galeus_arae:1.780975267):5.102239248,(Galeus_gracilis:6.829413204,Galeus_springeri:6.829413204):0.05380131062):21.48643347,Galeus_longirostris:28.36964798):3.164008547,((Galeus_mincaronei:21.53901611,(Galeus_schultzi:5.219641029,Galeus_nipponensis:5.219641029):16.31937509):5.282193644,(Galeus_polli:19.12528105,((Galeus_atlanticus:11.53266903,(Galeus_cadenati:7.510371514,Galeus_priapus:7.510371514):4.022297513):0.8642701572,Galeus_melastomus:12.39693918):6.728341865):7.695928709):4.712446773):36.23031213):31.89060977,((((Apristurus_canutus:70.89205627,(((((Apristurus_sibogae:15.72528345,Apristurus_brunneus:15.72528345):8.072755835,(Apristurus_laurussonii:22.44892634,(Apristurus_melanoasper:7.591394095,(Apristurus_platyrhynchus:7.397969596,Apristurus_sinensis:7.397969596):0.1934244992):14.85753325):1.349111944):5.252417669,Apristurus_japonicus:29.05045696):1.850256656,Apristurus_internatus:30.90071361):33.0284104,((((Apristurus_herklotsi:12.5794805,Apristurus_micropterygeus:12.5794805):3.125912202,Apristurus_spongiceps:15.7053927):8.299281952,Apristurus_macrorhynchus:24.00467365):29.21930485,Apristurus_exsanguis:53.2239785):10.70514451):6.962932256):10.20678597,Apristurus_saldanha:81.09884224):7.065326616,(((((Parmaturus_pilosus:3.863679753,(Parmaturus_campechiensis:3.347089727,Parmaturus_bigus:3.347089727):0.5165900253):9.086203566,(Parmaturus_albimarginatus:3.090710639,Parmaturus_sp.:3.090710639):9.859172679):6.719280529,((Parmaturus_melanobranchus:9.779649067,Parmaturus_albipenis:9.779649067):2.301776576,(Parmaturus_macmillani:0.8769423662,Parmaturus_xaniurus:0.8769423662):11.20448328):7.587738205):14.96656728,Parmaturus_lanatus:34.63573113):29.74324948,(Galeus_eastmani:48.47522869,Galeus_sauteri:48.47522869):15.90375093):23.78518824):7.380294526,(((Apristurus_investigatoris:34.35170453,(Apristurus_stenseni:8.949805272,Apristurus_riveri:8.949805272):25.40189926):20.73477259,((Apristurus_gibbosus:8.072209594,Apristurus_australis:8.072208594):40.36019453,(Apristurus_longicephalus:7.271468006,Apristurus_nasutus:7.271469006):41.16093512):6.654072992):2.202368655,(((Apristurus_macrostomus:8.42071245,(Apristurus_albisoma:3.608672727,Apristurus_bucephalus:3.608672727):4.812039722):21.03470219,((((Apristurus_indicus:3.430665966,Apristurus_fedorovi:3.430665966):6.973771697,Apristurus_kampae:10.40443766):6.466682086,(Apristurus_parvipinnis:4.236678078,Apristurus_aphyodes:4.236678078):12.63444167):4.764503466,(Apristurus_ampliceps:6.462678846,(Apristurus_manis:0.5912123198,Apristurus_microps:0.5912123198):5.871466526):15.17294437):7.819791425):2.653150503,(Apristurus_pinguis:12.64791336,Apristurus_profundorum:12.64791336):19.46065179):25.18028063):38.25561761):4.110116047):8.937149966):18.0806688,(((((Halaelurus_quagga:15.79295174,Halaelurus_maculosus:15.79295174):11.57301442,(Halaelurus_sellus:0.3148469232,Halaelurus_buergeri:0.3148469232):27.05111924):16.97671886,(Halaelurus_boesemani:30.06205835,(Halaelurus_natalensis:21.4140913,Halaelurus_lineatus:21.4140913):8.647967049):14.28062668):4.060016728,(Haploblepharus_fuscus:38.26163983,(Haploblepharus_edwardsii:11.7341343,(Haploblepharus_pictus:4.885439657,Haploblepharus_kistnasamyi:4.885439657):6.848694645):26.52750552):10.14106093):59.87827239,(Pentanchus_profundicolus:27.92186427,(Holohalaelurus_grennian:17.33014992,(Holohalaelurus_melanostigma:12.490452,((Holohalaelurus_favus:2.545224266,Holohalaelurus_regani:2.545224266):0.1340222516,Holohalaelurus_punctatus:2.679246517):9.811205485):4.839697922):10.59171434):80.35910989):18.39142204):1.328183538):21.28008339):18.2347998):10.53420754):17.50647266):28.13965111):14.9125452):15.88753659):22.86254789):98.15290602)Chondrichthyes; diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Holocephali_Inoue2010.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Holocephali_Inoue2010.PHY deleted file mode 100755 index 5c3332ec..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Holocephali_Inoue2010.PHY +++ /dev/null @@ -1,4 +0,0 @@ -[Holocephalans from: Evolutionary Origin and Phylogeny of the Modern Holocephalans (Chondrichthyes: Chimaeriformes): A Mitogenomic Perspective https://doi.org/10.1093/molbev/msq147 - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(Callorhinchidae_ott550645@:170,((Rhinochimaera_ott886233@:50,(Harriotta_ott776014@:50,Neoharriotta_ott195188@:50))Rhinochimaeridae:70,(Chimaera_ott29488@:80,Hydrolagus_ott29492@:80)Chimaeridae:40):50)Holocephali; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Carcharhinicae_minus.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Carcharhinicae_minus.PHY deleted file mode 100755 index 68c2bd62..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Carcharhinicae_minus.PHY +++ /dev/null @@ -1,9 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -The OpenTree sharks (draftversion 3) are hopelessly mixed - -Where possible, I have replaced monophyletic genera with their OToL equivalents, to increase the number of species present. - -To resolve the few polytomies, the opentree has been used. In the case of Triaenodon_obesus it has been shuffled to a new place, as per the OpenTree. - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(((((((((((Apristurus_ampliceps,Apristurus_manis),Apristurus_profundorum),Apristurus_kampae),Apristurus_australis),(((((Apristurus_brunneus,Apristurus_laurussonii),Apristurus_melanoasper),Apristurus_platyrhynchus),(Apristurus_exsanguis,Apristurus_macrorhynchus)),(Galeus_sauteri,Parmaturus_ott821052@))),((Galeus_arae,(Galeus_polli,Galeus_melastomus)),Galeus_murinus)),((Asymbolus_ott296645@,Figaro_ott105538@),Bythaelurus_ott903791@)),((((Halaelurus_lineatus,Halaelurus_natalensis),Haploblepharus_ott201994@),(Halaelurus_maculosus,Halaelurus_sellus)),Holohalaelurus_ott103472@)),Proscyllium_ott821049@),(((((((((((((Carcharhinus_acronotus,Nasolamia_velox),Carcharhinus_isodon),Isogomphodon_oxyrhynchus),((Carcharhinus_dussumieri,Carcharhinus_sealei),(Carcharhinus_borneensis,Carcharhinus_macloti))),((((Carcharhinus_amblyrhynchos,Carcharhinus_wheeleri),((Carcharhinus_falciformis,Prionace_glauca),Carcharhinus_albimarginatus)),(Carcharhinus_altimus,Carcharhinus_plumbeus)),((((((((Carcharhinus_amblyrhynchoides,(Carcharhinus_limbatus,Carcharhinus_tilstoni)),Carcharhinus_leiodon),Carcharhinus_fitzroyensis),(Carcharhinus_cautus,Carcharhinus_melanopterus)),Carcharhinus_signatus),Carcharhinus_sorrah),(((Carcharhinus_amboinensis,Carcharhinus_leucas),Triaenodon_obesus),(Carcharhinus_brachyurus,Carcharhinus_brevipinna))),(((Carcharhinus_galapagensis,Carcharhinus_obscurus),Carcharhinus_longimanus),Carcharhinus_perezii)))),Carcharhinus_porosus),((Glyphis_ott541142@,Lamiopsis_ott19958@),Negaprion_ott450140@)),((Loxodon_macrorhinus,(Scoliodon_laticaudus,Scoliodon_macrorhynchos)),Rhizoprionodon_ott846406@)),((Eusphyra_blochii,(Sphyrna_mokarran,Sphyrna_zygaena)),(((Sphyrna_tiburo,Sphyrna_tudes),Sphyrna_corona),Sphyrna_lewini))),Galeocerdo_cuvier),Leptocharias_smithii),(((Chaenogaleus, Hemigaleus), Paragaleus),Hemipristis_elongata)),(((((Furgaleus_ott401912@,Hemitriakis_ott32028@),(Triakis_scyllium,Triakis_semifasciata)),(((((((Mustelus_albipinnis,Mustelus_canis),Mustelus_henlei),Mustelus_norrisi),Mustelus_californicus),((Mustelus_mosis,(Mustelus_ravidus,Mustelus_widodoi)),Mustelus_mustelus)),Mustelus_lunulatus),(((((Mustelus_antarcticus,Mustelus_lenticulatus),Mustelus_stevensi),((Mustelus_asterias,Mustelus_palumbes),Mustelus_schmitti)),Mustelus_manazo),(Scylliogaleus_quecketti,Triakis_megalopterus)))),(Galeorhinus_ott29487@,Hypogaleus_ott1035202@)),Iago_ott1037426@))),((Gollum_ott1037449@,Pseudotriakis_ott261215@),Eridacnis_ott73188@)); \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Dalatiidae.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Dalatiidae.PHY deleted file mode 100755 index add0050d..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Dalatiidae.PHY +++ /dev/null @@ -1,4 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((Squaliolus_ott956134@,(Euprotomicroides_ott3595317@,Euprotomicrus_ott547469@)),(Dalatias_ott1027234@,Isistius_ott277031@))Dalatiidae; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Etmopteridae.phy b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Etmopteridae.phy deleted file mode 100755 index 60631bea..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Etmopteridae.phy +++ /dev/null @@ -1,5 +0,0 @@ -[Lantern shark arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' https://doi.org/10.1201/B11867-9 (see http://sharksrays.org) -Etmopteridae (esp Etmopterus) is scattered all over the OpenTree, so here we use the subtree from sharksrays.org, which is unfortunately missing many species - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((((((Centroscyllium_fabricii,Centroscyllium_ritteri),(Centroscyllium_granulatum,Centroscyllium_nigrum)),Centroscyllium_excelsum),Aculeola_nigra),Trigonognathus_kabeyai),((((Etmopterus_granulosus,((Etmopterus_spinax,Etmopterus_princeps),Etmopterus_unicolor)),(Etmopterus_gracilispinis,((Etmopterus_virens,Etmopterus_polli),Etmopterus_schultzi))),((Etmopterus_bigelowi,Etmopterus_pusillus),(Etmopterus_splendidus,Etmopterus_sentosus))),(Etmopterus_lucifer,((Etmopterus_molleri,Etmopterus_brachyurus),Etmopterus_sheikoi))))Etmopteridae; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Pristiophoridae.phy b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Pristiophoridae.phy deleted file mode 100755 index bded4d66..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Pristiophoridae.phy +++ /dev/null @@ -1,6 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -Pristiophoridae is non-monophyletic in the Opentree, but this is probably an error. So we use the subtree from sharksrays.org, -but unfortunately this is missing about 5 species of Pristiophorus - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(Pliotrema_warreni,((Pristiophorus_nudipinnis,Pristiophorus_cirratus),Pristiophorus_japonicus))Pristiophoridae; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae2.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae2.PHY deleted file mode 100755 index e445416c..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae2.PHY +++ /dev/null @@ -1,5 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -Note that this was labelled Scyliorhinidae-I in TimeTree. OpenTree has non-monophyletic Atelomycterus (catfish), so here we use the Naylor tree, which unfortunately is missing about 5 spp. - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(((Atelomycterus_marmoratus,Atelomycterus_marnkalha),Aulohalaelurus_ott541139@),Schroederichthys_ott618631@); \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae3.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae3.PHY deleted file mode 100755 index b771d697..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Scyliorhinidae3.PHY +++ /dev/null @@ -1,5 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -Note that this was labelled Scyliorhinidae-2 in TimeTree. OpenTree has non monophyletic Scyliorhinus (catfish), so here we use the Naylor tree, which unfortunately is missing about 10 spp. - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(Cephaloscyllium_ott481014@,((Poroderma_africanum,Poroderma_pantherinum),(Scyliorhinus_canicula,(Scyliorhinus_capensis,(Scyliorhinus_retifer,Scyliorhinus_stellaris))))); \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Selachimorpha.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Selachimorpha.PHY deleted file mode 100755 index 96f71918..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Selachimorpha.PHY +++ /dev/null @@ -1,37 +0,0 @@ -[Selachimorpha (Selachii) tree, The opentree (draftversion3) is rather screwed up for sharks, especially the Carcharhiniformes, so ths is the most complex of the trees and involved inclusion of many other files, all labelled Naylor2012***, as they are mostly based on Naylor et al (2012): 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' from Biology of Sharks and Their Relatives (https://doi.org/10.1201/B11867-9 see http://sharksrays.org). Relative dates from an earlier Naylor paper in the timetree book (http://hedgeslab.com/pubs/216.pdf), with some polytomies broken using https://doi.org/10.1186/s12862-015-0446-6 - -Then rescaled by path d8, using this: - -#sequence length should be irrelevant here, set it to something high -Sequence length = 1000000; - -I subtract by hand a few misleading species around the great white. - -(Chimaeridae:470.2079,(Rajidae:392.60970000000003,((((((CARCHARHINICAE_MINUS@,SCYLIORHINIDAE2@:178.291):47.1132,SCYLIORHINIDAE3@:225.4042):32.3326,(Lamniformes_minus_Mitsukurinidae__ott~32038-801828-760455@:183.8337,Mitsukurinidae_ott801828@:183.8337)Lamniformes_ott32038:73.903):29.5612,(((((Ginglymostoma_ott400230@:0,Nebrius_ott833444@:0)Ginglymostomatidae:52.6559,((Stegostomatidae_ott696403@:0,Pseudoginglymostoma_ott356286@:0),Rhincodontidae_ott738324@:0):52.6559):52.6559,Hemiscylliidae_ott438189@:105.3118):88.6836,(Brachaeluridae_ott274901@:141.3395,Orectolobidae_ott572732@:141.3395):53.5797):41.5704,Parascylliidae_ott154216@:236.4896)Orectolobiformes:50.8083):29.5612,Heterodontidae_ott335517@:315.9353):32.3326,(((Squalidae_ott856584@:168.1293,((DALATIIDAE@:135.7968,(SOMNIOSIDAEOXYNOTIDAE@:133.9492,ETMOPTERIDAE@:135.7968):0.0):0.0,Centrophoridae_ott852403@:133.9492):35.0):92.3788,(SQUATINIDAE@:213.3949,(Echinorhiniformes_ott340760@:213.3949,PRISTIOPHORIDAE@:213.3949):0):49):62.8175,(Chlamydoselachidae_ott1093534@:233.7182,Notorynchidae_plus_Hexanchidae__ott~32032-1093534@:233.7182)Hexanchiformes_ott32032:89.6074):24.0185):42.4942):78.5219); - - -mrca: Chimaeridae,Carcharhinicae, fixage=420; -mrca: Rajidae,Carcharhinicae, fixage=300; - -#from N.C. Aschliman et al. / Molecular Phylogenetics and Evolution 63 (2012) 28–42 Supp Mat https://doi.org/10.1016/j.ympev.2011.12.012 - -mrca:Squalidae_ott856584@,Heterodontidae_ott335517@, fixage=225; -mrca:Heterodontidae_ott335517@,SCYLIORHINIDAE3@, fixage=200; -mrca:CARCHARHINICAE_MINUS@,SCYLIORHINIDAE3@, fixage=170; - -#from Straube et al. BMC Evolutionary Biology (2015) 15:162 (http://www.biomedcentral.com/content/pdf/s12862-015-0446-6.pdf) Fig 2 https://doi.org/10.1186/s12862-015-0446-6 - -mrca:Squalidae_ott856584@,Chlamydoselachidae_ott1093534, fixage=202.8; -mrca:Squalidae_ott856584@,Centrophoridae_ott852403@, fixage=132.86; -mrca:Centrophoridae_ott852403@,DALATIIDAE@, fixage=126.68; -mrca:DALATIIDAE@,ETMOPTERIDAE@, fixage=116.1; -mrca:ETMOPTERIDAE@,SOMNIOSIDAEOXYNOTIDAE@, fixage=110.51; -mrca:Squalidae_ott856584@,SQUATINIDAE@, fixage=177.34; -mrca:SQUATINIDAE@,Echinorhiniformes_ott340760@, fixage=147.59; - -##this produces (Chimaeridae:420.000000,(Rajidae:300.000000,((((((CARCHARHINICAE_MINUS@:134.467193,SCYLIORHINIDAE2@:134.467193):35.532807,SCYLIORHINIDAE3@:170.000000):3.555331,(Lamniformes_minus_Mitsukurinidae__ott~32038-801828-760455@:116.193063,Mitsukurinidae_ott801828@:116.193063)Lamniformes_ott32038:57.362269):9.696336,(((((Ginglymostoma_ott400230@:0.000000,Nebrius_ott833444@:0.000000)Ginglymostomatidae:33.281440,((Stegostomatidae_ott696403@:0.000000,Pseudoginglymostoma_ott356286@:0.000000):0.000000,Rhincodontidae_ott738324@:0.000000):33.281440):33.281440,Hemiscylliidae_ott438189@:66.562880):56.198912,(Brachaeluridae_ott274901@:89.334379,Orectolobidae_ott572732@:89.334379):33.427413):26.323449,Parascylliidae_ott154216@:149.085241)Orectolobiformes:34.166426):16.748333,Heterodontidae_ott335517@:200.000000):25.000000,(((Squalidae_ott856584@:132.860000,((DALATIIDAE@:116.100000,(SOMNIOSIDAEOXYNOTIDAE@:110.510000,ETMOPTERIDAE@:110.510000):5.590000):10.580000,Centrophoridae_ott852403@:126.680000):6.180000):44.480000,(SQUATINIDAE@:147.590000,(Echinorhiniformes_ott340760@:147.590000,PRISTIOPHORIDAE@:147.590000):0.000000):29.750000):25.460000,(Chlamydoselachidae_ott1093534@:146.024768,Notorynchidae_plus_Hexanchidae__ott~32032-1093534@:146.024768)Hexanchiformes:56.775232):22.200000):75.000000):120.000000) - -From which the Chimaeridae and Rajidae branches have been removed - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((((((CARCHARHINICAE_MINUS@:134.467193,SCYLIORHINIDAE2@:134.467193):35.532807,SCYLIORHINIDAE3@:170.000000):3.555331,(Lamniformes_minus_Mitsukurinidae__ott~32038-801828-760455@:116.193063,Mitsukurinidae_ott801828@:116.193063)Lamniformes_ott32038:57.362269):9.696336,(((((Ginglymostoma_ott400230@:0.000000,Nebrius_ott833444@:0.000000)Ginglymostomatidae:33.281440,((Stegostomatidae_ott696403@:0.000000,Pseudoginglymostoma_ott356286@:0.000000):0.000000,Rhincodontidae_ott738324@:0.000000):33.281440):33.281440,Hemiscylliidae_ott438189@:66.562880):56.198912,(Brachaeluridae_ott274901@:89.334379,Orectolobidae_ott572732@:89.334379):33.427413):26.323449,Parascylliidae_ott154216@:149.085241)Orectolobiformes:34.166426):16.748333,Heterodontidae_ott335517@:200.000000):25.000000,(((Squalidae_ott856584@:132.860000,((DALATIIDAE@:116.100000,(SOMNIOSIDAEOXYNOTIDAE@:110.510000,ETMOPTERIDAE@:110.510000):5.590000):10.580000,Centrophoridae_ott852403@:126.680000):6.180000):44.480000,(SQUATINIDAE@:147.590000,(Echinorhiniformes_ott340760@:147.590000,PRISTIOPHORIDAE@:147.590000):0.000000):29.750000):25.460000,(Chlamydoselachidae_ott1093534@:146.024768,Notorynchidae_plus_Hexanchidae__ott~32032-1093534@:146.024768)Hexanchiformes:56.775232):22.200000)Selachii; diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Somniosidae_Oxynotidae.PHY b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Somniosidae_Oxynotidae.PHY deleted file mode 100755 index 275bb60a..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Somniosidae_Oxynotidae.PHY +++ /dev/null @@ -1,6 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org). NB, the Somniosidae/Oxynotidae -clade from sharksrays.org should also include Centroscymnus & Proscymnodon, but these are hard to include -from the opentree, as they are marked as non monophyletic, so have simply been omitted (yuck) - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -(Somniosus_ott442056@,(Scymnodalatias_ott3595305@,((Centroselachus_ott756559@,Zameus_ott399864@),(Scymnodon_ott956139@,Oxynotidae_ott250745@))))SomniosidaeOxynotidae; diff --git a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Squatinidae.phy b/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Squatinidae.phy deleted file mode 100755 index 603406f4..00000000 --- a/data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/Naylor2012Squatinidae.phy +++ /dev/null @@ -1,5 +0,0 @@ -[Arrangement from 'Elasmobranch Phylogeny: A Mitochondrial Estimate Based on 595 Species' (https://doi.org/10.1201/B11867-9 see http://sharksrays.org) -Angel sharks (Squatinidae) are non monophyletic in the Opentree, but this is probably an error. So we use the subtree from sharksrays.org, with additions from the opentree (based on VélezZuazo), but unfortunately this is missing about 20 species. - -#from https://github.com/OpenTreeOfLife/opentree/wiki/Open-Tree-of-Life-APIs#contexts to help add OTTids: context=Vertebrates] -((Squatina_albipunctata,(Squatina_californica,Squatina_dumeril)),(((Squatina_tergocellatoides,Squatina_japonica),(Squatina_legnota,Squatina_formosa)),((Squatina_squatina,Squatina_aculeata),Squatina_oculata)))Squatinidae; \ No newline at end of file diff --git a/data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/471706.nwk b/data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/471706.nwk new file mode 100644 index 00000000..8b7f34f4 --- /dev/null +++ b/data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/471706.nwk @@ -0,0 +1 @@ +(((((Adineta_oculata_ott4101,Adineta_barbata_ott122767,Adineta_tuberculosa_ott653951,Adineta_vaga_ott681215,Adineta_steineri_ott743723,Adineta_ricciae_ott743825,Adineta_gracilis_ott991846,Adineta_grandis_ott991851,Adineta_environmental_sample_ott4952810,Adineta_longicornis_ott4952811,Adineta_elongata_ott4952812,Adineta_glauca_ott4952813,Adineta_cuneata_ott4952814,Adineta_bartosi_ott4952815,Adineta_acuticornis_ott4952816,Adineta_editae_ott5974468,Adineta_emsliei_ott5974469,Adineta_fontanetoi_ott5974470,'Adineta_vaga_complex_sp._A_JFF-2016_ott5974472','Adineta_vaga_complex_sp._B_JFF-2016_ott5974473','Adineta_vaga_complex_sp._C_JFF-2016_ott5974474','Adineta_vaga_complex_sp._D_JFF-2016_ott5974475','Adineta_vaga_complex_sp._E_JFF-2016_ott5974476','Adineta_vaga_complex_sp._F_JFF-2016_ott5974477',Adineta_beysunae_ott7506288,Adineta_coatsi_ott7506289)Adineta_ott681218,(Bradyscela_clauda_ott991847,Bradyscela_granulosa_ott4952817,Bradyscela_hoonsooi_ott6367874)Bradyscela_ott991844)Adinetidae_ott787513)Adinetida_ott157975,(Rotaria_tardigrada_ott48499,Rotaria_neptunoida_ott48504,Rotaria_magnacalcarata_ott48505,Rotaria_sordida_ott48506,Rotaria_socialis_ott48507,Rotaria_citrina_ott621261,Rotaria_macrura_ott621262,Rotaria_macroceros_ott755138,Rotaria_rotatoria_ott1018353,Rotaria_neptunia_ott1018354,Rotaria_tridens_ott4953152,Rotaria_elongata_ott4953153,Rotaria_haptica_ott4953154,Rotaria_laticeps_ott4953155,Rotaria_montana_ott4953156,Rotaria_quadrangularis_ott4953157,Rotaria_spicata_ott4953158,Rotaria_trisecata_ott4953159,Rotaria_exoculis_ott4953160,Rotaria_mento_ott4953161,Rotaria_quadrioculata_ott4953162,Rotaria_ovata_ott4953163,Rotaria_murrayi_ott4953164,Rotaria_curtipes_ott4953165,'Rotaria_cf._rotatoria_DDF-2018_ott7506308',Rotaria_megarostris_ott7506309)Rotaria_ott1018360,(((Philodina_plena_ott122765,Philodina_flaviceps_ott124555,Philodina_roseola_ott236073,Philodina_citrina_ott342351,Philodina_megalotrocha_ott400740,Philodina_vorax_ott644416,Philodina_duplicalcar_ott726737,Philodina_rugosa_ott790563,Philodina_quadrata_ott4953037,Philodina_americana_ott4953038,Philodina_jeanneli_ott4953039,Philodina_eurystephana_ott4953040,Philodina_gregaria_ott4953041,Philodina_nitida_ott4953045,Philodina_calceata_ott4953046,Philodina_australis_ott4953047,Philodina_parvicalcar_ott4953048,Philodina_praelonga_ott4953049,Philodina_dobrogensis_ott4953050,Philodina_proterva_ott4953052,Philodina_indica_ott4953053,Philodina_amethystina_ott4953054,Philodina_lepta_ott4953055,Philodina_antarctica_ott4953056,Philodina_grandis_ott4953057,Philodina_alata_ott4953058,Philodina_inopinata_ott4953059,Philodina_scabra_ott4953060,Philodina_erythrophthalma_ott4953061,Philodina_rapida_ott4953062,Philodina_foissneri_ott4953063,Philodina_striata_ott4953064,Philodina_cristata_ott4953065,Philodina_nemoralis_ott4953066,Philodina_brevipes_ott4953067,Philodina_childi_ott4953068,Philodina_tridentata_ott4953069,Philodina_arndti_ott4953070,Philodina_squamosa_ott4953071,Philodina_tenuicalcar_ott4953072,Philodina_morigera_ott4953073,Philodina_patula_ott4953074,Philodina_tranquilla_ott4953075,Philodina_convergens_ott4953076,Philodina_dartnallis_ott5974492,(Philodina_acuticornis_odiosa_ott7506298)Philodina_acuticornis_ott831317,Philodina_koreana_ott7506299,Philodina_shackletoni_ott7506300)Philodina_ott831316,(Macrotrachela_papillosa_ott267232,Macrotrachela_musculosa_ott267233,Macrotrachela_bullata_ott267234,Macrotrachela_multispinosa_ott267236,Macrotrachela_habita_ott267237,Macrotrachela_latior_ott377601,Macrotrachela_ehrenbergi_ott570855,Macrotrachela_quadricornifera_ott681209,Macrotrachela_plicata_ott790558,Macrotrachela_extensa_ott4952961,Macrotrachela_bilfingeri_ott4952962,Macrotrachela_rostrata_ott4952963,Macrotrachela_aculeata_ott4952964,Macrotrachela_zichendrahti_ott4952965,Macrotrachela_nixa_ott4952966,Macrotrachela_ehrenbergii_ott4952967,Macrotrachela_cuthberti_ott4952968,Macrotrachela_smithi_ott4952969,Macrotrachela_angusta_ott4952970,Macrotrachela_herzigana_ott4952971,Macrotrachela_labiata_ott4952972,Macrotrachela_tenuis_ott4952973,Macrotrachela_inermis_ott4952974,Macrotrachela_concinna_ott4952975,Macrotrachela_induta_ott4952977,Macrotrachela_tuberilabris_ott4952978,Macrotrachela_longistyla_ott4952979,Macrotrachela_samali_ott4952980,Macrotrachela_allani_ott4952981,Macrotrachela_insulana_ott4952982,Macrotrachela_mariae_ott4952983,Macrotrachela_punctata_ott4952984,Macrotrachela_formosa_ott4952985,Macrotrachela_pilousi_ott4952986,Macrotrachela_muricata_ott4952987,Macrotrachela_verecunda_ott4952988,Macrotrachela_pacifica_ott4952989,Macrotrachela_compacta_ott4952990,Macrotrachela_hewitti_ott4952991,Macrotrachela_ligulifera_ott4952992,Macrotrachela_plicatula_ott4952993,Macrotrachela_magna_ott4952994,Macrotrachela_murrayi_ott4952995,Macrotrachela_ambigua_ott4952996,Macrotrachela_pinnigera_ott4952997,Macrotrachela_vesicularis_ott4952998,Macrotrachela_canadensis_ott4952999,Macrotrachela_gracillima_ott4953000,Macrotrachela_nana_ott4953001,Macrotrachela_serrulata_ott4953002,Macrotrachela_decora_ott4953003,Macrotrachela_microcornis_ott4953004,Macrotrachela_intermedia_ott4953005,Macrotrachela_macmillani_ott4953006,Macrotrachela_minuta_ott4953007,Macrotrachela_kallosoma_ott4953008,Macrotrachela_speciosa_ott4953009,Macrotrachela_brevilabris_ott4953010,Macrotrachela_sonorensis_ott4953011,Macrotrachela_natans_ott4953012,Macrotrachela_crucicornis_ott4953013,Macrotrachela_insolita_ott4953014,Macrotrachela_timida_ott4953015,Macrotrachela_faveolata_ott4953016,Macrotrachela_oblita_ott4953017,Macrotrachela_vanoyei_ott4953018,Macrotrachela_gunningi_ott4953019,Macrotrachela_petulans_ott4953020,Macrotrachela_obtusa_ott4953021,Macrotrachela_festinans_ott4953022,Macrotrachela_lata_ott4953023,Macrotrachela_cancrophila_ott4953024,Macrotrachela_zickendrahti_ott4953025,Macrotrachela_lepida_ott4953026,Macrotrachela_ornata_ott4953027,Macrotrachela_brachysoma_ott4953028,Macrotrachela_ligulata_ott4953029,Macrotrachela_fungicola_ott4953030,Macrotrachela_libera_ott4953031,Macrotrachela_armillata_ott4953032,Macrotrachela_asperula_ott4953033,Macrotrachela_donneri_ott5974487,Macrotrachela_ioannae_ott5974488,Macrotrachela_jankoi_ott5974489)Macrotrachela_ott681212,(Embata_parasitica_ott267238,Embata_hamata_ott267239,Embata_laticeps_ott400736,Embata_commensalis_ott841828,Embata_environmental_sample_ott4953078,Embata_laticornis_ott4953079)Embata_ott841827,(Anomopus_telphusae_ott368451,Anomopus_chasmagnathi_ott4953092)Anomopus_ott893424,((Dissotrocha_macrostyla_tuberculata_ott436785)Dissotrocha_macrostyla_ott621248,(Dissotrocha_aculeata_crystallina_ott621251,Dissotrocha_aculeata_medioaculeata_ott621252)Dissotrocha_aculeata_ott621254,Dissotrocha_pectinata_ott4952953,Dissotrocha_scutellata_ott4952954,Dissotrocha_hertzogi_ott4952955,Dissotrocha_guyanensis_ott4952956,Dissotrocha_decembullata_ott4952957,Dissotrocha_kostei_ott4952958,Dissotrocha_schlienzi_ott4952959,Dissotrocha_bjoerki_ott4952960,Dissotrocha_spinosa_ott6367877)Dissotrocha_ott621253,(Didymodactylos_carnosus_ott693524)Didymodactylos_ott693526,(Pleuretra_lineata_ott790553,Pleuretra_hystrix_ott790565,Pleuretra_similis_ott4953080,Pleuretra_bovicornis_ott4953081,Pleuretra_proxima_ott4953082,Pleuretra_humerosa_ott4953083,Pleuretra_alpium_ott4953084,Pleuretra_triangularis_ott4953085,Pleuretra_intermedia_ott4953086,Pleuretra_africana_ott4953087,Pleuretra_brycei_ott4953088,Pleuretra_costata_ott4953089,Pleuretra_reticulata_ott4953090,Pleuretra_sulcata_ott4953091)Pleuretra_ott841829,(Mniobia_incrassata_ott790559,Mniobia_russeola_ott961833,Mniobia_dentata_ott4953093,Mniobia_recurvicornis_ott4953094,Mniobia_setifera_ott4953095,Mniobia_mirabilis_ott4953096,Mniobia_granulosa_ott4953097,Mniobia_adhaerens_ott4953098,Mniobia_ocypetes_ott4953099,Mniobia_frankenbergeri_ott4953100,Mniobia_edmondsoni_ott4953101,Mniobia_scarlatina_ott4953102,Mniobia_tentans_ott4953103,Mniobia_obtusicalcar_ott4953104,Mniobia_iurensis_ott4953105,Mniobia_bdelloidea_ott4953106,Mniobia_burgeri_ott4953107,Mniobia_procera_ott4953108,Mniobia_lobata_ott4953109,Mniobia_armata_ott4953110,Mniobia_symbiotica_ott4953111,Mniobia_donneri_ott4953112,Mniobia_placida_ott4953113,Mniobia_incurata_ott4953114,Mniobia_barbatula_ott4953115,Mniobia_tarda_ott4953116,Mniobia_montium_ott4953117,Mniobia_scabrosa_ott4953118,Mniobia_circinata_ott4953119,Mniobia_loxocorona_ott4953120,Mniobia_lamellata_ott4953121,Mniobia_obtusicornis_ott4953122,Mniobia_tetraodon_ott4953123,Mniobia_magna_ott4953124,Mniobia_punctulata_ott4953125,Mniobia_conarus_ott4953126,Mniobia_branchicola_ott4953127,Mniobia_lineata_ott4953128,Mniobia_modesta_ott4953129,Mniobia_storkani_ott4953130,Mniobia_animosa_ott4953131,Mniobia_orta_ott4953132,Mniobia_variabilis_ott4953133,Mniobia_bredensis_ott4953134,Mniobia_discophora_ott4953135,Mniobia_vargai_ott4953136,Mniobia_lenta_ott4953137,Mniobia_brachypoda_ott4953138,Mniobia_punctata_ott4953139)Mniobia_ott961834,(Ceratotrocha_multiserialis_ott4953142,Ceratotrocha_cornigera_ott4953144,Ceratotrocha_franzi_ott4953145,Ceratotrocha_velata_ott4953146,Ceratotrocha_rodewaldi_ott4953147)Ceratotrocha_ott4953141,(Pseudoembata_acutipoda_ott4953149)Pseudoembata_ott4953148,(Zelinkiella_synaptae_ott4953151)Zelinkiella_ott4953150)Philodinidae_ott831315,((Habrotrocha_rosa_ott234664,Habrotrocha_lata_ott267235,Habrotrocha_bidens_ott267241,Habrotrocha_constricta_ott681211,Habrotrocha_modesta_ott4952824,Habrotrocha_pavida_ott4952825,Habrotrocha_komareki_ott4952826,Habrotrocha_gibbosa_ott4952827,Habrotrocha_recumbens_ott4952828,Habrotrocha_crenata_ott4952829,Habrotrocha_visa_ott4952830,Habrotrocha_roperi_ott4952831,Habrotrocha_novemdens_ott4952832,Habrotrocha_vicina_ott4952833,Habrotrocha_solitaria_ott4952834,Habrotrocha_levis_ott4952835,Habrotrocha_incola_ott4952836,Habrotrocha_praelonga_ott4952837,Habrotrocha_fuhrmanni_ott4952838,Habrotrocha_caudata_ott4952839,Habrotrocha_strangulata_ott4952840,Habrotrocha_scabropyga_ott4952841,Habrotrocha_placida_ott4952842,Habrotrocha_baradlana_ott4952843,Habrotrocha_soror_ott4952844,Habrotrocha_microcephala_ott4952845,Habrotrocha_tranquilla_ott4952846,Habrotrocha_insignis_ott4952847,Habrotrocha_leitgebii_ott4952848,Habrotrocha_solida_ott4952849,Habrotrocha_appendiculata_ott4952850,Habrotrocha_thienemanni_ott4952851,Habrotrocha_colliflectens_ott4952852,Habrotrocha_mediocris_ott4952853,Habrotrocha_diarthrantenna_ott4952854,Habrotrocha_valida_ott4952855,Habrotrocha_curvicollis_ott4952856,Habrotrocha_murrayi_ott4952857,Habrotrocha_pusilla_ott4952858,Habrotrocha_tridens_ott4952859,Habrotrocha_minuta_ott4952860,Habrotrocha_subtilis_ott4952861,Habrotrocha_angularis_ott4952862,Habrotrocha_tripus_ott4952863,Habrotrocha_porrecta_ott4952864,Habrotrocha_flexicollis_ott4952865,Habrotrocha_humilis_ott4952866,Habrotrocha_serpens_ott4952867,Habrotrocha_thermalis_ott4952868,Habrotrocha_maculata_ott4952869,Habrotrocha_heinisi_ott4952870,Habrotrocha_alacris_ott4952871,Habrotrocha_gracilis_ott4952872,Habrotrocha_roeperi_ott4952873,Habrotrocha_angusticollis_ott4952874,Habrotrocha_stenochlaena_ott4952875,Habrotrocha_nodosa_ott4952876,Habrotrocha_tridentata_ott4952877,Habrotrocha_flaviformis_ott4952878,Habrotrocha_scepanotrochoides_ott4952879,Habrotrocha_collaris_ott4952880,Habrotrocha_gulosa_ott4952881,Habrotrocha_flava_ott4952882,Habrotrocha_ligula_ott4952883,Habrotrocha_aspera_ott4952884,Habrotrocha_megalocephala_ott4952885,Habrotrocha_crassa_ott4952886,Habrotrocha_parvipes_ott4952887,Habrotrocha_elliptica_ott4952888,Habrotrocha_fuscochlaena_ott4952889,Habrotrocha_reclusa_ott4952890,Habrotrocha_plana_ott4952891,Habrotrocha_longicollis_ott4952892,Habrotrocha_brocklehursti_ott4952893,Habrotrocha_lamellata_ott4952894,Habrotrocha_eremita_ott4952895,Habrotrocha_cucullata_ott4952896,Habrotrocha_quinquedens_ott4952897,Habrotrocha_nodulata_ott4952898,Habrotrocha_trilobata_ott4952899,Habrotrocha_longula_ott4952900,Habrotrocha_sylvestris_ott4952901,Habrotrocha_annulata_ott4952902,Habrotrocha_schultei_ott4952903,Habrotrocha_acornis_ott4952904,Habrotrocha_amphichlaena_ott4952905,Habrotrocha_longicalcarata_ott4952906,Habrotrocha_bulbosa_ott4952907,Habrotrocha_cuneata_ott4952908,Habrotrocha_spicula_ott4952909,Habrotrocha_pulchra_ott4952910,Habrotrocha_perforata_ott4952911,Habrotrocha_minima_ott4952912,Habrotrocha_bartosi_ott4952913,Habrotrocha_rara_ott4952914,Habrotrocha_fusca_ott4952915,Habrotrocha_puella_ott4952916,Habrotrocha_sollicita_ott4952917,Habrotrocha_ampulla_ott4952918,Habrotrocha_iners_ott4952919,Habrotrocha_elegans_ott4952920,Habrotrocha_calosa_ott4952921,Habrotrocha_curva_ott4952922,Habrotrocha_stenostephana_ott4952923,Habrotrocha_longiceps_ott4952924,Habrotrocha_pertinax_ott4952925,Habrotrocha_filum_ott4952926,Habrotrocha_munda_ott4952927,Habrotrocha_granulata_ott4952928,(Habrotrocha_elusa_elusa_ott5706037)Habrotrocha_elusa_ott267240,Habrotrocha_antarctica_ott5974478,Habrotrocha_devetteri_ott5974479,Habrotrocha_vernadskii_ott5974486)Habrotrocha_ott681214,(Scepanotrocha_sp._SceS1_ott749559,Scepanotrocha_setifera_ott4952929,Scepanotrocha_delicata_ott4952930,Scepanotrocha_haueri_ott4952931,Scepanotrocha_impexa_ott4952932,Scepanotrocha_rubra_ott4952933,Scepanotrocha_simplex_ott4952934,Scepanotrocha_galeata_ott4952935,Scepanotrocha_parva_ott4952936,Scepanotrocha_semitecta_ott4952937,Scepanotrocha_corniculata_ott4952938)Scepanotrocha_ott749561,(Otostephanos_donneri_ott790561,Otostephanos_kostei_ott4952939,Otostephanos_torquatus_ott4952940,Otostephanos_monteti_ott4952941,Otostephanos_annulatus_ott4952942,Otostephanos_regalis_ott4952943,Otostephanos_macrantennus_ott4952944,Otostephanos_jersabeki_ott4952945,Otostephanos_mundiformis_ott4952946,Otostephanos_auriculatus_ott4952947,Otostephanos_cuspidilabris_ott4952948,Otostephanos_jolantae_ott5693687)Otostephanos_ott874457,Habrotrochidae_sp._nd_ott4153537)Habrotrochidae_ott681213)Philodinida_ott725949,((Philodinavus_paradoxus_ott633720,Philodinavus_aussiensis_ott4952805)Philodinavus_ott841830,(Abrochtha_meselsoni_ott688001,Abrochtha_sonneborni_ott785906,Abrochtha_kingi_ott785919,'Abrochtha_sp._CWB-2010_ott785920',Abrochtha_carnivora_ott4952803,Abrochtha_intermedia_ott4952804)Abrochtha_ott113845,(Henoceros_falcatus_ott4952807,Henoceros_caudatus_ott4952808)Henoceros_ott2942292)Philodinavidae_ott113846)Bdelloidea_ott662648,(((((Trichotria_tetractis_ott21309,Trichotria_curta_ott2941775,Trichotria_truncata_ott2941776,Trichotria_buchneri_ott2941777,Trichotria_zanclum_ott2941778,Trichotria_pseudocurta_ott2941779,Trichotria_eukosmeta_ott2941780,Trichotria_pocillum_ott2941781,Trichotria_brevidactyla_ott7992165)Trichotria_ott740411,(Macrochaetus_collinsi_ott563703,Macrochaetus_paggiensae_ott2941783,Macrochaetus_aspera_ott2941784,Macrochaetus_collinsii_ott2941785,Macrochaetus_altamirai_ott2941786,Macrochaetus_hauerianus_ott2941787,Macrochaetus_clavicornis_ott2941788,Macrochaetus_sericus_ott2941789,Macrochaetus_philopax_ott2941790,Macrochaetus_americanus_ott2941792,Macrochaetus_longipes_ott2941793,Macrochaetus_longisetus_ott2941795,Macrochaetus_subquadratus_ott2941796,Macrochaetus_aspinus_ott2941797,Macrochaetus_multispinosus_ott2941798,Macrochaetus_dispar_ott2941799,Macrochaetus_kostei_ott2941800,Macrochaetus_danneelae_ott4952788)Macrochaetus_ott563702,(Wolga_spinifera_ott2941803)Wolga_ott2941804,(Pulchritia_dorsicornuta_ott7992164)Pulchritia_ott7992163)Trichotriidae_ott563705,((Keratella_quadrata_ott59521,(Keratella_cochlearis_robusta_ott169321,Keratella_cochlearis_tecta_ott743644,Keratella_cochlearis_faluta_ott1021339)Keratella_cochlearis_ott169319,Keratella_morenoi_ott620072,Keratella_hiemalis_ott743643,Keratella_cochlearis_AEG1_ott820711,Keratella_tropica_ott1023631,Keratella_americana_ott1023633,Keratella_procurva_ott2941872,Keratella_cruciformis_ott2941874,Keratella_stipita_ott2941875,Keratella_mixta_ott2941877,Keratella_eichwaldi_ott2941878,Keratella_maliensis_ott2941879,Keratella_zhugeae_ott2941880,Keratella_crassa_ott2941881,Keratella_lenzi_ott2941882,Keratella_kostei_ott2941883,Keratella_serrulata_ott2941884,Keratella_ahlstromi_ott2941885,Keratella_slacki_ott2941886,Keratella_javana_ott2941887,Keratella_irregularis_ott2941888,Keratella_taurocephala_ott2941890,Keratella_mexicana_ott2941891,Keratella_yamana_ott2941892,Keratella_sinensis_ott2941893,Keratella_taksinensis_ott2941895,Keratella_paludosa_ott2941896,Keratella_reducta_ott2941897,Keratella_shieli_ott2941898,Keratella_earlinae_ott2941899,Keratella_sancta_ott2941900,Keratella_ona_ott2941901,Keratella_valga_ott2941902,Keratella_wangi_ott2941903,Keratella_nhamundaiensis_ott2941904,Keratella_edmondsoni_ott2941905,Keratella_ticinensis_ott2941906,Keratella_trapezoida_ott2941907,Keratella_testudo_ott2941908,Keratella_mongoliana_ott2941909,Keratella_armadura_ott2941910,Keratella_canadensis_ott2941911,Keratella_thomassoni_ott2941912,Keratella_australis_ott2941913,Keratella_tecta_ott7506423,Keratella_valdiviensis_ott7992119)Keratella_ott169316,((Mytilina_ventralis_macracantha_ott98572,Mytilina_ventralis_brevispina_ott98575,Mytilina_ventralis_macracantha_AEG3_ott826220,Mytilina_ventralis_macracantha_AEG4_ott871351,Mytilina_ventralis_macracantha_AEG2_ott871352)Mytilina_ventralis_ott937969,Mytilina_mucronata_ott937971,Mytilina_trigona_ott2941914,Mytilina_bicarinata_ott2941915,Mytilina_macrocera_ott2941916,Mytilina_crassipes_ott2941917,Mytilina_bisulcata_ott2941918,Mytilina_compressa_ott2941919,Mytilina_lobata_ott2941920,Mytilina_carpatica_ott2941922,Mytilina_mutica_ott2941924,Mytilina_macrocerca_ott2941925,Mytilina_unguipes_ott2941926,Mytilina_acanthophora_ott2941927,Mytilina_michelangellii_ott7992121)Mytilina_ott937972,(Brachionus_urceolaris_ott169314,Brachionus_caudatus_ott197558,'Brachionus_aff._plicatilis_LSE-2010_ott255994','Brachionus_aff._dimidiatus_LSE-2010_ott255995',Brachionus_orientalis_ott259976,Brachionus_patulus_ott263248,(Brachionus_quadridentatus_brevispinus_AEG1_ott296318,Brachionus_quadridentatus_cluniorbicularis_ott745906,Brachionus_quadridentatus_brevispinus_AEG2_ott820707,Brachionus_quadridentatus_cluniorbicularis_AEG1_ott820708,Brachionus_quadridentatus_brevispinus_ott890462)Brachionus_quadridentatus_ott1047970,'Brachionus_plicatilis_group_sp._MEG-2012_ott296319',Brachionus_koreanus_ott430689,Brachionus_plicatilis_ott471703,Brachionus_dimidiatus_ott533638,Brachionus_forficula_ott618412,Brachionus_diversicornis_ott618413,Brachionus_bidentatus_ott620076,Brachionus_variabilis_ott620078,Brachionus_havanaensis_ott700604,Brachionus_macracanthus_ott700607,Brachionus_falcatus_ott700608,Brachionus_rubens_ott700609,Brachionus_manjavacas_ott737934,Brachionus_sericus_ott797562,Brachionus_calyciflorus_AEG1_ott820710,Brachionus_leydigi_ott837283,(Brachionus_angularis_pseudodolabratus_ott890473)Brachionus_angularis_ott837275,Brachionus_calyciflorus_ott939320,Brachionus_rotundiformis_ott1009399,Brachionus_ibericus_ott1047966,'Brachionus_cf._urceolaris_MEG-2012_ott1094691',Brachionus_murphyi_ott2941942,Brachionus_sessilis_ott2941943,Brachionus_durgae_ott2941944,Brachionus_pseudonilsoni_ott2941945,Brachionus_ahlstromi_ott2941946,Brachionus_pterodinoides_ott2941947,Brachionus_mirus_ott2941948,Brachionus_keikoa_ott2941949,Brachionus_spatiosus_ott2941950,Brachionus_dolabratus_ott2941951,Brachionus_amsterdamensis_ott2941952,Brachionus_schwoerbeli_ott2941953,Brachionus_satanicus_ott2941955,Brachionus_zahniseri_ott2941956,Brachionus_adisi_ott2941957,Brachionus_srisumonae_ott2941959,Brachionus_postcurvatus_ott2941960,Brachionus_lyratus_ott2941961,Brachionus_kostei_ott2941962,Brachionus_donneri_ott2941963,Brachionus_incertus_ott2941964,Brachionus_austrogenitus_ott2941965,Brachionus_huangi_ott2941966,Brachionus_amazonicus_ott2941967,Brachionus_pinneenaus_ott2941968,Brachionus_budapestinensis_ott2941969,Brachionus_nilsoni_ott2941970,Brachionus_novaezealandiae_ott2941971,Brachionus_leydigii_ott2941972,Brachionus_africanus_ott2941973,Brachionus_bennini_ott2941974,Brachionus_baylyi_ott2941975,Brachionus_charini_ott2941976,Brachionus_kultrum_ott2941977,Brachionus_josefinae_ott2941978,Brachionus_asplanchnoides_ott2941979,Brachionus_dichotomus_ott2941980,Brachionus_araceliae_ott4952777,Brachionus_mirabilis_ott4952778,Brachionus_plicatilis_complex_sp._1CM13_ott5974504,'Brachionus_plicatilis_complex_sp._28_FCOIROT_ott5974505','Brachionus_plicatilis_complex_sp._AF3_6_Turkana_Lake_Kenya_ott5974506','Brachionus_plicatilis_complex_sp._AL_1_1_ott5974507','Brachionus_plicatilis_complex_sp._AL_1_10_ott5974508','Brachionus_plicatilis_complex_sp._AL_1_11_ott5974509','Brachionus_plicatilis_complex_sp._AL_1_2_ott5974510','Brachionus_plicatilis_complex_sp._AL_1_3_ott5974511','Brachionus_plicatilis_complex_sp._AL_1_4_ott5974512','Brachionus_plicatilis_complex_sp._AL_1_5_ott5974513','Brachionus_plicatilis_complex_sp._AL_1_6_ott5974514','Brachionus_plicatilis_complex_sp._AL_1_7_ott5974515','Brachionus_plicatilis_complex_sp._AL_1_8_ott5974516','Brachionus_plicatilis_complex_sp._AL_1_9_ott5974517',Brachionus_plicatilis_complex_sp._AUBUS001_ott5974518,Brachionus_plicatilis_complex_sp._AUCOL051_ott5974519,Brachionus_plicatilis_complex_sp._AUCOL075_ott5974520,Brachionus_plicatilis_complex_sp._AUCOL149_ott5974521,Brachionus_plicatilis_complex_sp._AUCOL155_ott5974522,Brachionus_plicatilis_complex_sp._AUDAM003_ott5974523,Brachionus_plicatilis_complex_sp._AUDAM004_ott5974524,Brachionus_plicatilis_complex_sp._AUDAM006_ott5974525,Brachionus_plicatilis_complex_sp._AUDAM007_ott5974526,Brachionus_plicatilis_complex_sp._AUDAM008_ott5974527,Brachionus_plicatilis_complex_sp._AUDAM009_ott5974528,Brachionus_plicatilis_complex_sp._AUDAM010_ott5974529,Brachionus_plicatilis_complex_sp._AUDAM011_ott5974530,Brachionus_plicatilis_complex_sp._AUDAM012_ott5974531,Brachionus_plicatilis_complex_sp._AUDAM013_ott5974532,Brachionus_plicatilis_complex_sp._AUDAM015_ott5974533,Brachionus_plicatilis_complex_sp._AUDAM018_ott5974534,Brachionus_plicatilis_complex_sp._AUDAM019_ott5974535,Brachionus_plicatilis_complex_sp._AUDAM021_ott5974536,Brachionus_plicatilis_complex_sp._AUDAM025_ott5974537,Brachionus_plicatilis_complex_sp._AUDAM028_ott5974538,Brachionus_plicatilis_complex_sp._AUDAM029_ott5974539,Brachionus_plicatilis_complex_sp._AUDAM030_ott5974540,Brachionus_plicatilis_complex_sp._AUDAM033_ott5974541,Brachionus_plicatilis_complex_sp._AUDAM034A_ott5974542,Brachionus_plicatilis_complex_sp._AUDAM034B_ott5974543,Brachionus_plicatilis_complex_sp._AUDAM040_ott5974544,Brachionus_plicatilis_complex_sp._AUDAM041_ott5974545,Brachionus_plicatilis_complex_sp._AUDAM042_ott5974546,Brachionus_plicatilis_complex_sp._AUDAM045_ott5974547,Brachionus_plicatilis_complex_sp._AUDAM046_ott5974548,Brachionus_plicatilis_complex_sp._AUDAM047_ott5974549,Brachionus_plicatilis_complex_sp._AUDAM048_ott5974550,Brachionus_plicatilis_complex_sp._AUDAM049_ott5974551,Brachionus_plicatilis_complex_sp._AUDAM056_ott5974552,Brachionus_plicatilis_complex_sp._AUDAM057_ott5974553,Brachionus_plicatilis_complex_sp._AUDAM058_ott5974554,Brachionus_plicatilis_complex_sp._AUDAM059_ott5974555,Brachionus_plicatilis_complex_sp._AUDAM060_ott5974556,Brachionus_plicatilis_complex_sp._AUDAM061_ott5974557,Brachionus_plicatilis_complex_sp._AUDUN001_ott5974558,Brachionus_plicatilis_complex_sp._AUDUN003_ott5974559,Brachionus_plicatilis_complex_sp._AULAT006_ott5974560,Brachionus_plicatilis_complex_sp._AULAT007_ott5974561,Brachionus_plicatilis_complex_sp._AULAT013_ott5974562,Brachionus_plicatilis_complex_sp._AULAT017_ott5974563,Brachionus_plicatilis_complex_sp._AULAT019_ott5974564,Brachionus_plicatilis_complex_sp._AULAT024_ott5974565,Brachionus_plicatilis_complex_sp._AULAT042_ott5974566,Brachionus_plicatilis_complex_sp._AUPEA002_ott5974567,Brachionus_plicatilis_complex_sp._AUPEA008_ott5974568,Brachionus_plicatilis_complex_sp._AUPEA011_ott5974569,Brachionus_plicatilis_complex_sp._AUPEA013_ott5974570,Brachionus_plicatilis_complex_sp._AUPEA015_ott5974571,Brachionus_plicatilis_complex_sp._AUPEA020_ott5974572,Brachionus_plicatilis_complex_sp._AUPEA021_ott5974573,Brachionus_plicatilis_complex_sp._AUPEA022_ott5974574,Brachionus_plicatilis_complex_sp._AUPEA025_ott5974575,Brachionus_plicatilis_complex_sp._AUPEA028_ott5974576,Brachionus_plicatilis_complex_sp._AUPEA030_ott5974577,Brachionus_plicatilis_complex_sp._AUPEA031_ott5974578,Brachionus_plicatilis_complex_sp._AUPIP011_ott5974579,Brachionus_plicatilis_complex_sp._AUTOW002_ott5974580,Brachionus_plicatilis_complex_sp._AUTOW003_ott5974581,Brachionus_plicatilis_complex_sp._AUTYEN080_ott5974582,Brachionus_plicatilis_complex_sp._AUWAR001_ott5974583,Brachionus_plicatilis_complex_sp._AUWAR002_ott5974584,Brachionus_plicatilis_complex_sp._AUWARCL_ott5974585,Brachionus_plicatilis_complex_sp._AUYEL003_ott5974586,Brachionus_plicatilis_complex_sp._AUYEL004_ott5974587,Brachionus_plicatilis_complex_sp._AUYEL005_ott5974588,Brachionus_plicatilis_complex_sp._AUYEN005_ott5974589,Brachionus_plicatilis_complex_sp._AUYEN010_ott5974590,Brachionus_plicatilis_complex_sp._AUYEN016_ott5974591,Brachionus_plicatilis_complex_sp._AUYEN020_ott5974592,Brachionus_plicatilis_complex_sp._AUYEN075_ott5974593,Brachionus_plicatilis_complex_sp._AUYEN076_ott5974594,Brachionus_plicatilis_complex_sp._AUYEN077_ott5974595,Brachionus_plicatilis_complex_sp._AUYEN078_ott5974596,Brachionus_plicatilis_complex_sp._AUYEN079_ott5974597,Brachionus_plicatilis_complex_sp._AUYEN080_ott5974598,Brachionus_plicatilis_complex_sp._AUYEN081_ott5974599,Brachionus_plicatilis_complex_sp._AUYEN082_ott5974600,Brachionus_plicatilis_complex_sp._AUYEN083_ott5974601,Brachionus_plicatilis_complex_sp._Almenara_ott5974602,Brachionus_plicatilis_complex_sp._Almenara2_ott5974603,Brachionus_plicatilis_complex_sp._Almenara3_ott5974604,Brachionus_plicatilis_complex_sp._Alvarado_ott5974605,Brachionus_plicatilis_complex_sp._BEARC001_ott5974606,Brachionus_plicatilis_complex_sp._BEARC002_ott5974607,Brachionus_plicatilis_complex_sp._BEARC003_ott5974608,Brachionus_plicatilis_complex_sp._BEARC004_ott5974609,Brachionus_plicatilis_complex_sp._BEARC005_ott5974610,Brachionus_plicatilis_complex_sp._BEARC006_ott5974611,Brachionus_plicatilis_complex_sp._BEARC007_ott5974612,Brachionus_plicatilis_complex_sp._BEARC008_ott5974613,Brachionus_plicatilis_complex_sp._BEARC009_ott5974614,Brachionus_plicatilis_complex_sp._BEARC010_ott5974615,Brachionus_plicatilis_complex_sp._BEARC011_ott5974616,Brachionus_plicatilis_complex_sp._BEARC012_ott5974617,Brachionus_plicatilis_complex_sp._BEARC013_ott5974618,Brachionus_plicatilis_complex_sp._BEARC014_ott5974619,Brachionus_plicatilis_complex_sp._BEARC015_ott5974620,Brachionus_plicatilis_complex_sp._BEARC016_ott5974621,Brachionus_plicatilis_complex_sp._BEARC017_ott5974622,Brachionus_plicatilis_complex_sp._BEARC018_ott5974623,Brachionus_plicatilis_complex_sp._BEARC019_ott5974624,Brachionus_plicatilis_complex_sp._BEARC020_ott5974625,Brachionus_plicatilis_complex_sp._BUL01_ott5974626,Brachionus_plicatilis_complex_sp._BUL03_ott5974627,Brachionus_plicatilis_complex_sp._BUL04_ott5974628,Brachionus_plicatilis_complex_sp._BUL05_ott5974629,Brachionus_plicatilis_complex_sp._BULO4_ott5974630,Brachionus_plicatilis_complex_sp._BUS06_ott5974631,Brachionus_plicatilis_complex_sp._BUS08_ott5974632,Brachionus_plicatilis_complex_sp._BUS20_ott5974633,Brachionus_plicatilis_complex_sp._BUSCL_ott5974634,Brachionus_plicatilis_complex_sp._COL01_ott5974635,Brachionus_plicatilis_complex_sp._COL03_ott5974636,Brachionus_plicatilis_complex_sp._COL05_ott5974637,Brachionus_plicatilis_complex_sp._COL07_ott5974638,Brachionus_plicatilis_complex_sp._COY01_ott5974639,Brachionus_plicatilis_complex_sp._COY03_ott5974640,Brachionus_plicatilis_complex_sp._COY05_ott5974641,Brachionus_plicatilis_complex_sp._COY08_ott5974642,Brachionus_plicatilis_complex_sp._COYCL_ott5974643,Brachionus_plicatilis_complex_sp._DAM01_ott5974644,Brachionus_plicatilis_complex_sp._DAM02_ott5974645,Brachionus_plicatilis_complex_sp._DAM03_ott5974646,Brachionus_plicatilis_complex_sp._DAM04_ott5974647,Brachionus_plicatilis_complex_sp._DAMCL_ott5974648,Brachionus_plicatilis_complex_sp._DUN01_ott5974649,Brachionus_plicatilis_complex_sp._DUN05_ott5974650,Brachionus_plicatilis_complex_sp._DUN08_ott5974651,Brachionus_plicatilis_complex_sp._DUN09_ott5974652,Brachionus_plicatilis_complex_sp._DUN13_ott5974653,Brachionus_plicatilis_complex_sp._DUNCL_ott5974654,Brachionus_plicatilis_complex_sp._ESUVE001_ott5974655,Brachionus_plicatilis_complex_sp._Esk_ott5974656,Brachionus_plicatilis_complex_sp._Figure8_ott5974657,Brachionus_plicatilis_complex_sp._GRKOR003_ott5974658,Brachionus_plicatilis_complex_sp._GRKOR021_ott5974659,Brachionus_plicatilis_complex_sp._GRKOR044_ott5974660,Brachionus_plicatilis_complex_sp._HAR01_ott5974661,Brachionus_plicatilis_complex_sp._HAR04_ott5974662,Brachionus_plicatilis_complex_sp._HAR11_ott5974663,Brachionus_plicatilis_complex_sp._HOT02_ott5974664,Brachionus_plicatilis_complex_sp._HOTCL_ott5974665,Brachionus_plicatilis_complex_sp._JPNAG001_ott5974666,Brachionus_plicatilis_complex_sp._JPNAG002_ott5974667,Brachionus_plicatilis_complex_sp._JPNAG003_ott5974668,Brachionus_plicatilis_complex_sp._JPNAG004_ott5974669,Brachionus_plicatilis_complex_sp._JPNAG005_ott5974670,Brachionus_plicatilis_complex_sp._JPNAG006_ott5974671,Brachionus_plicatilis_complex_sp._JPNAG007_ott5974672,Brachionus_plicatilis_complex_sp._JPNAG008_ott5974673,Brachionus_plicatilis_complex_sp._JPNAG009_ott5974674,Brachionus_plicatilis_complex_sp._JPNAG010_ott5974675,Brachionus_plicatilis_complex_sp._JPNAG011_ott5974676,Brachionus_plicatilis_complex_sp._JPNAG012_ott5974677,Brachionus_plicatilis_complex_sp._JPNAG013_ott5974678,Brachionus_plicatilis_complex_sp._JPNAG014_ott5974679,Brachionus_plicatilis_complex_sp._JPNAG015_ott5974680,Brachionus_plicatilis_complex_sp._JPNAG016_ott5974681,Brachionus_plicatilis_complex_sp._JPNAG017_ott5974682,Brachionus_plicatilis_complex_sp._JPNAG018_ott5974683,Brachionus_plicatilis_complex_sp._JPNAG019_ott5974684,Brachionus_plicatilis_complex_sp._JPNAG020_ott5974685,Brachionus_plicatilis_complex_sp._JPNAG022_ott5974686,Brachionus_plicatilis_complex_sp._JPNAG023_ott5974687,Brachionus_plicatilis_complex_sp._JPNAG024_ott5974688,Brachionus_plicatilis_complex_sp._JPNAG028_ott5974689,Brachionus_plicatilis_complex_sp._JPNAG030_ott5974690,Brachionus_plicatilis_complex_sp._JPNAG032_ott5974691,Brachionus_plicatilis_complex_sp._JPNAG033_ott5974692,Brachionus_plicatilis_complex_sp._JPNAG034_ott5974693,Brachionus_plicatilis_complex_sp._JPNAG035_ott5974694,Brachionus_plicatilis_complex_sp._JPNAG036_ott5974695,Brachionus_plicatilis_complex_sp._JPNAG037_ott5974696,Brachionus_plicatilis_complex_sp._JPNAG038_ott5974697,Brachionus_plicatilis_complex_sp._JPNAG039_ott5974698,Brachionus_plicatilis_complex_sp._JPNAG041_ott5974699,Brachionus_plicatilis_complex_sp._JPNAG042_ott5974700,Brachionus_plicatilis_complex_sp._JPNAG043_ott5974701,Brachionus_plicatilis_complex_sp._JPNAG044_ott5974702,Brachionus_plicatilis_complex_sp._JPNAG046_ott5974703,Brachionus_plicatilis_complex_sp._JPNAG047_ott5974704,Brachionus_plicatilis_complex_sp._JPNAG048_ott5974705,Brachionus_plicatilis_complex_sp._JPNAG049_ott5974706,Brachionus_plicatilis_complex_sp._JPNAG050_ott5974707,Brachionus_plicatilis_complex_sp._JPNAG051_ott5974708,Brachionus_plicatilis_complex_sp._JPNAG052_ott5974709,Brachionus_plicatilis_complex_sp._JPNAG053_ott5974710,Brachionus_plicatilis_complex_sp._JPNAG054_ott5974711,Brachionus_plicatilis_complex_sp._JPNAG055_ott5974712,Brachionus_plicatilis_complex_sp._JPNAG056_ott5974713,Brachionus_plicatilis_complex_sp._JPNAG057_ott5974714,Brachionus_plicatilis_complex_sp._JPNAG058_ott5974715,Brachionus_plicatilis_complex_sp._JPNAG059_ott5974716,Brachionus_plicatilis_complex_sp._JPNAG060_ott5974717,Brachionus_plicatilis_complex_sp._JPNAG061_ott5974718,Brachionus_plicatilis_complex_sp._JPNAG062_ott5974719,Brachionus_plicatilis_complex_sp._JPNAG063_ott5974720,Brachionus_plicatilis_complex_sp._JPNAG064_ott5974721,Brachionus_plicatilis_complex_sp._JPNAG065_ott5974722,Brachionus_plicatilis_complex_sp._JPNAG066_ott5974723,Brachionus_plicatilis_complex_sp._JPNAG067_ott5974724,Brachionus_plicatilis_complex_sp._JPNAG068_ott5974725,Brachionus_plicatilis_complex_sp._JPNAG069_ott5974726,Brachionus_plicatilis_complex_sp._JPNAG070_ott5974727,Brachionus_plicatilis_complex_sp._JPNAG071_ott5974728,Brachionus_plicatilis_complex_sp._JUS02_ott5974729,Brachionus_plicatilis_complex_sp._JUS04_ott5974730,'Brachionus_plicatilis_complex_sp._KS_1_1_ott5974731','Brachionus_plicatilis_complex_sp._KS_1_2_ott5974732','Brachionus_plicatilis_complex_sp._KS_1_3_ott5974733','Brachionus_plicatilis_complex_sp._KS_1_4_ott5974734','Brachionus_plicatilis_complex_sp._KS_1_5_ott5974735','Brachionus_plicatilis_complex_sp._KS_1_6_ott5974736','Brachionus_plicatilis_complex_sp._KS_1_7_ott5974737',Brachionus_plicatilis_complex_sp._LAT01_ott5974738,Brachionus_plicatilis_complex_sp._LAT02_ott5974739,Brachionus_plicatilis_complex_sp._LAT03_ott5974740,Brachionus_plicatilis_complex_sp._LAT04_ott5974741,Brachionus_plicatilis_complex_sp._MNCHU002_ott5974742,Brachionus_plicatilis_complex_sp._MNCHU003_ott5974743,Brachionus_plicatilis_complex_sp._MNCHU008_ott5974744,Brachionus_plicatilis_complex_sp._MNCHU010_ott5974745,Brachionus_plicatilis_complex_sp._MNCHU012_ott5974746,Brachionus_plicatilis_complex_sp._MNCHU020_ott5974747,Brachionus_plicatilis_complex_sp._MNCHU024_ott5974748,Brachionus_plicatilis_complex_sp._MNCHU031_ott5974749,Brachionus_plicatilis_complex_sp._MNCHU035_ott5974750,Brachionus_plicatilis_complex_sp._MNTSA011_ott5974751,Brachionus_plicatilis_complex_sp._MNTSA012_ott5974752,Brachionus_plicatilis_complex_sp._MOF01_ott5974753,Brachionus_plicatilis_complex_sp._MOR03_ott5974754,Brachionus_plicatilis_complex_sp._MOR05_ott5974755,Brachionus_plicatilis_complex_sp._MOR07_ott5974756,Brachionus_plicatilis_complex_sp._MOR10_ott5974757,Brachionus_plicatilis_complex_sp._MORCL_ott5974758,Brachionus_plicatilis_complex_sp._MUL02_ott5974759,Brachionus_plicatilis_complex_sp._MUL06_ott5974760,Brachionus_plicatilis_complex_sp._MUL08_ott5974761,Brachionus_plicatilis_complex_sp._MUL15_ott5974762,Brachionus_plicatilis_complex_sp._MULCL_ott5974763,Brachionus_plicatilis_complex_sp._MXALC001_ott5974764,Brachionus_plicatilis_complex_sp._MXALC002_ott5974765,Brachionus_plicatilis_complex_sp._MXALC004_ott5974766,Brachionus_plicatilis_complex_sp._MXALC008_ott5974767,Brachionus_plicatilis_complex_sp._MXALC010_ott5974768,Brachionus_plicatilis_complex_sp._MXALC013_ott5974769,Brachionus_plicatilis_complex_sp._MXALC018_ott5974770,Brachionus_plicatilis_complex_sp._MXALC021_ott5974771,Brachionus_plicatilis_complex_sp._MXALC022_ott5974772,Brachionus_plicatilis_complex_sp._MXALC024_ott5974773,Brachionus_plicatilis_complex_sp._MXALC025_ott5974774,Brachionus_plicatilis_complex_sp._MXALC026_ott5974775,Brachionus_plicatilis_complex_sp._MXALC028_ott5974776,Brachionus_plicatilis_complex_sp._MXALC029_ott5974777,Brachionus_plicatilis_complex_sp._MXALC031_ott5974778,Brachionus_plicatilis_complex_sp._MXALC033_ott5974779,Brachionus_plicatilis_complex_sp._MXALC034_ott5974780,Brachionus_plicatilis_complex_sp._MXALC036_ott5974781,Brachionus_plicatilis_complex_sp._MXALC037_ott5974782,Brachionus_plicatilis_complex_sp._MXALC038_ott5974783,Brachionus_plicatilis_complex_sp._MXALC039_ott5974784,Brachionus_plicatilis_complex_sp._MXALC042_ott5974785,Brachionus_plicatilis_complex_sp._MXALC045_ott5974786,Brachionus_plicatilis_complex_sp._MXALC046_ott5974787,Brachionus_plicatilis_complex_sp._MXALC047_ott5974788,Brachionus_plicatilis_complex_sp._MXALC049_ott5974789,Brachionus_plicatilis_complex_sp._MXALC054_ott5974790,Brachionus_plicatilis_complex_sp._MXATE005_ott5974791,Brachionus_plicatilis_complex_sp._MXPRE001_ott5974792,Brachionus_plicatilis_complex_sp._MXPRE004_ott5974793,Brachionus_plicatilis_complex_sp._MXPRE005_ott5974794,Brachionus_plicatilis_complex_sp._NOCCN001_ott5974795,Brachionus_plicatilis_complex_sp._Nakuru10_ott5974796,Brachionus_plicatilis_complex_sp._Nakuru11_ott5974797,Brachionus_plicatilis_complex_sp._Nakuru12_ott5974798,Brachionus_plicatilis_complex_sp._Nakuru13_ott5974799,Brachionus_plicatilis_complex_sp._Nakuru14_ott5974800,Brachionus_plicatilis_complex_sp._Nakuru15_ott5974801,Brachionus_plicatilis_complex_sp._Nakuru16_ott5974802,Brachionus_plicatilis_complex_sp._Nakuru3_ott5974803,Brachionus_plicatilis_complex_sp._Nakuru4_ott5974804,Brachionus_plicatilis_complex_sp._Nakuru5_ott5974805,Brachionus_plicatilis_complex_sp._Nakuru6_ott5974806,Brachionus_plicatilis_complex_sp._Nakuru7_ott5974807,Brachionus_plicatilis_complex_sp._Nakuru8_ott5974808,Brachionus_plicatilis_complex_sp._Nakuru9_ott5974809,Brachionus_plicatilis_complex_sp._OHJ10_ott5974810,Brachionus_plicatilis_complex_sp._OHJ100_ott5974811,Brachionus_plicatilis_complex_sp._OHJ101_ott5974812,Brachionus_plicatilis_complex_sp._OHJ102_ott5974813,Brachionus_plicatilis_complex_sp._OHJ103_ott5974814,Brachionus_plicatilis_complex_sp._OHJ104_ott5974815,Brachionus_plicatilis_complex_sp._OHJ105_ott5974816,Brachionus_plicatilis_complex_sp._OHJ11_ott5974817,Brachionus_plicatilis_complex_sp._OHJ13_ott5974818,Brachionus_plicatilis_complex_sp._OHJ19_ott5974819,Brachionus_plicatilis_complex_sp._OHJ1new_ott5974820,Brachionus_plicatilis_complex_sp._OHJ2_ott5974821,Brachionus_plicatilis_complex_sp._OHJ21_ott5974822,Brachionus_asplanchnoidis_ott5974823,Brachionus_plicatilis_complex_sp._OHJ23_ott5974824,Brachionus_plicatilis_complex_sp._OHJ24_ott5974825,Brachionus_plicatilis_complex_sp._OHJ25_ott5974826,Brachionus_plicatilis_complex_sp._OHJ26_ott5974827,Brachionus_plicatilis_complex_sp._OHJ28_ott5974828,Brachionus_plicatilis_complex_sp._OHJ29_ott5974829,Brachionus_plicatilis_complex_sp._OHJ30_ott5974830,Brachionus_plicatilis_complex_sp._OHJ31_ott5974831,Brachionus_plicatilis_complex_sp._OHJ32_ott5974832,Brachionus_plicatilis_complex_sp._OHJ34_ott5974833,Brachionus_plicatilis_complex_sp._OHJ36_ott5974834,Brachionus_plicatilis_complex_sp._OHJ38_ott5974835,Brachionus_plicatilis_complex_sp._OHJ40_ott5974836,Brachionus_plicatilis_complex_sp._OHJ41_ott5974837,Brachionus_plicatilis_complex_sp._OHJ42_ott5974838,Brachionus_plicatilis_complex_sp._OHJ43_ott5974839,Brachionus_plicatilis_complex_sp._OHJ44_ott5974840,Brachionus_plicatilis_complex_sp._OHJ45_ott5974841,Brachionus_plicatilis_complex_sp._OHJ46_ott5974842,Brachionus_plicatilis_complex_sp._OHJ47_ott5974843,Brachionus_plicatilis_complex_sp._OHJ48_ott5974844,Brachionus_plicatilis_complex_sp._OHJ49_ott5974845,Brachionus_plicatilis_complex_sp._OHJ4new_ott5974846,Brachionus_plicatilis_complex_sp._OHJ50_ott5974847,Brachionus_plicatilis_complex_sp._OHJ51_ott5974848,Brachionus_plicatilis_complex_sp._OHJ53_ott5974849,Brachionus_plicatilis_complex_sp._OHJ54_ott5974850,Brachionus_plicatilis_complex_sp._OHJ6_ott5974851,Brachionus_plicatilis_complex_sp._OHJ60_ott5974852,Brachionus_plicatilis_complex_sp._OHJ61_ott5974853,Brachionus_plicatilis_complex_sp._OHJ62_ott5974854,Brachionus_plicatilis_complex_sp._OHJ64_ott5974855,Brachionus_plicatilis_complex_sp._OHJ65_ott5974856,Brachionus_plicatilis_complex_sp._OHJ66_ott5974857,Brachionus_plicatilis_complex_sp._OHJ67_ott5974858,Brachionus_plicatilis_complex_sp._OHJ68_ott5974859,Brachionus_plicatilis_complex_sp._OHJ69_ott5974860,Brachionus_plicatilis_complex_sp._OHJ7_ott5974861,Brachionus_plicatilis_complex_sp._OHJ70_ott5974862,Brachionus_plicatilis_complex_sp._OHJ71_ott5974863,Brachionus_plicatilis_complex_sp._OHJ72_ott5974864,Brachionus_plicatilis_complex_sp._OHJ73_ott5974865,Brachionus_plicatilis_complex_sp._OHJ74_ott5974866,Brachionus_plicatilis_complex_sp._OHJ75_ott5974867,Brachionus_plicatilis_complex_sp._OHJ76_ott5974868,Brachionus_plicatilis_complex_sp._OHJ77_ott5974869,Brachionus_plicatilis_complex_sp._OHJ78_ott5974870,Brachionus_plicatilis_complex_sp._OHJ79_ott5974871,Brachionus_plicatilis_complex_sp._OHJ80_ott5974872,Brachionus_plicatilis_complex_sp._OHJ81_ott5974873,Brachionus_plicatilis_complex_sp._OHJ82_ott5974874,Brachionus_plicatilis_complex_sp._OHJ83_ott5974875,Brachionus_plicatilis_complex_sp._OHJ84_ott5974876,Brachionus_plicatilis_complex_sp._OHJ85_ott5974877,Brachionus_plicatilis_complex_sp._OHJ86_ott5974878,Brachionus_plicatilis_complex_sp._OHJ87_ott5974879,Brachionus_plicatilis_complex_sp._OHJ88_ott5974880,Brachionus_plicatilis_complex_sp._OHJ89_ott5974881,Brachionus_plicatilis_complex_sp._OHJ9_ott5974882,Brachionus_plicatilis_complex_sp._OHJ90_ott5974883,Brachionus_plicatilis_complex_sp._OHJ91_ott5974884,Brachionus_plicatilis_complex_sp._OHJ92_ott5974885,Brachionus_plicatilis_complex_sp._OHJ93_ott5974886,Brachionus_plicatilis_complex_sp._OHJ94_ott5974887,Brachionus_plicatilis_complex_sp._OHJ95_ott5974888,Brachionus_plicatilis_complex_sp._OHJ96_ott5974889,Brachionus_plicatilis_complex_sp._OHJ97_ott5974890,Brachionus_plicatilis_complex_sp._OHJ98_ott5974891,Brachionus_plicatilis_complex_sp._OHJ99_ott5974892,Brachionus_plicatilis_complex_sp._PEA02_ott5974893,Brachionus_plicatilis_complex_sp._PEA05_ott5974894,Brachionus_plicatilis_complex_sp._PEA09_ott5974895,Brachionus_plicatilis_complex_sp._PEA15_ott5974896,Brachionus_plicatilis_complex_sp._PEA18_ott5974897,Brachionus_plicatilis_complex_sp._PEACL_ott5974898,Brachionus_plicatilis_complex_sp._PIP02_ott5974899,Brachionus_plicatilis_complex_sp._PIP03_ott5974900,Brachionus_plicatilis_complex_sp._PIP04_ott5974901,'Brachionus_plicatilis_complex_sp._Poza_sur_ott5974902','Brachionus_plicatilis_complex_sp._Qo_L2_ott5974903','Brachionus_plicatilis_complex_sp._Qo_S1_ott5974904',Brachionus_plicatilis_complex_sp._SAN02_ott5974905,Brachionus_plicatilis_complex_sp._SAN07_ott5974906,Brachionus_plicatilis_complex_sp._SAN10_ott5974907,Brachionus_plicatilis_complex_sp._SAN11_ott5974908,Brachionus_plicatilis_complex_sp._SAN19_ott5974909,'Brachionus_plicatilis_complex_sp._Segu_L1_ott5974910','Brachionus_plicatilis_complex_sp._Sht_L1_ott5974911','Brachionus_plicatilis_complex_sp._Sht_L2_ott5974912','Brachionus_plicatilis_complex_sp._Sht_S1_ott5974913','Brachionus_plicatilis_complex_sp._Sht_S2_ott5974914',Brachionus_plicatilis_complex_sp._TOW01_ott5974915,Brachionus_plicatilis_complex_sp._TOW02_ott5974916,Brachionus_plicatilis_complex_sp._TOW15_ott5974917,Brachionus_plicatilis_complex_sp._TOWCL_ott5974918,Brachionus_plicatilis_complex_sp._USGET002_ott5974919,Brachionus_plicatilis_complex_sp._USGET003_ott5974920,Brachionus_plicatilis_complex_sp._USGET004_ott5974921,Brachionus_plicatilis_complex_sp._USGET005_ott5974922,Brachionus_plicatilis_complex_sp._USGET006_ott5974923,Brachionus_plicatilis_complex_sp._USGET007_ott5974924,Brachionus_plicatilis_complex_sp._USIND002_ott5974925,Brachionus_plicatilis_complex_sp._USIND092_ott5974926,Brachionus_plicatilis_complex_sp._USIND125_ott5974927,Brachionus_plicatilis_complex_sp._USIND168_ott5974928,Brachionus_plicatilis_complex_sp._USIND172_ott5974929,Brachionus_plicatilis_complex_sp._USIND182_ott5974930,Brachionus_plicatilis_complex_sp._USIND190_ott5974931,Brachionus_plicatilis_complex_sp._USIND237_ott5974932,Brachionus_plicatilis_complex_sp._USSAL020_ott5974933,Brachionus_plicatilis_complex_sp._USSAL024_ott5974934,Brachionus_plicatilis_complex_sp._USSAL033_ott5974935,Brachionus_plicatilis_complex_sp._USSAL042_ott5974936,Brachionus_plicatilis_complex_sp._USSAL049_ott5974937,Brachionus_plicatilis_complex_sp._WAN02_ott5974938,Brachionus_plicatilis_complex_sp._WAN11_ott5974939,Brachionus_plicatilis_complex_sp._WAN16_ott5974940,Brachionus_plicatilis_complex_sp._WANCL_ott5974941,Brachionus_plicatilis_complex_sp._WAR02_ott5974942,Brachionus_plicatilis_complex_sp._WAR05_ott5974943,Brachionus_plicatilis_complex_sp._WAR06_ott5974944,Brachionus_plicatilis_complex_sp._WAR07_ott5974945,Brachionus_plicatilis_complex_sp._WARCL_ott5974946,Brachionus_plicatilis_complex_sp._YEL02_ott5974947,Brachionus_plicatilis_complex_sp._YEL05_ott5974948,Brachionus_plicatilis_complex_sp._YEL11_ott5974949,Brachionus_plicatilis_complex_sp._YELCL_ott5974950,Brachionus_plicatilis_complex_sp._YEN01_ott5974951,Brachionus_plicatilis_complex_sp._YEN02_ott5974952,Brachionus_plicatilis_complex_sp._YEN03_ott5974953,Brachionus_plicatilis_complex_sp._YEN05_ott5974954,Brachionus_plicatilis_complex_sp._YEN06_ott5974955,Brachionus_plicatilis_complex_sp._YEN16_ott5974956,Brachionus_plicatilis_complex_sp._YENCL_ott5974957,'Brachionus_plicatilis_complex_sp._Zbl1_L_ott5974958','Brachionus_plicatilis_complex_sp._Zbl3_1_ott5974959','Brachionus_plicatilis_complex_sp._Zbl4_L_ott5974960',Brachionus_budapestensis_ott6367884,'Brachionus_cf._plicatilis_MEG-2019_ott7506334','Brachionus_plicatilis_complex_sp._CHILE_clone_1_ott7506335','Brachionus_plicatilis_complex_sp._CHILE_clone_12_ott7506336','Brachionus_plicatilis_complex_sp._CHILE_clone_13_ott7506337','Brachionus_plicatilis_complex_sp._CHILE_clone_14_ott7506338','Brachionus_plicatilis_complex_sp._CHILE_clone_16_ott7506339','Brachionus_plicatilis_complex_sp._CHILE_clone_17_ott7506340','Brachionus_plicatilis_complex_sp._CHILE_clone_18_ott7506341','Brachionus_plicatilis_complex_sp._CHILE_clone_1a_ott7506342','Brachionus_plicatilis_complex_sp._CHILE_clone_2_ott7506343','Brachionus_plicatilis_complex_sp._CHILE_clone_21_ott7506344','Brachionus_plicatilis_complex_sp._CHILE_clone_22_ott7506345','Brachionus_plicatilis_complex_sp._CHILE_clone_26_ott7506346','Brachionus_plicatilis_complex_sp._CHILE_clone_28_ott7506347','Brachionus_plicatilis_complex_sp._CHILE_clone_29_ott7506348','Brachionus_plicatilis_complex_sp._CHILE_clone_2a_ott7506349','Brachionus_plicatilis_complex_sp._CHILE_clone_3_ott7506350','Brachionus_plicatilis_complex_sp._CHILE_clone_3a_ott7506351','Brachionus_plicatilis_complex_sp._CHILE_clone_4_ott7506352','Brachionus_plicatilis_complex_sp._CHILE_clone_4a_ott7506353','Brachionus_plicatilis_complex_sp._CHILE_clone_5_ott7506354','Brachionus_plicatilis_complex_sp._CHILE_clone_6_ott7506355','Brachionus_plicatilis_complex_sp._CHILE_clone_7_ott7506356','Brachionus_plicatilis_complex_sp._CHILE_clone_7a_ott7506357','Brachionus_plicatilis_complex_sp._CHILE_clone_8_ott7506358','Brachionus_plicatilis_complex_sp._CHILE_clone_9_ott7506359',Brachionus_plicatilis_complex_sp._Chile_clone_1_ott7506360,Brachionus_plicatilis_complex_sp._Chile_clone_12_ott7506361,Brachionus_plicatilis_complex_sp._Chile_clone_13_ott7506362,Brachionus_plicatilis_complex_sp._Chile_clone_14_ott7506363,Brachionus_plicatilis_complex_sp._Chile_clone_15_ott7506364,Brachionus_plicatilis_complex_sp._Chile_clone_16_ott7506365,Brachionus_plicatilis_complex_sp._Chile_clone_17_ott7506366,Brachionus_plicatilis_complex_sp._Chile_clone_18_ott7506367,Brachionus_plicatilis_complex_sp._Chile_clone_1a_ott7506368,Brachionus_plicatilis_complex_sp._Chile_clone_2_ott7506369,Brachionus_plicatilis_complex_sp._Chile_clone_21_ott7506370,Brachionus_plicatilis_complex_sp._Chile_clone_24_ott7506371,Brachionus_plicatilis_complex_sp._Chile_clone_25_ott7506372,Brachionus_plicatilis_complex_sp._Chile_clone_26_ott7506373,Brachionus_plicatilis_complex_sp._Chile_clone_27_ott7506374,Brachionus_plicatilis_complex_sp._Chile_clone_28_ott7506375,Brachionus_plicatilis_complex_sp._Chile_clone_29_ott7506376,Brachionus_plicatilis_complex_sp._Chile_clone_2a_ott7506377,Brachionus_plicatilis_complex_sp._Chile_clone_3_ott7506378,Brachionus_plicatilis_complex_sp._Chile_clone_3a_ott7506379,Brachionus_plicatilis_complex_sp._Chile_clone_4a_ott7506380,Brachionus_plicatilis_complex_sp._Chile_clone_5_ott7506381,Brachionus_plicatilis_complex_sp._Chile_clone_6_ott7506382,Brachionus_plicatilis_complex_sp._Chile_clone_7_ott7506383,Brachionus_plicatilis_complex_sp._Chile_clone_7a_ott7506384,Brachionus_plicatilis_complex_sp._Chile_clone_8_ott7506385,Brachionus_plicatilis_complex_sp._Chile_clone_9_ott7506386,Brachionus_amphiceras_ott7992110,Brachionus_budapestiensis_ott7992111,Brachionus_tetracanthus_ott7992112,Brachionus_urcealaris_ott7992113,Brachionus_urceus_ott7992114)Brachionus_ott471702,(Notholca_acuminata_ott492241,Notholca_squamula_ott2941805,Notholca_bipalium_ott2941806,Notholca_triarthroides_ott2941807,Notholca_orbiculata_ott2941808,Notholca_verae_ott2941809,Notholca_ikaitophila_ott2941811,Notholca_japonica_ott2941815,Notholca_laurentiae_ott2941816,Notholca_foliacea_ott2941817,Notholca_jugosa_ott2941818,Notholca_guidoi_ott2941819,Notholca_labis_ott2941820,Notholca_olchonensis_ott2941821,Notholca_michiganensis_ott2941822,Notholca_lyrata_ott2941823,Notholca_latistyla_ott2941824,Notholca_angulata_ott2941826,Notholca_cornuta_ott2941827,Notholca_hollowdayi_ott2941828,Notholca_haueri_ott2941830,Notholca_beta_ott2941831,Notholca_lamellifera_ott2941832,Notholca_walterkostei_ott2941834,Notholca_cinetura_ott2941835,Notholca_liepetterseni_ott2941836,Notholca_kozhovi_ott2941837,Notholca_marina_ott2941839,Notholca_striata_ott2941840,Notholca_tibetica_ott2941841,Notholca_gaigalasi_ott2941842,Notholca_angakkoq_ott2941843,Notholca_psammarina_ott2941844,Notholca_caudata_ott2941845,Notholca_kostei_ott2941846,Notholca_jasnitskii_ott4952779,Notholca_rectospina_ott4952780,Notholca_baicalensis_ott4952781,Notholca_bythonoma_ott6367886)Notholca_ott492249,(Kellicottia_bostoniensis_ott672796,Kellicottia_longispina_ott2941812)Kellicottia_ott672795,(Platyias_quadricornis_ott937975,Platyias_leloupi_ott2941931,Platyias_latiscapularis_ott2941932)Platyias_ott937963,(Euchlanis_dilatata_ott1018357,Euchlanis_alata_ott1037814,Euchlanis_perpusilla_ott2941847,Euchlanis_mikropous_ott2941848,Euchlanis_mamorokaensis_ott2941849,Euchlanis_ligulata_ott2941850,Euchlanis_callimorpha_ott2941852,Euchlanis_phryne_ott2941853,Euchlanis_hyphidactyla_ott2941854,Euchlanis_semicarinata_ott2941855,Euchlanis_dactyliseta_ott2941856,Euchlanis_triquetra_ott2941857,Euchlanis_meneta_ott2941858,Euchlanis_incisa_ott2941859,Euchlanis_pyriformis_ott2941860,Euchlanis_callysta_ott2941861,Euchlanis_contorta_ott2941862,Euchlanis_lucksiana_ott2941863,Euchlanis_dapidula_ott2941864,Euchlanis_arenosa_ott2941865,Euchlanis_parameneta_ott2941866,Euchlanis_calpidia_ott2941867,Euchlanis_deflexa_ott2941868,Euchlanis_lyra_ott2941869,Euchlanis_parva_ott2941870)Euchlanis_ott492240,(Anuraeopsis_fissa_ott1023628,Anuraeopsis_urawensis_ott2941933,Anuraeopsis_quadriantennata_ott2941934,Anuraeopsis_lata_ott2941935,Anuraeopsis_navicula_ott2941936,Anuraeopsis_miracleae_ott2941937,Anuraeopsis_wulferti_ott2941938,Anuraeopsis_coelata_ott2941939,Anuraeopsis_cristata_ott2941940,Anuraeopsis_siolii_ott2941941,'Anuraeopsis_sp._WM-2017a_ott7506333')Anuraeopsis_ott1023630,((Plationus_patulus_macracanthus_ott1037810)Plationus_patulus_ott937964,Plationus_polyacanthus_ott2941928,Plationus_felicitas_ott2941929)Plationus_ott937966,(Lophocharis_oxysternon_ott2942212,Lophocharis_kutikovae_ott2942216,Lophocharis_rubens_ott2942217,Lophocharis_tutiurensis_ott2942219,Lophocharis_turanica_ott2942220,Lophocharis_parva_ott2942222,Lophocharis_curvata_ott2942223,Lophocharis_hutchinsoni_ott2942225,Lophocharis_salpina_ott2942334,Lophocharis_naias_ott2942335,Lophocharis_ambidentata_ott2942347)Lophocharis_ott2942213)Brachionidae_ott471705,((Eosphora_ehrenbergi_ott107179,Eosphora_therina_ott2940733,Eosphora_thoides_ott2940734,Eosphora_gibba_ott2940735,Eosphora_najas_ott2940736,Eosphora_anthadis_ott2940737,Eosphora_thoa_ott2940738)Eosphora_ott107180,(Notommata_copeus_ott171571,Notommata_cordonella_ott937967,Notommata_allantois_ott937968,Notommata_torulosa_ott2940662,Notommata_pachyura_ott2940664,Notommata_voigti_ott2940665,Notommata_grandis_ott2940666,Notommata_avena_ott2940667,Notommata_codonella_ott2940668,Notommata_cyrtopus_ott2940669,Notommata_megaladena_ott2940670,Notommata_weberi_ott2940671,Notommata_collaris_ott2940672,Notommata_omentata_ott2940673,Notommata_diasema_ott2940674,Notommata_peridia_ott2940675,Notommata_groenlandica_ott2940676,Notommata_myrmeleo_ott2940677,Notommata_veroleti_ott2940678,Notommata_reinhardti_ott2940679,Notommata_venusta_ott2940681,Notommata_brachyota_ott2940682,Notommata_falcinella_ott2940683,Notommata_potamis_ott2940684,Notommata_doneta_ott2940685,Notommata_angusta_ott2940686,Notommata_sulcata_ott2940687,Notommata_meganglena_ott2940688,Notommata_syrinx_ott2940689,Notommata_cherada_ott2940690,Notommata_rugosa_ott2940691,Notommata_haueri_ott2940692,Notommata_glyphura_ott2940693,Notommata_apochaeta_ott2940694,Notommata_aurita_ott2940695,Notommata_lenis_ott2940696,Notommata_mera_ott2940697,Notommata_saccigera_ott2940698,Notommata_fasciola_ott2940699,Notommata_tripus_ott2940700,Notommata_aethis_ott2940701,Notommata_stitista_ott2940702,Notommata_silphoides_ott2940703,Notommata_placida_ott2940704,Notommata_bennetchi_ott2940705,Notommata_thopica_ott2940708,Notommata_ovulum_ott2940709,Notommata_galena_ott2940710,Notommata_gisleni_ott2940711,Notommata_spinata_ott2940712,Notommata_longina_ott2940713,Notommata_contorta_ott2940714,Notommata_pseudocerberus_ott2940715,Notommata_endoxa_ott2940716,Notommata_prodota_ott2940717,Notommata_cerberus_ott2940718,Notommata_pygmaea_ott2940719,Notommata_cerebrus_ott2940720,Notommata_silpha_ott2940721,Notommata_bennetschi_ott2940722,Notommata_onisciformis_ott2940723,Notommata_limax_ott2940724,Notommata_paracyrtopus_ott4952790)Notommata_ott171572,('Cephalodella_cf._gibba_MEG-2012_ott296322',Cephalodella_gibba_ott492250,Cephalodella_forficula_ott513188,Cephalodella_brandorffi_ott2940502,Cephalodella_curiculata_ott2940503,Cephalodella_balatonica_ott2940504,Cephalodella_segersi_ott2940505,Cephalodella_boettgeri_ott2940506,Cephalodella_dora_ott2940507,Cephalodella_intuta_ott2940508,Cephalodella_maior_ott2940509,Cephalodella_mira_ott2940510,Cephalodella_mus_ott2940511,Cephalodella_ungulata_ott2940512,Cephalodella_decidua_ott2940513,Cephalodella_tantilla_ott2940514,Cephalodella_elegans_ott2940515,Cephalodella_hoodii_ott2940518,Cephalodella_monica_ott2940519,Cephalodella_wrighti_ott2940520,Cephalodella_panarista_ott2940521,Cephalodella_mucronata_ott2940522,Cephalodella_forficata_ott2940523,Cephalodella_irisae_ott2940524,Cephalodella_eva_ott2940525,Cephalodella_auriculata_ott2940526,Cephalodella_gibboides_ott2940527,Cephalodella_akrobeles_ott2940528,Cephalodella_rostrum_ott2940529,Cephalodella_hollowdayi_ott2940530,Cephalodella_collactea_ott2940531,Cephalodella_planera_ott2940532,Cephalodella_apocolea_ott2940533,Cephalodella_jakubskii_ott2940534,Cephalodella_anebodica_ott2940535,Cephalodella_carina_ott2940536,Cephalodella_cyclops_ott2940537,Cephalodella_tincaformis_ott2940538,Cephalodella_hiulca_ott2940539,Cephalodella_subsecunda_ott2940540,Cephalodella_celeris_ott2940541,Cephalodella_tenuis_ott2940542,Cephalodella_limosa_ott2940543,Cephalodella_angusta_ott2940544,Cephalodella_arcuata_ott2940545,Cephalodella_obvia_ott2940546,Cephalodella_paxi_ott2940547,Cephalodella_pentaplax_ott2940548,Cephalodella_inquilina_ott2940549,Cephalodella_vitella_ott2940550,Cephalodella_gusuleaci_ott2940551,Cephalodella_mineri_ott2940552,Cephalodella_derbyi_ott2940553,Cephalodella_volvocicola_ott2940554,Cephalodella_gobio_ott2940555,Cephalodella_latifulcrum_ott2940556,Cephalodella_elongata_ott2940557,Cephalodella_incila_ott2940558,Cephalodella_unguitata_ott2940559,Cephalodella_belone_ott2940560,Cephalodella_poitera_ott2940561,Cephalodella_delicata_ott2940562,Cephalodella_praelonga_ott2940563,Cephalodella_tantilloides_ott2940564,Cephalodella_hyalina_ott2940565,Cephalodella_gigantea_ott2940566,Cephalodella_bertonicensis_ott2940567,Cephalodella_glypha_ott2940568,Cephalodella_forceps_ott2940569,Cephalodella_compacta_ott2940570,Cephalodella_harringi_ott2940571,Cephalodella_friebei_ott2940572,Cephalodella_tachyphora_ott2940573,Cephalodella_catellina_ott2940574,Cephalodella_lindamayae_ott2940575,Cephalodella_pseudeva_ott2940576,Cephalodella_vacuna_ott2940577,Cephalodella_zeteta_ott2940578,Cephalodella_biungulata_ott2940579,Cephalodella_songkhlaensis_ott2940580,Cephalodella_paxilla_ott2940581,Cephalodella_dixonnuttalli_ott2940582,Cephalodella_trigona_ott2940583,Cephalodella_eupoda_ott2940584,Cephalodella_conica_ott2940585,Cephalodella_retusa_ott2940586,Cephalodella_licinia_ott2940587,Cephalodella_dorseyi_ott2940588,Cephalodella_speciosa_ott2940589,Cephalodella_eurynota_ott2940590,Cephalodella_ablusa_ott2940591,Cephalodella_abstrusa_ott2940592,Cephalodella_stenroosi_ott2940593,Cephalodella_qionghaiensis_ott2940594,Cephalodella_sterea_ott2940595,Cephalodella_megalotrocha_ott2940596,Cephalodella_lipara_ott2940597,Cephalodella_pheloma_ott2940598,Cephalodella_gisleni_ott2940599,Cephalodella_vittata_ott2940600,Cephalodella_licina_ott2940601,Cephalodella_montana_ott2940602,Cephalodella_tecta_ott2940603,Cephalodella_asarcia_ott2940604,Cephalodella_euderbyi_ott2940605,Cephalodella_tenuiseta_ott2940606,Cephalodella_rigida_ott2940607,Cephalodella_misgurnus_ott2940608,Cephalodella_exigua_ott2940609,Cephalodella_unquitara_ott2940610,Cephalodella_rotunda_ott2940611,Cephalodella_dentata_ott2940612,Cephalodella_melia_ott2940613,Cephalodella_pachyodon_ott2940614,Cephalodella_papillosa_ott2940615,Cephalodella_glandulosa_ott2940616,Cephalodella_astricta_ott2940617,Cephalodella_strigosa_ott2940618,Cephalodella_globata_ott2940619,Cephalodella_fluviatilis_ott2940620,Cephalodella_calosa_ott2940621,Cephalodella_xenica_ott2940622,Cephalodella_physalis_ott2940623,Cephalodella_minora_ott2940624,Cephalodella_clara_ott2940625,Cephalodella_edax_ott2940626,Cephalodella_tinca_ott2940627,Cephalodella_eunoma_ott2940628,Cephalodella_macrodactyla_ott2940629,Cephalodella_crassipes_ott2940630,Cephalodella_tenuior_ott2940631,Cephalodella_graciosa_ott2940632,Cephalodella_asta_ott2940633,Cephalodella_nelitis_ott2940634,Cephalodella_mucosa_ott2940635,Cephalodella_reimanni_ott2940636,Cephalodella_innesi_ott2940637,Cephalodella_somniculosa_ott2940638,Cephalodella_gracilis_ott2940639,Cephalodella_euknema_ott2940640,Cephalodella_parasitica_ott2940641,Cephalodella_tempesta_ott2940642,Cephalodella_pachydactyla_ott2940643,Cephalodella_nana_ott2940644,Cephalodella_paggiae_ott2940645,Cephalodella_compressa_ott2940646,Cephalodella_psammophila_ott2940647,Cephalodella_plicata_ott2940648,Cephalodella_megalocephala_ott2940649,Cephalodella_marina_ott2940650,Cephalodella_doryphora_ott2940651,Cephalodella_labiosa_ott2940652,Cephalodella_dorystoma_ott2940653,Cephalodella_theodora_ott2940654,Cephalodella_conjuncta_ott2940655,Cephalodella_lepida_ott2940656,Cephalodella_evabroedae_ott2940657,Cephalodella_ventripes_ott2940658,Cephalodella_obesa_ott2940659,Cephalodella_oxydactyla_ott2940660,Cephalodella_laisi_ott4952792,Cephalodella_inquila_ott5705992,Cephalodella_jersabeki_ott6367900,Cephalodella_teniuseta_ott6367901,Cephalodella_elouenteita_ott7992147,Cephalodella_promta_ott7992148,Cephalodella_pseudocuneata_ott7992149,Cephalodella_sasquatcha_ott7992150,Cephalodella_symbiotica_ott7992151)Cephalodella_ott492248,(Eothinia_elongata_ott460955,Eothinia_euklopa_ott2940725,Eothinia_eukolpa_ott2940726,Eothinia_triphaea_ott2940727,Eothinia_striata_ott2940729,Eothinia_argus_ott2940730,Eothinia_poitera_ott2940731,Eothinia_carogaensis_ott2940732)Eothinia_ott576380,(Monommata_maculata_ott563701,Monommata_actices_ott2940739,Monommata_pseudophoxa_ott2940740,Monommata_phoxa_ott2940741,Monommata_appendiculata_ott2940742,Monommata_enedra_ott2940743,Monommata_diaphora_ott2940744,Monommata_caeca_ott2940745,Monommata_dissimilis_ott2940746,Monommata_longiseta_ott2940747,Monommata_caudata_ott2940748,Monommata_viridis_ott2940749,Monommata_grandis_ott2940750,Monommata_astia_ott2940751,Monommata_aequalis_ott2940752,Monommata_aeschyna_ott2940753,Monommata_dentata_ott2940754,Monommata_hyalina_ott2940755)Monommata_ott563700,(Pleurotrocha_petromyzon_ott995259,Pleurotrocha_aurea_ott2940756,Pleurotrocha_thrua_ott2940758,Pleurotrocha_altila_ott2940761,Pleurotrocha_daphnicola_ott2940762,Pleurotrocha_chalicodis_ott2940763,Pleurotrocha_altanica_ott2940764,Pleurotrocha_channa_ott2940765,Pleurotrocha_atlantica_ott2940766,Pleurotrocha_larvarum_ott2940767,Pleurotrocha_elegans_ott2940768,Pleurotrocha_sigmoidea_ott4952791,Pleurotrocha_fontanetoi_ott6367902,Pleurotrocha_altilis_ott7992153)Pleurotrocha_ott418185,(Pleurata_vernalis_ott2940757,Pleurata_trypeta_ott2940769,Pleurata_tyleri_ott4153445,Pleurata_uroglenae_ott4153446,Pleurata_thura_ott4153447,Pleurata_tithasa_ott4153448,Pleurata_chalicodes_ott4153449)Pleurata_ott4153443,(Rousseletia_corniculata_ott2940770)Rousseletia_ott2940771,(Taphrocampa_levinseni_ott2940772,Taphrocampa_lemurensis_ott2940781,Taphrocampa_selenura_ott2940802,Taphrocampa_clavigera_ott2940803,Taphrocampa_annulosa_ott2940809)Taphrocampa_ott2940773,(Tylotrocha_monopus_ott2940774)Tylotrocha_ott2940775,(Pourriotia_carcharodonta_ott2940777,Pourriotia_werneckii_ott4952796)Pourriotia_ott2940776,(Drilophaga_delagei_ott2940778,Drilophaga_bucephalus_ott2940780,Drilophaga_judayi_ott2940786)Drilophaga_ott2940779,(Pleurotrochopsis_multispinosa_ott2940782)Pleurotrochopsis_ott2940783,(Resticula_vermiculus_ott2940784,Resticula_gelida_ott2940787,Resticula_nyssa_ott2940788,Resticula_plicata_ott2940791,Resticula_anceps_ott2940792,Resticula_vermisculus_ott2940797,Resticula_lestes_ott2940804,Resticula_melandocus_ott2940805)Resticula_ott2940785,(Dorystoma_caudata_ott2940789,Dorystoma_furcata_ott2940795)Dorystoma_ott2940790,(Enteroplea_lacustris_ott2940793,Enteroplea_similis_ott2940801)Enteroplea_ott2940794,(Sphyrias_lophuana_ott2940800,Sphyrias_lofuana_ott2940808)Sphyrias_ott2940798,(Pseudoharringia_similis_ott4153441,Pseudoharringia_romanica_ott4153442)Pseudoharringia_ott4153440,Notommatidae_environmental_sample_ott4952795)Notommatidae_ott71917,((Trichocerca_elongata_ott128964,Trichocerca_rattus_ott128965,'Trichocerca_cf._capucina_MEG-2012_ott296328',Trichocerca_longiseta_ott352613,Trichocerca_stylata_ott672791,Trichocerca_tenuior_ott740398,Trichocerca_murchiearum_ott2941163,Trichocerca_weberi_ott2941164,Trichocerca_iernis_ott2941165,Trichocerca_catellus_ott2941166,Trichocerca_scipio_ott2941167,Trichocerca_inermis_ott2941168,Trichocerca_tenuidens_ott2941170,Trichocerca_uncinata_ott2941171,Trichocerca_compressa_ott2941172,Trichocerca_gracilis_ott2941173,Trichocerca_intermedia_ott2941174,Trichocerca_minuta_ott2941175,Trichocerca_bambekei_ott2941176,Trichocerca_ripli_ott2941177,Trichocerca_rousseleti_ott2941178,Trichocerca_dixonnuttalli_ott2941179,Trichocerca_myersi_ott2941180,Trichocerca_sulcata_ott2941181,Trichocerca_bidens_ott2941182,Trichocerca_voluta_ott2941183,Trichocerca_porcellus_ott2941184,Trichocerca_cavia_ott2941185,Trichocerca_capucina_ott2941186,Trichocerca_similis_ott2941187,Trichocerca_agnatha_ott2941188,Trichocerca_pustilla_ott2941189,Trichocerca_maior_ott2941190,Trichocerca_euodonta_ott2941191,Trichocerca_cimolia_ott2941192,Trichocerca_brachydactyla_ott2941193,Trichocerca_pygocera_ott2941196,Trichocerca_abilioi_ott2941197,Trichocerca_macera_ott2941198,Trichocerca_brevidactyla_ott2941199,Trichocerca_obtusidens_ott2941200,Trichocerca_bicurvicornis_ott2941201,Trichocerca_ruttneri_ott2941202,Trichocerca_caspica_ott2941203,Trichocerca_sejunctipes_ott2941204,Trichocerca_platessa_ott2941205,Trichocerca_bicuspes_ott2941206,Trichocerca_multicrinis_ott2941207,Trichocerca_longistyla_ott2941208,Trichocerca_wanarra_ott2941209,Trichocerca_taurocephala_ott2941210,Trichocerca_edmondsoni_ott2941211,Trichocerca_unidens_ott2941212,Trichocerca_gillardi_ott2941213,Trichocerca_microstyla_ott2941214,Trichocerca_rectangularis_ott2941215,Trichocerca_braziliensis_ott2941216,Trichocerca_nitida_ott2941219,Trichocerca_lophoessa_ott2941220,Trichocerca_insolens_ott2941221,Trichocerca_flagellata_ott2941222,Trichocerca_antilopaea_ott2941223,Trichocerca_tigris_ott2941224,Trichocerca_chattoni_ott2941225,Trichocerca_collaris_ott2941226,Trichocerca_siamensis_ott2941227,Trichocerca_flava_ott2941228,Trichocerca_bilunaris_ott2941229,Trichocerca_insulana_ott2941230,Trichocerca_pusilla_ott2941231,Trichocerca_mus_ott2941232,Trichocerca_cylindrica_ott2941233,Trichocerca_heterodactyla_ott2941234,Trichocerca_vernalis_ott2941235,Trichocerca_lata_ott2941236,Trichocerca_cryptodus_ott2941237,Trichocerca_mucripes_ott2941238,Trichocerca_plaka_ott2941239,Trichocerca_pediculus_ott2941240,Trichocerca_parva_ott2941242,Trichocerca_insignis_ott2941244,Trichocerca_marina_ott2941245,Trichocerca_harveyensis_ott2941246,Trichocerca_brevistyla_ott2941247,Trichocerca_ornata_ott2941248,Trichocerca_mollis_ott2941249,Trichocerca_calypta_ott2941250,Trichocerca_vargai_ott2941251,Trichocerca_cuspidata_ott2941252,Trichocerca_vassilijevae_ott2941254,Trichocerca_musculus_ott2941255,Trichocerca_orca_ott2941256,Trichocerca_bicristata_ott2941257,Trichocerca_simoneae_ott2941258,Trichocerca_barsica_ott2941259,Trichocerca_kostei_ott2941260,Trichocerca_rotundata_ott2941261,Trichocerca_mucosa_ott2941262,Trichocerca_hollaerti_ott2941263,Trichocerca_helminthodes_ott2941265,Trichocerca_valga_ott2941266,Trichocerca_rosea_ott2941267,Trichocerca_brachyura_ott2941268,Trichocerca_joblotii_ott4952764,Trichocerca_bauthiemensis_ott6367906)Trichocerca_ott128969,(Elosa_worrallii_ott2941269,Elosa_spinifera_ott2941271)Elosa_ott2941270,(Ascomorphella_volvocicola_ott2941273)Ascomorphella_ott2941272,Trichocercidae_environmental_sample_ott4952766,Trichocercidae_sp._SHCX151203c_ott5974973)Trichocercidae_ott128966,((Scaridium_longicaudum_ott128980,Scaridium_grande_ott2941275,Scaridium_montanum_ott2941276,Scaridium_neglectum_ott2941277,Scaridium_bostjani_ott2941278,Scaridium_elegans_ott2941279,Scaridium_elongatum_ott2941280)Scaridium_ott128971)Scaridiidae_ott412239,((Synchaeta_grandis_ott154828,Synchaeta_kitina_ott390795,Synchaeta_lakowitziana_ott390797,Synchaeta_tremula_ott418177,Synchaeta_pectinata_ott418179,'Synchaeta_cf._tremula/oblonga_UO-2012_ott645986','Synchaeta_cf._cecilia_UO-2012_ott645987',Synchaeta_cf._pectinata_ott743639,Synchaeta_longipes_ott2940810,Synchaeta_neapolitana_ott2940811,Synchaeta_fennica_ott2940812,Synchaeta_gyrina_ott2940813,Synchaeta_oblonga_ott2940814,Synchaeta_curvata_ott2940815,Synchaeta_triophthalma_ott2940816,Synchaeta_cecilia_ott2940817,Synchaeta_lackowitziana_ott2940818,Synchaeta_bicornis_ott2940819,Synchaeta_atlantica_ott2940820,Synchaeta_bacillifera_ott2940822,Synchaeta_pachypoida_ott2940823,Synchaeta_stylata_ott2940824,Synchaeta_vorax_ott2940825,Synchaeta_elsteri_ott2940826,Synchaeta_tremuloida_ott2940827,Synchaeta_jollyae_ott2940828,Synchaeta_glacialis_ott2940829,Synchaeta_arcifera_ott2940830,Synchaeta_baltica_ott2940831,Synchaeta_rousseleti_ott2940832,Synchaeta_pachypoda_ott2940833,Synchaeta_littoralis_ott2940834,Synchaeta_tamara_ott2940835,Synchaeta_tavina_ott2940836,Synchaeta_prominula_ott2940837,Synchaeta_rufina_ott2940838,Synchaeta_hutchingsi_ott2940839,Synchaeta_squamadigitata_ott2940840,Synchaeta_hyperborea_ott2940841,Synchaeta_johanseni_ott2940842,Synchaeta_calva_ott2940843,Synchaeta_monopus_ott2940844,Synchaeta_verrucosa_ott2940845,Synchaeta_cylindrica_ott2940846,Synchaeta_grimpei_ott2940847,'Synchaeta_cf._tremula/oblonga_CQT-2013_ott5454060')Synchaeta_ott743640,(Ploesoma_hudsoni_ott412240,Ploesoma_truncatus_ott412241,Ploesoma_lenticulare_ott2940862,Ploesoma_molle_ott2940864,Ploesoma_triacanthum_ott2940866,Ploesoma_murrayi_ott2940867,Ploesoma_truncatum_ott2940868,Ploesoma_multispinatum_ott2940869,Ploesoma_africanum_ott2940870,Ploesoma_peipsiense_ott2940871)Ploesoma_ott937974,(Polyarthra_remata_ott412243,Polyarthra_dolichoptera_ott967150,Polyarthra_major_ott2940848,Polyarthra_dissimulans_ott2940849,Polyarthra_bicera_ott2940850,Polyarthra_hexaptera_ott2940851,Polyarthra_trigla_ott2940852,Polyarthra_platyptera_ott2940853,Polyarthra_vulgaris_ott2940854,Polyarthra_minor_ott2940855,Polyarthra_bicerca_ott2940856,Polyarthra_luminosa_ott2940857,Polyarthra_longiremis_ott2940858,Polyarthra_leleki_ott2940859,Polyarthra_indica_ott2940860,Polyarthra_euryptera_ott2940861,'Polyarthra_dolichoptera_complex_sp._UO-2013_ott5223858',Polyarthra_platensis_ott6367905)Polyarthra_ott412242,(Pseudoploesoma_formosum_ott2940865,Pseudoploesoma_greeni_ott2940874)Pseudoploesoma_ott2940872)Synchaetidae_ott743641,((Asplanchna_silvestrii_ott245637,Asplanchna_sieboldi_ott251968,'Asplanchna_cf._sieboldi_MEG-2012_ott871348',Asplanchna_silvestris_ott2940875,Asplanchna_herricki_ott2940876,Asplanchna_terminalis_ott2940877,Asplanchna_amphora_ott2940878,Asplanchna_sieboldii_ott2940879,Asplanchna_herrickii_ott2940880,Asplanchna_girodi_ott2940881,Asplanchna_intermedia_ott2940882,Asplanchna_seiboldi_ott2940883,Asplanchna_brightwellii_ott2940884,Asplanchna_priodonta_ott2940885,Asplanchna_tropica_ott2940886,Asplanchna_asymmetrica_ott2940887)Asplanchna_ott251970,(Asplanchnopus_dahlgreni_ott1037813,Asplanchnopus_multiceps_ott2940888,Asplanchnopus_syrinx_ott2940889,Asplanchnopus_hyalimus_ott2940891,Asplanchnopus_hyalinus_ott2940892,Asplanchnopus_bhimavaramensis_ott2940893)Asplanchnopus_ott1037806,(Harringia_rousseleti_ott2940894,Harringia_eupoda_ott2940896)Harringia_ott2940895)Asplanchnidae_ott251965,((Lecane_lunaris_AEG1_ott245955,Lecane_hastata_AEG1_ott245956,Lecane_ohioensis_ott296324,Lecane_luna_ott316622,Lecane_rhenana_ott349144,Lecane_lunaris_ott349181,Lecane_cornuta_ott429869,Lecane_crepida_ott429871,Lecane_hamata_ott429874,Lecane_bulla_ott532775,Lecane_quadridentata_ott581923,Lecane_spinulifera_ott581939,Lecane_papuana_ott581949,Lecane_monostyla_ott581951,Lecane_closterocerca_ott620074,Lecane_curvicornis_ott620080,Lecane_grandis_ott701403,Lecane_hastata_ott701563,Lecane_crepida_AEG1_ott820705,Lecane_curvicornis_AEG2_ott820706,Lecane_bulla_AEG10_ott820712,Lecane_bulla_AEG9_ott820713,Lecane_cornuta_AEG2_ott820714,Lecane_elsa_ott1090174,Lecane_leontina_ott1090177,Lecane_affinis_ott2940904,Lecane_boorali_ott2940905,Lecane_carpatica_ott2940906,Lecane_elliptoides_ott2940907,Lecane_opias_ott2940908,Lecane_pyrrha_ott2940909,Lecane_sympoda_ott2940910,Lecane_gallagherorum_ott2940911,Lecane_stichaea_ott2940912,Lecane_styrax_ott2940913,Lecane_levistyla_ott2940914,Lecane_tryphema_ott2940915,Lecane_thalera_ott2940916,Lecane_sinuosa_ott2940917,Lecane_superaculeata_ott2940918,Lecane_crenata_ott2940919,Lecane_verecunda_ott2940920,Lecane_flabellata_ott2940921,Lecane_noobijupi_ott2940922,Lecane_niwati_ott2940923,Lecane_subulata_ott2940924,Lecane_schraederi_ott2940925,Lecane_bidactyla_ott2940926,Lecane_rudescui_ott2940927,Lecane_venusta_ott2940928,Lecane_herzigi_ott2940929,Lecane_arcuata_ott2940930,Lecane_romeroi_ott2940931,Lecane_infula_ott2940932,Lecane_ungulata_ott2940933,Lecane_psammophila_ott2940934,Lecane_namatai_ott2940935,Lecane_pyriformis_ott2940936,Lecane_galeata_ott2940937,Lecane_jaintiaensis_ott2940938,Lecane_marshi_ott2940939,Lecane_braumi_ott2940940,Lecane_jessupi_ott2940941,Lecane_hornemanni_ott2940942,Lecane_tabida_ott2940943,Lecane_inconspicua_ott2940944,Lecane_donneri_ott2940945,Lecane_latissima_ott2940946,Lecane_myersi_ott2940947,Lecane_elongata_ott2940948,Lecane_ordwayi_ott2940949,Lecane_perpusilla_ott2940950,Lecane_abanica_ott2940951,Lecane_scutata_ott2940952,Lecane_eylesi_ott2940953,Lecane_bifastigata_ott2940954,Lecane_nitida_ott2940955,Lecane_althausi_ott2940956,Lecane_subtilis_ott2940957,Lecane_copeis_ott2940958,Lecane_ligona_ott2940960,Lecane_inquieta_ott2940961,Lecane_pomiformis_ott2940964,Lecane_kutikowa_ott2940965,Lecane_solfatara_ott2940966,Lecane_doryssa_ott2940967,Lecane_bryophila_ott2940968,Lecane_dysorata_ott2940969,Lecane_mitis_ott2940970,Lecane_aspasia_ott2940971,Lecane_pelatis_ott2940972,Lecane_thailandensis_ott2940973,Lecane_lungae_ott2940974,Lecane_sverigis_ott2940975,Lecane_kunthuleensis_ott2940976,Lecane_ruttneri_ott2940977,Lecane_sola_ott2940978,Lecane_nana_ott2940979,Lecane_mira_ott2940980,Lecane_imbricata_ott2940981,Lecane_pawlowskii_ott2940982,Lecane_plesia_ott2940983,Lecane_tenua_ott2940984,Lecane_haliclysta_ott2940985,Lecane_tuxeni_ott2940986,Lecane_rhytida_ott2940987,Lecane_pertica_ott2940988,Lecane_goniata_ott2940989,Lecane_sinuata_ott2940992,Lecane_depressa_ott2940993,Lecane_furcata_ott2940994,Lecane_broaensis_ott2940995,Lecane_acus_ott2940996,Lecane_lauterborni_ott2940997,Lecane_thienemanni_ott2940998,Lecane_pycina_ott2940999,Lecane_enowi_ott2941000,Lecane_climacois_ott2941001,Lecane_lamellata_ott2941002,Lecane_segersi_ott2941003,Lecane_deridderae_ott2941004,Lecane_candida_ott2941005,Lecane_stephensae_ott2941006,Lecane_matsaluensis_ott2941009,Lecane_flexilis_ott2941010,Lecane_asymmetrica_ott2941013,Lecane_pluto_ott2941014,Lecane_satyrus_ott2941016,Lecane_ludwigii_ott2941018,Lecane_bifurca_ott2941019,Lecane_insulaconae_ott2941020,Lecane_pusilla_ott2941021,Lecane_nwadiaroi_ott2941022,Lecane_pumila_ott2941023,Lecane_inopinata_ott2941024,Lecane_eutarsa_ott2941025,Lecane_stenroosi_ott2941026,Lecane_margalefi_ott2941027,Lecane_boettgeri_ott2941028,Lecane_isanensis_ott2941029,Lecane_whitfordi_ott2941030,Lecane_formosa_ott2941031,Lecane_gwileti_ott2941032,Lecane_calcaria_ott2941033,Lecane_paradoxa_ott2941034,Lecane_obtusa_ott2941035,Lecane_pustulosa_ott2941036,Lecane_donyanaensis_ott2941037,Lecane_tanganyikae_ott2941038,Lecane_blachei_ott2941039,Lecane_intrasinuata_ott2941040,Lecane_sibina_ott2941042,Lecane_arcula_ott2941043,Lecane_sulcata_ott2941044,Lecane_stokesii_ott2941045,Lecane_hospes_ott2941046,Lecane_eswari_ott2941047,Lecane_gracilis_ott2941048,Lecane_unguitata_ott2941049,Lecane_nigeriensis_ott2941050,Lecane_fadeevi_ott2941051,Lecane_armata_ott2941052,Lecane_rugosa_ott2941053,Lecane_fusilis_ott2941054,Lecane_clara_ott2941055,Lecane_mitella_ott2941056,Lecane_dumonti_ott2941058,Lecane_tabulifera_ott2941059,Lecane_symoensi_ott2941060,Lecane_melini_ott2941061,Lecane_punctata_ott2941062,Lecane_braziliensis_ott2941063,Lecane_tenuiseta_ott2941064,Lecane_junki_ott2941065,Lecane_paxiana_ott2941066,Lecane_remanei_ott2941067,Lecane_halsei_ott2941068,Lecane_acanthinula_ott2941069,Lecane_amazonica_ott2941070,Lecane_mucronata_ott2941071,Lecane_elegans_ott2941072,Lecane_perplexa_ott2941073,Lecane_palinacis_ott2941074,Lecane_gillardi_ott2941075,Lecane_undulata_ott2941076,Lecane_agilis_ott2941077,Lecane_spiniventris_ott2941078,Lecane_stichoclysta_ott2941079,Lecane_nelsoni_ott2941080,Lecane_elasma_ott2941081,Lecane_baimaii_ott2941082,Lecane_shieli_ott2941083,Lecane_margarethae_ott2941084,Lecane_ivli_ott2941085,Lecane_boliviana_ott2941086,Lecane_sagula_ott2941087,Lecane_leura_ott2941089,Lecane_robertsonae_ott2941090,Lecane_lateralis_ott2941091,Lecane_rhopalura_ott2941092,Lecane_aeganea_ott2941093,Lecane_balatonica_ott2941094,Lecane_rhacois_ott2941095,Lecane_aculeata_ott2941096,Lecane_serrata_ott2941098,Lecane_marchantaria_ott2941099,Lecane_ercodes_ott2941100,Lecane_minuta_ott2941101,Lecane_niothis_ott2941102,Lecane_simonneae_ott2941103,Lecane_kluchor_ott2941104,Lecane_uenoi_ott2941105,Lecane_proiecta_ott2941106,Lecane_signifera_ott2941107,Lecane_inermis_ott2941108,Lecane_batillifer_ott2941109,Lecane_branchicola_ott2941110,Lecane_difficilis_ott2941111,Lecane_syngenes_ott2941112,Lecane_sylviae_ott2941113,Lecane_decipiens_ott2941114,Lecane_urna_ott2941115,Lecane_martensi_ott4952772,Lecane_yatseni_ott4952773,Lecane_chinesensis_ott4952774,Lecane_dorysimilis_ott6367894,Lecane_phapi_ott6367895,Lecane_compta_ott7992133,Lecane_flexlils_ott7992134,Lecane_ohiensis_ott7992135,Lecane_zhanjiangensis_ott7992137)Lecane_ott236072,Monostyla_ott488319,Lecanidae_environmental_sample_ott4952771)Lecanidae_ott404255,(('Dicranophorus_sp._MEG-2012_ott296321',Dicranophorus_forcipatus_ott513197,Dicranophorus_difflugiarum_ott2941638,Dicranophorus_colastes_ott2941639,Dicranophorus_aspondus_ott2941641,Dicranophorus_edestes_ott2941642,Dicranophorus_myriophylli_ott2941643,Dicranophorus_grypus_ott2941644,Dicranophorus_ponerus_ott2941645,Dicranophorus_hauerianus_ott2941646,Dicranophorus_scotius_ott2941647,Dicranophorus_halbachi_ott2941648,Dicranophorus_thysanus_ott2941649,Dicranophorus_esox_ott2941650,Dicranophorus_sigmoides_ott2941651,Dicranophorus_corystis_ott2941652,Dicranophorus_capucinus_ott2941653,Dicranophorus_pennatus_ott2941654,Dicranophorus_robustus_ott2941655,Dicranophorus_strigosus_ott2941656,Dicranophorus_cambari_ott2941657,Dicranophorus_kostei_ott2941658,Dicranophorus_stultus_ott2941659,Dicranophorus_lenapensis_ott2941661,Dicranophorus_dolerus_ott2941662,Dicranophorus_rostratus_ott2941663,Dicranophorus_luetkeni_ott2941664,Dicranophorus_alcimus_ott2941665,Dicranophorus_siedleckii_ott2941666,Dicranophorus_facinus_ott2941668,Dicranophorus_minutes_ott2941669,Dicranophorus_isothes_ott2941670,Dicranophorus_pauliani_ott2941671,Dicranophorus_mesotis_ott2941672,Dicranophorus_haueri_ott2941673,Dicranophorus_tegillus_ott2941674,Dicranophorus_proclestes_ott2941675,Dicranophorus_facilis_ott2941676,Dicranophorus_leptodon_ott2941678,Dicranophorus_epicharis_ott2941679,Dicranophorus_macrostyla_ott2941680,Dicranophorus_hercules_ott2941682,Dicranophorus_proclastes_ott2941683,Dicranophorus_spiculatus_ott2941684,Dicranophorus_bulgaricus_ott2941685,Dicranophorus_grandis_ott2941686,Dicranophorus_artamus_ott2941687,Dicranophorus_sebastus_ott2941689,Dicranophorus_biastis_ott2941690,Dicranophorus_prionacis_ott2941691,Dicranophorus_riparius_ott2941692,Dicranophorus_saevus_ott2941693,Dicranophorus_semnus_ott2941694)Dicranophorus_ott513196,(Encentrum_astridae_ott1090172,Encentrum_tectipes_ott1090175,Encentrum_barti_ott2941527,Encentrum_stechlinensis_ott2941528,Encentrum_mariae_ott2941529,Encentrum_gibbosum_ott2941530,Encentrum_salsum_ott2941531,Encentrum_elongatum_ott2941532,Encentrum_permolle_ott2941533,Encentrum_sutoroides_ott2941534,Encentrum_goldschmidi_ott2941535,Encentrum_carlini_ott2941536,Encentrum_valkanovi_ott2941537,Encentrum_eulitorale_ott2941538,Encentrum_kulmatyckii_ott2941539,Encentrum_boreale_ott2941540,Encentrum_tobyhannaense_ott2941541,Encentrum_desmeti_ott2941542,Encentrum_parime_ott2941543,Encentrum_murrayi_ott2941544,Encentrum_longirostrum_ott2941545,Encentrum_umbonatum_ott2941546,Encentrum_insolitum_ott2941547,Encentrum_stechlinense_ott2941548,Encentrum_flexile_ott2941549,Encentrum_pachypus_ott2941550,Encentrum_fluviatile_ott2941551,Encentrum_uncinatum_ott2941553,Encentrum_lupus_ott2941554,Encentrum_striatum_ott2941555,Encentrum_rousseleti_ott2941556,Encentrum_belluinum_ott2941557,Encentrum_dieteri_ott2941558,Encentrum_nesites_ott2941559,Encentrum_moldavicum_ott2941560,Encentrum_enteromorphae_ott2941561,Encentrum_permutandum_ott2941562,Encentrum_sacculiforme_ott2941563,Encentrum_kostei_ott2941564,Encentrum_saundersiae_ott2941565,Encentrum_mustela_ott2941566,Encentrum_kutikovae_ott2941567,Encentrum_forcipatum_ott2941568,Encentrum_pornsilpi_ott2941569,Encentrum_eristes_ott2941570,Encentrum_bidentatum_ott2941571,Encentrum_algente_ott2941572,Encentrum_torvitum_ott2941573,Encentrum_nikor_ott2941574,Encentrum_villosum_ott2941575,Encentrum_glaucum_ott2941576,Encentrum_lacidum_ott2941577,Encentrum_acrodon_ott2941578,Encentrum_marinum_ott2941579,Encentrum_tenuidigitatum_ott2941580,Encentrum_oxyodon_ott2941582,Encentrum_asellicola_ott2941583,Encentrum_cruentum_ott2941584,Encentrum_remanei_ott2941585,Encentrum_arvicola_ott2941586,Encentrum_mucronatum_ott2941587,Encentrum_hofsteni_ott2941588,Encentrum_incisum_ott2941589,Encentrum_tyrphos_ott2941590,Encentrum_longidens_ott2941591,Encentrum_felis_ott2941592,Encentrum_spinosum_ott2941593,Encentrum_martoides_ott2941594,Encentrum_porsildi_ott2941595,Encentrum_walterkostei_ott2941596,Encentrum_listensoides_ott2941597,Encentrum_longipes_ott2941598,Encentrum_arenarium_ott2941599,Encentrum_gulo_ott2941600,Encentrum_zetetum_ott2941601,Encentrum_listense_ott2941602,Encentrum_simillimum_ott2941603,Encentrum_rapax_ott2941604,Encentrum_aquila_ott2941605,Encentrum_torvitoides_ott2941607,Encentrum_otois_ott2941608,Encentrum_eurycephalum_ott2941609,Encentrum_putorius_ott2941610,Encentrum_orthodactylum_ott2941611,Encentrum_spatiatum_ott2941612,Encentrum_matthesi_ott2941613,Encentrum_wiszniewskii_ott2941614,Encentrum_sorex_ott2941615,Encentrum_psammophilum_ott2941616,Encentrum_caratum_ott2941617,Encentrum_oculatum_ott2941618,Encentrum_sutor_ott2941619,Encentrum_kozminskii_ott2941620,Encentrum_minax_ott2941621,Encentrum_myersi_ott2941622,Encentrum_axi_ott2941623,Encentrum_ussuriensis_ott2941624,Encentrum_lutra_ott2941625,Encentrum_frenoti_ott2941627,Encentrum_parvum_ott2941628,Encentrum_diglandula_ott2941629,Encentrum_graingeri_ott2941631,Encentrum_alpinum_ott2941632,Encentrum_martes_ott2941633,Encentrum_voigti_ott2941634,Encentrum_limicola_ott2941635,Encentrum_semiplicatum_ott2941636,Encentrum_obesum_ott2941637,Encentrum_armatum_ott4952785,Encentrum_liepolti_ott4952786,Encentrum_aluligerum_ott6367887,Encentrum_foroiuliense_ott6367888,Encentrum_loefgreni_ott6367889,Encentrum_pugiodigitatum_ott6367890,Encentrum_spatitium_ott6367891,Encentrum_uncinatoides_ott6367892)Encentrum_ott1090173,(Paradicranophorus_sordidus_ott2941606,Paradicranophorus_wesenberglundi_ott2941756,Paradicranophorus_sinus_ott2941759,Paradicranophorus_aculeatus_ott2941760,Paradicranophorus_verae_ott2941761,Paradicranophorus_hudsoni_ott2941763,Paradicranophorus_halophilus_ott6367893)Paradicranophorus_ott2941757,(Dicranophoroides_caudatus_ott2941667,Dicranophoroides_venezueliensis_ott2941720,Dicranophoroides_australiensis_ott2941722,Dicranophoroides_claviger_ott2941723)Dicranophoroides_ott2941717,(Wierzejskiella_subulosa_ott2941696,Wierzejskiella_marina_ott2941724,Wierzejskiella_ricciae_ott2941730,Wierzejskiella_subterranea_ott2941731,Wierzejskiella_ambigua_ott2941732,Wierzejskiella_elongata_ott2941733,Wierzejskiella_velox_ott2941734,Wierzejskiella_sabulosa_ott2941735,Wierzejskiella_vagneri_ott2941737)Wierzejskiella_ott2941697,(Erignatha_longidentata_ott2941698,Erignatha_capula_ott2941712,Erignatha_sagittoides_ott2941716,Erignatha_tenuidens_ott2941718,Erignatha_clastopis_ott2941719,Erignatha_sagitta_ott2941736,Erignatha_thienemanni_ott7992128)Erignatha_ott2941699,(Albertia_woronkowi_ott2941701,Albertia_reicheltae_ott2941703,Albertia_naidis_ott2941704,Albertia_ovagranulata_ott2941705,Albertia_vermiculus_ott2941708,Albertia_crystallina_ott2941709,Albertia_typhlina_ott2941711,Albertia_vermisculus_ott7992125)Albertia_ott2941702,(Balatro_calvus_ott2941706,Balatro_anguiformis_ott2941710,Balatro_aciliatus_ott2941738,Balatro_fridericiae_ott2941739)Balatro_ott2941707,(Aspelta_imputa_ott2941715,Aspelta_reibischi_ott2941725,Aspelta_imbuta_ott2941726,Aspelta_psitta_ott2941727,Aspelta_egregia_ott2941728,Aspelta_tilba_ott2941729,Aspelta_clydona_ott2941750,Aspelta_curvidactyla_ott2941751,Aspelta_angusta_ott2941752,Aspelta_chorista_ott2941753,Aspelta_pachida_ott2941754,Aspelta_circinator_ott2941755,Aspelta_labri_ott2941764,Aspelta_intradentata_ott2941765,Aspelta_europaea_ott2941766,Aspelta_alastor_ott2941767,Aspelta_beltista_ott2941768,Aspelta_aper_ott2941769,Aspelta_lestes_ott2941770,Aspelta_bidentata_ott2941771,Aspelta_macra_ott2941772,Aspelta_harringi_ott2941773,Aspelta_secreta_ott4952787)Aspelta_ott2941714,(Parencentrum_lutetiae_ott2941741,Parencentrum_plicatum_ott2941743)Parencentrum_ott2941742,(Dorria_dalecarlica_ott2941744)Dorria_ott2941745,(Wigrella_depressa_ott2941746,Wigrella_amphora_ott2941774)Wigrella_ott2941700,(Pedipartia_gracilis_ott2941747)Pedipartia_ott2941748,(Streptognatha_lepta_ott2941749)Streptognatha_ott2941740,(Donneria_sudzukii_ott2941762)Donneria_ott2941695,(Kostea_wockei_ott4153456)Kostea_ott4153455,(Myersinella_tetraglena_ott4153458,Myersinella_uncodonta_ott4153459,Myersinella_belodon_ott4153460,Myersinella_longiforceps_ott4153461)Myersinella_ott4153457,(Inflatana_pomazkovae_ott4153463)Inflatana_ott4153462,(Glaciera_schabetsbergeri_ott4153466)Glaciera_ott4153465)Dicranophoridae_ott513189,(('Lepadella_cf._ovalis_MEG-2012_ott296329',(Lepadella_patella_oblonga_ott630011)Lepadella_patella_ott1018359,Lepadella_triba_ott630013,Lepadella_rhomboides_ott1090179,Lepadella_astacicola_ott2941342,Lepadella_cornuta_ott2941343,Lepadella_vitrea_ott2941344,Lepadella_rhodesiana_ott2941345,Lepadella_neboissi_ott2941346,Lepadella_strepta_ott2941347,Lepadella_apsicora_ott2941348,Lepadella_branchicola_ott2941349,Lepadella_decora_ott2941350,Lepadella_quadricarinata_ott2941351,Lepadella_pseudosimilis_ott2941352,Lepadella_discoidea_ott2941353,Lepadella_ovalis_ott2941354,Lepadella_latusinus_ott2941355,Lepadella_amphitropis_ott2941356,Lepadella_monodactyla_ott2941357,Lepadella_mica_ott2941358,Lepadella_triprojectus_ott2941359,Lepadella_minuscula_ott2941360,Lepadella_dorsalis_ott2941361,Lepadella_deridderae_ott2941362,Lepadella_latusimus_ott2941363,Lepadella_pterygoida_ott2941364,Lepadella_duvigneaudi_ott2941365,Lepadella_hyalina_ott2941366,Lepadella_salisburii_ott2941367,Lepadella_heterodactyla_ott2941368,Lepadella_pterygoides_ott2941369,Lepadella_pseudoacuminata_ott2941370,Lepadella_dactyliseta_ott2941371,Lepadella_parasitica_ott2941372,Lepadella_vanoyei_ott2941373,Lepadella_psammophila_ott2941374,Lepadella_angusta_ott2941375,Lepadella_vandenbrandei_ott2941376,Lepadella_zigzag_ott2941377,Lepadella_donneri_ott2941378,Lepadella_cryphaea_ott2941379,Lepadella_acuminata_ott2941380,Lepadella_pyriformis_ott2941381,Lepadella_minorui_ott2941382,Lepadella_pumilo_ott2941383,Lepadella_tenella_ott2941384,Lepadella_gelida_ott2941385,Lepadella_rhomboidula_ott2941386,Lepadella_adjuncta_ott2941387,Lepadella_canadaensis_ott2941388,Lepadella_intermedia_ott2941389,Lepadella_sali_ott2941390,Lepadella_longiseta_ott2941391,Lepadella_borealis_ott2941392,Lepadella_wrighti_ott2941393,Lepadella_berzinsi_ott2941394,Lepadella_bidentata_ott2941395,Lepadella_pejleri_ott2941396,Lepadella_minuta_ott2941397,Lepadella_monodi_ott2941398,Lepadella_venefica_ott2941399,Lepadella_serrata_ott2941400,Lepadella_heterostyla_ott2941401,Lepadella_crytopus_ott2941402,Lepadella_mascarensis_ott2941403,Lepadella_desmeti_ott2941404,Lepadella_favorita_ott2941405,Lepadella_obtusa_ott2941406,Lepadella_beyensi_ott2941408,Lepadella_cristata_ott2941409,Lepadella_abbei_ott2941410,Lepadella_pontica_ott2941411,Lepadella_eurysterna_ott2941412,Lepadella_glossa_ott2941413,Lepadella_parvula_ott2941414,Lepadella_ehrenbergii_ott2941415,Lepadella_princisi_ott2941416,Lepadella_biloba_ott2941417,Lepadella_cyrtopus_ott2941418,Lepadella_curvicaudata_ott2941419,Lepadella_lata_ott2941420,Lepadella_bicornis_ott2941421,Lepadella_amazonica_ott2941422,Lepadella_xenica_ott2941423,Lepadella_nympha_ott2941424,Lepadella_tricostata_ott2941425,Lepadella_quadricurvata_ott2941426,Lepadella_kostei_ott2941427,Lepadella_haueri_ott2941428,Lepadella_akrobeles_ott2941429,Lepadella_rottenburgi_ott2941430,Lepadella_tyleri_ott2941431,Lepadella_evaginata_ott2941432,Lepadella_costatoides_ott2941433,Lepadella_koniari_ott2941434,Lepadella_punctata_ott2941435,Lepadella_whitfordi_ott2941436,Lepadella_costata_ott2941437,Lepadella_chengalathi_ott2941438,Lepadella_margalefi_ott2941439,Lepadella_benjamini_ott2941440,Lepadella_myersi_ott2941441,Lepadella_quinquecostata_ott2941442,Lepadella_elongata_ott2941444,Lepadella_neglecta_ott2941445,Lepadella_degreefi_ott2941446,Lepadella_elliptica_ott2941447,Lepadella_persimilis_ott2941448,Lepadella_imbricata_ott2941449,Lepadella_tana_ott2941450,Lepadella_mataca_ott2941451,Lepadella_minoruoides_ott2941452,Lepadella_apsida_ott2941453,Lepadella_ptilota_ott2941454,Lepadella_riedeli_ott2941455,Lepadella_lindaui_ott2941456,Lepadella_triptera_ott2941457,Lepadella_nartiangensis_ott2941458,Lepadella_visenda_ott2941459,Lepadella_paparoa_ott2941460,'Lepadella_cf._quadricarinata_AGROT318-10_ott7506432','Lepadella_cf._quadricarinata_AGROT327-10_ott7506433','Lepadella_cf._quadricarinata_AGROT337-10_ott7506434','Lepadella_cf._quadricarinata_AGROT338-10_ott7506435')Lepadella_ott1018358,(Squatinella_rostrum_ott995274,Squatinella_retrospina_ott2941461,Squatinella_lamellaris_ott2941462,Squatinella_leydigii_ott2941463,Squatinella_lunata_ott2941464,Squatinella_macrodactyla_ott2941465,Squatinella_geleii_ott2941467,Squatinella_stylata_ott2941468,Squatinella_longispinata_ott2941469,Squatinella_pseudorostrum_ott2941470,Squatinella_microdactyla_ott2941472,Squatinella_bifurca_ott2941473,Squatinella_bisetata_ott7992143)Squatinella_ott995261,(Colurella_unicauda_ott2941474,Colurella_collaris_ott2941476,Colurella_denticauda_ott2941477,Colurella_halophila_ott2941478,Colurella_hindenburgi_ott2941479,Colurella_obtusa_ott2941481,Colurella_marinovi_ott2941482,Colurella_dicentra_ott2941483,Colurella_adriatica_ott2941484,Colurella_mucronulata_ott2941485,Colurella_psammophila_ott2941486,Colurella_sulcata_ott2941490,Colurella_geophila_ott2941493,Colurella_tesselata_ott2941494,Colurella_sanoamuangae_ott2941495,Colurella_oblonga_ott2941496,Colurella_colurus_ott2941497,Colurella_aquaeducti_ott2941498,Colurella_anodonta_ott2941499,Colurella_paludosa_ott2941500,Colurella_sinistra_ott2941501,Colurella_salina_ott2941502,Colurella_uncinata_ott2941503,Colurella_unicaudata_ott6367897,Colurella_aquaducti_ott7992140,Colurella_ovalis_ott7992141,Colurella_oxycauda_ott7992142)Colurella_ott2941475,(Paracolurella_aemula_ott2941487,Paracolurella_logima_ott2941491)Paracolurella_ott2941489,Lepadellidae_environmental_sample_ott4952798,Lepadellidae_sp._SHQC150123a_ott5974969)Lepadellidae_ott1090176,((Epiphanes_senta_ott368450,Epiphanes_pelagica_ott2941139,Epiphanes_clavatula_ott2941140,Epiphanes_clavulata_ott2941141,Epiphanes_brachionus_ott2941142,Epiphanes_macroura_ott2941143,Epiphanes_desmeti_ott2941144,Epiphanes_chihuahuaensis_ott4952767,Epiphanes_ukera_ott4952768,Epiphanes_hawaiensis_ott4952769)Epiphanes_ott368449,(Cyrtonia_tuba_ott958183)Cyrtonia_ott958180,(Mikrocodides_robustus_ott2940759,Mikrocodides_chlaena_ott2941158,Mikrocodides_hertha_ott2941162)Mikrocodides_ott2941152,(Rhinoglena_frontalis_ott2941146,Rhinoglena_tokioensis_ott2941150,Rhinoglena_fertoeensis_ott2941155,Rhinoglena_kutikovae_ott2941156,Rhinoglena_tokidensis_ott2941161,Rhinoglena_ovigera_ott7992131,Rhinoglena_texana_ott7992132)Rhinoglena_ott2941147,(Proalides_subtilis_ott2941148,Proalides_digitus_ott2941153,Proalides_tentaculatus_ott2941159)Proalides_ott2941149,(Microcodides_hertha_ott7992130)Microcodides_ott7992129)Epiphanidae_ott368452,((Proales_similis_ott412238,Proales_doliaris_ott412245,Proales_reinhardti_ott412247,Proales_daphnicola_ott958185,Proales_theodora_ott958188,Proales_fallaciosa_ott958190,Proales_werneckii_ott2940661,Proales_gonothyraeae_ott2941281,Proales_prehensor_ott2941282,Proales_palimmeka_ott2941283,Proales_othodon_ott2941284,Proales_sordida_ott2941285,Proales_minima_ott2941286,Proales_granulosa_ott2941287,Proales_adenodis_ott2941289,Proales_gladia_ott2941290,Proales_halophila_ott2941291,Proales_lenta_ott2941293,Proales_globulifera_ott2941294,Proales_phaeopis_ott2941295,Proales_gigantea_ott2941296,Proales_parasita_ott2941297,Proales_cryptopus_ott2941298,Proales_litoralis_ott2941299,Proales_commutata_ott2941300,Proales_alba_ott2941301,Proales_baradlana_ott2941302,Proales_syltensis_ott2941303,Proales_micropus_ott2941304,Proales_macrura_ott2941305,Proales_indirae_ott2941306,Proales_oculata_ott2941308,Proales_kostei_ott2941309,Proales_christinae_ott2941310,Proales_coryneger_ott2941311,Proales_paguri_ott2941312,Proales_wesenbergi_ott2941313,Proales_ornata_ott2941314,Proales_decipiens_ott2941315,Proales_simplex_ott2941316,Proales_germanica_ott2941317,Proales_provida_ott2941318,Proales_segnis_ott2941319,Proales_bemata_ott2941320,Proales_cognita_ott2941321,Proales_pugio_ott2941322,Proales_ardechensis_ott4952775,Proales_laticauda_ott4952776,Proales_francescae_ott6367903,Proales_gammaricola_ott6367904,'Proales_sp._EM-2017_ott7506438',Proales_tillyensis_ott7992156)Proales_ott412244,(Proalinopsis_caudatus_ott2940706,Proalinopsis_squamipes_ott2941331,Proalinopsis_lobatus_ott2941333,Proalinopsis_selene_ott2941334,Proalinopsis_gracilis_ott2941335,Proalinopsis_staurus_ott2941336,Proalinopsis_phacus_ott2941338,Proalinopsis_phagus_ott2941339,Proalinopsis_pellucida_ott2941340)Proalinopsis_ott2941330,(Wulfertia_ornata_ott2941323,Wulfertia_kindensis_ott2941325,Wulfertia_kivuensis_ott2941329)Wulfertia_ott2941324,(Bryceella_tenella_ott2941327,Bryceella_stylata_ott2941328,Bryceella_perpusilla_ott7992154,Bryceella_voigtii_ott7992155)Bryceella_ott2941326)Proalidae_ott412246,((Ascomorpha_ovalis_ott513186,Ascomorpha_ovalis_AEG1_ott820709,Ascomorpha_agilis_ott2941519,Ascomorpha_minima_ott2941520,Ascomorpha_ecaudis_ott2941521,Ascomorpha_saltans_ott2941522,Ascomorpha_dumonti_ott2941523,Ascomorpha_tundisii_ott2941524,Ascomorpha_minuta_ott2941525,Ascomorpha_klementi_ott2941526)Ascomorpha_ott513185)Gastropidae_ott513193,((Lindia_torulosa_ott563704,(Lindia_tecusa_ott1090165,Lindia_elsae_ott2941510,Lindia_gravitata_ott2941517)Halolindia_ott2941518,Lindia_gracilis_ott2941504,Lindia_producta_ott2941505,Lindia_janickii_ott2941506,Lindia_truncata_ott2941507,Lindia_annecta_ott2941508,Lindia_caerulea_ott2941509,Lindia_ecela_ott2941511,Lindia_fulva_ott2941512,Lindia_candida_ott2941513,Lindia_pallida_ott2941514,Lindia_deridderae_ott2941515,Lindia_euchromatica_ott2941516,Lindia_aequorea_ott6367898,Lindia_anebodica_ott7992144,Lindia_sphagnophila_ott7992145,Lindia_virgata_ott7992146)Lindia_ott1090166)Lindiidae_ott1090178,((Microcodon_clavus_ott563707)Microcodon_ott563706)Microcodonidae_ott563711,((Beauchampiella_eudactylota_ott738316)Beauchampiella_ott672794,(Tripleuchlanis_plicata_ott1096272)Tripleuchlanis_ott352056,(Dipleuchlanis_conradi_ott2942183,Dipleuchlanis_ornata_ott2942298,Dipleuchlanis_elegans_ott2942299,Dipleuchlanis_propatula_ott2942300)Dipleuchlanis_ott2942184,(Diplois_daviesiae_ott2942344)Diplois_ott2942343,(Pseudoeuchlanis_longipedes_ott4153481)Pseudoeuchlanis_ott4153480)Euchlanidae_ott672793,((Asciaporrecta_hyalina_ott2940760,Asciaporrecta_arcellicola_ott4153523,Asciaporrecta_difflugicola_ott4153524)Asciaporrecta_ott4153521)Asciaporrectidae_ott2942284,((Itura_globata_ott2942178,Itura_aurita_ott2942266,Itura_myersi_ott2942267,Itura_chamadis_ott2942268,Itura_viridis_ott2942269,Itura_symmetrica_ott2942270,Itura_deridderae_ott2942271)Itura_ott2942179)Ituridae_ott2942180,((Gastropus_stylifer_ott2942247,Gastropus_minor_ott2942293,Gastropus_hyptopus_ott2942342)Gastropus_ott2942248)Gastropodidae_ott5693675,((Tetrasiphon_hydrocora_ott2942255)Tetrasiphon_ott2942256)Tetrasiphonidae_ott2942257,((Birgea_enantia_ott2942261)Birgea_ott2942262)Birgeidae_ott2942263,((Cotylegaleata_perplexa_ott4153520,Cotylegaleata_iskenderunensis_ott7992124)Cotylegaleata_ott4153519)Cotylegaleatidae_ott2942286,((Claria_segmentata_ott4153526)Claria_ott4153525)Clariaidae_ott2942336)Ploima_ott251966)Pseudotrocha_ott5673589,((((Sinantherina_socialis_ott107177,Sinantherina_ariprepes_ott513192,Sinantherina_triglandularis_ott2942057,Sinantherina_spinosa_ott2942058,Sinantherina_procera_ott2942059,Sinantherina_semibullata_ott2942060)Sinantherina_ott107178,(Floscularia_melicerta_ott418175,Floscularia_decora_ott426467,Floscularia_longicauda_ott2942045,Floscularia_wallacei_ott2942046,Floscularia_bifida_ott2942047,Floscularia_conifera_ott2942048,Floscularia_janus_ott2942049,Floscularia_pedunculata_ott2942052,Floscularia_noodti_ott2942053,Floscularia_curvicornis_ott2942054,Floscularia_armata_ott2942055,Floscularia_rigens_ott2942056,Floscularia_environmental_sample_ott4952751)Floscularia_ott435984,(Ptygura_libera_ott513191,Ptygura_kostei_ott2942061,Ptygura_elsteri_ott2942062,Ptygura_brevis_ott2942063,Ptygura_cristata_ott2942064,Ptygura_pilula_ott2942065,Ptygura_crystallina_ott2942067,Ptygura_beauchampi_ott2942068,Ptygura_stygis_ott2942070,Ptygura_agassizi_ott2942071,Ptygura_longicornis_ott2942072,Ptygura_tacita_ott2942073,Ptygura_velata_ott2942074,Ptygura_tihanyensis_ott2942075,Ptygura_wilsonii_ott2942076,Ptygura_mucicola_ott2942077,Ptygura_pedunculata_ott2942078,Ptygura_intermedia_ott2942079,Ptygura_barbata_ott2942080,Ptygura_rotifer_ott2942081,Ptygura_tridorsicornis_ott2942082,Ptygura_stephanion_ott2942083,Ptygura_furcillata_ott2942084,Ptygura_socialis_ott2942085,Ptygura_brachiata_ott2942086,Ptygura_seminatans_ott2942087,Ptygura_spongicola_ott2942088,Ptygura_linguata_ott2942089,Ptygura_noodti_ott6367878,Ptygura_melicerta_ott7506321)Ptygura_ott513190,(Collotheca_campanulata_ott1039706,Collotheca_trilobata_ott2942050,Collotheca_vargai_ott2942090,Collotheca_moselii_ott2942092,Collotheca_tenuilobata_ott2942093,Collotheca_libera_ott2942094,Collotheca_crateriformis_ott2942095,Collotheca_ornata_ott2942096,Collotheca_annulata_ott2942097,Collotheca_calva_ott2942098,Collotheca_lettevalli_ott2942099,Collotheca_sessilis_ott2942100,Collotheca_heptabrachiata_ott2942101,Collotheca_ferox_ott2942102,Collotheca_quadrilobata_ott2942105,Collotheca_triloba_ott2942106,Collotheca_bilfingeri_ott2942107,Collotheca_cucullata_ott2942108,Collotheca_coronetta_ott2942110,Collotheca_tenera_ott2942111,Collotheca_riverai_ott2942112,Collotheca_atrochoides_ott2942113,Collotheca_torquilobata_ott2942114,Collotheca_undulata_ott2942115,Collotheca_monoceros_ott2942116,Collotheca_evansonii_ott2942117,Collotheca_spinata_ott2942118,Collotheca_pelagica_ott2942119,Collotheca_ambigua_ott2942120,Collotheca_discophora_ott2942121,Collotheca_mutabilis_ott2942122,Collotheca_balatonica_ott2942123,Collotheca_stephanochaeta_ott2942124,Collotheca_bulbosa_ott2942125,Collotheca_judayi_ott2942126,Collotheca_quadrinodosa_ott2942127,Collotheca_trifidlobata_ott2942128,Collotheca_edentata_ott2942129,Collotheca_paradoxa_ott2942130,Collotheca_wiszniewskii_ott2942131,Collotheca_thunmarki_ott2942132,Collotheca_edmondsoni_ott2942134,Collotheca_hepatabrachia_ott2942135,Collotheca_minuta_ott2942136,Collotheca_epizootica_ott2942138,Collotheca_rasmae_ott2942139,Collotheca_vargae_ott2942141,Collotheca_gosseii_ott4952754,Collotheca_polyphemus_ott4952755,Collotheca_tetralobata_ott4952756,Collotheca_hexalobata_ott4952757,Collotheca_hoodii_ott4952758)Collotheca_ott1039707,(Lacinularia_reticulata_ott2942145,Lacinularia_racemosa_ott2942147,Lacinularia_flosculosa_ott2942155,Lacinularia_ismailoviensis_ott2942156,Lacinularia_elongata_ott2942158,Lacinularia_striolata_ott2942159,Lacinularia_megalotrocha_ott2942160,Lacinularia_pedunculata_ott2942161,Lacinularia_elliptica_ott2942162,Lacinularia_ismaloviensis_ott2942166,Lacinularia_causeyae_ott7992099)Lacinularia_ott2942146,(Limnias_shiawasseensis_ott2942148,Limnias_melicerta_ott2942157,Limnias_cornuella_ott2942163,Limnias_nymphaea_ott2942164,Limnias_ceratophylli_ott2942165,Limnias_myriophylli_ott2942167,'Limnias_sp._PM-2016_ott7506320')Limnias_ott2942149,(Beauchampia_crucigera_ott2942150)Beauchampia_ott2942151,(Octotrocha_speciosa_ott2942153)Octotrocha_ott2942154,(Pentatrocha_gigantea_ott4153435)Pentatrocha_ott4153434,(Lacinularoides_coloniensis_ott5706031)Lacinularoides_ott5693683)Flosculariidae_ott681210,((Testudinella_patina_AEG5_ott245957,Testudinella_caeca_ott519380,(Testudinella_patina_intermedia_ott890418,Testudinella_patina_dendradena_ott2942015)Testudinella_patina_ott995276,Testudinella_clypeata_ott958181,Testudinella_berzinsi_ott2941982,Testudinella_truncata_ott2941983,Testudinella_brevicaudata_ott2941984,Testudinella_wuhanensis_ott2941985,Testudinella_striata_ott2941986,Testudinella_aspis_ott2941987,Testudinella_magna_ott2941988,Testudinella_robertsonae_ott2941989,Testudinella_angulata_ott2941990,Testudinella_reflexa_ott2941991,Testudinella_husseyi_ott2941992,Testudinella_obscura_ott2941993,Testudinella_haueriensis_ott2941994,Testudinella_unicornuta_ott2941995,Testudinella_triangularis_ott2941996,Testudinella_mucronata_ott2941997,Testudinella_subdiscoidea_ott2941998,Testudinella_andranomenensis_ott2941999,Testudinella_elliptica_ott2942000,Testudinella_neboisi_ott2942001,Testudinella_sphagnicola_ott2942002,Testudinella_parva_ott2942003,Testudinella_ahlstromi_ott2942004,Testudinella_carlini_ott2942005,Testudinella_tridentata_ott2942006,Testudinella_discoidea_ott2942007,Testudinella_kostei_ott2942008,Testudinella_walkeri_ott2942009,Testudinella_gillardi_ott2942010,Testudinella_epicopta_ott2942011,Testudinella_greeni_ott2942012,Testudinella_panonica_ott2942013,Testudinella_emarginula_ott2942014,Testudinella_stappersi_ott2942016,Testudinella_amphora_ott2942017,Testudinella_munda_ott2942018,Testudinella_brycei_ott2942019,Testudinella_vanoyei_ott2942020,Testudinella_ohlei_ott2942021,Testudinella_incisa_ott2942022,Testudinella_ovata_ott2942023,Testudinella_bicorniculata_ott4952759,Testudinella_zhujiangensis_ott4952760,Testudinella_elongata_ott4952761,Testudinella_crassa_ott6367881,Testudinella_pseudobscura_ott6367882,Testudinella_quadrilobata_ott6367883,Testudinella_bonneri_ott7992105,Testudinella_clypleata_ott7992106)Testudinella_ott995263,(Pompholyx_sulcata_ott2942024,Pompholyx_triloba_ott2942028,Pompholyx_complanata_ott2942029)Pompholyx_ott2942025,(Anchitestudinella_mekongensis_ott2942027)Anchitestudinella_ott2942026,'Testudinella_sp._GG-2003_ott4153429',(Pompholys_sulcata_ott7992104)Pompholys_ott7992103)Testudinellidae_ott124556,((Filinia_longiseta_ott424470,Filinia_brachiata_ott2942042,Filinia_terminalis_ott2942043,Filinia_cornuta_ott2942044)Filinia_ott424467,(Horaella_brehmi_ott2942227,Horaella_thomassoni_ott2942341)Horaella_ott2942204,(Trochosphaera_aequatorialis_ott2942250,Trochosphaera_solstitialis_ott2942252)Trochosphaera_ott2942251)Filinidae_ott1026312,((Conochilus_hippocrepis_ott1039708,Conochilus_unicornis_ott1039709,Conochilus_exiguus_ott2942169,Conochilus_deltaicus_ott2942170,Conochilus_coenobasis_ott2942171,Conochilus_natans_ott2942172)Conochilus_ott1039704,(Conochiloides_dossuarius_ott2942168,Conochiloides_exiguus_ott2942175,'Conochiloides_sp._WM-2017a_ott7506313')Conochiloides_ott2942174,(Conochilopsis_causeyae_ott4153438)Conochilopsis_ott4153437)Conochilidae_ott1039705,((Hexarthra_polydonta_ott2942200,Hexarthra_jenkinae_ott2942203,Hexarthra_oxyuris_ott2942208,Hexarthra_polychaeta_ott2942228,Hexarthra_mira_ott2942246,Hexarthra_longicornicula_ott2942253,Hexarthra_mollis_ott2942272,Hexarthra_propinqua_ott2942288,Hexarthra_bulgarica_ott2942289,Hexarthra_reducens_ott2942320,Hexarthra_brandorffi_ott2942322,Hexarthra_fennica_ott2942329,Hexarthra_polyodonta_ott2942330,Hexarthra_libica_ott2942346,(Hexarthra_intermedia_brasiliensis_ott5723812)Hexarthra_intermedia_ott2942196,Hexarthra_polyptera_ott6367880,'Hexarthra_sp._WM-2017a_ott7506323',Hexarthra_oxyure_ott7992101)Hexarthra_ott491882)Hexarthridae_ott2942198,Flosculariaceae_environmental_sample_ott7506316)Flosculariaceae_ott107174,(((Acyclus_inquietus_ott2942237,Acyclus_trilobus_ott2942243)Acyclus_ott2942238,(Atrochus_tentaculatus_ott2942241)Atrochus_ott2942242,(Cupelopagis_vorax_ott2942244,Cupelopagis_bipera_ott2942332)Cupelopagis_ott2942245)Atrochidae_ott2942239,(Stephanoceros_fimbriatus_ott2942283,Stephanoceros_vulgaris_ott2942294,Stephanoceros_millsii_ott7992096)Stephanoceros_ott2942282,'Collothecaceae_sp._EM-2017_ott7506311')Collothecaceae_ott5677241)Gnesiotrocha_ott5673590,'Monogononta_sp._R35_CoastL_ott7506325')Monogononta_ott641254,((((Seison_nebaliae_ott157976,Seison_africanus_ott2940501)Seison_ott779396,(Paraseison_annulatus_ott2940500)Paraseison_ott4153539)Seisonidae_ott5673588)Seisonacea_ott2942226)Seisonidea_ott779397)Rotifera_ott471706; diff --git a/dvc.lock b/dvc.lock index 81c07fe6..8a1e68ae 100644 --- a/dvc.lock +++ b/dvc.lock @@ -1,7 +1,7 @@ schema: '2.0' stages: download_and_filter_wikidata: - cmd: download_and_filter_wikidata --url "$(cat + cmd: .venv/bin/download_and_filter_wikidata --url "$(cat data/Wiki/wd_JSON/latest-all-json-bz2-url.txt)" -o data/filtered/OneZoom_latest-all.json deps: @@ -15,8 +15,8 @@ stages: md5: e6f69def9d6fa2bb4b90060509c079bb size: 1567956473 extract_wikidata_titles: - cmd: extract_wikidata_titles data/filtered/OneZoom_latest-all.json -o - data/filtered/wikidata_titles.txt + cmd: .venv/bin/extract_wikidata_titles data/filtered/OneZoom_latest-all.json + -o data/filtered/wikidata_titles.txt deps: - path: data/filtered/OneZoom_latest-all.json hash: md5 @@ -41,7 +41,7 @@ stages: md5: 541324eaa9f3f1a14bb6ddcf7ea95de6 size: 2397370114 download_and_filter_pageviews: - cmd: download_and_filter_pageviews --titles-file + cmd: .venv/bin/download_and_filter_pageviews --titles-file data/filtered/wikidata_titles.txt --months 12 -o data/filtered/pageviews deps: - path: data/filtered/wikidata_titles.txt @@ -55,7 +55,7 @@ stages: size: 128216689 nfiles: 13 filter_wikipedia_sql: - cmd: filter_wikipedia_sql data/Wiki/wp_SQL/enwiki-page.sql.gz + cmd: .venv/bin/filter_wikipedia_sql data/Wiki/wp_SQL/enwiki-page.sql.gz data/filtered/wikidata_titles.txt -o data/filtered/OneZoom_enwiki-latest-page.sql deps: @@ -73,30 +73,33 @@ stages: md5: f7437ba9e2a89ff658749e434a38b39a size: 21507159 download_opentree: - cmd: download_opentree --version v16.1 --output-dir data/OpenTree + cmd: + - rm -rf data/OpenTree/v16.1/ + - mkdir -p data/OpenTree/v16.1/ + - .venv/bin/download_opentree --version v16.1 --output-dir data/OpenTree params: params.yaml: ot_version: v16.1 outs: - path: data/OpenTree/v16.1/ hash: md5 - md5: 87ff995e9d5028efc185857f34448746.dir - size: 587064765 + md5: 8c9c0fadda5ac919e5573462208a5c8f.dir + size: 564222021 nfiles: 3 add_ott_numbers_to_trees: cmd: - rm -rf data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1 - mkdir -p data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1 - - add_ott_numbers_to_trees --savein + - .venv/bin/add_ott_numbers_to_trees --savein data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1 --output_info data/add_ott_numbers_to_trees.log data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/*.[pP][hH][yY] deps: - path: data/OZTreeBuild/AllLife/BespokeTree/include_noAutoOTT/ hash: md5 - md5: c3c1ebf2453c636e3ffdfcef58722d9c.dir - size: 1231291 - nfiles: 56 + md5: 4097a3d5af5e05a587882129df7882df.dir + size: 1271857 + nfiles: 45 params: params.yaml: ot_version: v16.1 @@ -104,9 +107,9 @@ stages: outs: - path: data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1/ hash: md5 - md5: c12514e92740949250ecdb4375d6c360.dir - size: 1534814 - nfiles: 55 + md5: 4d9484418e30d363e0c33ffb2e12e5fd.dir + size: 1586233 + nfiles: 44 get_open_trees_from_one_zoom: cmd: - cd data/OZTreeBuild/AllLife && get_open_trees_from_one_zoom @@ -153,7 +156,7 @@ stages: md5: cb59f793fb021063b7c35e821ee8086a size: 31 filter_eol: - cmd: filter_eol data/EOL/provider_ids.csv.gz + cmd: .venv/bin/filter_eol data/EOL/provider_ids.csv.gz data/OpenTree/v16.1/taxonomy.tsv -o data/filtered/OneZoom_provider_ids.csv deps: - path: data/EOL/provider_ids.csv.gz @@ -174,8 +177,8 @@ stages: size: 221682224 build_oz_tree: cmd: - - cd data/OZTreeBuild/AllLife && build_oz_tree - BespokeTree/include_OT_v16.1/Base.PHY OpenTreeParts/OpenTree_all/ + - cd data/OZTreeBuild/AllLife && build_oz_tree --nodeages + ../../node_ages.json BespokeTree/include_OT_v16.1/Base.PHY AllLife_full_tree.phy deps: - path: data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1/ @@ -264,7 +267,7 @@ stages: size: 1158320679 nfiles: 7 discover_latest_wikidata_dump_url: - cmd: discover_latest_wikidata_dump_url > + cmd: .venv/bin/discover_latest_wikidata_dump_url > data/Wiki/wd_JSON/latest-all-json-bz2-url.txt outs: - path: data/Wiki/wd_JSON/latest-all-json-bz2-url.txt @@ -272,10 +275,130 @@ stages: md5: aa1692624401f4a1c759bc9526787fd7 size: 90 discover_latest_enwiki_sql_url: - cmd: discover_latest_enwiki_sql_url > + cmd: .venv/bin/discover_latest_enwiki_sql_url > data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt outs: - path: data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt hash: md5 md5: 1018a5c664d01747fdd7e218190cb4ac size: 72 + download_node_ages: + cmd: .venv/bin/download_node_ages data/node_ages.json + outs: + - path: data/node_ages.json + hash: md5 + md5: 02b43f39966ac4c461bed8f1fa3247a9 + size: 13157027 + date_tree: + cmd: + - rm -rf data/dated_tree + - mkdir -p data/dated_tree + - .venv/bin/date_tree --output_folder=data/dated_tree --date_cache + data/node_ages.json --annotations data/OpenTree/v16.1/annotations.json + --taxonomy data/OpenTree/v16.1/taxonomy.tsv --supertree + data/OpenTree/v16.1/labelled_supertree_ottnames.tre + deps: + - path: data/node_ages.json + hash: md5 + md5: 02b43f39966ac4c461bed8f1fa3247a9 + size: 13157027 + params: + params.yaml: + ot_version: v16.1 + outs: + - path: data/dated_tree/dated_tree_pre.tre + hash: md5 + md5: 531c294f7e80f3d588369af73047ddfe + size: 134355939 + taxon_map: + cmd: .venv/bin/taxon_map --OpenTreeTaxonomy data/OpenTree/v16.1/taxonomy.tsv + --wikidataDumpFile data/filtered/OneZoom_latest-all.json + --wikipediaSQLDumpFile data/filtered/OneZoom_enwiki-latest-page.sql + --wikipedia_totals_bz2_pageviews data/filtered/pageviews/ --EOLidentifiers + data/filtered/OneZoom_provider_ids.csv --extra_source_file + data/OZTreeBuild/AllLife/BespokeTree/SupplementaryTaxonomy.tsv -o + data/taxon_map.csv + deps: + - path: data/OZTreeBuild/AllLife/BespokeTree/SupplementaryTaxonomy.tsv + hash: md5 + md5: 8e861649388bf88595b93c0199f2cc3a + size: 312 + isexec: true + - path: data/OpenTree/v16.1/taxonomy.tsv + hash: md5 + md5: d7a58eaaf132522b89a506e96ca5098f + size: 417054016 + - path: data/filtered/OneZoom_enwiki-latest-page.sql + hash: md5 + md5: f7437ba9e2a89ff658749e434a38b39a + size: 21507159 + - path: data/filtered/OneZoom_latest-all.json + hash: md5 + md5: e6f69def9d6fa2bb4b90060509c079bb + size: 1567956473 + - path: data/filtered/OneZoom_provider_ids.csv + hash: md5 + md5: f7c9bb8374957c07168bec36d6591347 + size: 221682224 + - path: data/filtered/pageviews/ + hash: md5 + md5: 701c03fc874aa8a4d5578a6a74267bee.dir + size: 128216689 + nfiles: 13 + params: + params.yaml: + ot_version: v16.1 + oz_tree: AllLife + outs: + - path: data/taxon_map.csv + hash: md5 + md5: 08e16ef58049999ecb1608be819976d4 + size: 230041544 + tree_build: + cmd: + - rm -rf data/out + - mkdir -p data/out + - .venv/bin/tree_build --bespoke_dir + data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1/ --orphan_dir + data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/ --opentree + data/dated_tree/dated_tree_pre.tre --taxon_map data/taxon_map.csv + --exclude Archosauria_ott335588 Dinosauria_ott90215 --out_dir data/out + deps: + - path: data/OZTreeBuild/AllLife/BespokeTree/include_OT_v16.1/ + hash: md5 + md5: 4d9484418e30d363e0c33ffb2e12e5fd.dir + size: 1586233 + nfiles: 44 + - path: data/OZTreeBuild/AllLife/OpenTreeParts/OT_required/ + hash: md5 + md5: ba7be58a908cdd00297088d0227ffc54.dir + size: 103205 + nfiles: 4 + - path: data/dated_tree/dated_tree_pre.tre + hash: md5 + md5: 531c294f7e80f3d588369af73047ddfe + size: 134355939 + - path: data/taxon_map.csv + hash: md5 + md5: 08e16ef58049999ecb1608be819976d4 + size: 230041544 + params: + params.yaml: + ot_version: v16.1 + oz_tree: AllLife + outs: + - path: data/out + hash: md5 + md5: 8681e904a6f6cc834f85b981bcdcb78c.dir + size: 774309948 + nfiles: 7 + versioned_outputs: + cmd: + - mkdir -p data/out_versioned + - .venv/bin/versioned_outputs --out_dir data/out_versioned data/out/* -vv + deps: + - path: data/out/ + hash: md5 + md5: 8681e904a6f6cc834f85b981bcdcb78c.dir + size: 774309948 + nfiles: 7 diff --git a/dvc.yaml b/dvc.yaml index c3b40492..7f8d9325 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -3,19 +3,45 @@ vars: stages: download_opentree: - cmd: download_opentree --version ${ot_version} --output-dir data/OpenTree + cmd: + - rm -rf data/OpenTree/${ot_version}/ + - mkdir -p data/OpenTree/${ot_version}/ + - .venv/bin/download_opentree --version ${ot_version} --output-dir data/OpenTree params: - ot_version outs: - data/OpenTree/${ot_version}/ + download_node_ages: + cmd: .venv/bin/download_node_ages data/node_ages.json + outs: + - data/node_ages.json + + date_tree: + cmd: + - rm -rf data/dated_tree + - mkdir -p data/dated_tree + - >- + .venv/bin/date_tree + --output_folder=data/dated_tree + --date_cache data/node_ages.json + --annotations data/OpenTree/${ot_version}/annotations.json + --taxonomy data/OpenTree/${ot_version}/taxonomy.tsv + --supertree data/OpenTree/${ot_version}/labelled_supertree_ottnames.tre + deps: + - data/node_ages.json + params: + - ot_version + outs: + - data/dated_tree/dated_tree_pre.tre + # ~20 secs add_ott_numbers_to_trees: cmd: - rm -rf data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version} - mkdir -p data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version} - >- - add_ott_numbers_to_trees + .venv/bin/add_ott_numbers_to_trees --savein data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version} --output_info data/add_ott_numbers_to_trees.log data/OZTreeBuild/${oz_tree}/BespokeTree/include_noAutoOTT/*.[pP][hH][yY] @@ -27,41 +53,41 @@ stages: outs: - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ - # ~a few secs - get_open_trees_from_one_zoom: + tree_build: cmd: + - rm -rf data/out + - mkdir -p data/out - >- - cd data/OZTreeBuild/${oz_tree} && - get_open_trees_from_one_zoom - ../../OpenTree/${ot_version}/draftversion.tre - OpenTreeParts/OpenTree_all/ - BespokeTree/include_OT_${ot_version}/*.PHY + .venv/bin/tree_build + --bespoke_dir data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ + --orphan_dir data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ + --opentree data/dated_tree/dated_tree_pre.tre + --taxon_map data/taxon_map.csv + --exclude ${exclude_from_popularity} + --out_dir data/out deps: - - data/OpenTree/${ot_version}/draftversion.tre - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OT_required/ + - data/dated_tree/dated_tree_pre.tre + - data/taxon_map.csv params: - oz_tree - ot_version outs: - - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OpenTree_all/ + - data/out - build_oz_tree: + versioned_outputs: cmd: + # NB: Don't delete it, we can keep old versions at this point + - mkdir -p data/out_versioned - >- - cd data/OZTreeBuild/${oz_tree} && - build_oz_tree - BespokeTree/include_OT_${ot_version}/Base.PHY - OpenTreeParts/OpenTree_all/ - ${oz_tree}_full_tree.phy + .venv/bin/versioned_outputs + --out_dir data/out_versioned + data/out/* + -vv deps: - - data/OZTreeBuild/${oz_tree}/BespokeTree/include_OT_${ot_version}/ - - data/OZTreeBuild/${oz_tree}/OpenTreeParts/OpenTree_all/ - params: - - oz_tree - - ot_version - outs: - - data/OZTreeBuild/${oz_tree}/${oz_tree}_full_tree.phy + - data/out/ + always_changed: true download_eol: # EOL doesn't version the provider ids file, so capture the last-modified header instead @@ -79,7 +105,7 @@ stages: filter_eol: cmd: >- - filter_eol + .venv/bin/filter_eol data/EOL/provider_ids.csv.gz data/OpenTree/${ot_version}/taxonomy.tsv -o data/filtered/OneZoom_provider_ids.csv @@ -93,14 +119,14 @@ stages: discover_latest_wikidata_dump_url: cmd: >- - discover_latest_wikidata_dump_url > data/Wiki/wd_JSON/latest-all-json-bz2-url.txt + .venv/bin/discover_latest_wikidata_dump_url > data/Wiki/wd_JSON/latest-all-json-bz2-url.txt outs: - data/Wiki/wd_JSON/latest-all-json-bz2-url.txt # >6 hours (streams ~90 GB dump from wikidata server) download_and_filter_wikidata: cmd: >- - download_and_filter_wikidata + .venv/bin/download_and_filter_wikidata --url "$(cat data/Wiki/wd_JSON/latest-all-json-bz2-url.txt)" -o data/filtered/OneZoom_latest-all.json deps: @@ -110,7 +136,7 @@ stages: extract_wikidata_titles: cmd: >- - extract_wikidata_titles + .venv/bin/extract_wikidata_titles data/filtered/OneZoom_latest-all.json -o data/filtered/wikidata_titles.txt deps: @@ -120,7 +146,7 @@ stages: discover_latest_enwiki_sql_url: cmd: >- - discover_latest_enwiki_sql_url > data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt + .venv/bin/discover_latest_enwiki_sql_url > data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt outs: - data/Wiki/wp_SQL/enwiki-page-sql-gz-url.txt @@ -135,7 +161,7 @@ stages: filter_wikipedia_sql: cmd: >- - filter_wikipedia_sql + .venv/bin/filter_wikipedia_sql data/Wiki/wp_SQL/enwiki-page.sql.gz data/filtered/wikidata_titles.txt -o data/filtered/OneZoom_enwiki-latest-page.sql @@ -148,7 +174,7 @@ stages: # ~several hours (streams 12 ~5GB monthly dumps) download_and_filter_pageviews: cmd: >- - download_and_filter_pageviews + .venv/bin/download_and_filter_pageviews --titles-file data/filtered/wikidata_titles.txt --months 12 -o data/filtered/pageviews @@ -158,46 +184,26 @@ stages: - data/filtered/pageviews/: persist: true - # ~10 mins - CSV_base_table_creator: - cmd: - - mkdir -p data/output_files - - >- - CSV_base_table_creator - data/OZTreeBuild/${oz_tree}/${oz_tree}_full_tree.phy - data/OpenTree/${ot_version}/taxonomy.tsv - data/filtered/OneZoom_provider_ids.csv - data/filtered/OneZoom_latest-all.json - data/filtered/OneZoom_enwiki-latest-page.sql - data/filtered/pageviews/OneZoom_pageviews* - -o data/output_files -v - --version ${build_version} - --exclude ${exclude_from_popularity} - --extra_source_file data/OZTreeBuild/${oz_tree}/BespokeTree/SupplementaryTaxonomy.tsv - 2> data/CSV_base_table_creator.log + # ~4 mins + taxon_map: + cmd: >- + .venv/bin/taxon_map + --OpenTreeTaxonomy data/OpenTree/${ot_version}/taxonomy.tsv + --wikidataDumpFile data/filtered/OneZoom_latest-all.json + --wikipediaSQLDumpFile data/filtered/OneZoom_enwiki-latest-page.sql + --wikipedia_totals_bz2_pageviews data/filtered/pageviews/ + --EOLidentifiers data/filtered/OneZoom_provider_ids.csv + --extra_source_file data/OZTreeBuild/${oz_tree}/BespokeTree/SupplementaryTaxonomy.tsv + -o data/taxon_map.csv deps: - - data/OZTreeBuild/${oz_tree}/${oz_tree}_full_tree.phy - data/OpenTree/${ot_version}/taxonomy.tsv - - data/filtered/OneZoom_provider_ids.csv - data/filtered/OneZoom_latest-all.json - data/filtered/OneZoom_enwiki-latest-page.sql - data/filtered/pageviews/ + - data/filtered/OneZoom_provider_ids.csv - data/OZTreeBuild/${oz_tree}/BespokeTree/SupplementaryTaxonomy.tsv params: - oz_tree - ot_version - - build_version - - exclude_from_popularity outs: - - data/output_files/ - - make_js_treefiles: - cmd: - - rm -r data/js_output ; mkdir -p data/js_output - - >- - make_js_treefiles - --outdir data/js_output - data/output_files/ordered_tree_*.poly - deps: - - data/output_files/ - always_changed: true + - data/taxon_map.csv diff --git a/oz_tree_build/_OZglobals.py b/oz_tree_build/_OZglobals.py index 71056637..714f21ff 100644 --- a/oz_tree_build/_OZglobals.py +++ b/oz_tree_build/_OZglobals.py @@ -14,7 +14,6 @@ current = type("", (), {})() # allow us to set e.g. current.OZglobals # bitwise flags for existence of different language wikipedia articles -# this variable is also used in construct_wiki_info in CSV_base_table_creator.py wikiflags = cache.ram( "wikiflags", lambda: { diff --git a/oz_tree_build/date_tree/date_tree.py b/oz_tree_build/date_tree/date_tree.py new file mode 100644 index 00000000..8a5e3101 --- /dev/null +++ b/oz_tree_build/date_tree/date_tree.py @@ -0,0 +1,335 @@ +# BSD 3-Clause License + +# Copyright (c) 2025, Jonathan David Duke + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import json +import os +import sys +import gc +import numpy as np +import datetime + +import ete4 + +from dated_complete_tree import tree_loading +from dated_complete_tree import tree_labelling +from dated_complete_tree import tree_fixing +from dated_complete_tree import tree_dating +from dated_complete_tree import tree_metrics + +import argparse +import logging +logger = logging.getLogger(__name__) + + +def nwk_write(tree, outfile): + """ + Tidy properties and write out newick tree + """ + for n in tree.traverse(): + # Tidy up attributes, only output useful values + if "date" in n.props and n.props["date"] is None: + n.del_prop("date") + tree.write( + outfile=outfile, + props=["date"], + parser=1, + ) + + +def nwk_read(infile): + """ + Re-reads trees written by nwk_write, used by downstream processes + """ + tree = ete4.Tree(infile, parser=1) + for n in tree.traverse(): + n.props["date"] = float(n.props["date"]) if "date" in n.props else None + return tree + + +def generate_trees(args): + if not os.path.exists(args.output_folder): + os.makedirs(args.output_folder) + + sys.setrecursionlimit(10000) + + rng = np.random.default_rng(seed=1) + date_interpolation_rng = np.random.default_rng(seed=1) + date_source_rng = np.random.default_rng(seed=1) + + ##################################################################################################################### + # Load and prune tree + + # Load metadata for tree from Open Tree and Chronosynth + with open(args.date_cache) as f: + dates = json.load(f) + phylogeny_nodes, taxa = tree_loading.load_metadata( + date_cache=None, + annotations=args.annotations, + taxonomy=args.taxonomy, + ) + + # Create ETE3 tree structure for entire Open Tree of Life, with my annotations + whole_tre_unmodified = tree_loading.build_and_annotate_tree(phylogeny_nodes, taxa, tree_filename=args.supertree) + + tree_fixing.strip_birds(whole_tre_unmodified) + tree_fixing.strip_turtles(whole_tre_unmodified) + + rng = np.random.default_rng(seed=1) + + tree_fixing.remove_subspecies(whole_tre_unmodified, rng) + tree_fixing.impute_species_into_empty_taxa(whole_tre_unmodified) + + tree_fixing.fix_taxonomy_ordering(whole_tre_unmodified) + + tree_labelling.add_anc_ranks(whole_tre_unmodified) + tree_labelling.add_desc_ranks(whole_tre_unmodified) + + tree_fixing.forced_taxa_moves(whole_tre_unmodified) + + if args.pd_clades: + topo_pd_clades = [cld.strip() for cld in list(open(args.pd_clades))] + topo_pd_dict = {} + topo_dates_dict = {} + topo_spp_dict = {} + for clade in topo_pd_clades: + topo_pd_dict[clade] = [] + topo_dates_dict[clade] = [] + topo_spp_dict[clade] = [] + + if args.num_date_samples > 0: + both_pd_clades = [cld.strip() for cld in list(open(args.pd_clades))] + both_pd_dict = {} + both_dates_dict = {} + both_spp_dict = {} + for clade in both_pd_clades: + both_pd_dict[clade] = [] + both_dates_dict[clade] = [] + both_spp_dict[clade] = [] + + if args.compute_ed: + topo_ed_scores = {} + if args.num_date_samples > 0: + both_ed_scores = {} + + itr_start = datetime.datetime.now() + + for n in range(args.num_trees): + print("Tree number", n+1, "/ projected end time:", itr_start + args.num_trees*(datetime.datetime.now() - itr_start)/n if n > 0 else "first iteration, no estimate yet") + + # Copy tree - we will change the copy, and keep the original unchanged so we can restore it next iteration without + # reloading everything + whole_tre = whole_tre_unmodified.copy() + + ##################################################################################################################### + # Fix topology + + # First, do labelling for steps 1-3: + # - 1-2 are independent of each other; step 3 collects up nodes not labelled in 1-2. + # - tree is only labelled at this stage; modifications are made in tree_fixing functions. + genus_dict = {} # step 1, nodes below genus nodes + nmp_genus_dict = {} # step 2, non-monophyletic genera + tree_labelling.populate_genus_dict(whole_tre, genus_dict, nmp_genus_dict, None) + + tofix_dict = {} # step 3, all other nodes from taxonomy (not phylogenies) to + # be moved to a suitable place in the tree, such that we + # generated a plausible hypothetical tree + tree_labelling.populate_tofix_dict(whole_tre, tofix_dict, nmp_genus_dict) + + # Second, fix the topology based on the labels. + # Fix steps 1 and 2. + tree_fixing.fix_polyphyly(genus_dict, rng) + tree_fixing.fix_polyphyly(nmp_genus_dict, rng) + + tree_fixing.remove_nonspecies_leaves(whole_tre) + + # Find and label backbone for step 3, after steps 1 an 2 already fixed. + tree_labelling.populate_tofix_bkb(whole_tre, tofix_dict, []) + fix_dict = tree_labelling.process_tofix_bkb(tofix_dict) + + # Finally, fix step 3. + tree_fixing.fix_polyphyly(fix_dict, rng, expand_parent_backbones=True) + + # tree_fixing.remove_nonspecies_leaves(whole_tre) + + # Last of all, polytomy resolution. + tree_fixing.fix_all_polytomies(whole_tre, rng) + + # Remove one-child nodes. Gives a fully bifurcating tree. + # whole_tre = tree_fixing.delete_one_child_nodes(whole_tre) + + ##################################################################################################################### + # Assign and interpolate median dates + + # Assign dates + tree_dating.assign_dates(whole_tre, dates) + + ############# Write out early, before imputing dates + nwk_write(whole_tre, "%s/%s_pre.tre" % (args.output_folder, args.output_tree_filename)) + sys.exit(0) + ######################## + + # Date cleaning to ensure time consistency down the tree + tree_dating.label_older_descendants(whole_tre) + tree_dating.dq_date_removal(whole_tre) + + # Date imputation + tree_dating.date_labelling(whole_tre) + if args.use_birth_model: + tree_dating.impute_missing_dates(whole_tre, use_birth_model=True, rng=date_interpolation_rng) + else: + tree_dating.impute_missing_dates(whole_tre, l=0.25) + + # All nodes now dated - set dists in ete and write out tree. + tree_dating.compute_branch_lengths(whole_tre) + tree_dating.write_tree_with_branch_lengths(whole_tre, filename="%s/%s_topo_sample_%d.tre" % (args.output_folder, args.output_tree_filename, n+1)) + + if args.compute_ed: + tree_metrics.compute_ed_scores(whole_tre, topo_ed_scores) + + if args.pd_clades: + tree_metrics.compute_pd(whole_tre) + tree_metrics.save_pd_for_clades(whole_tre, topo_pd_clades, topo_pd_dict, topo_dates_dict, topo_spp_dict) + + # Now do date sampling, if desired + for s in range(args.num_date_samples): + print(" Tree number", n+1, "; Date sample", s+1) + for node in whole_tre.traverse(strategy="preorder"): + # reset all dates + node.props["date"] = None + node.props["imputed_date"] = False + node.props["imputation_type"] = 0 + + # Assign dates + tree_dating.assign_dates(whole_tre, dates, sample_dates=True, rng=date_source_rng) + + # Date cleaning to ensure time consistency down the tree + tree_dating.label_older_descendants(whole_tre) + tree_dating.dq_date_removal(whole_tre) + + # Date imputation + tree_dating.date_labelling(whole_tre) + if args.use_birth_model: + tree_dating.impute_missing_dates(whole_tre, use_birth_model=True, rng=date_interpolation_rng) + else: + tree_dating.impute_missing_dates(whole_tre, l=0.25) + + # All nodes now dated - set dists in ete and write out tree. + tree_dating.compute_branch_lengths(whole_tre) + tree_dating.write_tree_with_branch_lengths(whole_tre, filename="%s/%s_both_sample_%d.tre" % (args.output_folder, args.output_tree_filename, n*args.num_date_samples+s+1)) + + if args.compute_ed: + tree_metrics.compute_ed_scores(whole_tre, both_ed_scores) + + if args.pd_clades: + tree_metrics.compute_pd(whole_tre) + tree_metrics.save_pd_for_clades(whole_tre, both_pd_clades, both_pd_dict, both_dates_dict, both_spp_dict) + + del whole_tre + gc.collect() + + + if args.compute_ed: + print("Writing out ED score distributions for all species (takes ~5 minutes)") + tree_metrics.write_ed_scores("%s/%s_topo_ed_scores.txt" % (args.output_folder, args.output_tree_filename), topo_ed_scores) + + if args.num_date_samples > 0: + tree_metrics.write_ed_scores("%s/%s_both_ed_scores.txt" % (args.output_folder, args.output_tree_filename), both_ed_scores) + + if args.pd_clades: + tree_metrics.write_pd_dists("%s/%s_topo" % (args.output_folder, args.output_tree_filename), topo_pd_dict, topo_dates_dict, topo_spp_dict) + + if args.num_date_samples > 0: + tree_metrics.write_pd_dists("%s/%s_both" % (args.output_folder, args.output_tree_filename), both_pd_dict, both_dates_dict, both_spp_dict) + + +def main(): + # NB: Only configure logging when run as a script, not on import, otherwise + # we clobber the logging config of anything importing us + logging.basicConfig(filename="main.log", filemode="w", force=True, level=logging.ERROR) + + parser = argparse.ArgumentParser( + description=( + "Generate a set of dated trees of all life, based on the Open Tree of Life and Chronosynth. " + "Optionally, also generate evolutionary distinctiveness scores for the trees" + ) + ) + + parser.add_argument("--num_trees", + help="How many trees to generate. Default: 1", + type=int, + default=1) + + parser.add_argument("--num_date_samples", + help="How many times to sample a set of dates for each tree (in addition to the tree using median dates). Default: 0", + type=int, + default=0) + + parser.add_argument("--output_folder", + help="Path of folder where output trees will be written in Newick format, Default: output", + default="output") + + parser.add_argument("--output_tree_filename", + help="Filename for output trees, e.g. the default 'dated_tree' would result in trees named 'dated_tree_topo_sample_1', '..._2' etc.", + default="dated_tree") + + parser.add_argument("--supertree", + help="Path of the labelled_supertree_ottnames.tre file from the Open Tree of Life. Default: opentree16.1_tree/labelled_supertree/labelled_supertree_ottnames.tre", + default="opentree16.1_tree/labelled_supertree/labelled_supertree_ottnames.tre") + + parser.add_argument("--date_cache", + help="Path of the date cache generated by Chronosynth. Default: chronosynth_date_info/node_ages.json", + default="chronosynth_date_info/node_ages.json") + + parser.add_argument("--annotations", + help="Path of the annotations.json file from the Open Tree of Life. Default: opentree16.1_tree/annotations.json", + default="opentree16.1_tree/annotations.json") + + parser.add_argument("--taxonomy", + help="Path of the taxonomy.tsv file from the Open Tree Taxonomy. Default: ott3.7.3/taxonomy.tsv", + default="ott3.7.3/taxonomy.tsv") + + parser.add_argument("--use_birth_model", + help="Flag: whether use the date interpolation method based on a birth model, rather than the EQS-LS method. Default: False", + action="store_true") + + parser.add_argument("--pd_clades", + help="Path of a text file containing a list of node names (one on each line) for which to output PD estimates. Default: None", + default=None) + + parser.add_argument("--compute_ed", + help="Flag: whether to compute a distribution of ED scores. A csv file summarising the scores will be placed in the output folder. Default: False", + action="store_true") + + args = parser.parse_args() + generate_trees(args) + + +if __name__ == "__main__": + main() diff --git a/oz_tree_build/download_node_ages/chronosynth_config.ini b/oz_tree_build/download_node_ages/chronosynth_config.ini new file mode 100644 index 00000000..e368e342 --- /dev/null +++ b/oz_tree_build/download_node_ages/chronosynth_config.ini @@ -0,0 +1,42 @@ +# Copied from https://github.com/OpenTreeOfLife/chronosynth/blob/main/default.config + +[paths] +cache_file_dir = /tmp/ + +[params] +ultrametricity_precision=0.01 + + +### +# logging configuration +# https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html +### + +[loggers] +keys = root + +[handlers] +keys = console, chronosynth + +[formatters] +keys = generic + +[logger_root] +level = DEBUG +handlers = chronosynth, console + +[handler_console] +class = StreamHandler +level = NOTSET +formatter = generic +args = (sys.stderr,) + +[handler_chronosynth] +class = FileHandler +args = ('chronosynth.log', 'a') +level = DEBUG +formatter = generic + + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s:%(lineno)s][%(threadName)s] %(message)s diff --git a/oz_tree_build/download_node_ages/download_node_ages.py b/oz_tree_build/download_node_ages/download_node_ages.py new file mode 100644 index 00000000..3bf14e06 --- /dev/null +++ b/oz_tree_build/download_node_ages/download_node_ages.py @@ -0,0 +1,77 @@ +""" +Populate node_ages.json by calling out to the OpenTree API via. chronosynth + +Usage: download_node_ages node_ages.json + +NB: The output is not based on the tree downloaded in other steps, +internally chronosynth will call out to the OpenTree API & +github.com/OpenTreeOfLife/phylesystem-1. + +""" + +import argparse +import json +import logging +import os +import os.path +import sys +import time + +os.environ["CHRONOSYNTH_CONFIG_FILE"] = os.path.join(os.path.dirname(__file__), "chronosynth_config.ini") +os.environ["PEYOTL_CONFIG_FILE"] = os.path.join(os.path.dirname(__file__), "peyotl_config.ini") + +import chronosynth.chronogram # noqa: E402 - we need to set env first + + +def download_node_ages(): + dates = chronosynth.chronogram.build_synth_node_source_ages(fresh=True) + + # Remove sources, from dated_complete_tree/tree_loading.py + sources_to_delete = set(["ot_1250@tree2"]) + deletions = [] + for ott_name in dates["node_ages"]: + for i, source in enumerate(dates["node_ages"][ott_name]): + if source["source_id"] in sources_to_delete: + deletions.append((ott_name, i)) + + deletions.sort(reverse=True) + + for ott_name, i in deletions: + del dates["node_ages"][ott_name][i] + if len(dates["node_ages"][ott_name]) == 0: + del dates["node_ages"][ott_name] + #### + + return dates + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--verbosity", + "-v", + action="count", + default=0, + help="verbosity level: output extra non-essential info", + ) + parser.add_argument("output_path", help="Path to where output data should be saved") + args = parser.parse_args() + + if args.verbosity == 0: + logging.basicConfig(stream=sys.stderr, level=logging.WARNING) + elif args.verbosity == 1: + logging.basicConfig(stream=sys.stderr, level=logging.INFO) + elif args.verbosity == 2: + logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + + start = time.time() + + with open(args.output_path, "w") as f: + json.dump(download_node_ages(), f) + + end = time.time() + logging.debug(f"Time taken: {end - start} seconds") + + +if __name__ == "__main__": + main() diff --git a/oz_tree_build/download_node_ages/peyotl_config.ini b/oz_tree_build/download_node_ages/peyotl_config.ini new file mode 100644 index 00000000..cff1a75a --- /dev/null +++ b/oz_tree_build/download_node_ages/peyotl_config.ini @@ -0,0 +1,15 @@ +# From: https://github.com/OpenTreeOfLife/peyotl/blob/master/peyotl/default.conf + +[logging] +level = info +filepath = /tmp/peyotl-log +formatter = simple + +[apis] +phylesystem_api = https://devapi.opentreeoflife.org +collections_api = https://devapi.opentreeoflife.org +amendments_api = https://devapi.opentreeoflife.org +oti = https://devapi.opentreeoflife.org +taxomachine = https://api.opentreeoflife.org +treemachine = https://api.opentreeoflife.org + diff --git a/oz_tree_build/newick/format_newick.py b/oz_tree_build/newick/format_newick.py index d1e2b08e..56151e2d 100755 --- a/oz_tree_build/newick/format_newick.py +++ b/oz_tree_build/newick/format_newick.py @@ -22,16 +22,26 @@ import re import sys -from ..tree_build.build_oz_tree import trim_tree - __author__ = "David Ebbo" # Token may be quoted or not whole_token_regex = re.compile("('[^']*'|[^(),;[]+)(:[0-9.]+)?") +def trim_tree(tree): + # Trim any whitespace + tree = tree.strip() + + # Skip the comment block at the start of the file, if any + if tree[0] == "[": + tree = tree[tree.index("]") + 1 :] + tree = tree.lstrip() + + return tree + + def format_nwk(newick_tree, output_stream, indent_spaces=2): - newick_tree = trim_tree(newick_tree, strip_semicolon=False) + newick_tree = trim_tree(newick_tree) indent_string = " " * indent_spaces diff --git a/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py b/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py deleted file mode 100755 index 21305361..00000000 --- a/oz_tree_build/taxon_mapping_and_popularity/CSV_base_table_creator.py +++ /dev/null @@ -1,1141 +0,0 @@ -""" -Creates the base files for the dynamically-loaded tree, on the basis of a -single newick tree with OTT numbers on the leaves and nodes. - -Final output is 2 csv files, two long newick string with braces and commas -(one with curly braces for polytomies) and a dates file. - -First we remove polytomies, subspecies, and leaves and nodes in two large python structures - -Using the Open Tree of Life taxonomy file (taxonomy.tsv), point leaves and nodes to -source ids (e.g. ncbi:1234, etc etc), adding any missing leaves/nodes/subspecies etc -from the OpenTree of life (this helps calculate phylogenetic popularity later). - -Then use these source ids to map against Encyclopedia of Life and WikiData ids, -using the EOL identifiers.csv file and the WikiData JSON dump. Using wikidata, -also flag whether or not a wikipedia english language page for that taxon exists -(this helps decide whether or not to show the wikipedia tab). Additionally, to get extra -EoL ids, we supplement the EOL identifiers list by EOL numbers gleaned from wikidata - -We also use wikidata and EoL lists to populate the IUCN id field (potentially problematic & -out of date, but at least means we don't need to do any taxonomic name matching) - -==Popularity== - -Wikidata ids are used to get pagesize and pageview count files, to calculate raw popularities. - -Finally, use the full OpenTree to calculate phylogenetic popularities from base popularities - -==Nested set structure, to get leaves from nodes=== - -At the end of the script, nodes & leaves from the original tree are placed into 2 CSV files. -Each node provides a lft and rgt bracket delimiting all the leaves descended from it. -This allows us to quickly find all terminal children of a given node. - -CSVs can be imported into a mysql database by removing the existing data via 'TRUNCATE TABLE' -then using the mysql command 'LOAD DATA INFILE', which is orders of magnitude faster than -using the builtin web2py import (https://groups.google.com/forum/#!topic/web2py/1bGR8ojrEfs). - - -The OTT id is matched using a reg expr for names such as Aptenodytes_forsteri_ott494370. -This script also allows for temporary node names such as _1234, which are taken as (arbitrary) -*negative* OTT IDs (i.e. -1234 in this case) in the database. This -allows us to find children of unnamed nodes too. - -Download: - * the OpenTree taxonomy from https://tree.opentreeoflife.org/about/taxonomy-version/ott2.9 - * an eol mapping file from http://beta.eol.org/uploads/data_search_files/identifiers.csv.gz - * the wikidata JSON dump from http://dumps.wikimedia.org/wikidatawiki/entities/ - -To test, try e.g. - -Usage: -OT_VERSION=9.1 -ServerScripts/TaxonMappingAndPopularity/CSV_base_table_creator.py \ - ../static/FinalOutputs/Life_full_tree.phy data/OpenTree/ott/taxonomy.tsv \ - data/EOL/identifiers.csv data/Wiki/wd_JSON/* data/Wiki/wp_SQL/* \ - data/Wiki/wp_pagecounts/pagecounts* \ - --OpenTreeFile data/OpenTree/draftversion${OT_VERSION}.tre\ - -o data/output_files/ordered -v --exclude Archosauria_ott335588 Dinosauria_ott90215 > \ - data/output_files/ordered_output.log - -ServerScripts/TaxonMappingAndPopularity/CSV_base_table_creator.py \ - ../static/FinalOutputs/Life_full_tree.phy data/OpenTree/ott/taxonomy.tsv \ - data/EOL/identifiers.csv data/Wiki/wd_JSON/* data/Wiki/wp_SQL/* \ - data/Wiki/wp_pagecounts/* - --OpenTreeFile data/OpenTree/draftversion${OT_VERSION}.tre -o data/output_files/ordered -n - -currently results in: - Out of 3526952 OTT taxa, 2394052 (67.88%) have EOL ids from EOL. \ - Supplementing these with 371071 EOL ids from wikidata gives a coverage of 78.4 %. - Populating IUCN IDs using EOL csv file (or if absent, wikidata) -""" - -import argparse -import csv -import json -import logging -import os.path -import random -import re -import sys -import time -from collections import OrderedDict, defaultdict -from math import log - -from dendropy import Node, Tree - -from ..utilities.debug_util import parse_args_and_add_logging_switch -from ..utilities.file_utils import open_file_based_on_extension -from ..utilities.wikidata_utils import get_qid_from_taxa_data -from . import OTT_popularity_mapping - -# local packages -from .dendropy_extras import write_pop_newick - -__author__ = "Yan Wong" -__license__ = """This is free and unencumbered software released into the public domain by the author, Yan Wong, for OneZoom CIO. - -Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. - -In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to """ # noqa E501 - -sql_subs_string = "" # ? for sqlite, %s for mysql - -# DendroPy performs lots of recursion when reading large trees, this is expected -# https://github.com/jeetsukumaran/DendroPy/issues/52 -sys.setrecursionlimit(3000) - - -def is_unnamed_OTT(OTTid): - """ - TO DO: I'm not sure when we use unnamed nodes with an OTT, so is this needed? - """ - try: - return OTTid < 0 - except TypeError: - return False - - -def get_OTT_species(taxonomy_filename): - with open(taxonomy_filename) as taxonomy_file: - species_list = set() - taxonomy_file.seek(0) - reader = csv.DictReader(taxonomy_file, delimiter="\t") - for row in reader: - if row["rank"] == "species": - species_list.add(int(row["uid"])) - return species_list - - -def parse_tree(tree_filename): - """ - Parses (tree_filename) and returns the DendroPy tree object - """ - try: - tree = Tree.get_from_path( - tree_filename, - schema="newick", - preserve_underscores=True, - suppress_leaf_node_taxa=True, - ) - return tree - except Exception as e: - sys.exit("Problem reading tree from " + tree_filename + ": " + str(e)) - logging.info(" > read tree from " + tree_filename) - - -def get_OTT_list(tree, sources): - """ - Takes a base tree and creates objects for each node and leaf, attaching them as 'data' - dictionaries to each node in the DendroPy tree. Nodes and leaves with an OTT id also - have pointers to their data dicts stored in an OTT-keyed dict, so that mappings to other - databases (ncbi id, etc etc) can be created. - - We want to allow duplicate leaf names, so for the entire procedure we ignore the Dendropy - concept of a taxon list and simply use labels. Returns the Dendropy tree and the OTT dict. - """ - indexed_by_ott = {} - - ott_node = re.compile(r"(.*) ott(\d+)(@\d*)?$") # matches the OTT number - mrca_ott_node = re.compile( - r"(.*) (mrcaott\d+ott\d+)(@\d*)?$" - ) # matches a node with an "mrca" node number (no unique OTT) - tot = 0 - for node in tree.preorder_node_iter(): - tot += 1 - node.data = {} - if node.label: - node.label = node.label.replace("_", " ") - m = ott_node.search(node.label) - if m is not None: - if m.group(3): - logging.warning( - "Node has an @ sign at the end ({node.label}), meaning it has " - "probably not been substituted by an OpenTree equivalent. You " - "may want to provide an alternative subtree from this node " - "downwards, as otherwise it will probably be deleted from the " - "main tree." - ) - node.label = m.group(1) - node.data["ott"] = int(m.group(2)) - indexed_by_ott[node.data["ott"]] = node.data - node.data["sources"] = {} - else: - m = mrca_ott_node.search(node.label) - if m is not None: - if m.group(3): - logging.warning( - f"Node has an @ sign at the end ({node.label}), meaning it " - "has probably not been substituted by an OpenTree " - "equivalent. You may want to provide an alternative subtree " - "from this node downwards, as otherwise it will probably be " - "deleted from the main tree." - ) - node.label = m.group(1) - # this is an 'mrca' node, so we want to save sources but *not* save - # the ott number in node.data - indexed_by_ott[m.group(2)] = node.data - node.data["sources"] = {} - elif node.is_leaf(): - logging.warning( - f"Leaf without an OTT id: '{node.label}'. " "This will not be associated with any other data" - ) - # Finally, put underscores at the start or the end of the new label back as these - # denote "fake" names that are hidden and only used for mapping. We could keep - # them as spaces, but leading/trailing underscores are easier to see by eye. - if node.label[0] == " ": - node.label = "_" + node.label[1:] - if node.label[-1] == " ": - node.label = node.label[:-1] + "_" - logging.info( - f"✔ extracted {len(indexed_by_ott)} otts from {tot} leaves & nodes. " - f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - return indexed_by_ott - - -def add_eol_IDs_from_EOL_table_dump(source_ptrs, identifiers_filename, source_mapping): - used = 0 - EOL2OTT = {v: k for k, v in source_mapping.items()} - with open_file_based_on_extension(identifiers_filename, "rt") as identifiers_file: - reader = csv.DictReader(identifiers_file) - for EOLrow in reader: - if reader.line_num % 1000000 == 0: - logging.info( - f"... {reader.line_num} rows read, {used} used, " - f"mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - provider = int(EOLrow["resource_id"]) - if provider in EOL2OTT: - src = source_ptrs[EOL2OTT[provider]] - if EOL2OTT[provider] == "gbif" and not EOLrow["resource_pk"].isdigit(): - # The EoL file has duplicate (non numeric) IDs for GBIF: ignore these - continue - providerid = EOLrow["resource_pk"] - EOLid = int(EOLrow["page_id"]) - try: - if int(providerid) in src: - used += 1 - src[int(providerid)]["EoL"] = EOLid - except ValueError: - if providerid in src: - used += 1 - src[providerid]["EoL"] = EOLid - logging.info( - f"✔ Matched {used} EoL entries in the EoL identifiers file. " - f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - - -def identify_best_EoLdata(OTT_ptrs, sources): - """ - Each OTT number may point to several EoL entries, one for the NCBI number, - another for the WORMS number, etc etc. Hopefully these will be the same entry, - but they may not be. If they are different we need to choose the best one - to use. We take the one with the most sources supporting this entry: - if there is a tie, we take the lowest, as recommended by JRice from EoL - """ - validOTTs = OTTs_with_EOLmatch = dups = 0 - for OTTid, data in OTT_ptrs.items(): - if is_unnamed_OTT(OTTid): - continue - validOTTs += 1 - choose = {} - for src in sources: - if src in data["sources"] and data["sources"][src] is not None: - if "EoL" in data["sources"][src]: - EOLid = int(data["sources"][src]["EoL"]) - if EOLid not in choose: - choose[EOLid] = [] - choose[EOLid] += [src] - if len(choose) == 0: - data["eol"] = None - else: - OTTs_with_EOLmatch += 1 - errstr = None - if len(choose) > 1: - # weed out those EOLids with the least support. - errstr = f"More than one EoL ID {choose} for taxon OTT: {OTTid}" - dups += 1 - max_refs = max([len(choose[i]) for i in choose]) - choose = [EOLid for EOLid in choose if len(choose[EOLid]) == max_refs] - best = min(choose) - data["eol"] = best - if errstr: - logging.debug(f" {errstr}, chosen {best}") - logging.info( - f" ✔ Of {validOTTs} OpenTree taxa, {OTTs_with_EOLmatch} " - f"({OTTs_with_EOLmatch / validOTTs * 100:.2f}%) have EoL entries in the EoL " - f"identifiers file, and {dups} have multiple possible EOL ids. " - f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - - -def set_wikidata(bz2_filename, source_ptrs, lang): - """ - Will alter the source_ptrs. - Returns WDitems (Q->WD), WPnames (name-WD), common_name_Qs (Q->Q) - """ - WDitems = {} - WPnames = {} - common_name_Qs = {} - sum_info = defaultdict(int) - - ( - Q_to_WD, - WPname_to_WD, - src_to_WD, - replace_Q, - info, - ) = OTT_popularity_mapping.wikidata_info(bz2_filename, source_ptrs, lang) - - WDitems.update(Q_to_WD) - WPnames.update(WPname_to_WD) - common_name_Qs.update(replace_Q) - # Add 'wd' item to source_ptrs - for src, ids in src_to_WD.items(): - for src_id, WD in ids.items(): - source_ptrs[src][src_id]["wd"] = WD - for k, v in info.items(): - sum_info[k] += v - - logging.info( - f"✔ {len(WDitems)} wikidata matches, of which " - f"{sum_info['n_eol']} have EOL ids, {sum_info['n_iucn']} have IUCN ids, " - f"{sum_info['n_ipni']} have IPNI, and {len(WPnames)} " - f"({(len(WPnames)/len(WDitems)*100):.2f}%) have titles that exist on " - f"{lang}.wikipedia. Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - return WDitems, WPnames, common_name_Qs - - -def set_wikipedia_pageviews(filenames, WPnames, lang): - names_found = 0 - for fn in filenames: - WPnames_views = OTT_popularity_mapping.pageviews_for_titles(fn, set(WPnames.keys()), lang) - for name, n_views in WPnames_views.items(): - if not hasattr(WPnames[name], "pageviews"): - names_found += 1 - WPnames[name].pageviews = [] - WPnames[name].pageviews.append(n_views) - logging.info( - f" ✔ Of {len(WPnames)} WikiData taxon entries, {names_found} " - f"({(names_found/len(WPnames) * 100):.2f}%) have pageview data for '{lang}' in " - f"{len(filenames)} files. Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - - -def supplement_from_wikidata(OTT_ptrs): - """ - If no OTT_ptrs[OTTid]['eol'] exists, but there is an - OTT_ptrs[OTTid]['wd']['initial_wiki_item']['EoL'] then put this into - OTT_ptrs[OTTid]['eol'] - Similarly for IPNI (although this is currently unpopulated) - """ - EOLalready = n_eol = n_ipni = n = 0 - for OTTid, data in OTT_ptrs.items(): - if is_unnamed_OTT(OTTid): - logging.info(f" unlabelled node (OTT: {OTTid}) when iterating through OTT_ptrs") - continue - n += 1 - if data.get("eol") is None: - try: - data["eol"] = int(data["wd"].EoL) - n_eol += 1 - except (AttributeError, KeyError, TypeError, ValueError): - pass - else: - EOLalready += 1 - if data.get("ipni") is None: - try: - data["ipni"] = int(data["wd"].ipni) - n_ipni += 1 - except (AttributeError, KeyError, TypeError, ValueError): - pass - logging.info( - f"✔ Out of {n} OTT taxa, {EOLalready} ({(EOLalready/n * 100):.2f}%) already " - f"have EOL ids from the EOL file. Supplementing these with {n_eol} EOL ids from " - f"wikidata gives a coverage of {((EOLalready + n_eol)/n * 100):.1f} %." - + (f" An addition {n_ipni} IPNI identifiers added via wikidata" if n_ipni else "") - ) - - -iucn_num = 5 - - -def populate_iucn(OTT_ptrs, identifiers_filename, verbosity=0): - """ - Port the IUCN number from both EoL and Wikidata, and keep both if there is a conflict - """ - used = 0 - - eol_mapping = {} # to store eol=>iucn - for OTTid, data in OTT_ptrs.items(): - if "eol" in data: - if data["eol"] in eol_mapping: - eol_mapping[data["eol"]].append(OTTid) - else: - eol_mapping[data["eol"]] = [OTTid] - - with open_file_based_on_extension(identifiers_filename, "rt") as identifiers_file: - reader = csv.DictReader(identifiers_file) - for EOLrow in reader: - if reader.line_num % 1000000 == 0: - logging.info( - f" - {reader.line_num} rows read, {used} used. " f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - if int(EOLrow["resource_id"]) == iucn_num and EOLrow["resource_pk"].isdigit(): - # there are lots of non-species IUCN rows with pk == str (e.g. Animalia) - try: - for ott in eol_mapping[int(EOLrow["page_id"])]: - OTT_ptrs[ott]["iucn"] = EOLrow["resource_pk"] - used += 1 - except LookupError: - pass # no equivalent eol id in eol_mapping - logging.info( - f" > matched {used} IUCN entries in the EoL identifiers file. " - f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" - ) - - # now go through and double-check against IUCN stored on wikidata - for OTTid, data in OTT_ptrs.items(): - try: - wd_iucn = str(int(data["wd"].iucn)) - if "iucn" not in data: - data["iucn"] = wd_iucn - used += 1 - else: - if wd_iucn not in data["iucn"].split("|"): - data["iucn"] += "|" + wd_iucn - logging.debug( - f' conflicting IUCN IDs for OTT {OTTid}: EoL = {data["iucn"]} ' - f'(via http://eol.org/pages/{data["eol"]}), wikidata = ' - f'{wd_iucn} (via http://http://wikidata.org/wiki/Q{data["wd"].Q}).' - ) - except ValueError: - logging.warning(f" Cannot convert wikidata IUCN ID {data['wd'].iucn} to integer.") - except (KeyError, AttributeError): - pass # can't find a wd instance or an iucn within the wd instance. Oh well. - - logging.info(f" > Increased IUCN coverage to {used} taxa using wikidata") - - -def popularity_function( - sum_of_all_ancestor_popularities, - sum_of_all_descendant_popularities, - number_of_ancestors, - number_of_descendants, -): - """ - a) Dividing by number_of_ancestors+number_of_descendants would mean averaging - popularity over all nodes, which would bias against taxa which have many - unvisited/unpopular children - b) Alternatively, dividing by a constant is equivalent to summing popularity over - all nodes, which biases towards taxa with many fine taxonomic divisions - We do something between the two by dividing by the log of the number of nodes. - """ - if ( - (sum_of_all_ancestor_popularities is None) - or (sum_of_all_descendant_popularities is None) - or (number_of_ancestors is None) - or (number_of_descendants is None) - ): - return None - elif number_of_ancestors + number_of_descendants == 1: - # Avoid a divide by zero error if this adds up to 1 - # Though the need for this makes me think that the log calculation - # may not be mathematically sound - return sum_of_all_ancestor_popularities + sum_of_all_descendant_popularities - else: - return (sum_of_all_ancestor_popularities + sum_of_all_descendant_popularities) / log( - number_of_ancestors + number_of_descendants - ) - - -def resolve_polytomies_add_popularity(tree, seed): - """ - If there are polytomies in the tree, resolve them, but make sure that the newly - created nodes get popularity values too. These can be recalculated from the - descendants and ancestors of the children - - """ - prev_num_nodes = sum(1 for i in tree.postorder_node_iter()) - random.seed(seed) # so we get the same bifurcations each time - - # We implement a slightly non-random resolution to group nodes with the same genus together - # See https://github.com/OneZoom/OZtree/issues/958 - tree.group_genera_in_polytomies() - tree.resolve_polytomies(rng=random) - num_new_nodes = sum(1 for i in tree.postorder_node_iter()) - prev_num_nodes - for node in tree.postorder_node_iter(): - if not hasattr(node, "data"): - # this is a new node - it should always have 2 children - try: - n = ancestor_pop_sum = descendant_pop_sum = n_ancestors_sum = n_descendants_sum = 0 - for c in node.child_node_iter(): - n += 1 - ancestor_pop_sum += c.ancestors_popsum - descendant_pop_sum += c.descendants_popsum - n_ancestors_sum += c.n_ancestors - n_descendants_sum += c.n_descendants - - node.data = { - "popularity": popularity_function( - ancestor_pop_sum / n, - descendant_pop_sum, - n_ancestors_sum / n, - n_descendants_sum, - ) - } - except AttributeError: - # probably popularity values undefined for one of the children - pass - return num_new_nodes - - -def create_leaf_popularity_rankings(tree): - """ - Make a rank of all existing leaves by phylogenetic popularity - Must be run once all invalid tips etc have been removed. - If there are no popularities, set all ranks to None - """ - leaf_popularities = defaultdict(int) - for node in tree.leaf_node_iter(): - leaf_popularities[node.data.get("popularity")] += 1 - cumsum = 1 - if None in leaf_popularities: - return - for k in sorted(leaf_popularities.keys(), reverse=True): - add_next = leaf_popularities[k] - leaf_popularities[k] = cumsum - cumsum += add_next - for leaf in tree.leaf_node_iter(): - leaf.data["popularity_rank"] = leaf_popularities[leaf.data.get("popularity")] - - -def write_popularity_tree(tree, outdir, filename, version, verbosity=0): - Node.write_pop_newick = write_pop_newick - with open(os.path.join(outdir, f"{filename}_{version}.nwk"), "w+") as popularity_newick: - tree.seed_node.write_pop_newick(popularity_newick) - - -def output_simplified_tree(tree, taxonomy_file, outdir, version, seed, save_sql=True, extinct_tree_mode=False): - """ - We should now have leaf entries attached to each node in the tree like - data = { - 'ott':, - 'wd': WikidataItem(Q=15478814, EoL=1100788, l={'en','fr'}), - 'pop_dscdt': 0, - 'pop_ancst': 220183.23395609166, - 'sources': {'ncbi': None, - 'worms': None, - 'gbif': {'wd': WikidataItem(Q=15478814, EoL=1100788), 'id': '2840414'}, - 'if': None, - 'irmng': None}, - 'popularity': 220183.23395609166, - 'eol': 1100788 - 'iucn':XXXXXXX} - - ... or, if we have managed to calculate popularity ... - - data = { - 'wd': WikidataItem( - Q=15478814, EoL=1100788, l={'en','fr'}, pageviews=[64, 47], pagesize=1465, raw_pop=285.1 - ), - 'pop_dscdt': 0, - 'pop_ancst': 392245.76075749274, - 'sources': { - 'ncbi': { - 'wd': WikidataItem( - pageviews=[64, 47], raw_popularity=285.1, Q=4672161, EoL=281897, pagesize=1465), - 'EoL': 281897, - 'id': '691616' - }, - 'worms': None, - 'gbif': { - 'wd': WikidataItem( - pageviews=[64, 47], raw_popularity=285.1, Q=4672161, EoL=281897, pagesize=1465), - 'id': '1968205' - } - 'if': None, - 'irmng': {'EoL': 281897, 'id': '10290975'} - }, - 'eol': 281897 - } - - Removes non-species from tips, outputs simplified versions. - """ - from .dendropy_extras import ( - group_genera_in_polytomies, - prune_children_of_otts, - prune_non_species, - remove_unifurcations_keeping_higher_taxa, - set_node_ages, - set_real_parent_nodes, - write_brief_newick, - write_preorder_to_csv, - ) - - # monkey patch the existing dendropy objects - Tree.prune_children_of_otts = prune_children_of_otts - Tree.prune_non_species = prune_non_species - Tree.set_node_ages = set_node_ages - Tree.set_real_parent_nodes = set_real_parent_nodes - Tree.remove_unifurcations_keeping_higher_taxa = remove_unifurcations_keeping_higher_taxa - Tree.write_preorder_to_csv = write_preorder_to_csv - Tree.group_genera_in_polytomies = group_genera_in_polytomies - - Tree.create_leaf_popularity_rankings = ( - create_leaf_popularity_rankings # not defined in dendropy_extras, but in this file - ) - Tree.resolve_polytomies_add_popularity = resolve_polytomies_add_popularity - Node.write_brief_newick = write_brief_newick - - # For the extinct tree, we don't want to remove any species - if not extinct_tree_mode: - logging.info(f" > removing children labeled species in '{taxonomy_file}'") - n = len(tree.prune_children_of_otts(get_OTT_species(taxonomy_file))) - logging.info(f" ✔ removed all children of {n} nodes") - - logging.info(" > removing tips that appear not to be species") - # species names containing these (even initially) are discarded - bad_sp = ["cf.", "aff.", "subsp.", "environmental sample"] - # species names containing these within the name are discarded: - bad_sp += [" cv.", " sp."] - n = {k: len(v) for k, v in tree.prune_non_species(bad_matches=bad_sp, extinct_tree_mode=extinct_tree_mode).items()} - logging.info( - f" ✔ removed {n['unlabelled']} blank leaves, {n['no_space']} lacking a space, " - f"& {n['bad_match']} containing {bad_sp} (assumed bad tips)" - ) - - logging.info(" > setting node ages & removing extinction props") - a, n = tree.set_node_ages() - logging.info(f" ✔ set ages on {a} nodes and leaves & removed {n} extinction props") - - logging.info(" > removing unary nodes, keeping monotypic species or highest taxon") - n_deleted_nodes = tree.remove_unifurcations_keeping_higher_taxa() - # see https://github.com/jeetsukumaran/DendroPy/issues/75 - logging.info(f" ✔ removed {n_deleted_nodes} unifurcations") - - logging.info(" > splitting polytomies by genera and assigning popularities to new nodes") - n_new = tree.resolve_polytomies_add_popularity(seed) - tree.create_leaf_popularity_rankings() - logging.info(f" ✔ polytomies split with seed={seed}: {n_new} extra nodes created") - - # NB: we shouldn't need to (re)set popularity or ages, since deleting nodes - # does not affect these, and both have been calculated *after* new - # nodes were created by resolve_polytomies. - logging.info(" > setting real parents and ranking leaf popularity") - tree.set_real_parent_nodes() - logging.info(" ✔ real parents and popularity ranks set") - - logging.info(" > ladderizing tree (groups with fewer leaves first)") - tree.ladderize(ascending=True) # warning: ladderize ascending is needed for the short OZ newick-like form - logging.info(" ✔ ladderized") - - logging.info(" > writing tree, dates, and csv to files") - with open(os.path.join(outdir, f"ordered_tree_{version}.nwk"), "w+") as condensed_newick: - tree.seed_node.write_brief_newick(condensed_newick) - with open(os.path.join(outdir, f"ordered_tree_{version}.poly"), "w+") as condensed_poly: - tree.seed_node.write_brief_newick(condensed_poly, "{}") - - # these are the extra columns output to the leaf csv file - leaf_extras = OrderedDict() - leaf_extras["ott"] = ["ott"] - leaf_extras["wikidata"] = ["wd", "Q"] - leaf_extras["wikipedia_lang_flag"] = ["wd", "wikipedia_lang_flag"] - leaf_extras["iucn"] = ["iucn"] - leaf_extras["eol"] = ["eol"] - leaf_extras["raw_popularity"] = ["wd", "raw_popularity"] - leaf_extras["popularity"] = ["popularity"] - leaf_extras["popularity_rank"] = ["popularity_rank"] - leaf_extras["price"] = None - leaf_extras["ncbi"] = ["sources", "ncbi", "id"] - leaf_extras["ifung"] = ["sources", "ifung", "id"] - leaf_extras["worms"] = ["sources", "worms", "id"] - leaf_extras["irmng"] = ["sources", "irmng", "id"] - leaf_extras["gbif"] = ["sources", "gbif", "id"] - leaf_extras["ipni"] = ["ipni"] - - # these are the extra columns output to the node csv file - node_extras = OrderedDict() - node_extras["ott"] = ["ott"] - node_extras["wikidata"] = ["wd", "Q"] - node_extras["wikipedia_lang_flag"] = ["wd", "wikipedia_lang_flag"] - node_extras["eol"] = ["eol"] - # We avoid using 'rank' as it is a reserved word in mysql - node_extras["rnk"] = ["rank"] - node_extras["raw_popularity"] = ["wd", "raw_popularity"] - node_extras["popularity"] = ["popularity"] - node_extras["ncbi"] = ["sources", "ncbi", "id"] - node_extras["ifung"] = ["sources", "ifung", "id"] - node_extras["worms"] = ["sources", "worms", "id"] - node_extras["irmng"] = ["sources", "irmng", "id"] - node_extras["gbif"] = ["sources", "gbif", "id"] - node_extras["ipni"] = ["ipni"] - node_extras["vern_synth"] = None - for representative_image_type in ["rep", "rtr", "rpd"]: - for i in [str(x + 1) for x in range(8)]: - node_extras[representative_image_type + i] = None - for iucn_type in ["NE", "DD", "LC", "NT", "VU", "EN", "CR", "EW", "EX"]: - node_extras["iucn" + iucn_type] = None - - with ( - open(os.path.join(outdir, f"ordered_leaves_{version}.csv"), "w+", encoding="utf-8") as leaves, - open(os.path.join(outdir, f"ordered_nodes_{version}.csv"), "w+", encoding="utf-8") as nodes, - ): - tree.write_preorder_to_csv(leaves, leaf_extras, nodes, node_extras, -version) - logging.info(f" ✔ written into {outdir}/ordered_..._{version}...") - - # make a copy of the csv file that can be imported into mySQL (has \\N for null values) - if save_sql: - from shutil import copyfile - from subprocess import call - - with open(os.path.join(outdir, f"import_{version}.sql"), "w", encoding="utf-8") as sql_f: - # make CSV files that can be imported into mySQL (subs \\N for null values) - logging.info(" > saving extra file copies in mySQL format: import them using:") - for tab in ["_leaves", "_nodes"]: - fn = os.path.join(outdir, "ordered" + tab + f"_{version}" + ".csv") - sqlfile = fn + ".mySQL" - copyfile(fn, sqlfile) - call(["perl", "-pi", "-e", r"s/,(?=(,|\n))/,\\N/g", sqlfile]) - sql_f.writelines( - [ - f"TRUNCATE TABLE ordered{tab};\n" - f"LOAD DATA LOCAL INFILE '{sqlfile}' REPLACE INTO TABLE `ordered{tab}` \n" - f" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' \n" - f" IGNORE 1 LINES ({open(fn).readline().rstrip()}) SET id = NULL;\n" - ] - ) - - -def display_WD_ott_stats(OTT_ptrs): - """ - Display some stats about OTTs coming from Wikidata - """ - matching_otts = 0 - mismatching_otts = 0 - no_wd_otts = 0 - for ott in OTT_ptrs: - try: - if OTT_ptrs[ott]["rank"] == "species": - if OTT_ptrs[ott]["wd"].get("wd_ott") is not None: - if ott == OTT_ptrs[ott]["wd"].wd_ott: - matching_otts += 1 - else: - logging.debug( - f"Q{OTT_ptrs[ott]['wd'].Q}: OTT {ott} does not match {OTT_ptrs[ott]['wd'].wd_ott}" - ) - mismatching_otts += 1 - else: - no_wd_otts += 1 - except (KeyError, AttributeError): - pass - - logging.info("✔ Stats on Wikidata OTT matching:") - logging.info(f" Leaves where the WD ott matches the ott: {matching_otts}") - logging.info(f" Leaves where the WD ott does not match the wd_ott: {mismatching_otts}") - logging.info(f" Leaves where WD does not have an ott: {no_wd_otts}") - - -def map_wiki_info( - source_ptrs, - source_order, - OTT_ptrs, - WD_filename, - lang, - WP_SQL_filename, - WP_pageviews_filenames, -): - """ - 1) use the wikidata JSON dump to map identifiers from the source_ptrs structure to - wikidata Qids, and then on to wikipedia pages - 2) if sql and pagevisits filenames are given - - Return True if popularity is mapped - """ - logging.debug(f"Processing wikidata json dump in parallel for {lang}") - popularity_steps = 0 - WDitems, WPnames, swap_Qs = set_wikidata(WD_filename, source_ptrs, lang) - - if WP_SQL_filename is not None: - logging.info(f" > Adding wikipedia page sizes from {WP_SQL_filename}") - # Can't easily parallelize this as it is gzip compressed (not a block format) - OTT_popularity_mapping.add_pagesize_for_titles(WPnames, WP_SQL_filename) - popularity_steps += 1 - - if len(WP_pageviews_filenames) > 0: - logging.info(f" > Adding wikipedia visit counts from {len(WP_pageviews_filenames)} files") - set_wikipedia_pageviews(WP_pageviews_filenames, WPnames, lang) - popularity_steps += 1 - - if popularity_steps == 2: - logging.info(" > Calculating raw popularity measures") - tot = 0 - for WDinstance in WDitems.values(): - if WDinstance.set_raw_popularity(): - tot += 1 - logging.info(f" ✔ Raw popularity measures set on {tot} wikidata items") - else: - logging.info(" x Skipping popularity calculations") - - # Here we might want to multiply up some taxa, e.g. plants, - # see https://github.com/OneZoom/OZtree/issues/130 - logging.info(" > Finding best wiki matches") - OTT_popularity_mapping.identify_best_wikidata(OTT_ptrs, lang, source_order) - - logging.info(" > Swapping vernacular wikidata items into taxon items") - OTT_popularity_mapping.overwrite_wd(WDitems, swap_Qs, only_if_more_popular=(popularity_steps == 2), check_lang=lang) - - logging.info(" > Supplementing ids (EOL/IPNI) with ones from wikidata") - supplement_from_wikidata(OTT_ptrs) - - logging.info("✔ Wikidata/wikipedia data mapped") - - display_WD_ott_stats(OTT_ptrs) - - return popularity_steps == 2 - - -def percolate_popularity( - tree, - exclude_taxa, - output_location, - popularity_file, - version, - info_on_focal_labels=None, -): - """ - NB: we must percolate popularities through the tree before deleting monotomies, - since these often contain useful popularity information. - This should also allocate popularities even for nodes that have been - created by polytomy resolving. - - We should also check that there are not multiple uses of the same Qid - (https://github.com/OneZoom/OZtree/issues/132) - """ - if info_on_focal_labels is None: - info_on_focal_labels = [] - OTT_popularity_mapping.sum_popularity_over_tree(tree, exclude=exclude_taxa) - # now apply the popularity function - Qids = set() - for node in tree.preorder_node_iter(): - try: - Q = node.data["wd"]["Q"] - if Q in Qids: - logging.warning( - f"duplicate wikidata Qids used (Q{Q}) - this will cause " - f"popularity double-counting for OTT {node.data['ott']}" - ) - else: - Qids.add(Q) - except KeyError: - pass - pop = popularity_function( - node.ancestors_popsum, - node.descendants_popsum, - node.n_ancestors, - node.n_descendants, - ) - - # Round to 2 decimal places - node.data["popularity"] = round(pop, 2) - - if popularity_file: - write_popularity_tree(tree, output_location, popularity_file, version) - # NB to examine a taxon for popularity contributions here, you could try - for focal_label in info_on_focal_labels: - focal_taxon = focal_label.replace("_", " ") - node = tree.find_node_with_label(focal_taxon) - try: - print( - "{}: own pop = {} (Q{}) descendant pop sum = {}".format( - focal_taxon, - node.pop_store, - node.data["wd"].get("Q", " absent"), - node.descendants_popsum, - ) - ) - try: - leaf_iter = node.leaf_node_iter() - except AttributeError: - leaf_iter = node.leaf_iter() - for t, tip in enumerate(leaf_iter): - print( - "Tip {} = {}: own_pop = {}, Qid = Q{}".format( - t, - tip.label, - getattr(tip, "pop_store", None), - tip.data["wd"].get("Q", " absent"), - ) - ) - if t > 100: - print("More tips exist, but have been omitted") - break - while node.parent_node: - node = node.parent_node - if node.pop_store: - print(f"Ancestors: {node.label} = {node.pop_store:.2f}") - except (IndexError, AttributeError) as e: - logging.warning(f"Problem reporting on focal taxon '{focal_taxon}': {e}") - - -def switch_otts_to_qids(taxa_data_file, tree): - """ - For the extinct tree, OTTs don't work well, as many species don't have one, or have the wrong one. - However, we trust the QIDs that we got from the previous extinct tree building phase. - So we switch all the OTTs to QIDs, and essentially pretend that they are OTTs. This works - because OneZoom doesn't actually rely on the ID being an OTT, just that it is unique. - """ - taxa_data = {} - with open(taxa_data_file) as f: - taxa_data = json.load(f) - - for node in tree.preorder_node_iter(): - try: - # Get the QID for this node's taxon - qid = get_qid_from_taxa_data(taxa_data, node.label) - if qid: - # Replace the node's OTT with the QID - node.data["ott"] = qid - # Also, update the QID in the wikidata item, as it may not have - # one at all, or the one it has may be wrong - if "wd" in node.data: - if isinstance(node.data["wd"], dict): - node.data["wd"]["Q"] = qid - else: - node.data["wd"].Q = qid - else: - node.data["wd"] = OTT_popularity_mapping.WikidataItem({"id": f"Q{qid}"}) - except Exception as e: - logging.warning(f"switch_otts_to_qids error: {node.label} qid={qid} Error: {e}") - - -def process_all(args): - random_seed_addition = 42 - start = time.time() - logging.info(f"OneZoom data generation started on {time.asctime(time.localtime(time.time()))}") - skip_popularity = ( - args.popularity_file is None - ) # Default is "": None is when popularity_file explictly specified with no name - - # From http://eol.org/api/docs/provider_hierarchies - # These need to be an ordered dict with the first being the preferred id used when getting - # a corresponding wikidata ID. - # All the ids for these are integers >= 0 - sources = ["ncbi", "if", "worms", "irmng", "gbif"] - eol_sources = { - "ncbi": 676, - "worms": 459, - "gbif": 767, - } # update when EoL has harvested index fungorum & IRMNG - # the ids for these sources may not be numbers (e.g. Silva has things like D11377/#1 - - logging.info("> Creating tree structure") - tree = parse_tree(args.Tree) - OTT_ptrs = get_OTT_list(tree, sources) - - logging.info("> Adding source IDs") - source_ptrs = OTT_popularity_mapping.create_from_taxonomy( - args.OpenTreeTaxonomy, sources, OTT_ptrs, args.extra_source_file - ) - - logging.info("> Adding EOL IDs from EOL csv file") - add_eol_IDs_from_EOL_table_dump(source_ptrs, args.EOLidentifiers, eol_sources) - logging.info("> Finding best EoL matches") - identify_best_EoLdata(OTT_ptrs, eol_sources) - - if args.wikidataDumpFile: - logging.info("> Adding wikidata info") - has_popularity = map_wiki_info( - source_ptrs, - sources, - OTT_ptrs, - args.wikidataDumpFile, - args.wikilang, - None if skip_popularity else args.wikipediaSQLDumpFile, - None if skip_popularity else args.wikipedia_totals_bz2_pageviews, - ) - if has_popularity: - logging.info("> Percolating popularity through the tree") - percolate_popularity( - tree, - args.exclude, - args.output_location, - args.popularity_file, - args.version, - args.info_on_focal_labels, - ) - else: - logging.info("No wikidataDumpFile given: skipping wiki mapping and popularity calc") - - logging.info("> Populating IUCN IDs using EOL csv file (or if absent, wikidata)") - populate_iucn(OTT_ptrs, args.EOLidentifiers) - - # If a taxa_data_file is passed in (typically for the extinct tree), we use it to - # switch the OTTs to QIDs, which work more reliably for the extinct tree - extinct_tree_mode = False - if args.taxa_data_file: - extinct_tree_mode = True - logging.info("> Switching OTTs to QIDs, which works better for the extinct tree") - switch_otts_to_qids(args.taxa_data_file, tree) - - logging.info(f"> Writing out results to {args.output_location}/xxx") - output_simplified_tree( - tree, - args.OpenTreeTaxonomy, - args.output_location, - args.version, - random_seed_addition, - extinct_tree_mode=extinct_tree_mode, - ) - t_fmt = "%H hrs %M min %S sec" - logging.info(f"✔ ALL DONE IN {time.strftime(t_fmt, time.gmtime(time.time()-start))}") - - -def main(): - parser = argparse.ArgumentParser( - description=( - "Convert a newick file with OpenTree labels into refined trees and CSV tables, " - "while mapping OpenTree Taxonomy IDs to other ids (including EoL & Wikidata)" - ) - ) - parser.add_argument("Tree", help="The newick format tree to use") - parser.add_argument( - "OpenTreeTaxonomy", - help="The OpenTree taxonomy.tsv file, from http://files.opentreeoflife.org/ott/", - ) - parser.add_argument( - "EOLidentifiers", - help=("The gzipped EOL identifiers file, from " "https://opendata.eol.org/dataset/identifiers-csv-gz"), - ) - parser.add_argument( - "wikidataDumpFile", - nargs="?", - help=( - "The very large wikidata JSON dump, " - "from https://dumps.wikimedia.org/wikidatawiki/entities/ (latest-all.json.bz2)." - "A filtered version can be used for faster processing." - ), - ) - parser.add_argument( - "wikipediaSQLDumpFile", - nargs="?", - help=( - "The gzipped >1GB wikipedia -latest-page.sql.gz dump, " - "from https://dumps.wikimedia.org/enwiki/latest/ (enwiki-page.sql.gz) " - ), - ) - parser.add_argument( - "wikipedia_totals_bz2_pageviews", - nargs="*", - help=( - 'One or more b2zipped "totals" pageview count files, ' - "from https://dumps.wikimedia.org/other/pagecounts-ez/merged/ " - "(e.g. pagecounts-2016-01-views-ge-5-totals.bz2, or pagecounts*totals.bz2)" - ), - ) - parser.add_argument( - "--popularity_file", - "-p", - nargs="?", - const=None, - default="", - help=( - "Save popularity as branch lengths in a tree under this filename. If no " - "filename given, skip the tedious process of calculating popularity altogether", - ), - ) - parser.add_argument( - "--exclude", - "-x", - nargs="*", - default=[], - help=( - "(Optional) taxa to exclude from calculation of phylogenetic popularities, " - "such as Dinosauria_ott90215, Archosauria_ott335588" - ), - ) - parser.add_argument( - "--output_location", - "-o", - default="output", - help="The directory to store the csv, newick, and date files", - ) - parser.add_argument( - "--wikilang", - "-l", - default="en", - help=( - 'The language wikipedia to check for popularity, e.g. "en". ' - "Where there are multiple Wikidata items for a taxon " - "(e.g. one under the common name, one under the scientific name), " - "then we also default to using the WD item with the sitelink in this language." - ), - ) - parser.add_argument( - "--version", - default=int(time.time() / 60.0), - type=int, - help=( - "A unique version number for the tree, to be saved in the DB tables & output " - "files. Defaults to minutes since epoch (time()/60)" - ), - ) - parser.add_argument( - "--extra_source_file", - default=None, - type=str, - help=( - "An optional additional file to supplement the taxonomy.tsv file, " - "providing additional mappings from OTTs to source ids (useful for overriding). " - 'The first line should be a header contining "uid" and "sourceinfo" column ' - "headers, similar to those in the taxonomy.tsv file. NB the OTT can be a " - 'number, or an ID of the form "mrcaott409215ott616649").' - ), - ) - parser.add_argument( - "--info_on_focal_labels", - nargs="*", - default=[], - help=('Output some extra information for these named taxa (e.g. "Canis_lupus"), ' "for debugging purposes"), - ) - parser.add_argument( - "--taxa-data-file", - default=None, - type=str, - help="JSON file with persisted data about taxa, typically used for the extinct tree", - ) - - args = parse_args_and_add_logging_switch(parser) - process_all(args) - - -if __name__ == "__main__": - main() diff --git a/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py b/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py index cc72d936..cf4baf70 100755 --- a/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py +++ b/oz_tree_build/taxon_mapping_and_popularity/OTT_popularity_mapping.py @@ -187,6 +187,16 @@ class WikidataItem: exclude_langs = frozenset(("species", "commons")) + def __repr__(self): + return ( + "WikidataItem[" + + " ".join( + (f"{a}={getattr(self, a)} " if hasattr(self, a) else "") + for a in ("Q", "ipni", "EoL", "iucn", "wd_ott", "raw_popularity", "l") + ) + + "]" + ) + def __init__(self, json_item): """ Create a basic item with an (integer) 'Q' attribute and an 'l' for sitelinks. @@ -340,83 +350,6 @@ def mem(): return mem -def create_from_taxonomy(OTTtax_filename, sources, OTT_ptrs, extra_taxonomy_file=None): - """ - Creates object data and a source_ptrs array pointing to elements within it. - Also fills out the OTT_ptrs array to point to the right place. OTT_ptrs can be - partially filled: new OTT numbers are simply appended OTT id in the taxonomy. - - src ids are ints where possible, although can be strings if they contain characters - - "extra_taxonomy_map" allows us to inject mappings that are missing from the OpenTree - e.g. - """ - - unused_sources = set() - source_ptrs = {s: {} for s in sources} - - # hack for NCBI_via_silva - # (see https://groups.google.com/d/msg/opentreeoflife/L2x3Ond16c4/CVp6msiiCgAJ) - silva_regexp = re.compile(r"ncbi:(\d+),silva:([^,$]+)") - # keep ncbi_id as ncbi_silva, but chop off the silva ID as it's not used in wikidata/EoL - silva_sub = r"ncbi_silva:\1" - - data_files = [OTTtax_filename] - if extra_taxonomy_file is not None: - try: - data_files.append(extra_taxonomy_file) - except FileNotFoundError: - logging.warning(f" Extra taxonomy file '{extra_taxonomy_file}' not found, so ignored") - - used = 0 - for fn in data_files: - with open(fn, encoding="utf-8") as f: - reader = csv.DictReader(f, delimiter="\t") - for OTTrow in reader: - # first 2 lines are header & blank in taxonomy.tsv - if ((reader.line_num - 2) % 1000000 == 0) and reader.line_num > 2: - logging.info( - f"Reading taxonomy file {fn}: {reader.line_num-2} rows read, " - f"{used} identifiers used, mem usage {mem():.1f} Mb" - ) - try: - OTTid = int(OTTrow["uid"]) - except ValueError: - OTTid = OTTrow["uid"] - logging.warning(f" Found an ott value which is not an integer: {OTTid}") - - sourceinfo = silva_regexp.sub(silva_sub, OTTrow["sourceinfo"]) - ncbi = False - for srcs in reversed(sourceinfo.split(",")): - # look at sources in reverse order, overwriting, so 1st ones take priority - src, src_id = srcs.split(":", 1) - if src == "ncbi": - ncbi = True - elif (src == "ncbi_silva") and (not ncbi): - # only use the ncbi_via_silva id if no 'normal' ncbi already set - src = "ncbi" - if src not in source_ptrs: - if src not in unused_sources: - logging.info(f" New and unused source: {src} (in '{srcs}')") - unused_sources.update([src]) - continue - used += 1 - if src_id.isdigit(): - src_id = int(src_id) - source_ptrs[src][src_id] = {"id": src_id} - try: - OTT_ptrs[OTTid]["sources"][src] = source_ptrs[src][src_id] - OTT_ptrs[OTTid]["rank"] = OTTrow["rank"] - except LookupError: - pass - - logging.info( - f"✔ created {used} source pointers for {len(source_ptrs)} sources " - f"{list(source_ptrs.keys())}. Mem usage {mem():.1f} Mb" - ) - return source_ptrs - - # P31 (instance of) values to search for # See https://en.wikipedia.org/wiki/Template:Taxonbar/whitelist for the full list match_taxa = { @@ -861,119 +794,3 @@ def pageviews_for_titles( pageviews[title] = int(views) return pageviews - - -def sum_popularity_over_tree(tree, OTT_ptrs=None, exclude=None, pop_store="pop", verbosity=0): - """ - Add popularity indices for branch lengths based on a phylogenetic tree (and return the - tree, or the number of root descendants). - We might want to exclude some names from the popularity metric (e.g. exclude archosaurs, - to ensure birds don't gather popularity intended for dinosaurs). This is done by passing - an array such as ['Dinosauria_ott90215', 'Archosauria_ott335588'] as the exclude argument. - - 'tree' can be the name of a tree file or a dendropy tree object - - 'pop_store' is the name of the attribute in which to store the popularity. If you wish to - create a tree with popularity on the branches, you can pass in pop_store='edge_length' - - NB: if OTT_ptrs is given, then the raw popularity is stored in the object pointed to by - OTT_ptrs[OTTid]['wd'], where OTTid can be extracted from the node label in the tree. - If OTT_ptrs is None, then the popularity is stored in the node object itself, in - Node.data['wd']['pop']. - - Popularity summed up and down the tree depends on the OpenTree structure, and is stored in - OTT_ptrs[OTTid]['pop_ancst'] (popularity summed upwards for all ancestors of this node) - and OTT_ptrs[OTTid]['pop_dscdt'] (popularity summed over all descendants). To get a - measure of the sum of both ancestor and descendant popularity, just add these together - - We also count up the *number* of edges above each node to the root and the number of those - that have a popularity measure. These are stored in - - OTT_ptrs[OTTid]['n_ancst'] and OTT_ptrs[OTTid]['n_pop_ancst'] - - we also flag up the poor seed plants (Spermatophyta_ott1007992) - we could add a little - to their pop value later - """ - from dendropy import Tree - - if exclude is None: - exclude = [] - if not isinstance(tree, Tree): - tree = Tree.get( - file=tree, - schema="newick", - suppress_edge_lengths=True, - preserve_underscores=True, - suppress_leaf_node_taxa=True, - ) - - logging.info(f" Tree read for phylogenetic popularity calc: mem usage {mem():.1f} Mb") - - # put popularity into the pop_store attribute - for node in tree.preorder_node_iter(): - if node.label in exclude: - node.pop_store = 0 - node.has_pop = False - else: - try: - node.pop_store = node.data["wd"].raw_popularity - node.has_pop = True - except (LookupError, AttributeError): - node.pop_store = 0 - node.has_pop = False - - # go up the tree from the tips, summing up the popularity indices beneath and - # adding the number of descendants - for node in tree.postorder_node_iter(): - if node.is_leaf(): - node.descendants_popsum = 0 - node.n_descendants = 0 - try: - node._parent_node.n_descendants += 1 + node.n_descendants - node._parent_node.descendants_popsum += node.pop_store + node.descendants_popsum - except AttributeError: # could be the first time we have checked the parent - try: - node._parent_node.n_descendants = 1 + node.n_descendants - node._parent_node.descendants_popsum = node.pop_store + node.descendants_popsum - except AttributeError: # this could be the root, with node._parent_node = None - pass - # root_descendants = node.n_descendants - - # go down the tree from the root, summing up the popularity indices above, - # and summing up numbers of nodes - for node in tree.preorder_node_iter(): - if node.parent_node is None: - # this is the root. - node.seedplant = False - node.n_ancestors = 0 - node.n_pop_ancestors = 0 - node.ancestors_popsum = 0.0 - else: - node.n_ancestors = node._parent_node.n_ancestors + 1 - node.ancestors_popsum = node._parent_node.ancestors_popsum + node.pop_store - if getattr(node, "has_pop", None): - node.n_pop_ancestors = node._parent_node.n_pop_ancestors + 1 - else: - node.n_pop_ancestors = node._parent_node.n_pop_ancestors - if node.label and node.label == "Spermatophyta": - node.seedplant = True - logging.info("Found plant root") - else: - node.seedplant = node._parent_node.seedplant - - # place these values into the OTT_ptrs structure - if OTT_ptrs: - for node in tree.preorder_node_iter(): - try: - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["pop_self"] = node.pop_store - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["pop_ancst"] = ( - node.ancestors_popsum - ) # nb, this includes popularity of self - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["pop_dscdt"] = node.descendants_popsum - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["n_ancst"] = node.n_ancestors - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["n_dscdt"] = node.n_descendants - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["n_pop_ancst"] = node.n_pop_ancestors - OTT_ptrs[int(node.label.rsplit("_ott", 1)[1])]["is_seed_plant"] = node.seedplant - except (LookupError, AttributeError): - pass - return tree diff --git a/oz_tree_build/taxon_mapping_and_popularity/dendropy_extras.py b/oz_tree_build/taxon_mapping_and_popularity/dendropy_extras.py deleted file mode 100755 index cacdf5f0..00000000 --- a/oz_tree_build/taxon_mapping_and_popularity/dendropy_extras.py +++ /dev/null @@ -1,526 +0,0 @@ -#!/usr/bin/env -S python3 -u -""" -A set of functions for monkey patching into dendropy objects. -These all assume that the tree has been loaded with suppress_leaf_node_taxa=True -""" - -# -# To be patched into the Tree object -# -import collections -import itertools -import logging - -import dendropy - - -def prune_children_of_otts(self, ott_species_list): - to_trim = set() - for nd in self.postorder_internal_node_iter(): - try: - if nd.data.get("ott") in ott_species_list: - trim_me = True - # check for extinction props - if ( - (nd.num_child_nodes() == 1) - and (next(iter(nd.child_nodes())).num_child_nodes() == 0) - and (next(iter(nd.child_nodes())).edge.length) - ): - trim_me = False # this is an extinction prop - # check for species within this group - for sub_nd in nd.postorder_internal_node_iter(): - if sub_nd in to_trim: - logging.warning( - f"Species {sub_nd.label} is contained within another species {nd.label}: not trimming it" - ) - trim_me = False - if trim_me: - to_trim.add(nd) - except AttributeError: - pass # nodes created by breaking polytomies will have no data attribute - for nd in to_trim: - nd.clear_child_nodes() - return to_trim - - -def prune_non_species( - self, - recursive=True, - bad_matches=None, # any strings in here indicate non-species (e.g. ' cf.') - update_bipartitions=False, - extinct_tree_mode=False, -): - """ - Removes all terminal nodes whose name is '' or does not contain a space. - Recursive=true means remove tips (which may create more tips) & keep going until - none left to prune. Extinction props should be unlabelled nodes with a length, e.g. - ((:65)Tyrannosaurus_rex,Birds) - """ - if bad_matches is None: - bad_matches = [] - nodes_removed = {"no_space": [], "unlabelled": [], "bad_match": []} - done = False - while not done: - nodes_to_remove = {k: [] for k in nodes_removed.keys()} - for nd in self.leaf_node_iter(): - if nd.label is None: - if ( - nd.edge.length - and nd.parent_node - and nd.parent_node.label - and (nd.parent_node.num_child_nodes() == 1) - ): - # only an extinction prop, if it has a length AND the parent - # node is a named unifurcation - pass - else: - nodes_to_remove["unlabelled"].append(nd) - elif " " not in nd.label and not extinct_tree_mode: - # For the extinct tree, we allow tips with no spaces, as we often end up with genera as tips - - # num_spaces is 0: a leaf, but prob not a species. Also catches label=='' - logging.info( - f"Removing '{nd.label}' since it does not seem to be a species " "(it does not contain a space)" - ) - nodes_to_remove["no_space"].append(nd) - elif any(match in nd.label for match in bad_matches): - logging.info(f"Removing '{nd.label}' since it contains one of {bad_matches}") - nodes_to_remove["bad_match"].append(nd) - for k, nodes in nodes_to_remove.items(): - for nd in nodes: - nd.edge.tail_node.remove_child(nd) - nodes_removed[k] += nodes - if not recursive: - done = True - if all([len(v) == 0 for v in nodes_to_remove.values()]): - done = True - - if update_bipartitions: - self.update_bipartitions() - return nodes_removed - - -def set_node_ages(self): - """ - Adds an attribute called 'age' to each node, with the value equal to - the sum of edge lengths from the node to the tips. Also adds the attribute - extinction_date to terminal nodes that have been propped to earlier in time than 0Ma - - By convention, null branch lengths are of unspecified length, whereas zero-length - branches (e.g. injected by resolving polytomies) are of a fixed length = 0. - Fossil species are denoted by a terminal unnamed monotomy (an 'extinction prop') - which allows us to set the extinction date of any taxon. That means the - entire tree is expected to be ultrametric. - - Returns number of nodes with age set, and number of deleted extinction props - """ - # Percolate age up the tree (go from tips upwards, assuming all tips at 0Ma) - # Where children disagree on the age of their parent, take the larger number - tot_ages = 0 - for node in self.postorder_node_iter(): - if node.is_leaf(): - node.age = 0 - if node.parent_node is not None: - l = node.edge.length - if getattr(node, "age", None) is not None and l is not None: - if l < 0: - logging.warning(f"length <0 for {node.label}") - l = 0 if l < 0 else l - if getattr(node.parent_node, "age", None) is None: - node.parent_node.age = node.age + l - tot_ages += 1 - else: - if node.parent_node.age < (node.age + l): - node.parent_node.age = node.age + l - if abs(node.parent_node.age - (node.age + l)) >= 1: - parent = [n.label for n in node.ancestor_iter() if n.label][:3] - logging.warning( - f"Age of node '{node.parent_node.label}' (child of {parent}) " - f"is {node.parent_node.age} via one route, but has a child " - f"node '{node.label}' of age {node.age}, attached by a " - f"branch of length {l}, which sums to {node.age+l}." - ) - # Round to 6 decimal places to prevent floating point errors - node.parent_node.age = round(node.parent_node.age, 6) - - # For newly fixed ages, percolate them down the tree if we know the age of a deeper node - for node in self.preorder_node_iter(): - if getattr(node, "age", None) is not None: - for ch in node.child_node_iter(): - if getattr(ch, "age", None) is None and ch.edge.length is not None: - ch.age = node.age - (ch.edge.length if ch.edge.length > 0 else 0) - tot_ages += 1 - - # Now we have calculated dates, remove 'extinction props', and flag up extinct species - removed = 0 - for leaf in self.leaf_node_iter(): - if (leaf.label is None) and leaf.edge.length > 0: - assert leaf.parent_node.num_child_nodes() == 1 - leaf.parent_node.extinction_date = getattr(leaf.parent_node, "age", None) - removed += 1 - # this is an extinction prop - remove the prop - leaf.parent_node.clear_child_nodes() - return tot_ages, removed - - -def set_real_parent_nodes(self): - """ - Adds an attribute called 'real_parent' to each leaf and node, which - represents the parent node ignoring randomly resolved polytomies. We can then find - all 'true' children by looking for leaves and nodes which have this as a 'real_parent'. - edge.length==0 indicates a node which is a polytomy. - """ - for node in self.preorder_node_iter(): - for ch in node.child_node_iter(): - if node.edge.length == 0 and node.real_parent_node: - ch.real_parent_node = node.real_parent_node - else: - ch.real_parent_node = node - - -def is_on_unifurcation_path(node): - """ - this is a node which is either a unifurcation or the first node in a path of - successive unifurcations - """ - return node.num_child_nodes() == 1 or (node.parent_node and node.parent_node.num_child_nodes() == 1) - - -def remove_unifurcations_keeping_higher_taxa(self): - """ - Does a more sophisticated pass than the remove_unifurcations flag in Dendropy4: - * If this is a unifurcation ending in a leaf, the lowest level (i.e. species) is - retained as the name of the leaf - * If this is a series of unifurcations within the tree, the node with the highest - raw popularity score should be kept. If there is a tie, any *named* nodes are - given priority, then the nodes highest in the tree. - """ - nd_iter = self.postorder_node_iter() - n_deleted = 0 - for k, g in itertools.groupby(nd_iter, is_on_unifurcation_path): - # k should alternate between 0 (not on unifurcation path) and 1 - if k: - # the group could consist of multiple unifurcation paths - # we need to separate them into groups themselves - node_lists = [[next(g)]] - for next_node in g: - if next_node == node_lists[-1][-1].parent_node: - node_lists[-1].append(next_node) - else: - node_lists.append([next_node]) - for sequential_unary_nodes in node_lists: - if sequential_unary_nodes[0].num_child_nodes() == 0: - # this ends in a tip, we can rely on the normal suppress_unifurcation - # behaviour (by default Dendropy keeps the lowest level taxa) - logging.debug( - "Unary nodes ending in tip left so that first is used: " - + ", ".join([(x.label or "None") for x in sequential_unary_nodes]) - ) - else: - # sort so that best is last - by popularity then presence of label, - # finally by existing position - sorted_unary_nodes = sorted( - sequential_unary_nodes, - key=lambda n: ( - n.data.get("raw_popularity"), - bool(getattr(n, "label", "")), - ), - ) - keep_node = sorted_unary_nodes[-1] - logging.debug( - "Unary nodes collapsed to last in this list: " - + ", ".join([(x.label or "None") for x in sorted_unary_nodes]) - ) - for nd in sequential_unary_nodes: - # these should still be in postorder - if nd != keep_node: - n_deleted += 1 - nd.edge.collapse(adjust_collapsed_head_children_edge_lengths=True) - n_deleted += len(self.suppress_unifurcations()) - return n_deleted - - -def write_preorder_to_csv( - self, - leaf_file, - extra_leaf_data_properties, - node_file, - extra_node_data_properties, - root_parent_id, -): - """ - Write the leaf and node info for this tree to csv files. - * for leaves, always write the parent, name, and extinction_data - * for nodes, always write the parent,node_rgt,leaf_lft,leaf_rgt, name, and age - - In addition to these, also write out the extra_leaf_data_properties and - extra_node_data_properties contained in the data property dictionary of each node - (or blank if the property does not exist), e.g. for leaves this might be - extinction_date, - ott, - wikidata, - wikipedia_lang_flag, - eol, - iucn, - popularity, - popularity_rank, - price, - ncbi, - ifung, - worms, - irmng, - gbif - - for nodes: - age, - ott, - wikidata, - wikipedia_lang_flag, - eol, - popularity, - ncbi, - ifung, - worms, - irmng, - gbif, - vern_synth, - rep1,..., - rtr1,..., - iucnNE,... - """ - import csv - from collections import OrderedDict - - leaf_csv = csv.writer(leaf_file, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") - leaf_csv.writerow(["parent", "real_parent", "name", "extinction_date", *extra_leaf_data_properties]) - node_csv = csv.writer(node_file, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") - node_csv.writerow( - [ - "parent", - "real_parent", - "node_rgt", - "leaf_lft", - "leaf_rgt", - "name", - "age", - *extra_node_data_properties, - ] - ) - - # allocate node numbers - internal_node_number = 0 - for node in self.preorder_internal_node_iter(): - # NB: increment first, since we use a 1-base numbering system, for mySQL row numbering - internal_node_number += 1 - node.id = internal_node_number - - # postorder traversal to allocate rgt side of ranges - internal_leaf_count = 0 - prev_node = None - for node in self.postorder_node_iter(): - # find rightmost leaf by postorder iteration. - # For rightmost node, if previously visited node is a leaf, then (because we ladderize - # ascending) the rightmost node must be self (i.e. this is a terminal internal node). - # Otherwise it is the previously visted node - if node.is_leaf(): - internal_leaf_count += 1 - else: - node.last_leaf = internal_leaf_count # should have counted all the internal leaves by now - if prev_node.is_leaf(): - node.node_rgt = node.id # node_rgt == self - else: - # the node_rgt should be the same as the node_rgt of the previous node - node.node_rgt = prev_node.node_rgt - prev_node = node - - leaf_count = 1 - extra_leaf_output = OrderedDict() - extra_node_output = OrderedDict() - for node in self.preorder_node_iter(): - if node.is_leaf(): - base_output = [ - node.parent_node.id, - # negative real_parent ids if this is a polytomy - (-node.real_parent_node.id if node.edge.length == 0 else node.real_parent_node.id), - node.label, - getattr(node, "extinction_date", None), - ] - for colname, keys in extra_leaf_data_properties.items(): - try: - extra_leaf_output[colname] = node.data - for k in keys: - extra_leaf_output[colname] = extra_leaf_output[colname][k] - except ( - KeyError, - TypeError, - AttributeError, - ): - # catch non-existent key name, None, or no data attribute - # (e.g. for polytomies) - extra_leaf_output[colname] = None - leaf_csv.writerow(base_output + list(extra_leaf_output.values())) - leaf_count += 1 - else: - base_output = [ - node.parent_node.id if node.parent_node else root_parent_id, - ( - (-node.real_parent_node.id if node.edge.length == 0 else node.real_parent_node.id) - if hasattr(node, "real_parent_node") - else 0 - ), - node.node_rgt, - leaf_count, - node.last_leaf, - node.label, - getattr(node, "age", None), - ] - for colname, keys in extra_node_data_properties.items(): - try: - extra_node_output[colname] = node.data - for k in keys: - extra_node_output[colname] = extra_node_output[colname][k] - except (KeyError, TypeError, AttributeError): - # catch none existent key name, None, or no data attribute - # (e.g. for polytomies) - extra_node_output[colname] = None - node_csv.writerow(base_output + list(extra_node_output.values())) - - -# -# To be patched into the Node object -# - - -def write_brief_newick(self, out, polytomy_braces="()", write_otts=False): - """ - Copied from the default dendropy 4 function Node._write_newick - The function requires a binary tree, and the tree should have been ladderized beforehand - It outputs a string consisting of braces only (no commas), such that the number of tips - (& therefore the number of nodes-1, since this is a binary tree) is equal to the number of - characters in the string. Edges lengths are omitted, but internal nodes that have an edge - length of 0 are represented by 'polytomy_braces' which can be specified, e.g. '{}' or '<>' - """ - child_nodes = self.child_nodes() - if child_nodes: - if self.edge and self.edge.length == 0: # if 0, this is a polytomy - out.write(polytomy_braces[0]) # added - else: - out.write("(") - assert len(child_nodes) == 2 # added - f_child = child_nodes[0] - for child in child_nodes: - if child is not f_child: - out.write(",") - child.write_brief_newick(out, polytomy_braces) - if self.edge and self.edge.length == 0: - out.write(polytomy_braces[1]) # added - if write_otts and "ott" in self.data: - out.write(str(self.data["ott"])) - else: - out.write(")") - if write_otts and "ott" in self.data: - out.write(str(self.data["ott"])) - - -def write_pop_newick(self, out): - """ - Copied from the default dendropy 4 function Node._write_newick - This returns the Node as a NEWICK statement but uses node.pop_store - instead of node.edge.length for the edge length values. - """ - child_nodes = self.child_nodes() - if child_nodes: - out.write("(") - f_child = child_nodes[0] - for child in child_nodes: - if child is not f_child: - out.write(",") - child.write_pop_newick(out) - out.write(")") - - label = self._get_node_token( - suppress_leaf_node_labels=False, - suppress_rooting=True, - unquoted_underscores=True, - ) - try: - ott = self.data["ott"] - if label.endswith("'"): - out.write(label[:-1] + f"_ott{ott}'") - else: - out.write(label + f"_ott{ott}") - except (AttributeError, KeyError): - out.write(label) - e = self.edge - if e: - sel = e.length - if sel is not None: - s = "" - try: - s = float(sel) - s = str(s) - except ValueError: - s = str(sel) - if s: - out.write(f":{s}") - - -def group_genera_in_polytomies(self): - """ - Group together nodes in a polytomy whose - tips have the same genus name. - """ - polytomies = [] - for node in self.postorder_node_iter(): - if len(node._child_nodes) > 2: - polytomies.append(node) - for node in polytomies: - # New code here to group children by genus - child_genera = collections.defaultdict(list) - for child in node._child_nodes: - genus = None - for leaf in child.leaf_iter(): - g = (leaf.label or "").split(" ")[0] - if genus is None: - genus = g - elif g != genus: - genus = "" - break - if genus: # ignore if None or empty string - child_genera[genus].append(child) - # first group children by genus - for _, children in child_genera.items(): - if len(children) > 1 and len(children) < len(node._child_nodes): - new_node = dendropy.Node() - node.add_child(new_node) - new_node.edge.length = 0.0 - for child in children: - node.remove_child(child) - new_node.add_child(child) - - -if __name__ == "__main__": - # test - import sys - from collections import OrderedDict - - from dendropy import Tree - - Tree.write_preorder_to_csv = write_preorder_to_csv - Tree.set_node_ages = set_node_ages - Tree.group_genera_in_polytomies = group_genera_in_polytomies - t = Tree.get_from_path( - sys.argv[1], - schema="newick", - suppress_internal_node_taxa=True, - suppress_leaf_node_taxa=True, - ) - for i, nd in enumerate(t.preorder_node_iter()): - nd.data = {"preorder_index": i} - t.set_node_ages() - # print(t.find_node_with_label("Primates").data) - t.ladderize(ascending=True) - with open("test_leaves.csv", "w+") as l, open("test_nodes.csv", "w+") as n: - node_extras = OrderedDict() - node_extras["preorder index"] = ["preorder_index"] - t.write_preorder_to_csv(l, {}, n, node_extras, -1) diff --git a/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py new file mode 100644 index 00000000..64185321 --- /dev/null +++ b/oz_tree_build/taxon_mapping_and_popularity/taxon_map.py @@ -0,0 +1,587 @@ +""" +Read in OpenTree taxonomy.csv, join it with wikidata / wikipedia data to +generate a CSV mapping OTT to various IDs, and raw popularity. +""" + +import argparse +import collections +import csv +import glob +import logging +import os.path + +from ..taxon_mapping_and_popularity import OTT_popularity_mapping +from ..utilities.debug_util import parse_args_and_add_logging_switch +from ..utilities.file_utils import open_file_based_on_extension + +logger = logging.getLogger(__name__) + +# The taxonomy sources we can map to wikidata, most trusted first (the order is used to +# choose between wikidata items when they disagree). Any other source in the taxonomy +# (silva, h2007, "additions-6520052-6520144", ...) is ignored. +SOURCES = ("ncbi", "if", "worms", "irmng", "gbif") + +# The EoL "resource_id" identifying IUCN rows in the EoL identifiers file +iucn_num = 5 + + +def map_wiki_info( + source_ptrs, + source_order, + OTT_ptrs, + WD_filename, + lang, + WP_SQL_filename, + WP_pageviews_filenames, +): + """ + - source_ptrs: + - OTT_ptrs: {ott: { ott: X, sources: {} }} (generated by get_OTT_list) + + 1) use the wikidata JSON dump to map identifiers from the source_ptrs structure to + wikidata Qids, and then on to wikipedia pages + 2) if sql and pagevisits filenames are given + + Return True if popularity is mapped + """ + logger.debug(f"Processing wikidata json dump in parallel for {lang}") + popularity_steps = 0 + WDitems, WPnames, swap_Qs = set_wikidata(WD_filename, source_ptrs, lang) + + if WP_SQL_filename is not None: + logger.info(f" > Adding wikipedia page sizes from {WP_SQL_filename}") + # Can't easily parallelize this as it is gzip compressed (not a block format) + OTT_popularity_mapping.add_pagesize_for_titles(WPnames, WP_SQL_filename) + popularity_steps += 1 + + if len(WP_pageviews_filenames) > 0: + logger.info(f" > Adding wikipedia visit counts from {len(WP_pageviews_filenames)} files") + set_wikipedia_pageviews(WP_pageviews_filenames, WPnames, lang) + popularity_steps += 1 + + if popularity_steps == 2: + logger.info(" > Calculating raw popularity measures") + tot = 0 + for WDinstance in WDitems.values(): + if WDinstance.set_raw_popularity(): + tot += 1 + logger.info(f" ✔ Raw popularity measures set on {tot} wikidata items") + else: + logger.info(" x Skipping popularity calculations") + + # Here we might want to multiply up some taxa, e.g. plants, + # see https://github.com/OneZoom/OZtree/issues/130 + logger.info(" > Finding best wiki matches") + OTT_popularity_mapping.identify_best_wikidata(OTT_ptrs, lang, source_order) + + logger.info(" > Swapping vernacular wikidata items into taxon items") + OTT_popularity_mapping.overwrite_wd(WDitems, swap_Qs, only_if_more_popular=(popularity_steps == 2), check_lang=lang) + + logger.info(" > Supplementing ids (EOL/IPNI) with ones from wikidata") + supplement_from_wikidata(OTT_ptrs) + + logger.info("✔ Wikidata/wikipedia data mapped") + + display_WD_ott_stats(OTT_ptrs) + + return popularity_steps == 2 + + +def display_WD_ott_stats(OTT_ptrs): + """ + Display some stats about OTTs coming from Wikidata + """ + matching_otts = 0 + mismatching_otts = 0 + no_wd_otts = 0 + for ott in OTT_ptrs: + try: + if OTT_ptrs[ott]["rank"] == "species": + if OTT_ptrs[ott]["wd"].get("wd_ott") is not None: + if ott == OTT_ptrs[ott]["wd"].wd_ott: + matching_otts += 1 + else: + logger.debug(f"Q{OTT_ptrs[ott]['wd'].Q}: OTT {ott} does not match {OTT_ptrs[ott]['wd'].wd_ott}") + mismatching_otts += 1 + else: + no_wd_otts += 1 + except (KeyError, AttributeError): + pass + + logger.info("✔ Stats on Wikidata OTT matching:") + logger.info(f" Leaves where the WD ott matches the ott: {matching_otts}") + logger.info(f" Leaves where the WD ott does not match the wd_ott: {mismatching_otts}") + logger.info(f" Leaves where WD does not have an ott: {no_wd_otts}") + + +def set_wikidata(bz2_filename, source_ptrs, lang): + """ + Will alter the source_ptrs. + Returns WDitems (Q->WD), WPnames (name-WD), common_name_Qs (Q->Q) + """ + WDitems = {} + WPnames = {} + common_name_Qs = {} + sum_info = collections.defaultdict(int) + + ( + Q_to_WD, + WPname_to_WD, + src_to_WD, + replace_Q, + info, + ) = OTT_popularity_mapping.wikidata_info(bz2_filename, source_ptrs, lang) + + WDitems.update(Q_to_WD) + WPnames.update(WPname_to_WD) + common_name_Qs.update(replace_Q) + # Add 'wd' item to source_ptrs + for src, ids in src_to_WD.items(): + for src_id, WD in ids.items(): + source_ptrs[src][src_id]["wd"] = WD + for k, v in info.items(): + sum_info[k] += v + + logger.info( + f"✔ {len(WDitems)} wikidata matches, of which " + f"{sum_info['n_eol']} have EOL ids, {sum_info['n_iucn']} have IUCN ids, " + f"{sum_info['n_ipni']} have IPNI, and {len(WPnames)} " + f"({(len(WPnames)/len(WDitems)*100):.2f}%) have titles that exist on " + f"{lang}.wikipedia. Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + return WDitems, WPnames, common_name_Qs + + +def set_wikipedia_pageviews(filenames, WPnames, lang): + names_found = 0 + for fn in filenames: + WPnames_views = OTT_popularity_mapping.pageviews_for_titles(fn, set(WPnames.keys()), lang) + for name, n_views in WPnames_views.items(): + if not hasattr(WPnames[name], "pageviews"): + names_found += 1 + WPnames[name].pageviews = [] + WPnames[name].pageviews.append(n_views) + logger.info( + f" ✔ Of {len(WPnames)} WikiData taxon entries, {names_found} " + f"({(names_found/len(WPnames) * 100):.2f}%) have pageview data for '{lang}' in " + f"{len(filenames)} files. Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + + +def supplement_from_wikidata(OTT_ptrs): + """ + If no OTT_ptrs[OTTid]['eol'] exists, but there is an + OTT_ptrs[OTTid]['wd']['initial_wiki_item']['EoL'] then put this into + OTT_ptrs[OTTid]['eol'] + Similarly for IPNI (although this is currently unpopulated) + """ + EOLalready = n_eol = n_ipni = n = 0 + for data in OTT_ptrs.values(): + n += 1 + if data.get("eol") is None: + try: + data["eol"] = int(data["wd"].EoL) + n_eol += 1 + except (AttributeError, KeyError, TypeError, ValueError): + pass + else: + EOLalready += 1 + if data.get("ipni") is None: + try: + data["ipni"] = int(data["wd"].ipni) + n_ipni += 1 + except (AttributeError, KeyError, TypeError, ValueError): + pass + logger.info( + f"✔ Out of {n} OTT taxa, {EOLalready} ({(EOLalready/n * 100):.2f}%) already " + f"have EOL ids from the EOL file. Supplementing these with {n_eol} EOL ids from " + f"wikidata gives a coverage of {((EOLalready + n_eol)/n * 100):.1f} %." + + (f" An addition {n_ipni} IPNI identifiers added via wikidata" if n_ipni else "") + ) + + +def add_eol_IDs_from_EOL_table_dump(source_ptrs, identifiers_filename, source_mapping): + used = 0 + EOL2OTT = {v: k for k, v in source_mapping.items()} + with open_file_based_on_extension(identifiers_filename, "rt") as identifiers_file: + reader = csv.DictReader(identifiers_file) + for EOLrow in reader: + if reader.line_num % 1000000 == 0: + logger.info( + f"... {reader.line_num} rows read, {used} used, " + f"mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + provider = int(EOLrow["resource_id"]) + if provider in EOL2OTT: + src = source_ptrs[EOL2OTT[provider]] + if EOL2OTT[provider] == "gbif" and not EOLrow["resource_pk"].isdigit(): + # The EoL file has duplicate (non numeric) IDs for GBIF: ignore these + continue + providerid = EOLrow["resource_pk"] + EOLid = int(EOLrow["page_id"]) + try: + if int(providerid) in src: + used += 1 + src[int(providerid)]["EoL"] = EOLid + except ValueError: + if providerid in src: + used += 1 + src[providerid]["EoL"] = EOLid + logger.info( + f"✔ Matched {used} EoL entries in the EoL identifiers file. " + f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + + +def identify_best_EoLdata(OTT_ptrs, sources): + """ + Each OTT number may point to several EoL entries, one for the NCBI number, + another for the WORMS number, etc etc. Hopefully these will be the same entry, + but they may not be. If they are different we need to choose the best one + to use. We take the one with the most sources supporting this entry: + if there is a tie, we take the lowest, as recommended by JRice from EoL + """ + validOTTs = OTTs_with_EOLmatch = dups = 0 + for OTTid, data in OTT_ptrs.items(): + validOTTs += 1 + choose = {} + for src in sources: + if src in data["sources"] and data["sources"][src] is not None: + if "EoL" in data["sources"][src]: + EOLid = int(data["sources"][src]["EoL"]) + if EOLid not in choose: + choose[EOLid] = [] + choose[EOLid] += [src] + if len(choose) == 0: + data["eol"] = None + else: + OTTs_with_EOLmatch += 1 + errstr = None + if len(choose) > 1: + # weed out those EOLids with the least support. + errstr = f"More than one EoL ID {choose} for taxon OTT: {OTTid}" + dups += 1 + max_refs = max([len(choose[i]) for i in choose]) + choose = [EOLid for EOLid in choose if len(choose[EOLid]) == max_refs] + best = min(choose) + data["eol"] = best + if errstr: + logger.debug(f" {errstr}, chosen {best}") + logger.info( + f" ✔ Of {validOTTs} OpenTree taxa, {OTTs_with_EOLmatch} " + f"({OTTs_with_EOLmatch / validOTTs * 100:.2f}%) have EoL entries in the EoL " + f"identifiers file, and {dups} have multiple possible EOL ids. " + f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + + +def populate_iucn(OTT_ptrs, identifiers_filename, verbosity=0, iucn_num=iucn_num): + """ + Port the IUCN number from both EoL and Wikidata, and keep both if there is a conflict + """ + used = 0 + + eol_mapping = {} # to store eol=>iucn + for OTTid, data in OTT_ptrs.items(): + if "eol" in data: + if data["eol"] in eol_mapping: + eol_mapping[data["eol"]].append(OTTid) + else: + eol_mapping[data["eol"]] = [OTTid] + + with open_file_based_on_extension(identifiers_filename, "rt") as identifiers_file: + reader = csv.DictReader(identifiers_file) + for EOLrow in reader: + if reader.line_num % 1000000 == 0: + logger.info( + f" - {reader.line_num} rows read, {used} used. " f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + if int(EOLrow["resource_id"]) == iucn_num and EOLrow["resource_pk"].isdigit(): + # there are lots of non-species IUCN rows with pk == str (e.g. Animalia) + try: + for ott in eol_mapping[int(EOLrow["page_id"])]: + OTT_ptrs[ott]["iucn"] = EOLrow["resource_pk"] + used += 1 + except LookupError: + pass # no equivalent eol id in eol_mapping + logger.info( + f" > matched {used} IUCN entries in the EoL identifiers file. " + f"Mem usage {OTT_popularity_mapping.mem():.1f} Mb" + ) + + # now go through and double-check against IUCN stored on wikidata + for OTTid, data in OTT_ptrs.items(): + try: + wd_iucn = str(int(data["wd"].iucn)) + if "iucn" not in data: + data["iucn"] = wd_iucn + used += 1 + else: + if wd_iucn not in data["iucn"].split("|"): + data["iucn"] += "|" + wd_iucn + logger.debug( + f' conflicting IUCN IDs for OTT {OTTid}: EoL = {data["iucn"]} ' + f'(via http://eol.org/pages/{data["eol"]}), wikidata = ' + f'{wd_iucn} (via http://http://wikidata.org/wiki/Q{data["wd"].Q}).' + ) + except ValueError: + logger.warning(f" Cannot convert wikidata IUCN ID {data['wd'].iucn} to integer.") + except (KeyError, AttributeError): + pass # can't find a wd instance or an iucn within the wd instance. Oh well. + + logger.info(f" > Increased IUCN coverage to {used} taxa using wikidata") + + +def parse_sourceinfo(sourceinfo): + """Deparse a sourceinfo string, e.g. "ncbi:1274384,gbif:8094325", into a dict of IDs""" + out = {} + for i in sourceinfo.split(","): + if not i: + continue + k, v = i.split(":", 1) + out[k] = int(v) if v.isdigit() else v + return out + + +def read_ot_taxonomy(path="./data/OpenTree/v16.1/taxonomy.tsv"): + """Yield each row of an OpenTree taxonomy file as a dict keyed by header. + + Fields are separated by ``\\t|\\t`` and each line ends with a trailing + ``\\t|\\t``, so this is not parseable as a plain TSV. + """ + with open(path, encoding="utf-8") as f: + header = next(f).rstrip("\n").split("\t|\t")[:-1] + for line in f: + fields = line.rstrip("\n").split("\t|\t")[:-1] + out = dict(zip(header, fields)) + + out["uid"] = int(out["uid"]) + out["parent_uid"] = None if out["parent_uid"] == "" else int(out["parent_uid"]) + out["sourceinfo"] = parse_sourceinfo(out["sourceinfo"]) + + yield out + + +def read_extra_source_file(path): + """Yield each row of an extra source file, in the same form as read_ot_taxonomy(). + + Unlike the OpenTree taxonomy this is a plain TSV, requiring only "uid" and + "sourceinfo" columns. The uid need not be a number (e.g. "mrcaott409215ott616649"). + """ + try: + with open(path, encoding="utf-8", newline="") as f: + for row in csv.DictReader(f, delimiter="\t"): + out = dict(row) + out["uid"] = int(row["uid"]) if row["uid"].isdigit() else row["uid"] + out["sourceinfo"] = parse_sourceinfo(row["sourceinfo"]) + yield out + except FileNotFoundError: + logger.warning(f" Extra source file '{path}' not found, so ignored") + + +def add_taxon_sources(OTT_ptrs, source_ptrs, ott, sourceinfo, rank=None, unused_sources=None): + """ + Point OTT_ptrs[ott] at an entry in source_ptrs for each source in sourceinfo, + adding the OTT if not already there, and overwriting any existing source of + the same name. Sources not in SOURCES are ignored, their names being added to + the unused_sources set, if given. + """ + ott_data = OTT_ptrs.setdefault(ott, {"ott": ott, "sources": {}}) + if rank is not None: + ott_data["rank"] = rank + + for src in reversed(sourceinfo.keys()): + # NB: look at sources in reverse order, overwriting, so 1st ones take priority + src_id = sourceinfo[src] + if src not in SOURCES: + if unused_sources is not None: + unused_sources.add(src) + continue + # NB: reuse any existing entry, so OTTs sharing a source id share its (wikidata) data + ott_data["sources"][src] = source_ptrs.setdefault(src, {}).setdefault(src_id, {"id": src_id}) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--OpenTreeTaxonomy", + default="data/OpenTree/v16.1/taxonomy.tsv", + help="The OpenTree taxonomy.tsv file, from http://files.opentreeoflife.org/ott/", + ) + parser.add_argument( + "--wikidataDumpFile", + default="data/filtered/OneZoom_latest-all.json", + help=( + "The very large wikidata JSON dump, " + "from https://dumps.wikimedia.org/wikidatawiki/entities/ (latest-all.json.bz2)." + "A filtered version can be used for faster processing." + ), + ) + parser.add_argument( + "--wikilang", + default="en", + help=( + 'The language wikipedia to check for popularity, e.g. "en". ' + "Where there are multiple Wikidata items for a taxon " + "(e.g. one under the common name, one under the scientific name), " + "then we also default to using the WD item with the sitelink in this language." + ), + ) + parser.add_argument( + "--wikipediaSQLDumpFile", + default="data/filtered/OneZoom_enwiki-latest-page.sql", + help=( + "The gzipped >1GB wikipedia -latest-page.sql.gz dump, " + "from https://dumps.wikimedia.org/enwiki/latest/ (enwiki-page.sql.gz) " + ), + ) + parser.add_argument( + "--wikipedia_totals_bz2_pageviews", + default="data/filtered/pageviews/", + help=( + 'Directory of b2zipped "totals" pageview count files, ' + "from https://dumps.wikimedia.org/other/pagecounts-ez/merged/ " + "(e.g. pagecounts-2016-01-views-ge-5-totals.bz2, or pagecounts*totals.bz2)" + ), + ) + parser.add_argument( + "--EOLidentifiers", + default="data/filtered/OneZoom_provider_ids.csv", + help=("EOL identifiers file, from " "https://opendata.eol.org/dataset/identifiers-csv-gz"), + ) + parser.add_argument( + "--extra_source_file", + default=None, + type=str, + help=( + "An optional additional file to supplement the taxonomy.tsv file, " + "providing additional mappings from OTTs to source ids (useful for overriding). " + 'The first line should be a header contining "uid" and "sourceinfo" column ' + "headers, similar to those in the taxonomy.tsv file. NB the OTT can be a " + 'number, or an ID of the form "mrcaott409215ott616649").' + ), + ) + parser.add_argument( + "-o", + type=argparse.FileType("w"), + default="-", + help=("File to output CSV OTT map to"), + ) + parser.add_argument( + "--skip_popularity", + action="store_true", + help="Skip popularity calculations (no wikipedia SQL dump or pageviews required)", + ) + args = parse_args_and_add_logging_switch(parser) + + # Generate both OTT_ptrs / source_ptrs from taxonomy + # Replaces get_OTT_list & OTT_popularity_mapping.create_from_taxonomy respectively + logger.info("Generating OTT_ptrs / source_ptrs from taxonomy") + OTT_ptrs = {} + source_ptrs = {s: {} for s in SOURCES} # NB: all sources must exist, even if empty + unused_sources = set() + for r in read_ot_taxonomy(args.OpenTreeTaxonomy): + add_taxon_sources(OTT_ptrs, source_ptrs, r["uid"], r["sourceinfo"], r["rank"], unused_sources) + + if args.extra_source_file is not None: + logger.info(f"Supplementing source IDs from {args.extra_source_file}") + extra_otts = 0 + for r in read_extra_source_file(args.extra_source_file): + add_taxon_sources(OTT_ptrs, source_ptrs, r["uid"], r["sourceinfo"], r.get("rank"), unused_sources) + extra_otts += 1 + logger.info(f"✔ {extra_otts} OTTs supplemented from {args.extra_source_file}") + + logger.info( + f"✔ {len(OTT_ptrs)} OTTs with sources {[f'{s}: {len(source_ptrs[s])}' for s in SOURCES]}. " + f"Ignored {len(unused_sources)} unused sources, e.g. {sorted(unused_sources)[:5]}" + ) + + eol_sources = { + "ncbi": 676, + "worms": 459, + "gbif": 767, + } # update when EoL has harvested index fungorum & IRMNG + add_eol_IDs_from_EOL_table_dump(source_ptrs, args.EOLidentifiers, eol_sources) + identify_best_EoLdata(OTT_ptrs, eol_sources) + + map_wiki_info( + source_ptrs=source_ptrs, + source_order=SOURCES, + OTT_ptrs=OTT_ptrs, + WD_filename=args.wikidataDumpFile, + lang=args.wikilang, + WP_SQL_filename=(None if args.skip_popularity else args.wikipediaSQLDumpFile), + WP_pageviews_filenames=( + None if args.skip_popularity else glob.glob(os.path.join(args.wikipedia_totals_bz2_pageviews, "*")) + ), + ) + populate_iucn(OTT_ptrs, args.EOLidentifiers) + + write_taxon_map(args.o, OTT_ptrs) + args.o.close() + + +def write_taxon_map(out_file, OTT_ptrs): + """Write the collated OTT data out as CSV, as read back by read_taxon_map() + + Source columns are named as the taxonomy names them, i.e. index fungorum is "if". + They are only renamed ("ifung") when written to the ordered_leaves/ordered_nodes + files, whose header is used as the column list when importing into the database. + """ + writer = csv.writer(out_file, dialect="excel") + writer.writerow( + [ + "ott", + "wikidata", + "wikipedia_lang_flag", + "iucn", + "eol", + "rank", + "raw_popularity", + *SOURCES, + "ipni", + ] + ) + for o in OTT_ptrs.values(): + writer.writerow( + [ + o["ott"], + o.get("wd", {}).get("Q"), + o.get("wd", {}).get("wikipedia_lang_flag"), + o.get("iucn"), + o.get("eol"), + o.get("rank"), + o.get("wd", {}).get("raw_popularity"), + *(o["sources"].get(src, {}).get("id") for src in SOURCES), + o.get("ipni"), + ] + ) + + +def read_taxon_map(path): + """Read a taxon map CSV (as written by :func:`main`) into a dict keyed by ott. + + Empty fields become ``None``. Numeric fields are converted to ``int`` or + ``float``; ``iucn`` is left as a string because multiple values may be + joined with ``|``. Source IDs (see ``SOURCES``) are converted to ``int`` + where possible and otherwise left as strings. + """ + int_fields = ("ott", "wikidata", "wikipedia_lang_flag", "eol", "ipni") + float_fields = ("raw_popularity",) + source_fields = SOURCES + out = {} + with open(path, encoding="utf-8", newline="") as f: + reader = csv.DictReader(f, dialect="excel") + for row in reader: + r = {k: (v if v != "" else None) for k, v in row.items()} + for k in int_fields: + if r.get(k) is not None: + r[k] = int(r[k]) + for k in float_fields: + if r.get(k) is not None: + r[k] = float(r[k]) + for k in source_fields: + if r.get(k) is not None and r[k].isdigit(): + r[k] = int(r[k]) + out[r["ott"]] = r + return out diff --git a/oz_tree_build/tree_build/build_oz_tree.py b/oz_tree_build/tree_build/build_oz_tree.py deleted file mode 100644 index ddb41dcb..00000000 --- a/oz_tree_build/tree_build/build_oz_tree.py +++ /dev/null @@ -1,164 +0,0 @@ -""" -Build the entire OneZoom tree from the saved parts. - -It does this in one pass, by starting with the base file (e.g. base.PHY) and -recursively expanding any OneZoom tokens it finds. -""" - -import argparse -import logging -import os -import sys - -from ..utilities.debug_util import parse_args_and_add_logging_switch -from .oz_tokens import enumerate_one_zoom_tokens - -__author__ = "David Ebbo" - - -def trim_tree(tree, strip_semicolon=True): - # Trim any whitespace - tree = tree.strip() - - # Skip the comment block at the start of the file, if any - if tree[0] == "[": - tree = tree[tree.index("]") + 1 :] - tree = tree.lstrip() - - # Strip the trailing semicolon - if strip_semicolon and tree[-1] == ";": - tree = tree[:-1] - - return tree - - -def build_oz_tree(base_file, ot_parts_folder, output_stream, print_file_tree): - """ - Do all the token replacement, starting with the base file - """ - - depth = 0 - - def process_newick( - file, - node_name_in_parent=None, - edge_length_in_parent=None, - override_edge_length=None, - override_taxon=None, - expand_nodes=False, - ): - """ - Copy the input file to the output file, recursively expanding any OneZoom tokens - """ - nonlocal depth - - logging.debug(f"Processing {file}") - - # If we're printing the file tree, print the current file - if print_file_tree and expand_nodes: - print(f"{' ' * depth}{node_name_in_parent}: {edge_length_in_parent} {override_edge_length or 0}") - - if not os.path.exists(file): - logging.warning(f"Subtree file {file} does not exist") - return False - - with open(file, encoding="utf8") as stream: - tree = stream.read() - - tree = trim_tree(tree) - index = 0 - - # We only need to look for children if it's a OneZoom file (i.e. .PHY extension) - if expand_nodes: - for result in enumerate_one_zoom_tokens( - tree, - dict( - ot=ot_parts_folder, - oz=oz_parts_folder, - ot_required=os.path.join(os.path.dirname(os.path.dirname(ot_parts_folder)), "OT_required"), - ), - ): - # Write the part of the tree before the child - output_stream.write(tree[index : result["start"]]) - - depth += 1 - if process_newick( - file=result["file"], - node_name_in_parent=result["node_name_in_parent"], - edge_length_in_parent=result["edge_length_in_parent"], - override_edge_length=result["override_edge_length"], - override_taxon=result["override_taxon"], - expand_nodes=result["expand_nodes"], - ): - index = result["end"] - else: - # If child file absent, we'll need to write the child token as-is - index = result["start"] - depth -= 1 - - # We've processed all the children, and we need to write the rest of the tree - last_chunk = tree[index:] - - # Write the last chunk, but exclude the last name:edge_length, - # which needs special handling - last_closed_bracket = last_chunk.rfind(")") - output_stream.write(last_chunk[: last_closed_bracket + 1]) - - # Parse the last token into the node name and edge length - last_token = last_chunk[last_closed_bracket + 1 :] - last_token_segments = last_token.split(":") - last_token_name = last_token_segments[0] - last_token_edge_length = last_token_segments[1] if len(last_token_segments) > 1 else None - - # Always favor the length from our mapping, falling back to the last token in the file - # Note that we never fall back to edge_length_in_parent here, following old code logic - # DISCUSS: should we? - edge_length = override_edge_length or last_token_edge_length - - if expand_nodes: - # Three levels of fallback for .PHY files: mapping, last token, parent - node_name = override_taxon or last_token_name or node_name_in_parent - else: - # NB: following old code logic, the above parent vs last logic is reversed here - # DISCUSS: is there a logical reason for this? - node_name = node_name_in_parent or last_token_name - - output_stream.write(node_name) - if edge_length: - output_stream.write(f":{edge_length}") - - return True - - # Assume that the base file is in the same folder as the OneZoom parts - oz_parts_folder = os.path.dirname(base_file) - - process_newick(base_file, expand_nodes=True) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--printfiletree", - action="store_true", - help="Print a tree of all the OneZoom included files", - ) - parser.add_argument("treefile", help="The base tree file in newick form") - parser.add_argument("ot_parts_folder", help="The folder containing the Open Tree parts") - parser.add_argument( - "outfile", - type=argparse.FileType("w"), - nargs="?", - default=sys.stdout, - help="The output tree file", - ) - args = parse_args_and_add_logging_switch(parser) - - build_oz_tree(args.treefile, args.ot_parts_folder, args.outfile, args.printfiletree) - - # Write out the ending semi-colon and flush the stream - args.outfile.write(";") - args.outfile.flush() - - -if __name__ == "__main__": - main() diff --git a/oz_tree_build/tree_build/get_open_trees_from_one_zoom.py b/oz_tree_build/tree_build/get_open_trees_from_one_zoom.py deleted file mode 100644 index 9dc42661..00000000 --- a/oz_tree_build/tree_build/get_open_trees_from_one_zoom.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -Create subtrees from the OpenTree, on the basis of ott numbers in a set of newick files. - -Usage: getOpenTreesFromOneZoom.py OpenTreeFile.tre output_dir file1.PHY file2.PHY ... - -This script places a set of inclusion files into output_dir, based on the names of nodes -in the input .PHY files. The input files should contain one or more node names in the OneZoom -@include format which is the scientific name + '_ott' + (an OTT id, optionally a ~ sign, and -optionally other OTT numbers separated by an minus sign) + '@', e.g. Brachiopoda_ott826261@ -This specifies that the node should be replaced with part of the OpenTree: namely the subtree -starting at ott node 826261. - -E.g. - foobar_ott123@ - create a node named foobar with ott 123, consisting of all descendants of - ott 123 in the opentree. - - foobar_ott123~456-789-111@ - create a node named foobar with ott 123, using ott456 minus the descendant - subtrees with ott 789 and 111 (the tilde sign can be read an a equals, used - as Dendropy doesn't like equals signs in taxon names. - - foobar_ott123~-789-111@ - shorthand for foobar_ott123~123-789-111@ - - foobar_ott~456-789-111@ - create a node named foobar without any OTT number, - using ott456 minus the descendant subtrees 789 and 111 - -The actual inclusion is done by the build_oz_tree.py. This script merely creates the -files to include. It does this by extracting the relevant subtree from the full OpenTree -""" - -import argparse -import logging -import os -import sys -import time - -from ..newick.extract_trees import extract_trees -from .oz_tokens import enumerate_one_zoom_tokens - -__author__ = "David Ebbo" - - -def get_inclusions_and_exclusions_from_one_zoom_file(file, all_included_otts, all_excluded_otts): - """ - Find all the included and excluded ott numbers in a OneZoom files & add them to the sets - """ - - with open(file, encoding="utf8") as stream: - tree = stream.read() - - for result in enumerate_one_zoom_tokens(tree): - # Check if the result has a base ott (won't have it if it's inserting another OZ file) - if result.get("base_ott") is not None: - all_included_otts.add(result["base_ott"]) - all_excluded_otts.update(result["excluded_otts"]) - - -def extract_trees_from_open_tree_file(open_tree_file, output_dir, all_included_otts, all_excluded_otts): - """ - Extract the subtrees from the Open Tree file, based on the list of included/excluded otts - """ - - # Read the contents of the open tree file into a string - with open(open_tree_file, encoding="utf8") as f: - fulltree = f.read() - - trees = extract_trees(fulltree, all_included_otts, excluded_taxa=all_excluded_otts) - - logging.info(f"Extracted {len(trees)} trees from Open Tree file") - - # Save each tree to a file named after the taxon - os.makedirs(output_dir, exist_ok=True) - for ott, tree in trees.items(): - file = os.path.join(output_dir, ott + ".phy") - logging.debug(f"Writing file: {file}") - with open(file, "w", encoding="utf8") as f: - f.write(tree) - f.write(";\n") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - parser.add_argument( - "--verbosity", - "-v", - action="count", - default=0, - help="verbosity level: output extra non-essential info", - ) - parser.add_argument("open_tree_file", help="Path to the Open Tree newick file") - parser.add_argument( - "output_dir", - help="Path to the directory in which to save the OpenTree subtrees", - ) - parser.add_argument( - "parse_files", - nargs="+", - help="A list of newick files to parse for OTT numbers, giving the subtrees to extract", - ) - args = parser.parse_args() - - if args.verbosity == 0: - logging.basicConfig(stream=sys.stderr, level=logging.WARNING) - elif args.verbosity == 1: - logging.basicConfig(stream=sys.stderr, level=logging.INFO) - elif args.verbosity == 2: - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - - start = time.time() - if not os.path.isfile(args.open_tree_file): - logging.warning(f"Could not find the OpenTree file {args.open_tree_file}") - - # Go through all the OneZoom files, and gather all the ott numbers to include and exclude. - # NB: excluded ott numbers don't need to be specifically given an included ott number - included_otts = set() - excluded_otts = set() - for file in args.parse_files: - logging.info(f"== Processing One Zoom file {file}") - get_inclusions_and_exclusions_from_one_zoom_file(file, included_otts, excluded_otts) - - extract_trees_from_open_tree_file(args.open_tree_file, args.output_dir, included_otts, excluded_otts) - - end = time.time() - logging.debug(f"Time taken: {end - start} seconds") - - -if __name__ == "__main__": - main() diff --git a/oz_tree_build/tree_build/oz_tokens.py b/oz_tree_build/tree_build/oz_tokens.py deleted file mode 100644 index 65242aec..00000000 --- a/oz_tree_build/tree_build/oz_tokens.py +++ /dev/null @@ -1,104 +0,0 @@ -__author__ = "David Ebbo" - -import logging -import os.path -import re - -from .token_to_oz_tree_file_mapping import token_to_file_map - -__author__ = "David Ebbo" - -full_ott_token = re.compile(r"'?([\w\-~]+)@'?(?::([\d\.]+))?") -ott_details = re.compile(r"(\w+)_ott(\d*)~?([-\d]*)$") - - -def parse_one_zoom_token(node_label, parts_folders=None): - """ - Parse a single OneZoom token from label name - """ - if parts_folders is None: - parts_folders = {} - - if not node_label: - return None - try: - return next(enumerate_one_zoom_tokens(node_label, parts_folders)) - except StopIteration: - return None - - -def enumerate_one_zoom_tokens(tree, parts_folders=None): - """ - Enumerates all the OneZoom tokens in a tree string (e.g. foobar_ott123~-789-111) - - Yields dicts with the keys: - - - start: Position in in string the match was found - - end: End of match - - node_name_in_parent: Node name from inclusion node, ignoring OZ inclusion syntax - - edge_length_in_parent: Edge length from inclusion node - - file: File path pointing to tree to substitute - - base_ott: OTT of root, if subtree is a OT tree - - excluded_otts: OTTs to exclude from subtree (as strings not ints) - - expand_nodes: Should we recurse and apply OZ inclusion rules to subtree? - - override_edge_length: Replace edge length from root node with this value - - override_taxon: Replace root node name with this value - """ - if parts_folders is None: - parts_folders = {} - - # Skip the comment block at the start of the file - start_index = tree.index("]") if "[" in tree else 0 - - for full_match in full_ott_token.finditer(tree, start_index): - result = { - "start": full_match.start(), - "end": full_match.end(), - "node_name_in_parent": full_match.group(1), - "edge_length_in_parent": float(full_match.group(2)) if full_match.group(2) else None, - } - - # Check if it matches our tilde (aka 'equal') exclusion syntax - match = ott_details.match(result["node_name_in_parent"]) - base_ott = None - if match: - # split by minus signs - result["excluded_otts"] = (match.group(3) or "").split("-") - - # If present, the first number after '=' is the tree to extract. - first_number_after_equal = result["excluded_otts"].pop(0) - base_ott = first_number_after_equal or match.group(2) - - # Note that we don't append the ott in the name if it came after the '=' - result["node_name_in_parent"] = match.group(1) - if not first_number_after_equal: - result["node_name_in_parent"] += f"_ott{base_ott}" - - # Check if OZ token has a base ott (e.g. 123 in foobar_ott123~456-789) - if base_ott is not None: - # It's an extracted Open Tree file, e.g. 123.phy - # NB: We can't make a valid path without parts_folder["ot"], but we probably don't care in this case - result["base_ott"] = base_ott - if os.path.exists( - os.path.join(parts_folders.get("ot_required", "/unconfiguredpath/ot_requried/"), f"{base_ott}.nwk") - ): - # An ot_required orphan OT file exists, use that - result["file"] = os.path.join( - parts_folders.get("ot_required", "/unconfiguredpath/ot_requried/"), f"{base_ott}.nwk" - ) - else: - result["file"] = os.path.join(parts_folders.get("ot") or ".", f"{base_ott}.phy") - result["override_edge_length"] = None - result["override_taxon"] = None - result["expand_nodes"] = False - else: - # Otherwise, it's a OneZoom file, e.g. AMORPHEA@ --> Amorphea.PHY - child_mapping_entry = token_to_file_map[result["node_name_in_parent"]] - result["base_ott"] = None - result["file"] = os.path.join(parts_folders.get("oz") or ".", child_mapping_entry["file"]) - result["override_edge_length"] = child_mapping_entry.get("edge_length", None) - result["override_taxon"] = child_mapping_entry.get("taxon", None) - result["expand_nodes"] = True - - logging.debug(result) - yield result diff --git a/oz_tree_build/tree_build/step_graft.py b/oz_tree_build/tree_build/step_graft.py new file mode 100644 index 00000000..c840fb16 --- /dev/null +++ b/oz_tree_build/tree_build/step_graft.py @@ -0,0 +1,202 @@ +import re + +from ..utilities.ete import node_get_ott + +OT_INCLUSION_SYNTAX_RE = re.compile(r"(\w+)[_ ]ott(\d*)~?([-\d]*)@$") + + +def graft_tree(t, additional_trees, prefer_subtree_name=False, disable_recursion=False): + """ + Resolve OneZoom inclusion syntax in ``t`` by grafting subtrees in place, + replacing from ``additional_trees`` (a dict inclusion string -> tree). + + Walks ``t`` and, for every node whose name matches the inclusion syntax (a + label ending in ``@``, see ``decypher_inclusion_syntax``). + ``additional_trees`` is a dict of inclusion labels to ete4 trees. + + Naming of the grafted node: + - default: use the name derived from the inclusion token (e.g. ``"Sub + ott1"``), falling back to the subtree's root name if the token has no + derived name + - ``prefer_subtree_name=True``: use the subtree's root name, falling + back to the token-derived name if the subtree root is unnamed + + By default, the function recurses into each grafted subtree so nested + inclusions are also resolved. Pass ``disable_recursion=True`` to graft + only one level, i.e. for OpenTree subtrees which won't contain inclusion syntax. + + Returns a list of inclusion labels that had no match in ``additional_trees`` + (including those found while recursing). Missing inclusions leave the + placeholder node unchanged in ``t``. + """ + missing_inclusions = [] + + def is_leaf_fn(n): + r = decypher_inclusion_syntax(n.name) + if r is None: + # No inclusion syntax, recurse + return n.is_leaf + + if n.name not in additional_trees: + # Not present, ignore for now + missing_inclusions.append(n.name) + return True + + # Graft sub_t at this point + sub_t = additional_trees[n.name] + if not disable_recursion: + missing_inclusions.extend(graft_tree(sub_t, additional_trees, prefer_subtree_name)) + if prefer_subtree_name: + n.name = sub_t.root.name or r["node_name"] + else: + n.name = r["node_name"] or sub_t.root.name + if sub_t.root.dist is not None: + n.dist = sub_t.root.dist + for key, val in sub_t.root.props.items(): + if val is not None: + n.props[key] = val + n.children = sub_t.root.children + + # Replaced children, no point recursing through the old ones + return True + + for _ in t.traverse(strategy="levelorder", is_leaf_fn=is_leaf_fn): + # NB: We do all the work in the is_leaf_fn, so we can influence whether to recurse + pass + + return missing_inclusions + + +def graft_extract_ot_subtrees(opentree_t, inclusions): + """ + Extract the subtrees needed to satisfy a list of OneZoom inclusion labels + from the full OpenTree tree. + + ``inclusions`` is a list of inclusion-syntax labels (e.g. ``"Sub_ott1@"``, + ``"Renamed_ott~5@"``); each is parsed for its base OTT + (see ``decypher_inclusion_syntax``). ``opentree_t`` is walked and any node + whose OTT matches one of those base OTTs is detached and returned as a + standalone subtree. ``opentree_t`` is mutated in place — every extracted + subtree is removed from it, along with any ancestor left with no + descendants as a result (see the loop below). + + The extraction recurses into each detached subtree, so a requested OTT + that lies inside another requested subtree is still extracted (and its + outer subtree no longer contains it). The returned dict is keyed by the + *original* inclusion label, preserving any rebase / exclusion syntax so + callers can pass the result straight to ``graft_tree``. + + Inclusions whose base OTT does not appear in ``opentree_t`` are silently + absent from the result. + """ + # Organise inclusions by base_ott (as string) + start_otts = {} + for i in inclusions: + r = decypher_inclusion_syntax(i) + start_otts[str(r["base_ott"])] = r + + def prune_ot_subtrees(ot_t): + out_trees = {} + + def is_leaf_fn(n): + # No point checking leaves + if n.is_leaf: + return True + + # Does this node have a required OTT? If not, ignore it + node_ott = node_get_ott(n) + if node_ott is None or node_ott not in start_otts: + return n.is_leaf + r = start_otts[node_ott] + del start_otts[node_ott] + + # Prune this tree, extract any required subtrees from this subtree + parent = n.up + sub_t = n.detach() + out_trees.update(prune_ot_subtrees(sub_t)) # NB: node_ott now removed from start_otts, so won't loop + out_trees[r["orig_name"]] = sub_t + + # Detaching can leave (parent) with nothing below it -- every one of its + # descendants was requested separately, so the whole clade now lives in + # the bespoke tree. An emptied node reads as a leaf from here on and + # would be written out as though it were a species, so drop it, and any + # ancestor it empties in turn. Only nodes emptied by *us* are removed: + # a node that was already a tip is a real taxon and is left alone. + while parent is not None and parent.up is not None and not parent.children: + emptied, parent = parent, parent.up + emptied.detach() + return True + + for _ in ot_t.traverse(strategy="preorder", is_leaf_fn=is_leaf_fn): + # NB: We do all the work in the is_leaf_fn, so we can influence whether to recurse + pass + return out_trees + + return prune_ot_subtrees(opentree_t) + + +def present_in_tree(t, inclusion): + """ + Is a node matching (inclusion) present anywhere in (t)? + """ + r = decypher_inclusion_syntax(inclusion) + to_find = "ott" + str(r["base_ott"]) + for n in t.traverse(): + if n.name.endswith(to_find): + return n + return None + + +def decypher_inclusion_syntax(node_name): + """ + Parse inclusion syntax from node label + Parse a single OneZoom token from label name + """ + if not node_name or not node_name.endswith("@"): + return None + + result = dict( + orig_name=node_name, + ) + + match = OT_INCLUSION_SYNTAX_RE.match(node_name) + if not match: + # Has an @, but not ott syntax. Assume bespoke + result["node_name"] = node_name[:-1] + return result + + # split by minus signs + result["excluded_otts"] = (match.group(3) or "").split("-") + + # If present, the first number after '=' is the tree to extract. + first_number_after_equal = result["excluded_otts"].pop(0) + result["base_ott"] = first_number_after_equal or match.group(2) + + # Note that we don't append the ott in the name if it came after the '=' + result["node_name"] = match.group(1) + if not first_number_after_equal: + result["node_name"] += f" ott{result['base_ott']}" + return result + + +def remove_exclusions(t, exclusion_otts): + """ + Given (t), prune any (exclusion_otts) from tree + """ + orphan_ns = [] + + def is_leaf_fn(n): + node_ott = node_get_ott(n) + if node_ott is None or node_ott not in exclusion_otts: + return n.is_leaf + + orphan_ns.append(n.detach()) + return True + + if len(exclusion_otts) == 0: + return orphan_ns + exclusion_otts = set(str(x) for x in exclusion_otts) + for _ in t.traverse(strategy="preorder", is_leaf_fn=is_leaf_fn): + # NB: We do all the work in the is_leaf_fn, so we can influence whether to recurse + pass + return orphan_ns diff --git a/oz_tree_build/tree_build/step_jsnewick.py b/oz_tree_build/tree_build/step_jsnewick.py new file mode 100644 index 00000000..afffc74c --- /dev/null +++ b/oz_tree_build/tree_build/step_jsnewick.py @@ -0,0 +1,176 @@ +""" +Tree-to-string converters used by the JS frontend. + +OneZoom's frontend slurps a "bracket-only" newick that has had every +leaf name, branch length, comma, and semicolon stripped — only ``(`` +and ``)`` characters remain. Internal nodes are identified by their +character position in that string, and a cut-position map tells the +frontend where to split each internal node's two children. + +Historically these strings were built by reading a named-leaf newick +back off disk and string-munging it (see +``oz_tree_build.utilities.make_js_treefiles.tidy_newick`` and +``generate_binary_cut_position_map``). This module produces the same +output directly from an ete4 tree, with no on-disk round-trip. +""" + +from .step_tidy import POLYTOMY_PROP + + +def jsnewick_brief_newick(tree, polytomy_braces="()"): + """ + Return the bracket-only string for ``tree``. + + Equivalent to writing the tree as newick, dropping leaf names and + branch lengths, then stripping commas, semicolons, and newlines: + each internal node contributes a matching ``(`` and ``)``, leaves + contribute nothing. The ete4 tree built from ``((A,B),C);`` yields + ``"(())"``. + + ``polytomy_braces`` is a two-character string overriding the + brackets used for any *non-root* internal carrying the ``polytomy`` + prop — the marker ``tidy_resolve_polytomies`` leaves on an artificial + split. Pass e.g. ``"{}"`` to flag those nodes for the frontend. + """ + parts = [] + for node, action in _walk_internal(tree): + braces = polytomy_braces if (node.up is not None and node.props.get(POLYTOMY_PROP)) else "()" + parts.append(braces[0] if action == "open" else braces[1]) + return "".join(parts) + + +def jsnewick_cutpositionmap_binary(tree, threshold=10000): + """ + Cut-position map for an ete4 ``tree``. + + Keys are the tidy-string position of an internal node's ``)``; + the value is the position of the last character of that node's + first child — i.e. where the frontend should split the subtree + string into its two children. An internal whose first child is a + leaf records the parent's ``(`` position as the cut (the leaf + occupies an empty range immediately after). + + Only the root and any internal node whose own subtree contributes + more than ``threshold`` characters to the tidied string get an + entry; the recursion mirrors ``make_js_treefiles`` exactly. An + internal with two leaf children produces no entry (there is no + further split for the frontend to make). + + The tree is assumed to be binary; nodes with fewer than two + children are skipped and any third-or-later child of a polytomy + is ignored — callers should resolve polytomies first. + + Equivalent to ``make_js_treefiles.generate_binary_cut_position_map`` + """ + start_pos = {} + end_pos = {} + for pos, (node, action) in enumerate(_walk_internal(tree)): + if action == "open": + start_pos[id(node)] = pos + else: + end_pos[id(node)] = pos + + cut_map = {} + worklist = [tree] + while worklist: + node = worklist.pop(0) + children = list(node.children) + if len(children) < 2: + continue + c1, c2 = children[0], children[1] + + if c1.is_leaf and c2.is_leaf: + continue + + cut = start_pos[id(node)] if c1.is_leaf else end_pos[id(c1)] + cut_map[end_pos[id(node)]] = cut + + for child in (c1, c2): + if child.is_leaf: + continue + if end_pos[id(child)] - start_pos[id(child)] + 1 > threshold: + worklist.append(child) + + return cut_map + + +def jsnewick_cutpositionmap_polytomy(tree, threshold=10000): + """ + Cut-position map for an ete4 ``tree``. + + Keys are the tidy-string position of an internal node's ``)``; + the value is a flat ``[start1, end1, start2, end2]`` list describing + each of the node's two children: + - An internal child contributes its bracket range ``(start, end)``. + - A leaf child contributes an inverted range marking the empty + position the leaf occupies between siblings (``start > end``). + - An internal whose children are *both* leaves falls back to the + degenerate ``[start, start, end, end]`` form — using the + parent's own bracket positions — matching the original + algorithm's cut-point-not-found case. + + Only the root and any internal node whose own subtree contributes + more than ``threshold + 1`` characters to the tidied string get an + entry; the (off-by-one) threshold check mirrors + ``make_js_treefiles`` exactly. + + The tree is assumed to be binary; nodes with fewer than two + children are skipped and any third-or-later child of a polytomy + is ignored — callers should resolve polytomies first. + + Equivalent to ``make_js_treefiles.generate_polytomy_cut_position_map`` + """ + start_pos = {} + end_pos = {} + for pos, (node, action) in enumerate(_walk_internal(tree)): + if action == "open": + start_pos[id(node)] = pos + else: + end_pos[id(node)] = pos + + cut_map = {} + worklist = [tree] + while worklist: + node = worklist.pop(0) + children = list(node.children) + if len(children) < 2: + continue + c1, c2 = children[0], children[1] + s_n = start_pos[id(node)] + e_n = end_pos[id(node)] + + if c1.is_leaf and c2.is_leaf: + cut_map[e_n] = [s_n, s_n, e_n, e_n] + else: + pair1 = [s_n + 1, s_n] if c1.is_leaf else [start_pos[id(c1)], end_pos[id(c1)]] + pair2 = [e_n, e_n - 1] if c2.is_leaf else [start_pos[id(c2)], end_pos[id(c2)]] + cut_map[e_n] = pair1 + pair2 + + for child in (c1, c2): + if child.is_leaf: + continue + if end_pos[id(child)] - start_pos[id(child)] > threshold: + worklist.append(child) + + return cut_map + + +def _walk_internal(tree): + """ + Iterative DFS yielding ``(node, action)`` for every internal node + in the order tidy_newick would emit them. ``action`` is ``"open"`` + on first visit and ``"close"`` on return. Leaves contribute no + character to the tidied string and are skipped. + """ + stack = [(tree, False)] + while stack: + node, visited = stack.pop() + if node.is_leaf: + continue + if visited: + yield node, "close" + else: + yield node, "open" + stack.append((node, True)) + for child in reversed(list(node.children)): + stack.append((child, False)) diff --git a/oz_tree_build/tree_build/step_output.py b/oz_tree_build/tree_build/step_output.py new file mode 100644 index 00000000..1959113a --- /dev/null +++ b/oz_tree_build/tree_build/step_output.py @@ -0,0 +1,304 @@ +import csv +import json +import os.path +import struct + +from ..utilities.ete import node_name_without_ott +from .step_tidy import POLYTOMY_PROP + + +def output_add_prop_ids(tree): + """ + Annotate every internal node with the integer ids the OneZoom MySQL + schema uses for nested-set descendant queries. + + Sets four props on each internal node: + - ``id`` : 1-based preorder index among internal nodes. + - ``leaf_lft`` : preorder position of the leftmost leaf in the subtree. + - ``leaf_rgt`` : preorder position of the rightmost leaf in the subtree. + - ``node_rgt`` : id of the rightmost internal-node descendant + (equals ``id`` when every child is a leaf). + + Leaves are not annotated — they are implicitly numbered by their + preorder position. Ids and leaf positions both start at 1 to line up + with MySQL row numbering. + + Assumes the tree is ladderized *ascending* (smallest subtree first): + ``node_rgt`` is derived by walking postorder and trusting that the + last-visited child sits at the right of its parent, which only holds + when the rightmost child carries the largest subtree. A descending + tree where a leaf sits to the right of an internal sibling will get + the wrong ``node_rgt`` (the parent will look terminal). + """ + # allocate node numbers + internal_node_number = 0 + leaf_count = 1 + for node in tree.traverse("preorder"): + if node.is_leaf: + leaf_count += 1 + else: + # NB: increment first, since we use a 1-base numbering system, for mySQL row numbering + internal_node_number += 1 + node.props["id"] = internal_node_number + node.props["leaf_lft"] = leaf_count + + # postorder traversal to allocate rgt side of ranges + internal_leaf_count = 0 + prev_node = None + for node in tree.traverse("postorder"): + # find rightmost leaf by postorder iteration. + # For rightmost node, if previously visited node is a leaf, then (because we ladderize + # ascending) the rightmost node must be self (i.e. this is a terminal internal node). + # Otherwise it is the previously visted node + if node.is_leaf: + internal_leaf_count += 1 + else: + node.props["leaf_rgt"] = internal_leaf_count # should have counted all the internal leaves by now + if prev_node.is_leaf: + node.props["node_rgt"] = node.props["id"] # node_rgt == self + else: + # the node_rgt should be the same as the node_rgt of the previous node + node.props["node_rgt"] = prev_node.props["node_rgt"] + prev_node = node + + +def mysql_null(value): + """ + Map a missing value to the ``\\N`` marker MySQL's ``LOAD DATA`` reads as NULL + + Values are missing if either absent or None: read_taxon_map() produces a full + set of keys for every taxon, with None for the columns that were empty, so + ``dict.get(key, "\\N")`` is not enough on its own. + """ + return "\\N" if value is None else value + + +def output_mysqlexport(tree, out_dir): + """ + Write the three files needed to load the tree into the OneZoom MySQL + database: + + - ``ordered_leaves.csv`` : one row per leaf, in preorder. + - ``ordered_nodes.csv`` : one row per internal node, in preorder. + - ``import.sql`` : TRUNCATE + ``LOAD DATA LOCAL INFILE`` + script that loads both CSVs. + + Preconditions + ------------- + Every node must carry ``props["taxon"]``, a dict of taxon-derived + columns (``ott``, ``wikidata``, ``ncbi``, ...); keys that are absent + or None are written as ``\\N``. Every internal node must additionally carry + ``id`` / ``node_rgt`` / ``leaf_lft`` / ``leaf_rgt`` as produced by + `output_add_prop_ids`. + + Encoding conventions + -------------------- + - ``\\N`` is the marker for missing values (MySQL ``LOAD DATA`` + treats it as NULL). + - ``real_parent`` walks past randomly-resolved polytomies: any + ancestor carrying the ``polytomy`` prop is skipped so the column + points at the nearest biologically meaningful parent. The raw + ``parent`` column still references the immediate parent. + - A node that is itself a polytomy resolution (has the ``polytomy`` + prop) records its ``real_parent`` as the *negative* of the resolved + parent's id, flagging the relationship as artificial. + - The leaf ``name`` column has any trailing ``_ottNNN`` suffix + stripped (the OTT is carried separately in its own column). + - An internal node's ``date`` prop is exposed via the ``age`` column. + - The root has no parent, but the ``parent`` column is NOT NULL, so it + gets the placeholder ``-999`` for the import script to replace. Its + ``real_parent`` is the sentinel ``0``. + - Every leaf is required to have a parent; a parentless leaf raises + ``ValueError``. + """ + + with ( + open(os.path.join(out_dir, "ordered_leaves.csv"), "w+", encoding="utf-8") as leaf_file, + open(os.path.join(out_dir, "ordered_nodes.csv"), "w+", encoding="utf-8") as node_file, + ): + leaf_csv = csv.writer(leaf_file, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") + node_csv = csv.writer(node_file, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") + leaf_csv.writerow( + [ + "parent", + "real_parent", + "name", + "extinction_date", + "ott", + "wikidata", + "wikipedia_lang_flag", + "iucn", + "eol", + "raw_popularity", + "popularity", + "popularity_rank", + "price", + "ncbi", + "ifung", # NB: the DB's name for the taxonomy's "if" source + "worms", + "irmng", + "gbif", + "ipni", + ] + ) + node_csv.writerow( + [ + "parent", + "real_parent", + "node_rgt", + "leaf_lft", + "leaf_rgt", + "name", + "age", + "ott", + "wikidata", + "wikipedia_lang_flag", + "eol", + "rnk", # We avoid using 'rank' as it is a reserved word in mysql + "raw_popularity", + "popularity", + "ncbi", + "ifung", # NB: the DB's name for the taxonomy's "if" source + "worms", + "irmng", + "gbif", + "ipni", + "vern_synth", + ] + + [rit + str(i + 1) for rit in ("rep", "rtr", "rpd") for i in range(8)] + + ["iucn" + t for t in ("NE", "DD", "LC", "NT", "VU", "EN", "CR", "EW", "EX")] + ) + + for node in tree.traverse("preorder"): + # Find our real parent, ignoring randomly resolved polytomies + real_parent = node.parent + while real_parent and real_parent.props.get(POLYTOMY_PROP): + real_parent = real_parent.parent + + if not real_parent: + real_parent_id = 0 + elif node.props.get(POLYTOMY_PROP): + # real_parent is negative iff we're a polytomy + real_parent_id = -real_parent.props["id"] + else: + real_parent_id = real_parent.props["id"] + + if node.is_leaf: + if not node.parent: + raise ValueError(f"Leaf {node} has no parent") + leaf_csv.writerow( + [ + node.parent.props["id"], # "parent" + # TODO: negative real_parent ids if this is a polytomy + real_parent_id, + node_name_without_ott(node), + mysql_null(node.props.get("extinction_date")), + mysql_null(node.props["taxon"].get("ott")), + mysql_null(node.props["taxon"].get("wikidata")), + mysql_null(node.props["taxon"].get("wikipedia_lang_flag")), + mysql_null(node.props["taxon"].get("iucn")), + mysql_null(node.props["taxon"].get("eol")), + mysql_null(node.props["taxon"].get("raw_popularity")), + mysql_null(node.props.get("popularity")), + mysql_null(node.props.get("popularity_rank")), + "\\N", # "price" + mysql_null(node.props["taxon"].get("ncbi")), + mysql_null(node.props["taxon"].get("if")), # NB: "ifung" in the DB + mysql_null(node.props["taxon"].get("worms")), + mysql_null(node.props["taxon"].get("irmng")), + mysql_null(node.props["taxon"].get("gbif")), + mysql_null(node.props["taxon"].get("ipni")), + ] + ) + else: + node_csv.writerow( + [ + # NB: This has to be NOT NULL, but root doesn't have a parent. Bodge temporary value + node.parent.props["id"] if node.parent else "-999", # "parent" + real_parent_id, + node.props["node_rgt"], + node.props["leaf_lft"], + node.props["leaf_rgt"], + node_name_without_ott(node), + mysql_null(node.props.get("date")), # TODO: But only if it's not imputed + mysql_null(node.props["taxon"].get("ott")), + mysql_null(node.props["taxon"].get("wikidata")), + mysql_null(node.props["taxon"].get("wikipedia_lang_flag")), + mysql_null(node.props["taxon"].get("eol")), + mysql_null(node.props["taxon"].get("rank")), # NB: "rnk" in the DB + mysql_null(node.props["taxon"].get("raw_popularity")), + mysql_null(node.props.get("popularity")), + mysql_null(node.props["taxon"].get("ncbi")), + mysql_null(node.props["taxon"].get("if")), # NB: "ifung" in the DB + mysql_null(node.props["taxon"].get("worms")), + mysql_null(node.props["taxon"].get("irmng")), + mysql_null(node.props["taxon"].get("gbif")), + mysql_null(node.props["taxon"].get("ipni")), + "\\N", # "vern_synth" + ] + + ["\\N" for _ in ("rep", "rtr", "rpd") for _ in range(8)] + + ["\\N" for _ in ("NE", "DD", "LC", "NT", "VU", "EN", "CR", "EW", "EX")] + ) + + with open(os.path.join(out_dir, "import.sql"), "w", encoding="utf-8") as sql_f: + for csvfile in ("ordered_leaves.csv", "ordered_nodes.csv"): + table = os.path.splitext(csvfile)[0] + sql_f.writelines( + [ + f"TRUNCATE TABLE {table};\n" + f"LOAD DATA LOCAL INFILE '{csvfile}' REPLACE INTO TABLE `{table}` \n" + f" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' \n" + f" IGNORE 1 LINES ({open(os.path.join(out_dir,csvfile)).readline().rstrip()}) SET id = NULL;\n" + ] + ) + + +def output_jssource(tree, out_dir, file_name, data): + """ + Turn ``data`` dict into a js source file that defines it's keys as variables, with JSON encoded values + """ + with open(os.path.join(out_dir, file_name), "w") as f: + for k, v in data.items(): + f.write(f"var {k} = ") + json.dump(v, f) + f.write(";\n") + + +def output_proparray(tree, out_dir, prop_name): + """ + Given an ete4 tree and prop_name, write out 2 packed arrays to out_dir: + + (prop_name)_leaves_(pack format).dat + (prop_name)_nodes_(pack format).dat + + The ordering will match ordered_leaves/ordered_nodes + """ + PROP_FORMAT_TO_PACK = dict( + c8="c", # 8-bit chars + i8="b", # Signed 8-bit ints + u8="B", # Unsigned 8-bit ints + f16=".nwk`` files, one per OpenTree + subtree that lives outside the main OpenTree synthesis (the "orphan" + pieces). Each ``inclusions`` label is parsed for its base OTT + (see ``decypher_inclusion_syntax``); if a file named + ``.nwk`` exists, it is loaded and returned, keyed by the + *original* inclusion label so callers can pass the result straight to + ``graft_tree``. + + Any ``excluded_otts`` carried by the inclusion syntax are pruned from the + loaded subtree via ``remove_exclusions`` before it is returned. Orphan + files that don't correspond to any requested inclusion are skipped, and + inclusions with no matching orphan file are silently absent from the + result. + """ + # Organise inclusions by base_ott (as string) + start_otts = {} + for i in inclusions: + r = decypher_inclusion_syntax(i) + start_otts[str(r["base_ott"])] = r + + # Find orphan trees that match inclusion points + out_trees = {} + for orphan_path in glob.glob(os.path.join(orphan_dir, "*.nwk")): + node_ott = os.path.splitext(os.path.basename(orphan_path))[0] + if node_ott not in start_otts: + continue + r = start_otts[node_ott] + del start_otts[node_ott] + + sub_t = ete4.Tree(orphan_path, parser=NWK_READ_PARSER) + remove_exclusions(sub_t, r["excluded_otts"]) + out_trees[r["orig_name"]] = sub_t + return out_trees + + +def parse_bespoke_trees(bespoke_dir, base_name="Base.PHY"): + """ + Load the base tree and every hand-curated ("bespoke") subtree referenced + by ``token_to_file_map`` from ``bespoke_dir``. + + Returns a ``(base_t, bespoke_t)`` tuple: + - ``base_t`` is the tree parsed from ``/`` and + forms the trunk that everything else hangs off. + - ``bespoke_t`` is a dict keyed by ``"@"`` (matching the + inclusion syntax used inside ``base_t``) mapping to the parsed + subtree. For each token, the entry in ``token_to_file_map`` may + override the subtree's root name (``taxon``) and the length of the + edge connecting it to its parent (``edge_length``). + + The result is suitable for passing straight to ``graft_tree`` as the + ``additional_trees`` argument. Tokens whose file is missing from + ``bespoke_dir`` are logged as errors and omitted from ``bespoke_t`` + rather than raising. + """ + base_t = ete4.Tree(os.path.join(bespoke_dir, base_name), parser=NWK_READ_PARSER) + bespoke_t = {} + for key, x in token_to_file_map.items(): + key = key + "@" + sub_path = os.path.join(bespoke_dir, x["file"]) + if not os.path.exists(sub_path): + logger.error(f"Sub-tree {x['file']} referenced in token_to_oz_tree_file_mapping missing") + continue + bespoke_t[key] = ete4.Tree(sub_path, parser=NWK_READ_PARSER) + if x.get("taxon") is not None: + bespoke_t[key].root.name = x["taxon"] + if x.get("edge_length") is not None: + bespoke_t[key].root.dist = x["edge_length"] + return base_t, bespoke_t diff --git a/oz_tree_build/tree_build/step_popularity.py b/oz_tree_build/tree_build/step_popularity.py new file mode 100644 index 00000000..c4fc35ea --- /dev/null +++ b/oz_tree_build/tree_build/step_popularity.py @@ -0,0 +1,270 @@ +import collections +import logging +from math import log + +from ..utilities.ete import node_get_ott + +logger = logging.getLogger(__name__) + + +def popularity_add_prop( + tree, + exclude_taxa=None, +): + """ + Compute a phylogenetic popularity score for every node and store it on + ``node.props["popularity"]`` (rounded to 2 dp). + + The raw per-node popularity is taken from ``node.props["taxon"]["raw_popularity"]`` + (see ``sum_popularity_over_tree``). Each node's score combines the raw popularity + of its ancestors and descendants — so a node inherits some weight from its + relatives, not just from its own ``raw_popularity``. See ``popularity_function`` + for the exact combination. + + ``exclude_taxa`` is forwarded to ``sum_popularity_over_tree`` and lets the + caller zero out the raw popularity of named nodes before summation (e.g. + excluding Dinosauria so its popularity is not credited to birds). + + Wikidata Qids appearing on more than one node are counted and summarised in a + single warning; ``sum_popularity_over_tree`` splits their raw popularity + between the nodes sharing them, so this is a data-quality note rather than a + miscalculation. Set debug logging to list the individual nodes. + + Must be run before monotomies / unary nodes are removed: those nodes often + carry useful popularity that needs to percolate to their relatives first. + Nodes synthesised by polytomy resolution are handled the same way as any + other node. + """ + sum_popularity_over_tree(tree, exclude_taxa=exclude_taxa) + + # now apply the popularity function + Qids = set() + duplicate_qid_nodes = 0 + for node in tree.traverse(strategy="preorder"): + Q = node.props["taxon"].get("wikidata") + if Q is not None: + if Q in Qids: + duplicate_qid_nodes += 1 + logger.debug( + f"duplicate wikidata Qid (Q{Q}) on OTT {node_get_ott(node)} - " + f"its raw popularity is shared with the other nodes using this Qid" + ) + else: + Qids.add(Q) + pop = popularity_function( + node.props["ancestors_popsum"], + node.props["descendants_popsum"], + node.props["n_ancestors"], + node.props["n_descendants"], + ) + + # Round to 2 decimal places + node.props["popularity"] = round(pop, 2) + + if duplicate_qid_nodes: + logger.warning( + f"{duplicate_qid_nodes} nodes share a wikidata Qid with an earlier node, usually " + f"because OTT holds the same taxon more than once. Their raw popularity has been " + f"divided between the nodes sharing each Qid; enable debug logging to list them." + ) + + +def popularity_add_rank(tree): + """ + Rank every leaf by its ``node.props["popularity"]`` and write the position + to ``node.props["popularity_rank"]``. Rank 1 is the most popular leaf. + + Ties use standard competition ranking ("1224"): tied leaves share the lower + rank and the next distinct value skips ahead by the size of the tie. For + example, popularities [100, 50, 50, 50, 1] produce ranks [1, 2, 2, 2, 5]. + + Internal nodes are neither ranked nor used as tie-breakers, and their + ``popularity`` prop (if any) is ignored. + + Should be run after invalid tips and unary nodes have been removed, so the + ranking reflects the final set of leaves. + """ + leaf_popularities = collections.defaultdict(int) + for node in tree.traverse(): + if node.is_leaf: + leaf_popularities[node.props.get("popularity")] += 1 + cumsum = 1 + if None in leaf_popularities: + return + for k in sorted(leaf_popularities.keys(), reverse=True): + add_next = leaf_popularities[k] + leaf_popularities[k] = cumsum + cumsum += add_next + for node in tree.traverse(): + if node.is_leaf: + node.props["popularity_rank"] = leaf_popularities[node.props.get("popularity")] + + +def popularity_function( + sum_of_all_ancestor_popularities, + sum_of_all_descendant_popularities, + number_of_ancestors, + number_of_descendants, +): + """ + a) Dividing by number_of_ancestors+number_of_descendants would mean averaging + popularity over all nodes, which would bias against taxa which have many + unvisited/unpopular children + b) Alternatively, dividing by a constant is equivalent to summing popularity over + all nodes, which biases towards taxa with many fine taxonomic divisions + We do something between the two by dividing by the log of the number of nodes. + """ + if ( + (sum_of_all_ancestor_popularities is None) + or (sum_of_all_descendant_popularities is None) + or (number_of_ancestors is None) + or (number_of_descendants is None) + ): + return None + elif number_of_ancestors + number_of_descendants == 1: + # Avoid a divide by zero error if this adds up to 1 + # Though the need for this makes me think that the log calculation + # may not be mathematically sound + return sum_of_all_ancestor_popularities + sum_of_all_descendant_popularities + else: + return (sum_of_all_ancestor_popularities + sum_of_all_descendant_popularities) / log( + number_of_ancestors + number_of_descendants + ) + + +def popularity_add_info( + tree, + focal_labels, +): + """ + Print debug info for ete4 nodes whose names appear in ``focal_labels``: + each node's own popularity, its descendant popularity sum, a sample of + its leaves, and the chain of ancestors with non-zero popularity. + """ + remaining = set(focal_labels) + for node in tree.traverse(): + if not remaining: + break + if node.name not in remaining: + continue + remaining.discard(node.name) + print( + "{}: own pop = {} (Q{}) descendant pop sum = {}".format( + node.name, + node.props["pop"], + node.props["taxon"].get("wikidata", " absent"), + node.props["descendants_popsum"], + ) + ) + for t, tip in enumerate(node.leaves()): + print( + "Tip {} = {}: own_pop = {}, Qid = Q{}".format( + t, + tip.name, + tip.props.get("pop"), + tip.props["taxon"].get("wikidata", " absent"), + ) + ) + if t > 100: + print("More tips exist, but have been omitted") + break + ancestor = node.up + while ancestor: + if ancestor.props.get("pop"): + print(f"Ancestors: {ancestor.name} = {ancestor.props['pop']:.2f}") + ancestor = ancestor.up + for missing in remaining: + logger.warning(f"Problem reporting on focal taxon '{missing}': not found") + + +def sum_popularity_over_tree(tree, exclude_taxa=None): + """ + Sum raw popularity values up and down an ete4 phylogenetic tree. + + Each node's raw popularity is taken from ``node.props["taxon"]["raw_popularity"]`` + It is copied onto ``node.props["pop"]`` and then summed across ancestors and + descendants. + + Where several nodes share a Wikidata Qid they each hold the same raw + popularity, so it is divided between them before summing -- see the comment + on ``qid_counts`` below. + + We might want to exclude some names from the popularity metric (e.g. exclude + archosaurs, to ensure birds don't gather popularity intended for dinosaurs). + This is done by passing an array such as + ``['Dinosauria_ott90215', 'Archosauria_ott335588']`` as the ``exclude_taxa`` argument + -- the names are matched against ``node.name``. + + After running, the following props are set on every node: + pop raw popularity for this node + has_pop whether raw popularity was available + descendants_popsum popularity summed over all descendants + n_descendants number of descendants + ancestors_popsum popularity summed over all ancestors + n_ancestors number of ancestors + n_pop_ancestors number of ancestors with a popularity measure + """ + exclude_taxa = set(exclude_taxa or []) + + logger.info("Tree read for phylogenetic popularity calc") + + def has_own_pop(node): + return node.name not in exclude_taxa and node.props["taxon"].get("raw_popularity") is not None + + # A Qid on several nodes is nearly always one taxon held twice by OTT (usually + # split across source taxonomies), and every copy carries the *same* full + # raw_popularity, so summing them unaltered counts that popularity once per + # copy. Share it out evenly instead: that keeps the tree's total popularity + # right without having to pick which copy is the "real" one -- which we can't + # do sensibly anyway, as the copies are frequently in quite distant clades. + # Only nodes that actually contribute popularity are counted, so an excluded + # or unpopulated duplicate doesn't dilute its twin. + qid_counts = collections.Counter( + node.props["taxon"]["wikidata"] + for node in tree.traverse(strategy="preorder") + if has_own_pop(node) and node.props["taxon"].get("wikidata") is not None + ) + + # put popularity into the "pop" attribute + for node in tree.traverse(strategy="preorder"): + if not has_own_pop(node): + node.props["pop"] = 0 + node.props["has_pop"] = False + else: + Q = node.props["taxon"].get("wikidata") + raw_popularity = node.props["taxon"]["raw_popularity"] + node.props["pop"] = raw_popularity if Q is None else raw_popularity / qid_counts[Q] + node.props["has_pop"] = True + + # go up the tree from the tips, summing up the popularity indices beneath and + # adding the number of descendants + for node in tree.traverse(strategy="postorder"): + if node.is_leaf: + node.props["descendants_popsum"] = 0 + node.props["n_descendants"] = 0 + parent = node.up + if parent is None: + continue + parent.props["n_descendants"] = parent.props.get("n_descendants", 0) + 1 + node.props["n_descendants"] + parent.props["descendants_popsum"] = ( + parent.props.get("descendants_popsum", 0) + node.props["pop"] + node.props["descendants_popsum"] + ) + + # go down the tree from the root, summing up the popularity indices above, + # and summing up numbers of nodes + for node in tree.traverse(strategy="preorder"): + parent = node.up + if parent is None: + # this is the root. + node.props["n_ancestors"] = 0 + node.props["n_pop_ancestors"] = 0 + node.props["ancestors_popsum"] = 0.0 + else: + node.props["n_ancestors"] = parent.props["n_ancestors"] + 1 + node.props["ancestors_popsum"] = parent.props["ancestors_popsum"] + node.props["pop"] + if node.props.get("has_pop"): + node.props["n_pop_ancestors"] = parent.props["n_pop_ancestors"] + 1 + else: + node.props["n_pop_ancestors"] = parent.props["n_pop_ancestors"] + + return tree diff --git a/oz_tree_build/tree_build/step_taxon.py b/oz_tree_build/tree_build/step_taxon.py new file mode 100644 index 00000000..afd58718 --- /dev/null +++ b/oz_tree_build/tree_build/step_taxon.py @@ -0,0 +1,16 @@ +from ..utilities.ete import node_get_ott + + +def taxon_add_prop( + tree, + taxon_map, +): + """ + Add references to relevant lines in taxon_map to tree nodes + + We should also check that there are not multiple uses of the same Qid + (https://github.com/OneZoom/OZtree/issues/132) + """ + for n in tree.traverse(): + node_ott = node_get_ott(n) + n.props["taxon"] = taxon_map.get(int(node_ott), {}) if node_ott else {} diff --git a/oz_tree_build/tree_build/step_tidy.py b/oz_tree_build/tree_build/step_tidy.py new file mode 100644 index 00000000..50174685 --- /dev/null +++ b/oz_tree_build/tree_build/step_tidy.py @@ -0,0 +1,131 @@ +import re + +# Prop marking a node as an artificial split inserted to break up a polytomy. +# Its value says *how* the topology was chosen, since the two stages that +# resolve polytomies do so very differently. Any node carrying the prop is +# artificial, so consumers that only care about that can test it for truth. +POLYTOMY_PROP = "polytomy" + +# ete4's resolve_polytomy: deterministic, and not a sample of anything -- it +# pairs children off in whatever order they happen to be in, so a polytomy of +# n children always becomes the same left-nested comb. +POLYTOMY_COMB = "comb" + +# dated_complete_tree.tree_fixing.fix_polytomy: draws uniformly at random from +# the possible topologies, using a seeded rng. +POLYTOMY_RANDOM = "random" + +# Name fix_polytomy gives the nodes it inserts, and the only trace of them that +# survives into the newick date_tree writes out +OT_POLYTOMY_NAME = "mrcapoly" + +# Names belonging to synthetic nodes that stand in for an ancestor rather than +# naming a taxon: OpenTree's own MRCA labels, plus those dated_complete_tree +# gives the nodes it inserts resolving polyphyly ("mrcaimp") and polytomies +# ("mrcapoly"). None of them is ever a taxon in its own right. +SYNTHETIC_NAME_RE = re.compile(r"^mrca(ott\d+ott\d+|imp|poly)$") + + +def tidy_resolve_polytomies(tree, kind=POLYTOMY_COMB): + """ + Resolve any polytomies in ``tree`` via ete4's ``resolve_polytomy``, marking + each artificially inserted node with ``POLYTOMY_PROP`` set to ``kind``. + + ``resolve_polytomy`` leaves inserted nodes with ``dist == 0``, but branch + lengths are regenerated from dates later in the pipeline, so the resolution + has to be recorded as a prop to survive that. + + Return number of nodes inserted. + """ + pre_existing = {id(n) for n in tree.traverse()} + tree.resolve_polytomy() + + count = 0 + for node in tree.traverse(): + if id(node) not in pre_existing: + node.props[POLYTOMY_PROP] = kind + count += 1 + return count + + +def tidy_mark_resolved_polytomies(tree, kind=POLYTOMY_RANDOM, name=OT_POLYTOMY_NAME): + """ + Mark nodes in an already-resolved ``tree`` with ``POLYTOMY_PROP`` set to + ``kind``, so they match those ``tidy_resolve_polytomies`` marks itself. + + OpenTree subtrees arrive polytomy-resolved by + ``dated_complete_tree.tree_fixing.fix_polytomy``, which identifies the nodes + it inserts by giving them the name ``mrcapoly``. That name is all we have to + go on: ``date_tree.nwk_write`` emits only the ``date`` prop, and the tree is + written before ``compute_branch_lengths`` runs, so the nodes arrive with no + branch length either. + + Return number of nodes marked. + """ + count = 0 + for node in tree.traverse(): + if node.name == name: + node.props[POLYTOMY_PROP] = kind + count += 1 + return count + + +def tidy_prune_synthetic_leaves(tree, name_re=SYNTHETIC_NAME_RE): + """ + Drop childless synthetic nodes, repeating until none are left. + + ``graft_extract_ot_subtrees`` detaches each requested subtree from the + OpenTree tree in place and does not tidy up the ancestors that empties. An + emptied node has no children, so it reads as a leaf from then on and is + written out as though it were a species -- keeping the date it had as an + internal node, which is what makes ``date_labelling`` complain that a leaf + has a non-zero date. + + Removing one can empty its parent in turn (an OT MRCA node above a + ``mrcaimp`` node above two extracted subtrees, say), hence the fixed point. + Nodes that were *already* childless in the input are debris for the same + reason and go the same way. + + Only synthetic names are pruned. A named taxon left childless is left alone + deliberately: it would be a real taxon losing its whole subtree, which is + worth noticing rather than quietly deleting. + + Return number of nodes removed. + """ + removed = 0 + while True: + emptied = [n for n in tree.traverse() if n.is_leaf and n.up is not None and name_re.match(n.name or "")] + if not emptied: + return removed + for node in emptied: + node.detach() + removed += len(emptied) + + +def tidy_infill_dates_bottomup(tree): + """ + Working bottom-upwards, fill in missing date properties based on branch lengths. + """ + for node in tree.traverse(strategy="postorder"): + if node.is_leaf: + node.props["date"] = 0 + else: + for c in node.children: + if c.props.get("date") is not None and c.dist is not None: + c_date = c.props["date"] + c.dist + if node.props.get("date") is None or node.props["date"] < c_date: + node.props["date"] = c_date + + +def tidy_clear_conflicting_dates_topdown(parent, mrad=None): + """ + Work through tree, removing dates older than their most recent ancestor. + """ + if parent.props.get("date") is not None: + if mrad is not None and (parent.props["date"] - mrad) > 1e-5: + # date is greater than mrad, this shouldn't happen + parent.props["date"] = None + else: + mrad = parent.props["date"] + for c in parent.children: + tidy_clear_conflicting_dates_topdown(c, mrad) diff --git a/oz_tree_build/tree_build/step_treeprop.py b/oz_tree_build/tree_build/step_treeprop.py new file mode 100644 index 00000000..70962ed6 --- /dev/null +++ b/oz_tree_build/tree_build/step_treeprop.py @@ -0,0 +1,98 @@ +import logging + +logger = logging.getLogger(__name__) + + +# Sourced from https://stratigraphy.org/supplementary#data +# fmt: off +GEOLOGICAL_PERIODS = [ + {"eon": "Unknown","era": "Unknown","period": "Unknown","epoch": "Unknown","short_text": "Unknown","long_text": "Unknown","color": "#999999","mya_start": -1e9,"number": 0}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Anthropocene","short_text": "Anthropocene","long_text": "Anthropocene mass extinction event","color": "#1A1A1A","mya_start": 0.000246,"number": 1}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Holocene","short_text": "Holocene Epoch","long_text": "Holocene Epoch, Quaternary Period","color": "#74746C","mya_start": 0.0117,"number": 2}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Quaternary","epoch": "Pleistocene","short_text": "Pleistocene Epoch","long_text": "Pleistocene Epoch, Quaternary Period","color": "#818179","mya_start": 2.58,"number": 3}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Neogene","epoch": "Pliocene","short_text": "Neogene Period","long_text": "Pliocene Epoch, Neogene Period","color": "#9B7B4D","mya_start": 5.333,"number": 4}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Neogene","epoch": "Miocene","short_text": "Neogene Period","long_text": "Miocene Epoch, Neogene Period","color": "#A98555","mya_start": 23.04,"number": 5}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Oligocene","short_text": "Paleogene Period","long_text": "Oligocene Epoch, Paleogene Period","color": "#826333","mya_start": 33.9,"number": 6}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Eocene","short_text": "Paleogene Period","long_text": "Eocene Epoch, Paleogene Period","color": "#8C6C3D","mya_start": 56,"number": 7}, # noqa: E501 + {"eon": "Phanerozoic","era": "Cenozoic","period": "Paleogene","epoch": "Paleocene","short_text": "Paleogene Period","long_text": "Paleocene Epoch, Paleogene Period","color": "#967447","mya_start": 66,"number": 8}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Upper","short_text": "Cretaceous–Paleogene extinction","long_text": "Cretaceous–Paleogene extinction","color": "#1A1A1A","mya_start": 65.9999,"number": 9}, # noqa: E501 RUF001 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Upper","short_text": "Cretaceous Period","long_text": "(Upper) Cretaceous Period","color": "#657347","mya_start": 100.5,"number": 10}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Cretaceous","epoch": "Lower","short_text": "Cretaceous Period","long_text": "(Lower) Cretaceous Period","color": "#6C7A4D","mya_start": 143.1,"number": 11}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Upper","short_text": "Jurassic Period","long_text": "(Upper) Jurassic Period","color": "#385536","mya_start": 161.5,"number": 12}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Middle","short_text": "Jurassic Period","long_text": "(Middle) Jurassic Period","color": "#3E5B3A","mya_start": 174.7,"number": 13}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Jurassic","epoch": "Lower","short_text": "Jurassic Period","long_text": "(Lower) Jurassic Period","color": "#45613F","mya_start": 201.4,"number": 14}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Upper","short_text": "Triassic–Jurassic extinction event","long_text": "Triassic–Jurassic extinction event","color": "#1A1A1A","mya_start": 201.3,"number": 15}, # noqa: E501 RUF001 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Upper","short_text": "Triassic Period","long_text": "(Upper) Triassic Period","color": "#704329","mya_start": 237,"number": 16}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Middle","short_text": "Triassic Period","long_text": "(Middle) Triassic Period","color": "#7A4A2B","mya_start": 246.7,"number": 17}, # noqa: E501 + {"eon": "Phanerozoic","era": "Mesozoic","period": "Triassic","epoch": "Lower","short_text": "Triassic Period","long_text": "(Lower) Triassic Period","color": "#7A4A2B","mya_start": 251.902,"number": 18}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Lopingian","short_text": "Permian–Triassic extinction event","long_text": "Permian–Triassic extinction event \"Great dying\"","color": "#1A1A1A","mya_start": 252,"number": 19}, # noqa: E501 RUF001 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Lopingian","short_text": "Permian Period","long_text": "Lopingian Epoch, Permian Period","color": "#62572F","mya_start": 259.51,"number": 20}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Guadalupian","short_text": "Permian Period","long_text": "Guadalupian Epoch, Permian Period","color": "#6A5D35","mya_start": 274.4,"number": 21}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Permian","epoch": "Cisuralian","short_text": "Permian Period","long_text": "Cisuralian Epoch, Permian Period","color": "#71643A","mya_start": 298.9,"number": 22}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Carboniferous","epoch": "Pennsylvanian","short_text": "Carboniferous Period","long_text": "Pennsylvanian Epoch, Carboniferous Period","color": "#294A2B","mya_start": 323.4,"number": 23}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Carboniferous","epoch": "Mississippian","short_text": "Carboniferous Period","long_text": "Mississippian Epoch, Carboniferous Period","color": "#2F4F2F","mya_start": 358.86,"number": 24}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Upper","short_text": "Late Devonian mass extinction","long_text": "Late Devonian mass extinction","color": "#1A1A1A","mya_start": 372,"number": 25}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Upper","short_text": "Devonian Period","long_text": "(Upper Epoch, Devonian Period","color": "#62632B","mya_start": 382.31,"number": 26}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Middle","short_text": "Devonian Period","long_text": "(Middle) Devonian Period","color": "#6B6B2F","mya_start": 393.47,"number": 27}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Devonian","epoch": "Lower","short_text": "Devonian Period","long_text": "(Lower) Devonian Period","color": "#727333","mya_start": 419.62,"number": 28}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Pridoli","short_text": "Silurian Period","long_text": "Pridoli Epoch, Silurian Period","color": "#536534","mya_start": 422.7,"number": 29}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Ludlow","short_text": "Silurian Period","long_text": "Ludlow Epoch, Silurian Period","color": "#5A6A3A","mya_start": 426.7,"number": 30}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Wenlock","short_text": "Silurian Period","long_text": "Wenlock Epoch, Silurian Period","color": "#60713D","mya_start": 432.9,"number": 31}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Silurian","epoch": "Llandovery","short_text": "Silurian Period","long_text": "Llandovery Epoch, Silurian Period","color": "#566F3A","mya_start": 443.1,"number": 32}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Upper","short_text": "Late Ordovician mass extinction","long_text": "Late Ordovician mass extinction","color": "#1A1A1A","mya_start": 445,"number": 33}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Upper","short_text": "Ordovician Period","long_text": "(Upper) Ordovician Period","color": "#426846","mya_start": 458.2,"number": 34}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Middle","short_text": "Ordovician Period","long_text": "(Middle) Ordovician Period","color": "#486B4A","mya_start": 471.3,"number": 35}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Ordovician","epoch": "Lower","short_text": "Ordovician Period","long_text": "(Lower) Ordovician Period","color": "#4E704C","mya_start": 486.85,"number": 36}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Furongian","short_text": "Cambrian Period","long_text": "Furongian Epoch, Cambrian Period","color": "#28594D","mya_start": 497,"number": 37}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Miaolingian","short_text": "Cambrian Period","long_text": "Miaolingian Epoch, Cambrian Period","color": "#2E5D50","mya_start": 506.5,"number": 38}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Series 2","short_text": "Cambrian Period","long_text": "Series 2 Epoch, Cambrian Period","color": "#336353","mya_start": 521,"number": 39}, # noqa: E501 + {"eon": "Phanerozoic","era": "Paleozoic","period": "Cambrian","epoch": "Terreneuvian","short_text": "Cambrian Period","long_text": "Terreneuvian Epoch, Cambrian Period","color": "#2A6255","mya_start": 538.8,"number": 40}, # noqa: E501 + {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Ediacaran","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Ediacaran Period, Neo-proterozoic Era","color": "#35464F","mya_start": 635,"number": 41}, # noqa: E501 + {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Cryogenian","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Cryogenian Period, Neo-proterozoic Era","color": "#3B4A52","mya_start": 720,"number": 42}, # noqa: E501 + {"eon": "Proterozoic","era": "Neo-proterozoic","period": "Tonian","epoch": "-","short_text": "Neo-proterozoic Era","long_text": "Tonian Period, Neo-proterozoic Era","color": "#40505A","mya_start": 1000,"number": 43}, # noqa: E501 + {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Stenian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Stenian Period, Meso-proterozoic Era","color": "#3A4D58","mya_start": 1200,"number": 44}, # noqa: E501 + {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Ectasian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Ectasian Period, Meso-proterozoic Era","color": "#45525A","mya_start": 1400,"number": 45}, # noqa: E501 + {"eon": "Proterozoic","era": "Meso-proterozoic","period": "Calymmian","epoch": "-","short_text": "Meso-proterozoic Era","long_text": "Calymmian Period, Meso-proterozoic Era","color": "#394953","mya_start": 1600,"number": 46}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Statherian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Statherian Period, Paleo-proterozoic Era","color": "#46535A","mya_start": 1800,"number": 47}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Orosirian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Orosirian Period, Paleo-proterozoic Era","color": "#33444D","mya_start": 2050,"number": 48}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Rhyacian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Rhyacian Period, Paleo-proterozoic Era","color": "#3E4B55","mya_start": 2300,"number": 49}, # noqa: E501 + {"eon": "Proterozoic","era": "Paleo-proterozoic","period": "Siderian","epoch": "-","short_text": "Paleo-proterozoic Era","long_text": "Siderian Period, Paleo-proterozoic Era","color": "#2F414A","mya_start": 2500,"number": 50}, # noqa: E501 + {"eon": "Archean","era": "Neo-Archean","period": "-","epoch": "-","short_text": "Neo-Archean Era","long_text": "Neo-Archean Era","color": "#302D2A","mya_start": 2800,"number": 51}, # noqa: E501 + {"eon": "Archean","era": "Meso-Archean","period": "-","epoch": "-","short_text": "Meso-Archean Era","long_text": "Meso-Archean Era","color": "#35312D","mya_start": 3200,"number": 52}, # noqa: E501 + {"eon": "Archean","era": "Paleo-Archean","period": "-","epoch": "-","short_text": "Paleo-Archean Era","long_text": "Paleo-Archean Era","color": "#2B2927","mya_start": 3600,"number": 53}, # noqa: E501 + {"eon": "Archean","era": "Eo-Archean","period": "-","epoch": "-","short_text": "Eo-Archean Era","long_text": "Eo-Archean Era","color": "#272624","mya_start": 4031,"number": 54}, # noqa: E501 + {"eon": "Hadean","era": "-","period": "-","epoch": "-","short_text": "Hadean Eon","long_text": "Hadean Eon","color": "#1A1A1A","mya_start": 4567,"number": 55}, # noqa: E501 +] +# fmt: on + + +def treeprop_geological(tree): + """ + Given an ete4 tree object, add a "geological" prop to each node, + representing a 1-based period index. + + Assumes the tree already has a "date" prop representing an absolute age in Mya. + + Return name of prop just added. + """ + # Turn array into (mya, idx) pairs + lookup = [(p["mya_start"], idx) for idx, p in enumerate(GEOLOGICAL_PERIODS)] + + for node in tree.traverse("preorder"): + n_age = node.props.get("date") + if n_age is None: + logger.warning(f"Node {node.name} has no date property") + node.props["geological"] = 0 + else: + for mya_start, idx in lookup: # noqa: B007 # idx is used outside the lookup, not inside + if n_age <= mya_start: + break + else: + # Fell off end + idx = 0 + node.props["geological"] = idx + + prop_format = tree.root.props.setdefault("prop_format", {}) + prop_format["geological"] = "u8" + + return "geological" diff --git a/oz_tree_build/tree_build/token_to_oz_tree_file_mapping.py b/oz_tree_build/tree_build/token_to_oz_tree_file_mapping.py index f5d53c98..209d8f18 100644 --- a/oz_tree_build/tree_build/token_to_oz_tree_file_mapping.py +++ b/oz_tree_build/tree_build/token_to_oz_tree_file_mapping.py @@ -30,67 +30,15 @@ "GNATHOSTOMATA": {"file": "BonyFishOpenTree.PHY", "edge_length": 65, "taxon": None}, # for fewer species but with dates, try deepfin2, with Concestor 20 @ ~ 430Ma # tree.substitute('GNATHOSTOMATA@', 'BespokeTree/include_files/Deepfin2.phy', 37.6) + # Species-level chondrichthyan tree, crown at 375.511333. 460 - 375.511333 = 84.488667 keeps + # Gnathostomata at 460Mya, agreeing with the Euteleostomi side of BonyFishOpenTree.PHY. + # It is self-contained, so the old Renz2013 / Inoue2010 / Aschliman2012 / Naylor2012* split + # (and their HOLOCEPHALI@, BATOIDEA@, SELACHII@, ... tokens) are no longer needed. "CHONDRICHTHYES": { - "file": "Chondrichthyes_Renz2013.phy", - "edge_length": 40, + "file": "Chondrichthyes_Stein2018.PHY", + "edge_length": 84.488667, "taxon": None, }, - "HOLOCEPHALI": { - "file": "Holocephali_Inoue2010.PHY", - "edge_length": 250, - "taxon": None, - }, - "BATOIDEA": { - "file": "Batoids_Aschliman2012.PHY", - "edge_length": 100, - "taxon": None, - }, - # sharks are problematic in OToL v3 & 4, hence lots of files included here - "SELACHII": { - "file": "Naylor2012Selachimorpha.PHY", - "edge_length": 75, - "taxon": None, - }, - "DALATIIDAE": { - "file": "Naylor2012Dalatiidae.PHY", - "edge_length": 116.1, - "taxon": None, - }, - "SOMNIOSIDAEOXYNOTIDAE": { - "file": "Naylor2012Somniosidae_Oxynotidae.PHY", - "edge_length": 110.51, - "taxon": None, - }, - "ETMOPTERIDAE": { - "file": "Naylor2012Etmopteridae.phy", - "edge_length": 110.51, - "taxon": None, - }, - "SQUATINIDAE": { - "file": "Naylor2012Squatinidae.phy", - "edge_length": 147.59, - "taxon": None, - }, - "PRISTIOPHORIDAE": { - "file": "Naylor2012Pristiophoridae.phy", - "edge_length": 147.59, - "taxon": None, - }, - "SCYLIORHINIDAE3": { - "file": "Naylor2012Scyliorhinidae3.PHY", - "edge_length": 170, - "taxon": None, - }, - "SCYLIORHINIDAE2": { - "file": "Naylor2012Scyliorhinidae2.PHY", - "edge_length": 134.467193, - "taxon": None, - }, - "CARCHARHINICAE_MINUS": { - "file": "Naylor2012Carcharhinicae_minus.PHY", - "edge_length": 134.467193, - "taxon": "Most_Carcharhinicae_", - }, # Choanoflagellates: http://www.pnas.org/content/105/43/16641.short ## NB: to use the original deepfin tree, substitute these text strings back in instead ## # tree.substitute('TETRAPODA@', '(Xenopus_tropicalis:335.4,(Monodelphis_domestica:129,(Mus_musculus:71.12,Homo_sapiens:71.12):57.88):206.4)Tetrapodomorpha:46.5'); # noqa E501 diff --git a/oz_tree_build/tree_build/tree_build.py b/oz_tree_build/tree_build/tree_build.py new file mode 100644 index 00000000..0caa2b81 --- /dev/null +++ b/oz_tree_build/tree_build/tree_build.py @@ -0,0 +1,195 @@ +""" +Splice together a set of trees using OZ inclusion syntax +""" + +import argparse +import json +import logging + +from dated_complete_tree import tree_dating, tree_fixing + +from ..date_tree import date_tree +from ..taxon_mapping_and_popularity.taxon_map import read_taxon_map +from ..utilities.debug_util import parse_args_and_add_logging_switch +from .step_graft import graft_extract_ot_subtrees, graft_tree +from .step_jsnewick import jsnewick_brief_newick, jsnewick_cutpositionmap_binary, jsnewick_cutpositionmap_polytomy +from .step_output import ( + output_add_prop_ids, + output_jssource, + output_mysqlexport, + output_proparray, +) +from .step_parse import parse_bespoke_trees, parse_ot_orphans +from .step_popularity import popularity_add_prop, popularity_add_rank +from .step_taxon import taxon_add_prop +from .step_tidy import ( + tidy_clear_conflicting_dates_topdown, + tidy_infill_dates_bottomup, + tidy_mark_resolved_polytomies, + tidy_prune_synthetic_leaves, + tidy_resolve_polytomies, +) +from .step_treeprop import ( + treeprop_geological, +) + +logger = logging.getLogger(__name__) + +ROOT_DATE_MYA = 4000 # Last Universal Common Ancestor estimate + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--bespoke_dir", + help=("Directory containing bespoke trees, including Base.PHY, the root"), + ) + parser.add_argument( + "--orphan_dir", + help=("Directory containing orphan OpenTree subtrees"), + ) + parser.add_argument( + "--opentree", + help=("Newick tree with OpenTree"), + ) + parser.add_argument( + "--taxon_map", + default="data/taxon_map.csv", + help=("Taxon map CSV as generated by taxon_mapping_and_popularity.taxon_map"), + ) + parser.add_argument( + "--exclude", + "-x", + nargs="*", + default=[], + help=( + "(Optional) taxa to exclude from calculation of phylogenetic popularities, " + "such as Dinosauria_ott90215, Archosauria_ott335588" + ), + ) + parser.add_argument( + "--out_dir", + default="data/out", + help=("Directory to write output files to"), + ) + args = parse_args_and_add_logging_switch(parser) + + logger.info("Parse & graft bespoke tree together") + base_t, bespoke_ts = parse_bespoke_trees(args.bespoke_dir) + missing_inclusions = graft_tree(base_t, additional_trees=bespoke_ts, prefer_subtree_name=True) + + logger.info("Resolve polytomies in bespoke trees") + # https://etetoolkit.org/docs/latest/reference/reference_tree.html#ete3.TreeNode.resolve_polytomy + # NB: Despite the DendroPy equivalent this replaces, this isn't a random draw: it + # doesn't shuffle children, so every polytomy becomes the same comb. Marked as + # POLYTOMY_COMB to distinguish it from the OT trees' random resolution below. + logger.info(f"Resolved bespoke polytomies with {tidy_resolve_polytomies(base_t)} new nodes") + + logger.info("Resolve branch lengths to dates bottom-up. Remove (or not care about) branch lengths") + tidy_infill_dates_bottomup(base_t) + + logger.info("Top-down conflict resolution in bespoke tree, delete entries that conflict with higher ages") + tidy_clear_conflicting_dates_topdown(base_t) + + logger.info("Graft OT subtrees onto our trees. Already polytomy-resolved & date pins from chronosynth applied") + opentree_t = date_tree.nwk_read(args.opentree) + logger.info(f"Marked {tidy_mark_resolved_polytomies(opentree_t)} pre-resolved OT polytomy nodes") + opentree_ts = graft_extract_ot_subtrees(opentree_t, missing_inclusions) + opentree_ts.update(parse_ot_orphans(args.orphan_dir, missing_inclusions)) + missing_inclusions = graft_tree( + base_t, additional_trees=opentree_ts, prefer_subtree_name=False, disable_recursion=True + ) + for i in missing_inclusions: + logger.error(f"No subtree found for {i}") + + logger.info("Attach taxon information to nodes") + taxon_add_prop(base_t, read_taxon_map(args.taxon_map)) + + logger.info( + "Popularity calculations for entire tree (including any remaining subspecies from bespoke tree, " + "Jonathan's will have them already removed) (stop caring about polytomy vs. popularity calculations, " + "and just apply them post-resolution). Apply popularity based on OTT -> popularity map, percolate " + "using existing rules (which preserves popularity from removed subspecies)" + ) + popularity_add_prop(base_t, exclude_taxa=args.exclude) + + logger.info("Remove subspecies (now popularity has percolated)") + # tidy_remove_subspecies(base_t) + + logger.info( + "Top-down conflict resolution. If there's conflict with higher ages, remove ages until conflict goes away" + ) + tidy_clear_conflicting_dates_topdown(base_t) + + logger.info("Bin synthetic mrca nodes left childless by the grafting process") + # NB: Has to happen before delete_one_child_nodes. Detaching a leaf leaves its parent + # with one child, and there is no second unary-node pass to tidy those up afterwards. + logger.info(f"Pruned {tidy_prune_synthetic_leaves(base_t)} childless synthetic nodes") + + logger.info("Remove unary nodes (they are likely uninteresting, and make a mess of the tree rendering)") + # NB: Returns the tree, which is a *new* root if the old one was itself unary + base_t = tree_fixing.delete_one_child_nodes(base_t) + + logger.info("Re-interpoltate missing dates") + base_t.root.props.setdefault("date", ROOT_DATE_MYA) + for n in base_t.traverse(): # First do some tidying to force tree_dating to work + if not n.name: + # dated-complete-tree will assume all nodes have a name + n.name = "" + + if n.props.get("date") is None: + # Ensure we have a date property on every leaf + n.props["date"] = 0 if n.is_leaf else None + else: + # If we do have a date, we also have to set imputed date + n.props["imputed_date"] = n.props.get("imputed_date") or False + tree_dating.date_labelling(base_t) + tree_dating.impute_missing_dates(base_t, l=0.25) + + logger.info("Regenerate branch lengths from the final dates") + # Up to here 'dist' is whatever the input newicks happened to carry, which is + # stale (or missing) now dates have been infilled/cleared/imputed. Everything + # downstream that uses branch lengths wants them to agree with 'date'. + tree_dating.compute_branch_lengths(base_t) + + logger.info("Rank popularities, post-node removal") + popularity_add_rank(base_t) + + logging.info("ladderizing tree (groups with fewer leaves first)") + # warning: ladderize ascending is needed for the short OZ newick-like form + base_t.ladderize(topological=False, reverse=False) + + logger.info("Output MySQL CSV files") + output_add_prop_ids(base_t) + output_mysqlexport(base_t, args.out_dir) + + logger.info("Output JS newick / cut position map") + output_jssource( + base_t, + args.out_dir, + "completetree.js", + dict( + rawData=jsnewick_brief_newick(base_t, polytomy_braces="{}"), + ), + ) + cutmap_threshold = 10000 + output_jssource( + base_t, + args.out_dir, + "cut_position_map.js", + dict( + # NB: For legacy reasons the variable contains a JSON string, not JSON + cut_position_map_json_str=json.dumps(jsnewick_cutpositionmap_binary(base_t, threshold=cutmap_threshold)), + polytomy_cut_position_map_json_str=json.dumps( + jsnewick_cutpositionmap_polytomy(base_t, threshold=cutmap_threshold) + ), + threshold=cutmap_threshold, + ), + ) + + logger.info("Generate tree properties and output arrays") + output_proparray(base_t, args.out_dir, treeprop_geological(base_t)) + + +if __name__ == "__main__": + main() diff --git a/oz_tree_build/utilities/debug_util.py b/oz_tree_build/utilities/debug_util.py index cea334d4..29c07df4 100644 --- a/oz_tree_build/utilities/debug_util.py +++ b/oz_tree_build/utilities/debug_util.py @@ -51,15 +51,26 @@ def parse_args_and_add_logging_switch(parser): args = parser.parse_args() if args.verbosity == 0: - logging.basicConfig(stream=sys.stderr, level=logging.WARNING) + level = logging.WARNING elif args.verbosity == 1: - logging.basicConfig(stream=sys.stderr, level=logging.INFO) - elif args.verbosity == 2: - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + level = logging.INFO + else: + level = logging.DEBUG + + # NB: force=True, since importing ete4 configures the root logger for us + # (see ete4/smartview/explorer.py), which would make this call a no-op. + # For the same reason we spell out the format rather than inheriting one. + logging.basicConfig( + stream=sys.stderr, + level=level, + format="%(asctime)s %(levelname)s %(module)s: %(message)s", + force=True, + ) if _error_handler is None: _error_handler = _ErrorCountingHandler() - logging.getLogger().addHandler(_error_handler) atexit.register(_exit_if_errors_logged) + # NB: Added after basicConfig(), which removes any pre-existing handlers + logging.getLogger().addHandler(_error_handler) return args diff --git a/oz_tree_build/utilities/download_opentree.py b/oz_tree_build/utilities/download_opentree.py index 4105a908..e592546c 100644 --- a/oz_tree_build/utilities/download_opentree.py +++ b/oz_tree_build/utilities/download_opentree.py @@ -14,7 +14,7 @@ import argparse import os -import re +import os.path import shutil import tarfile import tempfile @@ -42,36 +42,19 @@ def find_synthesis_entry(synthesis_json, version): raise SystemExit(f"Version '{version}' not found in synthesis.json. " f"Available versions: {', '.join(available)}") -def strip_mrca_prefixes(content: str) -> str: - # Clean up synthetically named mrca (most recent common ancestor) node labels, no use to us - content = re.sub(r"\)mrcaott\d+ott\d+", ")", content) - # Also clean up unwanted spaces - content = re.sub(r"[ _]+", "_", content) - return content - - -def download_tree(version, output_dir): +def download_file(version, output_dir, download_file="/labelled_supertree/labelled_supertree_ottnames.tre"): """Download the labelled supertree and produce the processed draftversion.""" assert version.startswith("v") version_without_v = version[1:] - tree_url = ( - f"https://files.opentreeoflife.org/synthesis/opentree{version_without_v}" - f"/output/labelled_supertree/labelled_supertree_simplified_ottnames.tre" - ) - print(f"Downloading tree from {tree_url} ...") - response = requests.get(tree_url, verify=OT_SSL_VERIFY) + url = f"https://files.opentreeoflife.org/synthesis/opentree{version_without_v}/output/{download_file}" + out_path = os.path.join(output_dir, os.path.basename(url)) + + print(f"Downloading {url} -> {out_path} ...") + response = requests.get(url, verify=OT_SSL_VERIFY) response.raise_for_status() - raw_path = os.path.join(output_dir, "labelled_supertree_simplified_ottnames.tre") - with open(raw_path, "w") as f: + with open(out_path, "w") as f: f.write(response.text) - print(f" Saved raw tree to {raw_path}") - - draft_path = os.path.join(output_dir, "draftversion.tre") - print(" Stripping mrca prefixes ...") - with open(draft_path, "w") as f: - f.write(strip_mrca_prefixes(response.text)) - print(f" Saved processed tree to {draft_path}") def download_taxonomy(ott_version_raw, output_dir): @@ -130,7 +113,8 @@ def main(): output_dir = os.path.join(args.output_dir, version) os.makedirs(output_dir, exist_ok=True) - download_tree(version, output_dir) + download_file(version, output_dir, "/labelled_supertree/labelled_supertree_ottnames.tre") + download_file(version, output_dir, "/annotated_supertree/annotations.json") download_taxonomy(entry["OTT_version"], output_dir) print(f"Done. All files written to {output_dir}/") diff --git a/oz_tree_build/utilities/ete.py b/oz_tree_build/utilities/ete.py new file mode 100644 index 00000000..e7650099 --- /dev/null +++ b/oz_tree_build/utilities/ete.py @@ -0,0 +1,24 @@ +import re + +NODE_OTT_RE = re.compile(r"[_ ]ott(\d+)$") + + +def node_get_ott(n): + """ + Extract OTT from node if present, None otherwise + + NB: OTT is returned as string, not int + """ + if not n.name: + return None + m = NODE_OTT_RE.search(n.name) + return m.group(1) if m else None + + +def node_name_without_ott(n): + """ + Remove any OTT at the end of the label, return node name + """ + if not n.name: + return None + return NODE_OTT_RE.sub("", n.name) diff --git a/oz_tree_build/utilities/filter_eol.py b/oz_tree_build/utilities/filter_eol.py index 99a92095..b756e404 100644 --- a/oz_tree_build/utilities/filter_eol.py +++ b/oz_tree_build/utilities/filter_eol.py @@ -4,7 +4,7 @@ import logging import sys -from ..taxon_mapping_and_popularity.CSV_base_table_creator import iucn_num +from ..taxon_mapping_and_popularity.taxon_map import iucn_num from .file_utils import open_file_based_on_extension from .filter_common import read_taxonomy_source_ids diff --git a/oz_tree_build/utilities/make_js_treefiles.py b/oz_tree_build/utilities/make_js_treefiles.py deleted file mode 100755 index fb33af22..00000000 --- a/oz_tree_build/utilities/make_js_treefiles.py +++ /dev/null @@ -1,248 +0,0 @@ -import argparse -import fileinput -import json -import os -import re -import shutil -from subprocess import call - -from ..utilities.debug_util import parse_args_and_add_logging_switch - - -# string -> string -# Given newick filepath(string), return a string without comma and semi comma -# Input: '../../data/output_files/ordered_tree_test.nwk' -> '((,),)' -# Output: '(())' -def tidy_newick(newick_filepath): - res = "" - for line in fileinput.input(files=(newick_filepath)): - res += line.replace(",", "").replace(";", "").replace("\n", "") - return res - - -# String -> String -# Given tidied newick string, return rawData in completetree.js -# Input: (()) -# Output:var rawData = '(())'; -def generate_completetree_js(newick_str): - return "var rawData = '" + newick_str + "';" - - -# String, Number -> String -# Given tidied newick(polytomy) string, return stringified cut position map for -# binary tree and polytomy tree -def generate_cut_position_map(newick_str, threshold): - binary_cut_map = generate_binary_cut_position_map(newick_str, threshold) - polytomy_cut_map = generate_polytomy_cut_position_map(newick_str, threshold) - cut_threshold = "var cut_threshold = " + str(threshold) + ";" - return binary_cut_map + "\n\n" + polytomy_cut_map + "\n\n" + cut_threshold - - -# String, Number -> String -# Given tidied newick string, return stringified cut_position_map object. -# Output example: -# '{ -# "4203700":1302201,"4203701":685684,"4203702":685609,"4203703":683568, -# "4203704":7901,"4203705":7900,"4203706":6417,"4203707":6396 -# }' -def generate_binary_cut_position_map(newick_str, threshold): - count_arr = [None] * len(newick_str) - count = 0 - for index, c in enumerate(reversed(newick_str)): - index = len(newick_str) - index - 1 - if c == "(" or c == "{": - count = count - 1 - elif c == ")" or c == "}": - count = count + 1 - else: - raise ValueError("newick str contains non bracket character: " + c) - count_arr[index] = count - - start_end_arr = [0, len(count_arr) - 1] - cut_position_map = {} - while len(start_end_arr) > 0: - start = start_end_arr.pop(0) - end = start_end_arr.pop(0) - build_cut_position_map(start, end, start_end_arr, count_arr, cut_position_map, threshold) - cut_position_map = json.dumps(cut_position_map) - cut_position_map = "var cut_position_map_json_str = '" + cut_position_map + "';" - return cut_position_map - - -# String, Number -> String -# Given tidied newick(polytomy) string, return stringified cut_position_map object. -# Output example: -# '{ -# "4203700":{685684, 79999, 1302201, 4203701}, -# "4203702":{685609, 4203703}, -# "4203704": {7901,4203705,7900,4203706} -# }' -# The key of the output json string is the end position of a string in the newick_str, the -# value is an array: [start_sub1, end_sub1, start_sub2, end_sub2, ..., start_subN, end_subN]. -# start_subN is the start pos of its nth child, end_subN is the end pos of its nth child. -def generate_polytomy_cut_position_map(newick_str, threshold): - start_end_arr = [0, len(newick_str) - 1] - cut_position_map = {} - while len(start_end_arr) > 0: - start = start_end_arr.pop(0) - end = start_end_arr.pop(0) - cut_position_map[end] = get_polytomy_substring_pos(start, end, start_end_arr, threshold, newick_str) - cut_position_map = json.dumps(cut_position_map) - cut_position_map = "var polytomy_cut_position_map_json_str = '" + cut_position_map + "';" - return cut_position_map - - -# Number, Number, Array, Array, Map, Number -# start, end represent indices of a node A on rawData. -# this function finds cut position of node A on rawData, then store it in cut_position_map -# and put its children start and end position in start_end_arr -def build_cut_position_map(start, end, start_end_arr, count_arr, cut_position_map, threshold): - endValue = count_arr[end] - for index in reversed(range(start, end)): - if count_arr[index] == endValue: - cut_position_map[end] = index - 1 - if (index - start - 2) >= threshold: - start_end_arr.append(start + 1) - start_end_arr.append(index - 1) - if (end - index - 1) >= threshold: - start_end_arr.append(index) - start_end_arr.append(end - 1) - break - - -# Find substring start & end position given a string representing a polytomous tree. -# The start and end position is pushed into start_end_arr if its distance is > than threshold -def get_polytomy_substring_pos(start, end, start_end_arr, threshold, newick_str, called_by_self=False): - res = [] - if end <= start or (called_by_self and newick_str[end] == ")"): - res += [start, end] - if (end - start) > threshold: - start_end_arr.append(start) - start_end_arr.append(end) - return res - - cut_point = None - bracket_count = 0 - for index in reversed(range(start, end + 1)): - c = newick_str[index] - if c == ")" or c == "}": - bracket_count = bracket_count + 1 - elif c == "(" or c == "{": - bracket_count = bracket_count - 1 - if bracket_count == 1: - cut_point = index - 1 - break - if cut_point is not None: - res = res + get_polytomy_substring_pos(start + 1, cut_point, start_end_arr, threshold, newick_str, True) - res = res + get_polytomy_substring_pos(cut_point + 1, end - 1, start_end_arr, threshold, newick_str, True) - else: - res += [start, start, end, end] - return res - - -def write_js_file(outdir, input_path, version_number, args): - # Output to versioned path - input_name = os.path.basename(input_path) - output_path = os.path.join( - outdir, - re.sub( - # Extract any existing version number / extension from filename - r"(_\d+)?(\.[a-zA-Z]+)$", - # Replace with verison number / extension - "_" + str(version_number) + r"\2", - input_name, - ), - ) - - if input_name.startswith("ordered_tree_"): - output_path = re.sub(r"ordered_tree_", "completetree_", output_path) - output_path = re.sub(r"\.(nwk|poly)$", ".js", output_path) - - print(f"{input_path} -> {output_path}") - newick_str = tidy_newick(input_path) - with open(output_path, "w") as out_f: - out_f.write(generate_completetree_js(newick_str)) - - # Generate derived cut-position-map - cut_path = re.sub(r"completetree_", r"cut_position_map_", output_path) - with open(cut_path, "w") as out_f: - out_f.write(generate_cut_position_map(newick_str, args.threshold)) - # Trigger write_js_file for cut map so we gzip it - write_js_file(outdir, cut_path, version_number, args) - - elif input_path == output_path: - # Nothing to do, already in output_path - pass - else: - # By default we just copy file - print(f"{input_path} -> {output_path}") - shutil.copyfile(input_path, output_path) - print(f"{output_path} -> {output_path}.gz") - call(["gzip", "-9fk", output_path]) - - -def main(): - # rawData string + metadata -> output result into file. - - # produce cut_position_map.js and completetree.js given newick tree. - parser = argparse.ArgumentParser( - description="Generate rawData, metadata and cut_position_map given newick string", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - - # pick the most recent ordered_tree_XXX.nwk file - import re - - parser.add_argument( - "--outdir", - "-o", - default=os.path.join( - os.path.dirname(__file__), - "..", - "..", - "..", - "OZtree", - "static", - "FinalOutputs", - "data", - ), - help="output filepath of cut_position_map", - ) - parser.add_argument( - "in_files", - nargs="+", - metavar="FILE", - help="Files to move to outdir, with versions appended if not present", - ) - parser.add_argument( - "--threshold", - default=10000, - type=int, - help=("Threshold for deciding if a node and its descendants needs to be" "recorded in cut_position_map"), - ) - parser.add_argument( - "--version", - type=int, - help=("Version number / serial to append to file names, if not provided assume present on at least one file"), - ) - - args = parse_args_and_add_logging_switch(parser) - - if args.version: - version_number = args.version - else: - # Find higest version number in files present, use that as version - version_number = 0 - for f in args.in_files: - m = re.search(r"_(\d+)\.(\w+)$", f) - if m and int(m.group(1)) > version_number: - version_number = int(m.group(1)) - - for f in args.in_files: - write_js_file(args.outdir, f, version_number, args) - - print("Done") - - -if __name__ == "__main__": - main() diff --git a/oz_tree_build/versioned_outputs/versioned_outputs.py b/oz_tree_build/versioned_outputs/versioned_outputs.py new file mode 100644 index 00000000..2f532ea3 --- /dev/null +++ b/oz_tree_build/versioned_outputs/versioned_outputs.py @@ -0,0 +1,99 @@ +""" +Add version number / gzip output files as a final stage of the pipeline + +In addition to versioning data files, the SQL import script is also modified. +CSV filenames have their version added, and as a final step the version number +is inserted into the DB as the parent of the root. +""" + +import argparse +import logging +import os +import re +import shutil +import subprocess + +from ..utilities.debug_util import parse_args_and_add_logging_switch + +logger = logging.getLogger(__name__) + + +def process(in_files, out_dir, version_number): + """ + Copy list of ``in_files`` to ``out_dir``, with ``version_number`` appended to name + """ + + def add_version(f_name): + return re.sub( + # Extract any existing version number / extension from filename + r"(_\d+)?(\.[a-zA-Z]+)$", + # Replace with verison number / extension + "_" + str(version_number) + r"\2", + f_name, + ) + + for input_path in in_files: + input_name = os.path.basename(input_path) + output_path = os.path.join(out_dir, add_version(input_name)) + + logger.info(f"{input_path} -> {output_path}") + if input_name == "import.sql": + with open(input_path) as in_f, open(output_path, "w") as out_f: + for l in in_f: + # Replace any instance of an input filename with it's versioned equivalent + for repl_path in in_files: + repl_name = os.path.basename(repl_path) + l = l.replace("'" + repl_name + "'", "'" + add_version(repl_name) + "'") + out_f.write(l) + # Extra command to bodge version number into root's parent + out_f.writelines(f"UPDATE ordered_nodes SET parent = -{version_number} WHERE id = 1;\n") + else: + shutil.copyfile(input_path, output_path) + subprocess.call(["gzip", "-9fk", output_path]) + logger.info("Done") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument( + "--outdir", + "-o", + default=os.path.join( + os.path.dirname(__file__), + "..", + "..", + "..", + "OZtree", + "static", + "FinalOutputs", + "data", + ), + help="output filepath of cut_position_map", + ) + parser.add_argument( + "in_files", + nargs="+", + metavar="FILE", + help="Files to move to outdir, with versions appended if not present", + ) + parser.add_argument( + "--version", + type=int, + help=("Version number / serial to append to file names, if not provided use mtime of first in_file"), + ) + parser.add_argument( + "--out_dir", + default="data/out", + help=("Directory to write output files to"), + ) + args = parse_args_and_add_logging_switch(parser) + + process( + args.in_files, + args.out_dir, + int(os.path.getmtime(args.in_files[0])) if args.version is None else args.version, + ) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index bcfbac61..e0701055 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,11 @@ dependencies = [ "mwparserfromhell>=0.6.6", "requests-cache>=1.2.1", "dvc[s3]>=3.0", + "dated_complete_tree", + "chronosynth", + # Undeclared chronosynth dependencies + "peyotl", + "opentree", ] [project.optional-dependencies] @@ -33,8 +38,6 @@ dev = [ [project.scripts] add_ott_numbers_to_trees = "oz_tree_build.tree_build.ott_mapping.add_ott_numbers_to_trees:main" -build_oz_tree = "oz_tree_build.tree_build.build_oz_tree:main" -get_open_trees_from_one_zoom = "oz_tree_build.tree_build.get_open_trees_from_one_zoom:main" generate_filtered_files = "oz_tree_build.utilities.generate_filtered_files:main" filter_eol = "oz_tree_build.utilities.filter_eol:main" filter_wikidata = "oz_tree_build.utilities.filter_wikidata:main" @@ -46,11 +49,9 @@ download_and_filter_wikidata = "oz_tree_build.utilities.download_and_filter_wiki discover_latest_wikidata_dump_url = "oz_tree_build.utilities.download_and_filter_wikidata:discover_main" discover_latest_enwiki_sql_url = "oz_tree_build.utilities.filter_wikipedia_sql:discover_main" download_opentree = "oz_tree_build.utilities.download_opentree:main" -CSV_base_table_creator = "oz_tree_build.taxon_mapping_and_popularity.CSV_base_table_creator:main" get_wiki_images = "oz_tree_build.images.get_wiki_images:main" get_wiki_vernaculars = "oz_tree_build.vernaculars.get_wiki_vernaculars:main" process_image_bits = "oz_tree_build.images.process_image_bits:main" -make_js_treefiles = "oz_tree_build.utilities.make_js_treefiles:main" format_newick = "oz_tree_build.newick.format_newick:main" extract_minimal_tree = "oz_tree_build.newick.extract_minimal_tree:main" extract_trees = "oz_tree_build.newick.extract_trees:main" @@ -60,6 +61,11 @@ find_in_file = "oz_tree_build.utilities.find_in_file:main" wiki_clade_extractor = "oz_tree_build.wiki_extraction.wiki_clade_extractor:main" newick_combiner = "oz_tree_build.wiki_extraction.newick_combiner:main" add_dates_and_species_to_tree = "oz_tree_build.wiki_extraction.add_dates_and_species_to_tree:main" +download_node_ages = "oz_tree_build.download_node_ages.download_node_ages:main" +date_tree = "oz_tree_build.date_tree.date_tree:main" +tree_build = "oz_tree_build.tree_build.tree_build:main" +taxon_map = "oz_tree_build.taxon_mapping_and_popularity.taxon_map:main" +versioned_outputs = "oz_tree_build.versioned_outputs.versioned_outputs:main" [tool.setuptools] packages = ["oz_tree_build"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c0369395 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +dated_complete_tree @ git+https://github.com/OneZoom/dated-complete-tree.git +chronosynth @ git+https://github.com/OpenTreeOfLife/chronosynth@7fc31d2bb3bfbf0786d31530579c916499f3616a +# Undeclared chronosynth dependencies +peyotl @ git+https://github.com/OpenTreeOfLife/peyotl.git@72ccb5369bef07b76f57eb6852f5b90fe677b09b +opentree @ git+https://github.com/OpenTreeOfLife/python-opentree@9b9afce7d0a526a3328af0f9c328ef1194aec1b9#egg=opentree + +-e .[dev] diff --git a/ruff.toml b/ruff.toml index 36d9229a..64ce5172 100644 --- a/ruff.toml +++ b/ruff.toml @@ -28,6 +28,8 @@ exclude = [ "*.PHY", "*.md", "*.markdown", + # date_tree is actually from https://github.com/jdduke24/dated-complete-tree/blob/main/main.py, preserve its formatting + "oz_tree_build/date_tree/date_tree.py", ] line-length = 120 diff --git a/tests/test_debug_util.py b/tests/test_debug_util.py index f1bf131e..c2082b8b 100644 --- a/tests/test_debug_util.py +++ b/tests/test_debug_util.py @@ -72,7 +72,40 @@ def test_default_verbosity_suppresses_info(): assert "hello info" not in result.stderr +def test_verbose_flag_suppresses_debug(): + result = _run("logging.debug('hello debug')", "-v") + assert result.returncode == 0 + assert "hello debug" not in result.stderr + + +def test_v_still_emits_errors(): + result = _run("logging.error('boom')", "-v") + assert result.returncode == 1 + assert "boom" in result.stderr + assert "Exiting with status 1: 1 error(s) were logged" in result.stderr + + def test_vv_enables_debug_output(): result = _run("logging.debug('hello debug')", "-vv") assert result.returncode == 0 assert "hello debug" in result.stderr + + +def test_vv_still_emits_errors(): + result = _run("logging.error('boom')", "-vv") + assert result.returncode == 1 + assert "boom" in result.stderr + assert "Exiting with status 1: 1 error(s) were logged" in result.stderr + + +def test_vvv_enables_debug_output(): + result = _run("logging.debug('hello debug')", "-vvv") + assert result.returncode == 0 + assert "hello debug" in result.stderr + + +def test_vvv_still_emits_errors(): + result = _run("logging.error('boom')", "-vvv") + assert result.returncode == 1 + assert "boom" in result.stderr + assert "Exiting with status 1: 1 error(s) were logged" in result.stderr diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv b/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv deleted file mode 100644 index bf2216c3..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv +++ /dev/null @@ -1,41 +0,0 @@ -parent,real_parent,name,extinction_date,ott,wikidata,wikipedia_lang_flag,iucn,eol,raw_popularity,popularity,popularity_rank,price,ncbi,ifung,worms,irmng,gbif,ipni -1,1,Acinonyx jubatus,,752759,23907,1048575,219,328680,128090.45,128090.45,2,,32536,,,10856341,2435270, -3,3,Puma yagouaroundi,,86162,182304,1048575,,1053885,30183.81,42905.99,9,,1608482,,,10910448,2435146, -3,3,Puma concolor,,42307,35255,1048575,18868,311910,108261.25,113975.14,3,,9696,,,10212875,2435099, -5,5,Prionailurus planiceps,,86166,274177,1040383,18148,311659,7675.98,9545.35,29,,61403,,,11429819,2434917, -6,6,Prionailurus rubiginosus,,507541,309274,1048575,18149,312856,17500.57,14326.27,25,,61387,,,11039646,2434895, -7,7,Prionailurus viverrinus,,862641,190674,1048575,18150,1037335,27649.05,18532.47,22,,61388,,,10594861,2434899, -8,8,Prionailurus bengalensis,,280108,42627,1048575,18146|223138747,1041047,27344.18,16907.7,23,,37029,,,10210660,2434903, -8,8,Prionailurus iriomotensis,,418475,19829419,0,18151,1053884,0,2855.57,40,,37030,,,,2434901, -10,10,Leptailurus serval,,86170,42699,1048575,11638,1041048,42607.28,26473.39,17,,61405,,,11216434,2435172, -11,11,Caracal caracal,,1033549,30847,1048575,3847,312855,63025.4,35175.15,11,,61394,,,11060971,2435010, -12,12,Felis manul,,86183,166794,1048575,15640,328665,52482.91,34320.86,13,,61408,,,10859216,2435023, -14,14,Felis bieti,,54743,204322,1048575,8539,328664,7804.55,10061.31,27,,458418,,,11269571,2435040, -14,14,Felis chaus,,983181,42623,1048575,8540,328671,26429.95,18538.09,21,,61376,,,10591049,2435066, -16,16,Felis margarita,,983177,175329,1048575,8541,328670,34349.37,21129.2,19,,61378,,,11270420,2435028, -16,16,Felis nigripes,,983179,204814,1048575,8542,328666,36443.03,22038.46,18,,61379,,,10785265,2435037, -17,17,Felis silvestris,,563163,43576,1048575,60354712|181049859,328605,33214.97,20636.53,20,,9683,,,10201332,7964291, -17,17,Felis catus,,563166,146,1048575,,1037781,307566.59,139785.93,1,,9685,,,,8625722, -20,20,Leopardus pardalis,,752746,33261,1048575,11509,313991,52797.12,31945.02,14,,32538,,,10212871,2434982, -20,20,Leopardus wiedii,,507553,192421,1048575,11511,311954,14868.56,12453.59,26,,61382,,,10592818,2434950, -22,22,Leopardus geoffroyi,,774303,42682,1048575,15310,925988,6828.5,7787.43,34,,46844,,,10765089,2434942, -23,22,Leopardus tigrinus,,774309,205948,1048575,54012637,311661,9685.65,8670.33,31,,46842,,,11116645,2434930, -23,22,Leopardus guigna,,507542,211042,1048575,15311,1053887,12068.12,9754.64,28,,61386,,,11170429,2434923, -24,24,Leopardus jacobitus,,904397,213047,1048575,15452,1053886,7787.0,8248.37,33,,713925,,,10909680,2434979, -25,25,Leopardus colocolo,,86175,210314,1039871,15309,47054070,8894.45,8310.24,32,,61406,,,10534600,2434919, -26,25,Leopardus braccatus,,3613208,133763,1048559,,1053889,1345.42,4651.48,37,,,,,,2434927, -26,25,Leopardus pajeros,,3613206,311417,1044463,,925987,0,4067.17,39,,,,,,2434935, -28,28,Profelis aurata,,660447,192231,1048575,,311555,9315.95,4787.45,36,,61412,,,11097445,9546725, -29,29,Catopuma badia,,763032,213044,1048575,4037,311552,7483.18,4898.87,35,,61454,,,10224214,5787235, -29,29,Catopuma temminckii,,763025,192233,1048575,4038,311553,15930.33,8961.09,30,,61455,,,10999935,2435094, -31,31,Lynx rufus,,507545,131907,1048575,12521,328602,80838.36,57974.43,7,,61384,,,10199263,2435246, -32,32,Lynx pardinus,,442049,129727,1048575,12520,347432,25562.25,29709.45,16,,191816,,,10229763,2435261, -33,33,Lynx canadensis,,507549,146457,1048575,12518,328604,39433.81,34374.36,12,,61383,,,10201330,2435263, -33,33,Lynx lynx,,886829,43375,1048575,12519,328603,42295.46,35617.16,10,,13125,,,10199264,2435240, -34,34,Pardofelis marmorata,,660452,80191,1048575,16218,311554,8700.5,4184.06,38,,61410,,,10733346,2435089, -35,35,Neofelis nebulosa,,763016,36135,1048575,14519,328675,32620.44,14846.2,24,,61452,,,10200770,2435079, -36,36,Uncia uncia,,532117,30197,1048575,22732,328676,70362.32,30557.97,15,,29064,,,11222977,2435238, -37,37,Panthera tigris,,42314,19939,1048575,15955,328674,167475.09,88532.08,4,,9694,,,10762914,5219416, -38,38,Panthera onca,,42322,35694,1048575,15953,328606,110310.18,62427.19,6,,9690,,,10201333,5219426, -39,39,Panthera leo,,563151,140,1048575,15951,328672,164462.75,81591.6,5,,9689,,,10196306,5219404, -39,39,Panthera pardus,,42324,34706,1048575,15954,328673,103768.13,57928.51,8,,9691,,,10200769,5219436, diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv.mySQL b/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv.mySQL deleted file mode 100644 index 06151b02..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_leaves_0.csv.mySQL +++ /dev/null @@ -1,41 +0,0 @@ -parent,real_parent,name,extinction_date,ott,wikidata,wikipedia_lang_flag,iucn,eol,raw_popularity,popularity,popularity_rank,price,ncbi,ifung,worms,irmng,gbif,ipni -1,1,Acinonyx jubatus,\N,752759,23907,1048575,219,328680,128090.45,128090.45,2,\N,32536,\N,\N,10856341,2435270,\N -3,3,Puma yagouaroundi,\N,86162,182304,1048575,\N,1053885,30183.81,42905.99,9,\N,1608482,\N,\N,10910448,2435146,\N -3,3,Puma concolor,\N,42307,35255,1048575,18868,311910,108261.25,113975.14,3,\N,9696,\N,\N,10212875,2435099,\N -5,5,Prionailurus planiceps,\N,86166,274177,1040383,18148,311659,7675.98,9545.35,29,\N,61403,\N,\N,11429819,2434917,\N -6,6,Prionailurus rubiginosus,\N,507541,309274,1048575,18149,312856,17500.57,14326.27,25,\N,61387,\N,\N,11039646,2434895,\N -7,7,Prionailurus viverrinus,\N,862641,190674,1048575,18150,1037335,27649.05,18532.47,22,\N,61388,\N,\N,10594861,2434899,\N -8,8,Prionailurus bengalensis,\N,280108,42627,1048575,18146|223138747,1041047,27344.18,16907.7,23,\N,37029,\N,\N,10210660,2434903,\N -8,8,Prionailurus iriomotensis,\N,418475,19829419,0,18151,1053884,0,2855.57,40,\N,37030,\N,\N,\N,2434901,\N -10,10,Leptailurus serval,\N,86170,42699,1048575,11638,1041048,42607.28,26473.39,17,\N,61405,\N,\N,11216434,2435172,\N -11,11,Caracal caracal,\N,1033549,30847,1048575,3847,312855,63025.4,35175.15,11,\N,61394,\N,\N,11060971,2435010,\N -12,12,Felis manul,\N,86183,166794,1048575,15640,328665,52482.91,34320.86,13,\N,61408,\N,\N,10859216,2435023,\N -14,14,Felis bieti,\N,54743,204322,1048575,8539,328664,7804.55,10061.31,27,\N,458418,\N,\N,11269571,2435040,\N -14,14,Felis chaus,\N,983181,42623,1048575,8540,328671,26429.95,18538.09,21,\N,61376,\N,\N,10591049,2435066,\N -16,16,Felis margarita,\N,983177,175329,1048575,8541,328670,34349.37,21129.2,19,\N,61378,\N,\N,11270420,2435028,\N -16,16,Felis nigripes,\N,983179,204814,1048575,8542,328666,36443.03,22038.46,18,\N,61379,\N,\N,10785265,2435037,\N -17,17,Felis silvestris,\N,563163,43576,1048575,60354712|181049859,328605,33214.97,20636.53,20,\N,9683,\N,\N,10201332,7964291,\N -17,17,Felis catus,\N,563166,146,1048575,\N,1037781,307566.59,139785.93,1,\N,9685,\N,\N,\N,8625722,\N -20,20,Leopardus pardalis,\N,752746,33261,1048575,11509,313991,52797.12,31945.02,14,\N,32538,\N,\N,10212871,2434982,\N -20,20,Leopardus wiedii,\N,507553,192421,1048575,11511,311954,14868.56,12453.59,26,\N,61382,\N,\N,10592818,2434950,\N -22,22,Leopardus geoffroyi,\N,774303,42682,1048575,15310,925988,6828.5,7787.43,34,\N,46844,\N,\N,10765089,2434942,\N -23,22,Leopardus tigrinus,\N,774309,205948,1048575,54012637,311661,9685.65,8670.33,31,\N,46842,\N,\N,11116645,2434930,\N -23,22,Leopardus guigna,\N,507542,211042,1048575,15311,1053887,12068.12,9754.64,28,\N,61386,\N,\N,11170429,2434923,\N -24,24,Leopardus jacobitus,\N,904397,213047,1048575,15452,1053886,7787.0,8248.37,33,\N,713925,\N,\N,10909680,2434979,\N -25,25,Leopardus colocolo,\N,86175,210314,1039871,15309,47054070,8894.45,8310.24,32,\N,61406,\N,\N,10534600,2434919,\N -26,25,Leopardus braccatus,\N,3613208,133763,1048559,\N,1053889,1345.42,4651.48,37,\N,\N,\N,\N,\N,2434927,\N -26,25,Leopardus pajeros,\N,3613206,311417,1044463,\N,925987,0,4067.17,39,\N,\N,\N,\N,\N,2434935,\N -28,28,Profelis aurata,\N,660447,192231,1048575,\N,311555,9315.95,4787.45,36,\N,61412,\N,\N,11097445,9546725,\N -29,29,Catopuma badia,\N,763032,213044,1048575,4037,311552,7483.18,4898.87,35,\N,61454,\N,\N,10224214,5787235,\N -29,29,Catopuma temminckii,\N,763025,192233,1048575,4038,311553,15930.33,8961.09,30,\N,61455,\N,\N,10999935,2435094,\N -31,31,Lynx rufus,\N,507545,131907,1048575,12521,328602,80838.36,57974.43,7,\N,61384,\N,\N,10199263,2435246,\N -32,32,Lynx pardinus,\N,442049,129727,1048575,12520,347432,25562.25,29709.45,16,\N,191816,\N,\N,10229763,2435261,\N -33,33,Lynx canadensis,\N,507549,146457,1048575,12518,328604,39433.81,34374.36,12,\N,61383,\N,\N,10201330,2435263,\N -33,33,Lynx lynx,\N,886829,43375,1048575,12519,328603,42295.46,35617.16,10,\N,13125,\N,\N,10199264,2435240,\N -34,34,Pardofelis marmorata,\N,660452,80191,1048575,16218,311554,8700.5,4184.06,38,\N,61410,\N,\N,10733346,2435089,\N -35,35,Neofelis nebulosa,\N,763016,36135,1048575,14519,328675,32620.44,14846.2,24,\N,61452,\N,\N,10200770,2435079,\N -36,36,Uncia uncia,\N,532117,30197,1048575,22732,328676,70362.32,30557.97,15,\N,29064,\N,\N,11222977,2435238,\N -37,37,Panthera tigris,\N,42314,19939,1048575,15955,328674,167475.09,88532.08,4,\N,9694,\N,\N,10762914,5219416,\N -38,38,Panthera onca,\N,42322,35694,1048575,15953,328606,110310.18,62427.19,6,\N,9690,\N,\N,10201333,5219426,\N -39,39,Panthera leo,\N,563151,140,1048575,15951,328672,164462.75,81591.6,5,\N,9689,\N,\N,10196306,5219404,\N -39,39,Panthera pardus,\N,42324,34706,1048575,15954,328673,103768.13,57928.51,8,\N,9691,\N,\N,10200769,5219436,\N diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv b/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv deleted file mode 100644 index fd633893..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv +++ /dev/null @@ -1,40 +0,0 @@ -parent,real_parent,node_rgt,leaf_lft,leaf_rgt,name,age,ott,wikidata,wikipedia_lang_flag,eol,rnk,raw_popularity,popularity,ncbi,ifung,worms,irmng,gbif,ipni,vern_synth,rep1,rep2,rep3,rep4,rep5,rep6,rep7,rep8,rtr1,rtr2,rtr3,rtr4,rtr5,rtr6,rtr7,rtr8,rpd1,rpd2,rpd3,rpd4,rpd5,rpd6,rpd7,rpd8,iucnNE,iucnDD,iucnLC,iucnNT,iucnVU,iucnEN,iucnCR,iucnEW,iucnEX -0,0,39,1,40,Felidae,13.882716,563159,25265,1048575,7674,family,57879.95,476707.47,9681,,,104889,9703,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -1,1,39,2,40,,12.358025,,,,,,,448635.47,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -2,2,3,2,3,Puma,10.191358,86161,270748,1046527,34428,genus,16953.24,112096.18,146712,,,1405111,2435098,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -2,2,39,4,40,,12.197531,,,,,,,416672.81,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -4,4,8,4,8,Prionailurus,8.265432,570215,42592,1048575,27870,genus,5556.68,35750.71,37028,,,1040948,2434894,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -5,5,8,5,8,,6.5,,,,,,,33896.89,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -6,6,8,6,8,,6.179012,,,,,,,27557.45,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -7,7,8,7,8,,3.37037,,,,,,,15821.97,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -4,4,39,9,40,,9.067901,,,,,,,409080.5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -9,9,17,9,17,,7.54321,,,,,,,206369.06,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -10,10,17,10,17,,7.382716,,,,,,,195493.67,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -11,11,17,11,17,Felis,5.938272,563165,228283,1048575,20189,genus,14302.4,177345.27,9682,,,1179423,2435022,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -12,12,17,12,17,,4.092593,,,,,,,162398.95,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -13,13,14,12,13,,2.006173,,,,,,,21079.31,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -13,13,17,14,17,,4.012346,,,,,,,161374.43,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -15,15,16,14,15,,2.888889,,,,,,,35487.29,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -15,15,17,16,17,,1.925926,,,,,,,148081.51,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -9,9,39,18,40,,8.987654,,,,,,,281420.15,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -18,18,26,18,26,Leopardus,7.54321,774314,318414,1048575,14320,genus,9365.01,40610.58,46841,,,1384113,2434918,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -19,19,20,18,19,,4.975309,,,,,,,37043.93,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -19,19,26,20,26,,6.259259,,,,,,,19365.73,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -21,21,23,20,22,,6.098765,,,,,,,15825.24,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -22,-22,23,21,22,,6.098765,,,,,,,13514.71,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -21,21,26,23,26,,4.333333,,,,,,,10679.31,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -24,24,26,24,26,,3.450617,,,,,,,7889.58,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -25,-25,26,25,26,,3.450617,,,,,,,4466.6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -18,18,39,27,40,,8.907407,,,,,,,281245.7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -27,27,29,27,29,,6.259259,,,,,,,15388.44,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -28,28,29,28,29,Catopuma,5.537037,763015,1419858,1048567,35821,genus,2703.74,11886.47,61453,,,1294543,2435092,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -27,27,39,30,40,,8.746913,,,,,,,285553.52,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -30,30,33,30,33,Lynx,7.141975,886828,677014,1048575,18767,genus,39716.08,88830.59,13124,,,1026710,2435239,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -31,31,33,31,33,,4.333333,,,,,,,59160.21,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -32,32,33,32,33,,2.888889,,,,,,,50646.64,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -30,30,39,34,40,,6.660493,,,,,,,238590.44,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -34,34,39,35,40,Clade7728_,6.580246,,,,,,,240043.34,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -35,35,39,36,40,,4.895061,,,,,,,233372.49,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -36,36,39,37,40,Panthera,4.814814,563154,127960,1048575,14134,genus,44815.57,213097.5,9688,,,1330102,2435194,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -37,37,39,38,40,,4.734567,,,,,,,156332.64,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -38,38,39,39,40,,4.65432,,,,,,,118620.56,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv.mySQL b/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv.mySQL deleted file mode 100644 index c54df5ac..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_nodes_0.csv.mySQL +++ /dev/null @@ -1,40 +0,0 @@ -parent,real_parent,node_rgt,leaf_lft,leaf_rgt,name,age,ott,wikidata,wikipedia_lang_flag,eol,rnk,raw_popularity,popularity,ncbi,ifung,worms,irmng,gbif,ipni,vern_synth,rep1,rep2,rep3,rep4,rep5,rep6,rep7,rep8,rtr1,rtr2,rtr3,rtr4,rtr5,rtr6,rtr7,rtr8,rpd1,rpd2,rpd3,rpd4,rpd5,rpd6,rpd7,rpd8,iucnNE,iucnDD,iucnLC,iucnNT,iucnVU,iucnEN,iucnCR,iucnEW,iucnEX -0,0,39,1,40,Felidae,13.882716,563159,25265,1048575,7674,family,57879.95,476707.47,9681,\N,\N,104889,9703,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -1,1,39,2,40,\N,12.358025,\N,\N,\N,\N,\N,\N,448635.47,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -2,2,3,2,3,Puma,10.191358,86161,270748,1046527,34428,genus,16953.24,112096.18,146712,\N,\N,1405111,2435098,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -2,2,39,4,40,\N,12.197531,\N,\N,\N,\N,\N,\N,416672.81,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -4,4,8,4,8,Prionailurus,8.265432,570215,42592,1048575,27870,genus,5556.68,35750.71,37028,\N,\N,1040948,2434894,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -5,5,8,5,8,\N,6.5,\N,\N,\N,\N,\N,\N,33896.89,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -6,6,8,6,8,\N,6.179012,\N,\N,\N,\N,\N,\N,27557.45,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -7,7,8,7,8,\N,3.37037,\N,\N,\N,\N,\N,\N,15821.97,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -4,4,39,9,40,\N,9.067901,\N,\N,\N,\N,\N,\N,409080.5,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -9,9,17,9,17,\N,7.54321,\N,\N,\N,\N,\N,\N,206369.06,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -10,10,17,10,17,\N,7.382716,\N,\N,\N,\N,\N,\N,195493.67,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -11,11,17,11,17,Felis,5.938272,563165,228283,1048575,20189,genus,14302.4,177345.27,9682,\N,\N,1179423,2435022,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -12,12,17,12,17,\N,4.092593,\N,\N,\N,\N,\N,\N,162398.95,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -13,13,14,12,13,\N,2.006173,\N,\N,\N,\N,\N,\N,21079.31,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -13,13,17,14,17,\N,4.012346,\N,\N,\N,\N,\N,\N,161374.43,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -15,15,16,14,15,\N,2.888889,\N,\N,\N,\N,\N,\N,35487.29,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -15,15,17,16,17,\N,1.925926,\N,\N,\N,\N,\N,\N,148081.51,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -9,9,39,18,40,\N,8.987654,\N,\N,\N,\N,\N,\N,281420.15,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -18,18,26,18,26,Leopardus,7.54321,774314,318414,1048575,14320,genus,9365.01,40610.58,46841,\N,\N,1384113,2434918,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -19,19,20,18,19,\N,4.975309,\N,\N,\N,\N,\N,\N,37043.93,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -19,19,26,20,26,\N,6.259259,\N,\N,\N,\N,\N,\N,19365.73,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -21,21,23,20,22,\N,6.098765,\N,\N,\N,\N,\N,\N,15825.24,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -22,-22,23,21,22,\N,6.098765,\N,\N,\N,\N,\N,\N,13514.71,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -21,21,26,23,26,\N,4.333333,\N,\N,\N,\N,\N,\N,10679.31,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -24,24,26,24,26,\N,3.450617,\N,\N,\N,\N,\N,\N,7889.58,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -25,-25,26,25,26,\N,3.450617,\N,\N,\N,\N,\N,\N,4466.6,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -18,18,39,27,40,\N,8.907407,\N,\N,\N,\N,\N,\N,281245.7,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -27,27,29,27,29,\N,6.259259,\N,\N,\N,\N,\N,\N,15388.44,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -28,28,29,28,29,Catopuma,5.537037,763015,1419858,1048567,35821,genus,2703.74,11886.47,61453,\N,\N,1294543,2435092,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -27,27,39,30,40,\N,8.746913,\N,\N,\N,\N,\N,\N,285553.52,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -30,30,33,30,33,Lynx,7.141975,886828,677014,1048575,18767,genus,39716.08,88830.59,13124,\N,\N,1026710,2435239,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -31,31,33,31,33,\N,4.333333,\N,\N,\N,\N,\N,\N,59160.21,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -32,32,33,32,33,\N,2.888889,\N,\N,\N,\N,\N,\N,50646.64,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -30,30,39,34,40,\N,6.660493,\N,\N,\N,\N,\N,\N,238590.44,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -34,34,39,35,40,Clade7728_,6.580246,\N,\N,\N,\N,\N,\N,240043.34,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -35,35,39,36,40,\N,4.895061,\N,\N,\N,\N,\N,\N,233372.49,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -36,36,39,37,40,Panthera,4.814814,563154,127960,1048575,14134,genus,44815.57,213097.5,9688,\N,\N,1330102,2435194,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -37,37,39,38,40,\N,4.734567,\N,\N,\N,\N,\N,\N,156332.64,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N -38,38,39,39,40,\N,4.65432,\N,\N,\N,\N,\N,\N,118620.56,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N,\N diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.nwk b/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.nwk deleted file mode 100644 index 2c296378..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.nwk +++ /dev/null @@ -1 +0,0 @@ -(,((,),((,(,(,(,)))),((,(,(,((,),((,),(,)))))),(((,),((,(,)),(,(,(,))))),((,(,)),((,(,(,))),(,(,(,(,(,(,))))))))))))) \ No newline at end of file diff --git a/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.poly b/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.poly deleted file mode 100644 index de137a6a..00000000 --- a/tests/test_files_felidae/expected_output_files_generation/ordered_tree_0.poly +++ /dev/null @@ -1 +0,0 @@ -(,((,),((,(,(,(,)))),((,(,(,((,),((,),(,)))))),(((,),((,{,}),(,(,{,})))),((,(,)),((,(,(,))),(,(,(,(,(,(,))))))))))))) \ No newline at end of file diff --git a/tests/test_full_generation.py b/tests/test_full_generation.py deleted file mode 100644 index b61e819b..00000000 --- a/tests/test_full_generation.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import types - -from oz_tree_build.taxon_mapping_and_popularity import CSV_base_table_creator -from oz_tree_build.utilities.file_utils import check_identical_files - -from .felidae_helpers import get_felidae_test_folders - - -def test_full_felidae_generation(): - """ - This is more of a functional test than a unit test. It runs the full pipeline - on a small clade. It then compares the output to the expected output. - """ - - args = types.SimpleNamespace() - - ( - input_path, - expected_output_path, - args.output_location, - ) = get_felidae_test_folders("generation") - - # Set all the arguments, to mimic the command line - args.Tree = os.path.join(input_path, "Felidae_AllLife_full_tree.phy") - args.OpenTreeTaxonomy = os.path.join(input_path, "Felidae_taxonomy.tsv") - args.EOLidentifiers = os.path.join(input_path, "Felidae_provider_ids.csv") - args.wikidataDumpFile = os.path.join(input_path, "Felidae_latest-all.json") - args.wikipediaSQLDumpFile = os.path.join(input_path, "Felidae_enwiki-latest-page.sql") - args.wikipedia_totals_bz2_pageviews = [ - os.path.join(input_path, f) for f in os.listdir(input_path) if f.startswith("Felidae_pageviews") - ] - - # Sort the list of pagecount files so that the order is consistent - args.wikipedia_totals_bz2_pageviews.sort() - - args.verbosity = 0 - args.version = 0 - args.wikilang = "en" - args.popularity_file = "" - args.extra_source_file = None - args.taxa_data_file = None - args.exclude = [] - args.info_on_focal_labels = [] - - CSV_base_table_creator.process_all(args) - - # Check that the output files are the same as the expected files - check_identical_files(args.output_location, expected_output_path) diff --git a/tests/test_make_js_treefiles.py b/tests/test_make_js_treefiles.py deleted file mode 100644 index 9814764f..00000000 --- a/tests/test_make_js_treefiles.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Unit tests for make_js_treefiles -""" - -from oz_tree_build.utilities import make_js_treefiles - - -def test_generate_completetree_js(): - assert ( - make_js_treefiles.generate_completetree_js("(())") - == """ -var rawData = '(())'; - """.strip() - ) - assert ( - make_js_treefiles.generate_completetree_js("((()()))") - == """ -var rawData = '((()()))'; - """.strip() - ) diff --git a/tests/test_taxon_map.py b/tests/test_taxon_map.py new file mode 100644 index 00000000..3dfd3f9d --- /dev/null +++ b/tests/test_taxon_map.py @@ -0,0 +1,267 @@ +""" +Unit tests for the taxonomy-reading parts of taxon_map +""" + +from oz_tree_build.taxon_mapping_and_popularity.taxon_map import ( + add_taxon_sources, + parse_sourceinfo, + read_extra_source_file, + read_ot_taxonomy, + read_taxon_map, + write_taxon_map, +) + + +def write_ot_taxonomy(path, rows): + """Write rows (dicts) out in OpenTree taxonomy.tsv format, i.e. "\t|\t"-separated""" + header = ["uid", "parent_uid", "name", "rank", "sourceinfo", "uniqname", "flags"] + with open(path, "w", encoding="utf-8") as f: + for r in [{k: k for k in header}, *rows]: + f.write("".join(f"{r.get(k, '')}\t|\t" for k in header) + "\n") + + +def write_extra_source_file(path, rows, header=("uid", "name", "sourceinfo", "notes")): + with open(path, "w", encoding="utf-8") as f: + f.write("\t".join(header) + "\n") + for r in rows: + f.write("\t".join(str(r.get(k, "")) for k in header) + "\n") + + +class TestParseSourceinfo: + def test_numeric_ids_become_ints(self): + assert parse_sourceinfo("ncbi:1274384,gbif:8094325") == { + "ncbi": 1274384, + "gbif": 8094325, + } + + def test_non_numeric_ids_stay_strings(self): + # e.g. SILVA accessions, and GBIF ids of the form "D11377/#1" + assert parse_sourceinfo("silva:JX948102,gbif:D11377/#1") == { + "silva": "JX948102", + "gbif": "D11377/#1", + } + + def test_only_first_colon_separates(self): + assert parse_sourceinfo("silva:AB:CD") == {"silva": "AB:CD"} + + def test_zero_ids_are_kept(self): + # "life" is silva:0,ncbi:1,gbif:0,irmng:0, so 0 must not be treated as absent + assert parse_sourceinfo("silva:0,ncbi:1,gbif:0") == {"silva": 0, "ncbi": 1, "gbif": 0} + + def test_empty_sourceinfo(self): + assert parse_sourceinfo("") == {} + + def test_order_is_preserved(self): + # add_taxon_sources relies on the order for source priority + assert list(parse_sourceinfo("silva:0,ncbi:1,gbif:0,irmng:0")) == [ + "silva", + "ncbi", + "gbif", + "irmng", + ] + + +class TestReadOtTaxonomy: + def test_fields_are_split_and_converted(self, tmp_path): + path = tmp_path / "taxonomy.tsv" + write_ot_taxonomy( + path, + [ + {"uid": 805080, "name": "life", "rank": "no rank", "sourceinfo": "ncbi:1"}, + { + "uid": 93302, + "parent_uid": 805080, + "name": "cellular organisms", + "rank": "no rank", + "sourceinfo": "ncbi:131567", + }, + ], + ) + rows = list(read_ot_taxonomy(path)) + + assert [r["uid"] for r in rows] == [805080, 93302] + assert [r["parent_uid"] for r in rows] == [None, 805080] + assert [r["name"] for r in rows] == ["life", "cellular organisms"] + assert [r["rank"] for r in rows] == ["no rank", "no rank"] + assert [r["sourceinfo"] for r in rows] == [{"ncbi": 1}, {"ncbi": 131567}] + + def test_trailing_separator_is_not_a_field(self, tmp_path): + # Each line ends with "\t|\t", which must not yield an extra empty column + path = tmp_path / "taxonomy.tsv" + write_ot_taxonomy(path, [{"uid": 1, "sourceinfo": "ncbi:1", "flags": "sibling_higher"}]) + (row,) = list(read_ot_taxonomy(path)) + + assert set(row) == {"uid", "parent_uid", "name", "rank", "sourceinfo", "uniqname", "flags"} + assert row["flags"] == "sibling_higher" + + +class TestReadExtraSourceFile: + def test_row_is_parsed_like_a_taxonomy_row(self, tmp_path): + path = tmp_path / "SupplementaryTaxonomy.tsv" + write_extra_source_file( + path, + [ + { + "uid": 809432, + "name": "Strigops habroptilus", + "sourceinfo": "ncbi:2489341,irmng:11435975", + "notes": "Add in missing kakapo", + } + ], + ) + (row,) = list(read_extra_source_file(path)) + + assert row["uid"] == 809432 + assert row["sourceinfo"] == {"ncbi": 2489341, "irmng": 11435975} + assert row["name"] == "Strigops habroptilus" + assert row["notes"] == "Add in missing kakapo" + + def test_non_numeric_uid_stays_a_string(self, tmp_path): + path = tmp_path / "extra.tsv" + write_extra_source_file(path, [{"uid": "mrcaott409215ott616649", "sourceinfo": "ncbi:1"}]) + (row,) = list(read_extra_source_file(path)) + + assert row["uid"] == "mrcaott409215ott616649" + + def test_only_uid_and_sourceinfo_are_required(self, tmp_path): + path = tmp_path / "extra.tsv" + write_extra_source_file(path, [{"uid": 1, "sourceinfo": "gbif:2"}], header=("uid", "sourceinfo")) + (row,) = list(read_extra_source_file(path)) + + assert row == {"uid": 1, "sourceinfo": {"gbif": 2}} + + def test_missing_file_is_ignored_with_a_warning(self, tmp_path, caplog): + path = tmp_path / "nonexistent.tsv" + + assert list(read_extra_source_file(path)) == [] + assert "not found" in caplog.text + + +class TestAddTaxonSources: + def test_adds_a_new_ott_with_its_sources(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 770315, {"ncbi": 9999, "gbif": 1234}, "species") + + assert OTT_ptrs == { + 770315: { + "ott": 770315, + "rank": "species", + "sources": {"ncbi": {"id": 9999}, "gbif": {"id": 1234}}, + } + } + assert source_ptrs == {"ncbi": {9999: {"id": 9999}}, "gbif": {1234: {"id": 1234}}} + + def test_ott_and_source_entries_are_the_same_object(self): + # Wikidata data is added via source_ptrs, and read back out via OTT_ptrs, + # so the two must point at one shared dict + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5}) + + assert OTT_ptrs[1]["sources"]["ncbi"] is source_ptrs["ncbi"][5] + + def test_otts_sharing_a_source_id_share_its_entry(self): + # Otherwise the first OTT is left pointing at an orphaned dict, which never + # gets the wikidata item that is added via source_ptrs + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5}) + add_taxon_sources(OTT_ptrs, source_ptrs, 2, {"ncbi": 5}) + + source_ptrs["ncbi"][5]["wd"] = "Q123" + assert OTT_ptrs[1]["sources"]["ncbi"] is OTT_ptrs[2]["sources"]["ncbi"] + assert OTT_ptrs[1]["sources"]["ncbi"]["wd"] == "Q123" + + def test_reading_a_source_keeps_data_added_to_its_entry(self): + # An extra_source_file row re-stating an id must not wipe out its wikidata item + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5}) + source_ptrs["ncbi"][5]["wd"] = "Q123" + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 5}) + + assert OTT_ptrs[1]["sources"]["ncbi"] == {"id": 5, "wd": "Q123"} + + def test_non_numeric_source_ids_are_usable(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"gbif": "D11377/#1"}) + + assert source_ptrs["gbif"]["D11377/#1"] == {"id": "D11377/#1"} + + def test_unknown_sources_are_ignored(self): + # The taxonomy carries ~150 source names we can't map to wikidata + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": 0, "h2007": 1, "additions-6520052-6520144": 2}) + + assert OTT_ptrs == {1: {"ott": 1, "sources": {}}} + assert source_ptrs == {} + + def test_unknown_sources_are_collected_when_asked(self): + OTT_ptrs, source_ptrs = {}, {} + unused = set() + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": 0, "ncbi": 1}, unused_sources=unused) + add_taxon_sources(OTT_ptrs, source_ptrs, 2, {"silva": 3, "h2007": 4}, unused_sources=unused) + + assert unused == {"silva", "h2007"} + assert set(source_ptrs) == {"ncbi"} + + def test_a_second_call_overrides_only_the_sources_given(self): + # i.e. how an extra_source_file supplements the OpenTree taxonomy + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 9999, "gbif": 1234}, "species") + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 2489341, "irmng": 11435975}) + + assert OTT_ptrs[1]["sources"] == { + "ncbi": {"id": 2489341}, + "gbif": {"id": 1234}, + "irmng": {"id": 11435975}, + } + + def test_rank_is_kept_when_not_given(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 1}, "species") + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"gbif": 2}) + + assert OTT_ptrs[1]["rank"] == "species" + + def test_rank_is_absent_if_never_given(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"ncbi": 1}) + + assert "rank" not in OTT_ptrs[1] + + def test_a_silva_derived_ncbi_id_is_used_like_any_other(self): + # NCBI ids from SILVA-sourced rows used to be singled out as "ncbi_silva" + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {"silva": "JX948102", "ncbi": 1274384}) + + assert OTT_ptrs[1]["sources"] == {"ncbi": {"id": 1274384}} + + def test_empty_sourceinfo_still_adds_the_ott(self): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, {}, "species") + + assert OTT_ptrs == {1: {"ott": 1, "rank": "species", "sources": {}}} + assert source_ptrs == {} + + +class TestWriteTaxonMap: + def build(self, tmp_path, sourceinfo): + OTT_ptrs, source_ptrs = {}, {} + add_taxon_sources(OTT_ptrs, source_ptrs, 1, parse_sourceinfo(sourceinfo), "species") + path = tmp_path / "taxon_map.csv" + with open(path, "w", encoding="utf-8", newline="") as f: + write_taxon_map(f, OTT_ptrs) + return path + + def test_every_source_reaches_its_column(self, tmp_path): + # NB: sources are named as the taxonomy names them, i.e. index fungorum is "if" + path = self.build(tmp_path, "ncbi:1,if:2,worms:3,irmng:4,gbif:5") + row = read_taxon_map(path)[1] + + assert (row["ncbi"], row["if"], row["worms"], row["irmng"], row["gbif"]) == (1, 2, 3, 4, 5) + + def test_absent_sources_are_empty(self, tmp_path): + path = self.build(tmp_path, "ncbi:1") + row = read_taxon_map(path)[1] + + assert row["ncbi"] == 1 + assert (row["if"], row["worms"], row["irmng"], row["gbif"]) == (None, None, None, None) + assert (row["ott"], row["rank"]) == (1, "species") diff --git a/tests/test_tree_build_step_graft.py b/tests/test_tree_build_step_graft.py new file mode 100644 index 00000000..8ba73ad9 --- /dev/null +++ b/tests/test_tree_build_step_graft.py @@ -0,0 +1,210 @@ +import ete4 + +from oz_tree_build.tree_build.step_graft import graft_extract_ot_subtrees, graft_tree, present_in_tree + + +class TestGraftTree: + def test_no_inclusions_is_noop(self): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + missing = graft_tree(t, {}) + assert missing == [] + assert t.write() == "(A_ott1,B_ott2);" + + def test_simple_graft(self): + # The grafted node inherits the subtree's children and its dist. + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Y_ott12)SubRoot_ott1;", parser=1) + sub.root.dist = 3.5 + + missing = graft_tree(t, {"Sub_ott1@": sub}) + + assert missing == [] + assert t.write() == "(A_ott99,(X_ott11,Y_ott12):3.5);" + + def test_missing_inclusion_reported(self): + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + missing = graft_tree(t, {}) + assert missing == ["Sub_ott1@"] + # Original tree untouched at the inclusion point. + assert t.write() == "(A_ott99,Sub_ott1@);" + + def test_multiple_missing_inclusions(self): + t = ete4.Tree("(A_ott1@,(B_ott2@,C_ott3@)Sub)Root;", parser=1) + missing = graft_tree(t, {}) + assert sorted(missing) == ["A_ott1@", "B_ott2@", "C_ott3@"] + + def test_prefer_subtree_name_uses_subroot_name(self): + # With prefer_subtree_name=True, the subtree's root name wins. + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Y_ott12)SubRoot_ott1;", parser=1) + + graft_tree(t, {"Sub_ott1@": sub}, prefer_subtree_name=True) + + grafted_names = [n.name for n in t.traverse() if not n.is_leaf and n.name] + assert "SubRoot_ott1" in grafted_names + + def test_prefer_subtree_name_falls_back_when_subroot_unnamed(self): + # If the subtree's root is unnamed, fall back to the inclusion's + # derived node_name ("Sub ott1"). + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Y_ott12);", parser=1) + + graft_tree(t, {"Sub_ott1@": sub}, prefer_subtree_name=True) + + grafted_names = [n.name for n in t.traverse() if not n.is_leaf and n.name] + assert "Sub ott1" in grafted_names + + def test_recursion_resolves_nested_inclusions(self): + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Inner_ott2@)SubRoot;", parser=1) + inner = ete4.Tree("(I1_ott21,I2_ott22)InnerRoot;", parser=1) + + missing = graft_tree(t, {"Sub_ott1@": sub, "Inner_ott2@": inner}) + + assert missing == [] + assert t.write() == "(A_ott99,(X_ott11,(I1_ott21,I2_ott22)));" + + def test_disable_recursion_leaves_nested_inclusions(self): + # disable_recursion=True skips the inner graft_tree call, so the + # nested inclusion token is left in place and not reported missing + # (since the traversal treats the grafted node as a leaf). + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Inner_ott2@)SubRoot;", parser=1) + inner = ete4.Tree("(I1_ott21,I2_ott22)InnerRoot;", parser=1) + + missing = graft_tree( + t, + {"Sub_ott1@": sub, "Inner_ott2@": inner}, + disable_recursion=True, + ) + + assert missing == [] + assert t.write() == "(A_ott99,(X_ott11,Inner_ott2@));" + + def test_recursion_reports_missing_from_nested(self): + # A nested inclusion that has no provided subtree is reported. + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Missing_ott2@)SubRoot;", parser=1) + + missing = graft_tree(t, {"Sub_ott1@": sub}) + + assert missing == ["Missing_ott2@"] + + def test_subtree_props_copied(self): + # Non-None props from the subtree root are copied onto the grafted node. + t = ete4.Tree("(A_ott99,Sub_ott1@)Root;", parser=1) + sub = ete4.Tree("(X_ott11,Y_ott12)SubRoot_ott1;", parser=1) + sub.root.props["custom_prop"] = "hello" + + graft_tree(t, {"Sub_ott1@": sub}) + + grafted = next(n for n in t.traverse() if n.props.get("custom_prop") == "hello") + assert grafted is not None + + +class TestGraftExtractOtSubtrees: + def test_extracts_subtrees_by_base_ott(self, tmp_path): + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((X1_ott11,X2_ott12)Sub1_ott1,(Y1_ott21,Y2_ott22)Sub2_ott2)Inner_ott3,Z_ott4)Root_ott99;") + result = graft_extract_ot_subtrees(ete4.Tree(str(ot_file), parser=1), ["Sub1_ott1@", "Sub2_ott2@"]) + assert set(result.keys()) == {"Sub1_ott1@", "Sub2_ott2@"} + assert result["Sub1_ott1@"].write() == "(X1_ott11,X2_ott12);" + assert result["Sub2_ott2@"].write() == "(Y1_ott21,Y2_ott22);" + + def test_missing_otts_not_in_result(self, tmp_path): + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("((A_ott1,B_ott2)Sub_ott3)Root_ott4;") + result = graft_extract_ot_subtrees(ete4.Tree(str(ot_file), parser=1), ["NotThere_ott99@"]) + assert result == {} + + def test_renaming_inclusion_uses_orig_name_as_key(self, tmp_path): + # The key is the original inclusion string, including any rebase syntax. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("((A_ott1,B_ott2)Sub_ott5,C_ott3)Root_ott99;") + result = graft_extract_ot_subtrees(ete4.Tree(str(ot_file), parser=1), ["Renamed_ott~5@"]) + assert list(result.keys()) == ["Renamed_ott~5@"] + assert result["Renamed_ott~5@"].write() == "(A_ott1,B_ott2);" + + def test_recurses_into_extracted_subtrees(self, tmp_path): + # If an extracted subtree itself contains another requested OTT, that + # nested subtree is also extracted. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((((I1_ott11,I2_ott12)Inner_ott1)Filler_ott99)Sub_ott2)Outer_ott3)Root_ott99;") + result = graft_extract_ot_subtrees(ete4.Tree(str(ot_file), parser=1), ["Outer_ott2@", "Nested_ott1@"]) + assert set(result.keys()) == {"Outer_ott2@", "Nested_ott1@"} + assert result["Nested_ott1@"].write() == "(I1_ott11,I2_ott12);" + # NB: Outer tree no longer contains inner tree. Filler_ott99 held nothing + # but the nested subtree, so extracting it emptied Filler too and Filler + # goes with it -- leaving the outer subtree with nothing in it at all. + assert result["Outer_ott2@"].write() == ";" + + def test_ancestor_emptied_by_extraction_is_removed(self, tmp_path): + # Every child of Inner_ott3 is requested separately, so Inner is left + # with nothing below it. It would otherwise survive as a childless node, + # reading as a leaf and being written out as though it were a species. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((X_ott11)Sub1_ott1,(Y_ott21)Sub2_ott2)Inner_ott3,Z_ott4)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub1_ott1@", "Sub2_ott2@"]) + assert [n.name for n in ot_t.traverse()] == ["Root_ott99", "Z_ott4"] + + def test_emptying_cascades_up_the_ancestry(self, tmp_path): + # Removing Inner empties Middle, which empties Outer. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("((((((X_ott11)Sub_ott1)Inner_ott5)Middle_ott6)Outer_ott7),Z_ott4)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub_ott1@"]) + assert not any(n.name.startswith(("Inner", "Middle", "Outer")) for n in ot_t.traverse()) + assert "Z_ott4" in [n.name for n in ot_t.traverse()] + + def test_ancestor_keeping_a_child_is_left_alone(self, tmp_path): + # Inner still holds Keep_ott9, so it is a genuine ancestor, not debris. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((X_ott11)Sub_ott1,Keep_ott9)Inner_ott3,Z_ott4)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub_ott1@"]) + names = [n.name for n in ot_t.traverse()] + assert "Inner_ott3" in names + assert "Keep_ott9" in names + + def test_pre_existing_tips_are_not_pruned(self, tmp_path): + # A node that was already a tip is a real taxon, not something we + # emptied, so it must survive even though it has no children. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("(((X_ott11)Sub_ott1,Tip_ott8)Inner_ott3,Z_ott4)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub_ott1@"]) + assert "Tip_ott8" in [n.name for n in ot_t.traverse()] + + def test_root_is_never_detached(self, tmp_path): + # Extracting everything leaves the root childless; it must stay put. + ot_file = tmp_path / "ot.nwk" + ot_file.write_text("((X_ott11)Sub_ott1)Root_ott99;") + ot_t = ete4.Tree(str(ot_file), parser=1) + graft_extract_ot_subtrees(ot_t, ["Sub_ott1@"]) + assert ot_t.up is None + assert ot_t.name == "Root_ott99" + + +class TestPresentInTree: + def test_finds_matching_node(self): + t = ete4.Tree("(A_ott99,(Sub_ott1,C_ott4)B_ott7)Root_ott42;", parser=1) + n = present_in_tree(t, "Anything_ott1@") + assert n is not None + assert n.name == "Sub_ott1" + + def test_matches_internal_node(self): + t = ete4.Tree("(A_ott99,(Sub_ott1,C_ott4)B_ott7)Root_ott42;", parser=1) + n = present_in_tree(t, "Anything_ott7@") + assert n is not None + assert n.name == "B_ott7" + + def test_returns_none_when_absent(self): + t = ete4.Tree("(A_ott99,(Sub_ott1,C_ott4)B_ott7)Root_ott42;", parser=1) + assert present_in_tree(t, "Anything_ott999@") is None + + def test_matches_root(self): + t = ete4.Tree("(A_ott99,B_ott7)Root_ott42;", parser=1) + n = present_in_tree(t, "Anything_ott42@") + assert n is not None + assert n.name == "Root_ott42" diff --git a/tests/test_tree_build_step_jsnewick.py b/tests/test_tree_build_step_jsnewick.py new file mode 100644 index 00000000..6ae47957 --- /dev/null +++ b/tests/test_tree_build_step_jsnewick.py @@ -0,0 +1,189 @@ +import ete4 + +from oz_tree_build.tree_build.step_jsnewick import ( + jsnewick_brief_newick, + jsnewick_cutpositionmap_binary, + jsnewick_cutpositionmap_polytomy, +) +from oz_tree_build.tree_build.step_tidy import POLYTOMY_COMB, POLYTOMY_PROP + + +def _mark_polytomies(tree, *names): + """Flag the named internal nodes as polytomy resolutions.""" + for node in tree.traverse(): + if node.name in names: + node.props[POLYTOMY_PROP] = POLYTOMY_COMB + + +######################################## +# jsnewick_brief_newick +######################################## + + +def test_brief_newick_single_internal(): + """A single internal with two leaf children collapses to a bare ``()``.""" + t = ete4.Tree("(A,B);", parser=1) + assert jsnewick_brief_newick(t) == "()" + + +def test_brief_newick_nested_left(): + """``((A,B),C)`` and ``(C,(A,B))`` both yield ``(())`` — leaves are invisible.""" + t = ete4.Tree("((A,B),C);", parser=1) + assert jsnewick_brief_newick(t) == "(())" + t = ete4.Tree("(C,(A,B));", parser=1) + assert jsnewick_brief_newick(t) == "(())" + + +def test_brief_newick_two_subtrees(): + t = ete4.Tree("((A,B),(C,D));", parser=1) + assert jsnewick_brief_newick(t) == "(()())" + + +def test_brief_newick_deep_caterpillar(): + t = ete4.Tree("(A,(B,(C,(D,E))));", parser=1) + assert jsnewick_brief_newick(t) == "(((())))" + + +def test_brief_newick_polytomy_braces_default(): + """With the default polytomy_braces the polytomy prop is invisible.""" + t = ete4.Tree("((A:1,B:1)P:1,C:2);", parser=1) + _mark_polytomies(t, "P") + assert jsnewick_brief_newick(t) == "(())" + + +def test_brief_newick_polytomy_braces_overridden(): + """An internal with the polytomy prop gets the override braces; one without does not.""" + t = ete4.Tree("((A:1,B:1)P:1,C:2);", parser=1) + _mark_polytomies(t, "P") + assert jsnewick_brief_newick(t, polytomy_braces="{}") == "({})" + + t_unmarked = ete4.Tree("((A:1,B:1)P:1,C:2);", parser=1) + assert jsnewick_brief_newick(t_unmarked, polytomy_braces="{}") == "(())" + + +def test_brief_newick_polytomy_ignores_zero_dist(): + """A zero-length branch is no longer a polytomy marker on its own.""" + t = ete4.Tree("((A:1,B:1)P:0,C:2);", parser=1) + assert jsnewick_brief_newick(t, polytomy_braces="{}") == "(())" + + +def test_brief_newick_polytomy_root_excluded(): + """The root is never treated as a polytomy resolution, even if marked.""" + t = ete4.Tree("(A:1,B:1)R:1;", parser=1) + _mark_polytomies(t, "R") + assert jsnewick_brief_newick(t, polytomy_braces="{}") == "()" + + +def test_brief_newick_polytomy_braces_nested(): + """Multiple marked ancestors each get the polytomy braces.""" + t = ete4.Tree("(((A:1,B:1)P:1,C:2)Q:1,D:1);", parser=1) + _mark_polytomies(t, "P", "Q") + assert jsnewick_brief_newick(t, polytomy_braces="{}") == "({{}})" + + +######################################## +# jsnewick_cutpositionmap_binary +######################################## + + +def test_cutmap_binary_two_leaves_empty(): + """A single internal with two leaf children produces no entry — nothing to split.""" + t = ete4.Tree("(A,B);", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {} + + +def test_cutmap_binary_internal_then_leaf(): + """First child internal → cut is the position of that child's ``)``.""" + # brief = '(())': root open=0, inner open=1, inner close=2, root close=3. + t = ete4.Tree("((A,B),C);", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {3: 2} + + +def test_cutmap_binary_leaf_then_internal(): + """First child leaf → cut is the parent's ``(`` position.""" + t = ete4.Tree("(C,(A,B));", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {3: 0} + + +def test_cutmap_binary_two_internals(): + """Both children internal → cut is the first child's ``)`` position.""" + # brief = '(()())': positions root=0/5, (A,B)=1/2, (C,D)=3/4. + t = ete4.Tree("((A,B),(C,D));", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {5: 2} + + +def test_cutmap_binary_caterpillar(): + """Recursive descent records each non-trivial internal along the spine.""" + # (A,(B,(C,(D,E)))) → brief = '(((())))', root close at pos 7. + t = ete4.Tree("(A,(B,(C,(D,E))));", parser=1) + assert jsnewick_cutpositionmap_binary(t, threshold=0) == {7: 0, 6: 1, 5: 2} + + +def test_cutmap_binary_threshold_skips_small_subtrees(): + """Only subtrees whose bracket span exceeds the threshold get recursed into.""" + t = ete4.Tree("(A,(B,(C,(D,E))));", parser=1) + # Subtree spans: root=8, (B,(C,(D,E)))=6, (C,(D,E))=4, (D,E)=2. + # threshold=5 admits the first two; threshold=6 admits only the root. + assert jsnewick_cutpositionmap_binary(t, threshold=5) == {7: 0, 6: 1} + assert jsnewick_cutpositionmap_binary(t, threshold=6) == {7: 0} + + +######################################## +# jsnewick_cutpositionmap_polytomy +######################################## + + +def test_cutmap_polytomy_two_leaves_degenerate(): + """An internal with two leaf children falls back to the parent's own positions.""" + t = ete4.Tree("(A,B);", parser=1) + # threshold=0 still enqueues the root (the worklist always seeds with it). + assert jsnewick_cutpositionmap_polytomy(t, threshold=0) == {1: [0, 0, 1, 1]} + + +def test_cutmap_polytomy_internal_then_leaf(): + """Internal child gets its (start, end); trailing leaf gets the inverted + [parent_close, parent_close-1] empty range.""" + t = ete4.Tree("((A,B),C);", parser=1) + # Top-level root cut: c1=(A,B) spans [1,2], c2=C trails at [3,2]. + # Inner two-leaf node also gets degenerate entry at threshold=0. + assert jsnewick_cutpositionmap_polytomy(t, threshold=0) == { + 3: [1, 2, 3, 2], + 2: [1, 1, 2, 2], + } + + +def test_cutmap_polytomy_leaf_then_internal(): + """Leading leaf gets the inverted [parent_open+1, parent_open] empty range.""" + t = ete4.Tree("(C,(A,B));", parser=1) + assert jsnewick_cutpositionmap_polytomy(t, threshold=0) == { + 3: [1, 0, 1, 2], + 2: [1, 1, 2, 2], + } + + +def test_cutmap_polytomy_two_internals(): + """Both children internal → flat list of both children's spans.""" + t = ete4.Tree("((A,B),(C,D));", parser=1) + assert jsnewick_cutpositionmap_polytomy(t, threshold=0) == { + 5: [1, 2, 3, 4], + 2: [1, 1, 2, 2], + 4: [3, 3, 4, 4], + } + + +def test_cutmap_polytomy_threshold_off_by_one_vs_binary(): + """The polytomy recursion uses ``span > threshold`` (raw span), while binary + uses ``span + 1 > threshold``; at the same threshold the polytomy map admits + one fewer level than the binary map.""" + t = ete4.Tree("(A,(B,(C,(D,E))));", parser=1) + # The (B,(C,(D,E))) child of root has bracket span 6-1 = 5. + # Binary admits it (5+1 > 5); polytomy does not (5 > 5 is false), so only + # the root entry survives in polytomy at threshold=5. + assert jsnewick_cutpositionmap_binary(t, threshold=5) == {7: 0, 6: 1} + assert jsnewick_cutpositionmap_polytomy(t, threshold=5) == {7: [1, 0, 1, 6]} + + # Dropping the threshold below the span lets polytomy recurse one more level. + assert jsnewick_cutpositionmap_polytomy(t, threshold=3) == { + 7: [1, 0, 1, 6], + 6: [2, 1, 2, 5], + } diff --git a/tests/test_tree_build_step_output.py b/tests/test_tree_build_step_output.py new file mode 100644 index 00000000..4d37dbeb --- /dev/null +++ b/tests/test_tree_build_step_output.py @@ -0,0 +1,737 @@ +import csv +import json +import os +import struct + +import ete4 +import pytest + +from oz_tree_build.tree_build.step_output import ( + output_add_prop_ids, + output_jssource, + output_mysqlexport, + output_proparray, +) +from oz_tree_build.tree_build.step_tidy import POLYTOMY_COMB, POLYTOMY_PROP + + +def _by_name(tree): + return {n.name: n for n in tree.traverse()} + + +def _prep(tree, taxon_overrides=None): + """ + Give every node the minimum props output_mysqlexport requires: + a (possibly empty) taxon dict and the id props from output_add_prop_ids. + `taxon_overrides` is `{node_name: {key: value, ...}}`. + """ + taxon_overrides = taxon_overrides or {} + for n in tree.traverse(): + n.props["taxon"] = dict(taxon_overrides.get(n.name, {})) + output_add_prop_ids(tree) + + +def _read_csv(out_dir, name): + with open(os.path.join(out_dir, name), encoding="utf-8") as f: + return list(csv.reader(f)) + + +class TestOutputAddPropIds: + def test_minimal_two_leaf_tree(self): + # Single internal node with two leaves. The root is the only node + # that receives id/leaf_lft/leaf_rgt/node_rgt. + t = ete4.Tree("(A,B)R;", parser=1) + output_add_prop_ids(t) + assert t.props["id"] == 1 + assert t.props["leaf_lft"] == 1 + assert t.props["leaf_rgt"] == 2 + # No internal node sits below the root, so its rightmost-internal + # descendant is itself. + assert t.props["node_rgt"] == 1 + + def test_leaves_get_no_id_props(self): + # Leaves are implicitly numbered by their preorder position; the + # function must not write any of the id props onto leaf nodes. + t = ete4.Tree("((A,B)I,(C,D)J)R;", parser=1) + output_add_prop_ids(t) + for leaf in t.leaves(): + assert "id" not in leaf.props + assert "leaf_lft" not in leaf.props + assert "leaf_rgt" not in leaf.props + assert "node_rgt" not in leaf.props + + def test_internal_ids_are_one_based_preorder(self): + # Internal nodes receive ids 1..N in preorder. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + ids = [n.props["id"] for n in t.traverse("preorder") if not n.is_leaf] + assert ids == [1, 2, 3, 4] + nodes = _by_name(t) + assert nodes["R"].props["id"] == 1 + assert nodes["I"].props["id"] == 2 + assert nodes["K"].props["id"] == 3 + assert nodes["J"].props["id"] == 4 + + def test_leaf_lft_is_position_of_leftmost_descendant_leaf(self): + # leaf_lft is the 1-based preorder position of the subtree's + # leftmost leaf. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + # Preorder leaf order is A, B, C, D, E → positions 1..5. + assert nodes["R"].props["leaf_lft"] == 1 # leftmost descendant is A + assert nodes["I"].props["leaf_lft"] == 1 # leftmost descendant is A + assert nodes["K"].props["leaf_lft"] == 3 # leftmost descendant is C + assert nodes["J"].props["leaf_lft"] == 4 # leftmost descendant is D + + def test_leaf_rgt_is_position_of_rightmost_descendant_leaf(self): + # leaf_rgt is the 1-based preorder position of the subtree's + # rightmost leaf. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + assert nodes["R"].props["leaf_rgt"] == 5 # rightmost descendant is E + assert nodes["I"].props["leaf_rgt"] == 2 # rightmost descendant is B + assert nodes["K"].props["leaf_rgt"] == 5 # rightmost descendant is E + assert nodes["J"].props["leaf_rgt"] == 5 # rightmost descendant is E + + def test_node_rgt_for_terminal_internal_is_self(self): + # If every child of an internal node is a leaf, its rightmost + # internal descendant is itself. + t = ete4.Tree("((A,B)I,(C,D)J)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + assert nodes["I"].props["node_rgt"] == nodes["I"].props["id"] + assert nodes["J"].props["node_rgt"] == nodes["J"].props["id"] + + def test_node_rgt_walks_rightmost_internal_descendant(self): + # For a non-terminal internal node, node_rgt is the id of the + # rightmost internal descendant (assuming ascending ladderization + # so the rightmost child holds the biggest subtree). + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + # R's rightmost internal descendant is J (id=4) via K. + assert nodes["R"].props["node_rgt"] == 4 + # K's rightmost internal descendant is J (id=4). + assert nodes["K"].props["node_rgt"] == 4 + + def test_unnamed_internal_nodes_still_get_props(self): + # The function keys off is_leaf, not the node name, so unnamed + # internal nodes still receive the id props. + t = ete4.Tree("(C,(A,B));", parser=1) + output_add_prop_ids(t) + internals = [n for n in t.traverse("preorder") if not n.is_leaf] + assert [n.props["id"] for n in internals] == [1, 2] + # Root (id=1) spans all three leaves; its rightmost internal + # descendant is the (A,B) subtree (id=2). + assert internals[0].props["leaf_lft"] == 1 + assert internals[0].props["leaf_rgt"] == 3 + assert internals[0].props["node_rgt"] == 2 + # (A,B) (id=2) spans leaves 2..3; terminal, so node_rgt is itself. + assert internals[1].props["leaf_lft"] == 2 + assert internals[1].props["leaf_rgt"] == 3 + assert internals[1].props["node_rgt"] == 2 + + def test_relies_on_ascending_ladderization(self): + # node_rgt is computed by walking postorder and trusting that the + # last-visited child sits at the right of its parent — which holds + # only when the tree is ladderized ascending (small subtree first, + # so the rightmost child carries the largest subtree). If a leaf + # sits to the right of an internal sibling, the function wrongly + # treats the parent as terminal. This test pins that assumption. + t = ete4.Tree("((A,B)I,C)R;", parser=1) # descending: leaf C on the right + output_add_prop_ids(t) + nodes = _by_name(t) + # R's true rightmost-internal descendant is I (id=2), but because + # C is the rightmost child the function records R as terminal. + assert nodes["R"].props["id"] == 1 + assert nodes["R"].props["node_rgt"] == 1 + + def test_deeply_nested_ladder(self): + # A right-leaning ladder: each level's rightmost child is the + # bigger subtree, so node_rgt should chain down to the deepest + # internal node. + t = ete4.Tree("(A,(B,(C,(D,E)J)K)L)R;", parser=1) + output_add_prop_ids(t) + nodes = _by_name(t) + # Preorder of internals: R, L, K, J → ids 1, 2, 3, 4. + assert nodes["R"].props["id"] == 1 + assert nodes["L"].props["id"] == 2 + assert nodes["K"].props["id"] == 3 + assert nodes["J"].props["id"] == 4 + # Every ancestor's rightmost internal descendant is J. + assert nodes["R"].props["node_rgt"] == 4 + assert nodes["L"].props["node_rgt"] == 4 + assert nodes["K"].props["node_rgt"] == 4 + assert nodes["J"].props["node_rgt"] == 4 # terminal → self + # leaf_lft / leaf_rgt span the leaves below each node. + assert nodes["R"].props["leaf_lft"] == 1 + assert nodes["R"].props["leaf_rgt"] == 5 + assert nodes["L"].props["leaf_lft"] == 2 + assert nodes["L"].props["leaf_rgt"] == 5 + assert nodes["K"].props["leaf_lft"] == 3 + assert nodes["K"].props["leaf_rgt"] == 5 + assert nodes["J"].props["leaf_lft"] == 4 + assert nodes["J"].props["leaf_rgt"] == 5 + + def test_leaf_lft_leq_leaf_rgt_for_every_internal(self): + # Sanity invariant — leftmost-leaf position never exceeds the + # rightmost-leaf position within the same subtree. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + for n in t.traverse(): + if n.is_leaf: + continue + assert n.props["leaf_lft"] <= n.props["leaf_rgt"] + + def test_node_rgt_never_exceeds_max_internal_id(self): + # node_rgt always points at an existing internal node id, so it + # cannot exceed the count of internal nodes. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + output_add_prop_ids(t) + internals = [n for n in t.traverse() if not n.is_leaf] + max_id = max(n.props["id"] for n in internals) + for n in internals: + assert 1 <= n.props["node_rgt"] <= max_id + + +# Header columns in the order written by output_mysqlexport. +LEAF_HEADER = [ + "parent", + "real_parent", + "name", + "extinction_date", + "ott", + "wikidata", + "wikipedia_lang_flag", + "iucn", + "eol", + "raw_popularity", + "popularity", + "popularity_rank", + "price", + "ncbi", + "ifung", + "worms", + "irmng", + "gbif", + "ipni", +] + +NODE_HEADER = ( + [ + "parent", + "real_parent", + "node_rgt", + "leaf_lft", + "leaf_rgt", + "name", + "age", + "ott", + "wikidata", + "wikipedia_lang_flag", + "eol", + "rnk", + "raw_popularity", + "popularity", + "ncbi", + "ifung", + "worms", + "irmng", + "gbif", + "ipni", + "vern_synth", + ] + + [rit + str(i + 1) for rit in ("rep", "rtr", "rpd") for i in range(8)] + + ["iucn" + t for t in ("NE", "DD", "LC", "NT", "VU", "EN", "CR", "EW", "EX")] +) + + +class TestOutputMysqlExport: + def test_creates_three_output_files(self, tmp_path): + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + assert (tmp_path / "ordered_leaves.csv").exists() + assert (tmp_path / "ordered_nodes.csv").exists() + assert (tmp_path / "import.sql").exists() + + def test_leaf_header_matches_expected_columns(self, tmp_path): + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + rows = _read_csv(tmp_path, "ordered_leaves.csv") + assert rows[0] == LEAF_HEADER + + def test_node_header_matches_expected_columns(self, tmp_path): + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + rows = _read_csv(tmp_path, "ordered_nodes.csv") + assert rows[0] == NODE_HEADER + + def test_leaf_and_node_row_widths_match_their_headers(self, tmp_path): + # Each emitted row must have exactly as many fields as its header, + # otherwise MySQL's LOAD DATA INFILE will reject it. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + for row in leaves[1:]: + assert len(row) == len(LEAF_HEADER) + for row in nodes[1:]: + assert len(row) == len(NODE_HEADER) + + def test_leaves_go_to_leaf_csv_internals_to_node_csv(self, tmp_path): + # Five leaves under four internal nodes → 5 data rows in leaves, + # 4 data rows in nodes. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + leaf_names = [r[LEAF_HEADER.index("name")] for r in leaves[1:]] + node_names = [r[NODE_HEADER.index("name")] for r in nodes[1:]] + assert sorted(leaf_names) == ["A", "B", "C", "D", "E"] + assert sorted(node_names) == ["I", "J", "K", "R"] + + def test_leaf_rows_are_in_preorder(self, tmp_path): + # The function traverses preorder; leaf rows should reflect that. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + leaf_names = [r[LEAF_HEADER.index("name")] for r in leaves[1:]] + assert leaf_names == ["A", "B", "C", "D", "E"] + + def test_leaf_name_strips_ott_suffix(self, tmp_path): + # The "_ottNNN" suffix carries the OTT id and is removed from the + # name written to the CSV. + t = ete4.Tree("(A_ott1234,B_ott5678)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + names = [r[LEAF_HEADER.index("name")] for r in leaves[1:]] + assert names == ["A", "B"] + + def test_root_parent_field_is_placeholder(self, tmp_path): + # Root has no parent, but "parent" is NOT NULL in the DB, so it gets a + # placeholder for the import script to replace rather than \N. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root_row = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + assert root_row[NODE_HEADER.index("parent")] == "-999" + + def test_parentless_leaf_is_rejected(self, tmp_path): + # A single-leaf tree has a leaf at the root. There's no placeholder for + # this case, so it's an error rather than a bad "parent" value. + t = ete4.Tree("A;", parser=1) + _prep(t) + with pytest.raises(ValueError, match="no parent"): + output_mysqlexport(t, str(tmp_path)) + + def test_root_real_parent_is_zero(self, tmp_path): + # Root has no parent, so real_parent is the sentinel 0. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root_row = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + assert root_row[NODE_HEADER.index("real_parent")] == "0" + + def test_non_root_parent_is_parent_id(self, tmp_path): + # A leaf's "parent" field is the id of its (internal) parent. + t = ete4.Tree("((A,B)I,(C,D)J)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + nodes = _by_name(t) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + i_id = str(nodes["I"].props["id"]) + j_id = str(nodes["J"].props["id"]) + rows = {r[LEAF_HEADER.index("name")]: r for r in leaves[1:]} + assert rows["A"][LEAF_HEADER.index("parent")] == i_id + assert rows["B"][LEAF_HEADER.index("parent")] == i_id + assert rows["C"][LEAF_HEADER.index("parent")] == j_id + assert rows["D"][LEAF_HEADER.index("parent")] == j_id + + def test_node_row_writes_id_range_columns(self, tmp_path): + # Internal nodes carry node_rgt / leaf_lft / leaf_rgt across to + # their CSV row. + t = ete4.Tree("((A,B)I,(C,(D,E)J)K)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + nodes_by_name = _by_name(t) + rows = _read_csv(tmp_path, "ordered_nodes.csv") + by_name = {r[NODE_HEADER.index("name")]: r for r in rows[1:]} + for nm in ("R", "I", "K", "J"): + row = by_name[nm] + n = nodes_by_name[nm] + assert row[NODE_HEADER.index("node_rgt")] == str(n.props["node_rgt"]) + assert row[NODE_HEADER.index("leaf_lft")] == str(n.props["leaf_lft"]) + assert row[NODE_HEADER.index("leaf_rgt")] == str(n.props["leaf_rgt"]) + + def test_taxon_props_are_written_to_leaf_row(self, tmp_path): + # Values supplied via node.props["taxon"] are projected onto the + # matching CSV columns; absent keys become \N. + t = ete4.Tree("(A,B)R;", parser=1) + _prep( + t, + taxon_overrides={ + "A": { + "ott": "111", + "wikidata": "Q1", + "iucn": "LC", + "eol": "42", + "ncbi": "999", + "if": "531546", + }, + }, + ) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + rows = {r[LEAF_HEADER.index("name")]: r for r in leaves[1:]} + a = rows["A"] + assert a[LEAF_HEADER.index("ott")] == "111" + assert a[LEAF_HEADER.index("wikidata")] == "Q1" + assert a[LEAF_HEADER.index("iucn")] == "LC" + assert a[LEAF_HEADER.index("eol")] == "42" + assert a[LEAF_HEADER.index("ncbi")] == "999" + # The taxonomy's "if" source is the DB's "ifung" column + assert a[LEAF_HEADER.index("ifung")] == "531546" + # B had no overrides → \N everywhere taxon-derived. + b = rows["B"] + assert b[LEAF_HEADER.index("ott")] == "\\N" + assert b[LEAF_HEADER.index("ncbi")] == "\\N" + + def test_taxon_props_are_written_to_node_row(self, tmp_path): + # Internal nodes get the same taxon projection — but with the rank + # in place of the leaf-only `iucn`/`extinction_date` columns. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t, taxon_overrides={"R": {"ott": "777", "rank": "family", "if": "9257"}}) + output_mysqlexport(t, str(tmp_path)) + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + assert root[NODE_HEADER.index("ott")] == "777" + # The taxon map's "rank" is the DB's "rnk" column + assert root[NODE_HEADER.index("rnk")] == "family" + assert root[NODE_HEADER.index("ifung")] == "9257" + + def test_none_taxon_values_are_backslash_N(self, tmp_path): + # read_taxon_map() gives every taxon a full set of keys, using None for + # the columns that were empty, so a present-but-None value has to be + # written as \N just like an absent key would be. + t = ete4.Tree("(A,B)R;", parser=1) + none_taxon = dict.fromkeys( + ( + "ott", + "wikidata", + "wikipedia_lang_flag", + "iucn", + "eol", + "rank", + "raw_popularity", + "ncbi", + "if", + "worms", + "irmng", + "gbif", + "ipni", + ) + ) + _prep(t, taxon_overrides={"A": none_taxon, "R": none_taxon}) + output_mysqlexport(t, str(tmp_path)) + + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + a = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "A") + for col in ("ott", "wikidata", "wikipedia_lang_flag", "iucn", "eol", "raw_popularity"): + assert a[LEAF_HEADER.index(col)] == "\\N", col + for col in ("ncbi", "ifung", "worms", "irmng", "gbif", "ipni"): + assert a[LEAF_HEADER.index(col)] == "\\N", col + + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + for col in ("ott", "wikidata", "wikipedia_lang_flag", "eol", "rnk", "raw_popularity"): + assert root[NODE_HEADER.index(col)] == "\\N", col + for col in ("ncbi", "ifung", "worms", "irmng", "gbif", "ipni"): + assert root[NODE_HEADER.index(col)] == "\\N", col + + def test_none_node_props_are_backslash_N(self, tmp_path): + # Ditto for props set on the node itself rather than its taxon. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + nodes_by_name = _by_name(t) + nodes_by_name["A"].props["extinction_date"] = None + nodes_by_name["A"].props["popularity"] = None + nodes_by_name["A"].props["popularity_rank"] = None + nodes_by_name["R"].props["date"] = None + nodes_by_name["R"].props["popularity"] = None + output_mysqlexport(t, str(tmp_path)) + + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + a = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "A") + for col in ("extinction_date", "popularity", "popularity_rank"): + assert a[LEAF_HEADER.index(col)] == "\\N", col + + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + for col in ("age", "popularity"): + assert root[NODE_HEADER.index(col)] == "\\N", col + + def test_unpopulated_columns_are_backslash_N(self, tmp_path): + # Columns tree_build doesn't (yet) generate are still NULL, not empty + # strings — "price" for leaves, "vern_synth" and the rep/rtr/rpd/iucn* + # summary columns for internal nodes. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + for row in leaves[1:]: + assert row[LEAF_HEADER.index("price")] == "\\N" + + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + for row in nodes[1:]: + assert row[NODE_HEADER.index("vern_synth")] == "\\N" + assert row[NODE_HEADER.index("rep1")] == "\\N" + assert row[NODE_HEADER.index("iucnEX")] == "\\N" + + def test_missing_extinction_date_and_popularity_are_backslash_N(self, tmp_path): + # Leaf-only props (extinction_date, popularity, popularity_rank) + # default to \N when not set. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + for row in leaves[1:]: + assert row[LEAF_HEADER.index("extinction_date")] == "\\N" + assert row[LEAF_HEADER.index("popularity")] == "\\N" + assert row[LEAF_HEADER.index("popularity_rank")] == "\\N" + + def test_leaf_extinction_date_and_popularity_are_emitted(self, tmp_path): + # Values on the leaf itself flow through unchanged. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + nodes = _by_name(t) + nodes["A"].props["extinction_date"] = "2020" + nodes["A"].props["popularity"] = 1.5 + nodes["A"].props["popularity_rank"] = 3 + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + a = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "A") + assert a[LEAF_HEADER.index("extinction_date")] == "2020" + assert a[LEAF_HEADER.index("popularity")] == "1.5" + assert a[LEAF_HEADER.index("popularity_rank")] == "3" + + def test_internal_node_date_written_as_age(self, tmp_path): + # An internal node's "date" property is exposed via the "age" column. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + t.props["date"] = 12.5 + output_mysqlexport(t, str(tmp_path)) + nodes = _read_csv(tmp_path, "ordered_nodes.csv") + root = next(r for r in nodes[1:] if r[NODE_HEADER.index("name")] == "R") + assert root[NODE_HEADER.index("age")] == "12.5" + + def test_polytomy_parent_is_skipped_for_real_parent(self, tmp_path): + # A non-polytomy node whose immediate parent carries the polytomy + # prop (a randomly-resolved polytomy node) should attribute its + # real_parent to the next unmarked ancestor. + # Tree shape: G -> P (polytomy) -> X (leaf). + # X's real_parent must be G, not P. + t = ete4.Tree("((X:1,Y:1)P:1,Z:1)G:1;", parser=1) + nodes = _by_name(t) + nodes["P"].props[POLYTOMY_PROP] = POLYTOMY_COMB + _prep(t) + output_mysqlexport(t, str(tmp_path)) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + x = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "X") + # parent (raw) is still P; real_parent skips it up to G. + assert x[LEAF_HEADER.index("parent")] == str(nodes["P"].props["id"]) + assert x[LEAF_HEADER.index("real_parent")] == str(nodes["G"].props["id"]) + + def test_polytomy_self_emits_negative_real_parent(self, tmp_path): + # A node that is itself a polytomy resolution writes a negative + # real_parent_id, flagging the relationship as artificial. + t = ete4.Tree("((X:1,Y:1)P:1,Z:1)G:1;", parser=1) + nodes = _by_name(t) + nodes["P"].props[POLYTOMY_PROP] = POLYTOMY_COMB + _prep(t) + output_mysqlexport(t, str(tmp_path)) + node_rows = _read_csv(tmp_path, "ordered_nodes.csv") + p = next(r for r in node_rows[1:] if r[NODE_HEADER.index("name")] == "P") + # P is marked as a polytomy; G is its (unmarked) parent. + assert p[NODE_HEADER.index("real_parent")] == str(-nodes["G"].props["id"]) + + def test_zero_dist_is_not_a_polytomy_marker(self, tmp_path): + # Branch lengths are regenerated from dates, so a zero-length branch + # no longer implies an artificial split. + t = ete4.Tree("((X:1,Y:1)P:0,Z:1)G:1;", parser=1) + nodes = _by_name(t) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + node_rows = _read_csv(tmp_path, "ordered_nodes.csv") + p = next(r for r in node_rows[1:] if r[NODE_HEADER.index("name")] == "P") + assert p[NODE_HEADER.index("real_parent")] == str(nodes["G"].props["id"]) + leaves = _read_csv(tmp_path, "ordered_leaves.csv") + x = next(r for r in leaves[1:] if r[LEAF_HEADER.index("name")] == "X") + assert x[LEAF_HEADER.index("real_parent")] == str(nodes["P"].props["id"]) + + def test_import_sql_contains_load_data_for_both_tables(self, tmp_path): + # The SQL script should truncate-and-load both CSV files. The + # column list inside `LOAD DATA INFILE` is read from the first + # line of the CSV, so it must match the CSV header exactly. + t = ete4.Tree("(A,B)R;", parser=1) + _prep(t) + output_mysqlexport(t, str(tmp_path)) + sql = (tmp_path / "import.sql").read_text(encoding="utf-8") + assert "TRUNCATE TABLE ordered_leaves;" in sql + assert "TRUNCATE TABLE ordered_nodes;" in sql + assert "LOAD DATA LOCAL INFILE 'ordered_leaves.csv'" in sql + assert "LOAD DATA LOCAL INFILE 'ordered_nodes.csv'" in sql + # Header echoed inside the LOAD DATA column list. + assert "(" + ",".join(LEAF_HEADER) + ")" in sql + assert "(" + ",".join(NODE_HEADER) + ")" in sql + # `id` is auto-assigned by MySQL, not loaded from CSV. + assert "SET id = NULL;" in sql + + +######################################## +# output_jssource +######################################## + + +class TestOutputJsSource: + def test_writes_file_at_given_path(self, tmp_path): + # The file is created at out_dir/file_name. + output_jssource(None, str(tmp_path), "out.js", {"x": 1}) + assert (tmp_path / "out.js").exists() + + def test_single_key_emits_var_declaration(self, tmp_path): + # Each dict key becomes a `var = ;` line. + output_jssource(None, str(tmp_path), "out.js", {"rawData": "abc"}) + assert (tmp_path / "out.js").read_text() == 'var rawData = "abc";\n' + + def test_multiple_keys_emit_separate_lines(self, tmp_path): + # Every dict entry produces its own line, in insertion order. + data = {"a": 1, "b": "two", "c": [3, 4]} + output_jssource(None, str(tmp_path), "out.js", data) + assert (tmp_path / "out.js").read_text() == ("var a = 1;\n" 'var b = "two";\n' "var c = [3, 4];\n") + + def test_values_are_json_encoded(self, tmp_path): + # Non-trivial Python values are serialised through json.dump, so + # nested dicts/lists and unicode survive a JSON round-trip. + value = {"nested": [1, 2, {"k": "v"}], "u": "café"} + output_jssource(None, str(tmp_path), "out.js", {"obj": value}) + text = (tmp_path / "out.js").read_text() + # Strip the `var obj = ` prefix and trailing `;\n` to recover the JSON. + assert text.startswith("var obj = ") + assert text.endswith(";\n") + json_payload = text[len("var obj = ") : -len(";\n")] + assert json.loads(json_payload) == value + + def test_string_values_are_quoted(self, tmp_path): + # JSON encoding wraps strings in double quotes — without it the + # generated JS would reference an undefined identifier. + output_jssource(None, str(tmp_path), "out.js", {"s": "hello"}) + assert (tmp_path / "out.js").read_text() == 'var s = "hello";\n' + + def test_empty_dict_writes_empty_file(self, tmp_path): + # An empty data dict still creates the file, just with no content. + output_jssource(None, str(tmp_path), "out.js", {}) + assert (tmp_path / "out.js").read_text() == "" + + def test_special_characters_in_strings_are_escaped(self, tmp_path): + # Quotes/backslashes/newlines inside values must be JSON-escaped so + # the emitted JS parses. + output_jssource(None, str(tmp_path), "out.js", {"s": 'a"b\\c\nd'}) + assert (tmp_path / "out.js").read_text() == 'var s = "a\\"b\\\\c\\nd";\n' + + +###### + + +def build_tree(nwk, prop_name, prop_values, prop_format="u8"): + """Parse (nwk), assigning node.props[prop_name] = prop_values[node.name] where set.""" + t = ete4.Tree(nwk, parser=1) + for node in t.traverse("preorder"): + if node.name in prop_values: + node.props[prop_name] = prop_values[node.name] + + t.root.props.setdefault("prop_format", {})[prop_name] = prop_format + + return t + + +def read_packed(path, pack_format): + sz = struct.calcsize("<" + pack_format) + with open(path, "rb") as f: + data = f.read() + assert len(data) % sz == 0 + n = len(data) // sz + return list(struct.unpack("<" + pack_format * n, data)) + + +class TestOutputPropArray: + def test_int_packing(self, tmp_path): + """Integer properties pack as unsigned bytes; leaves and internals split into separate files in preorder.""" + t = build_tree( + "((a,b)x,c)root;", + "myprop", + {"root": 5, "x": 12, "a": 1, "b": 2, "c": 3}, + prop_format="u8", + ) + leaf_path, node_path = output_proparray(t, str(tmp_path), "myprop") + + assert leaf_path == str(tmp_path / "myprop_leaves_u8.dat") + assert node_path == str(tmp_path / "myprop_nodes_u8.dat") + + # Preorder: root, x, a, b, c -> leaves [a, b, c], internals [root, x] + assert read_packed(leaf_path, "B") == [1, 2, 3] + assert read_packed(node_path, "B") == [5, 12] + + def test_float_packing(self, tmp_path): + """Float properties pack as 2-byte half-floats.""" + t = build_tree( + "((a,b)x,c)root;", + "myprop", + {"root": 5.0, "x": 12.0, "a": 1.0, "b": 2.0, "c": 3.0}, + prop_format="f32", + ) + leaf_path, node_path = output_proparray(t, str(tmp_path), "myprop") + + assert leaf_path == str(tmp_path / "myprop_leaves_f32.dat") + assert node_path == str(tmp_path / "myprop_nodes_f32.dat") + + assert struct.calcsize(" has_pop=False at Mid + {"ott": 3, "raw_popularity": 7.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + # Root counts as a "pop ancestor" only when its own has_pop counts up + # via descendants — n_pop_ancestors counts populated nodes on the way down. + assert by_name["Mid_ott2"].props["n_pop_ancestors"] == 0 # mid itself missing + assert by_name["A_ott1"].props["n_pop_ancestors"] == 1 # only A itself + + def test_exclude_taxa_zeroes_pop_for_named_node(self, tmp_path): + # Excluded node's own pop becomes 0 / has_pop=False, but descendants + # still contribute to its descendants_popsum (used by children below). + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "raw_popularity": 20.0}, + {"ott": 3, "raw_popularity": 999.0}, + ], + ) + + sum_popularity_over_tree(t, exclude_taxa=["Root_ott3"]) + root = next(n for n in t.traverse() if n.name == "Root_ott3") + + assert root.props["pop"] == 0 + assert root.props["has_pop"] is False + # Children's pop still aggregates upwards. + assert root.props["descendants_popsum"] == 30.0 + + +class TestSharedWikidataQids: + """ + OTT frequently holds one taxon twice (split across source taxonomies), and + every copy carries the same raw_popularity. It is shared between them rather + than counted once per copy. + """ + + def test_shared_qid_splits_raw_popularity(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["A_ott1"].props["pop"] == 50.0 + assert by_name["B_ott2"].props["pop"] == 50.0 + # The clade's total is what one copy of the taxon is worth, not two. + assert by_name["Root_ott3"].props["descendants_popsum"] == 100.0 + + def test_distinct_qids_are_untouched(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 43, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["A_ott1"].props["pop"] == 100.0 + assert by_name["B_ott2"].props["pop"] == 100.0 + + def test_node_without_a_qid_is_untouched(self, tmp_path): + # No Qid means nothing to share with, so no division. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 100.0}, + {"ott": 2, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["A_ott1"].props["pop"] == 100.0 + assert by_name["B_ott2"].props["pop"] == 100.0 + + def test_qid_shared_by_three_nodes_splits_three_ways(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2,C_ott4)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 90.0}, + {"ott": 2, "wikidata": 42, "raw_popularity": 90.0}, + {"ott": 4, "wikidata": 42, "raw_popularity": 90.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["A_ott1"].props["pop"] == 30.0 + assert by_name["Root_ott3"].props["descendants_popsum"] == 90.0 + + def test_excluded_duplicate_does_not_dilute_its_twin(self, tmp_path): + # An excluded node contributes no popularity, so the remaining node + # should keep the full score rather than be halved against a zero. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t, exclude_taxa=["B_ott2"]) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["B_ott2"].props["pop"] == 0 + assert by_name["A_ott1"].props["pop"] == 100.0 + + def test_duplicate_without_raw_popularity_does_not_dilute_its_twin(self, tmp_path): + # Likewise for a duplicate that has a Qid but no popularity to give. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 42}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + sum_popularity_over_tree(t) + by_name = {n.name: n for n in t.traverse()} + + assert by_name["B_ott2"].props["has_pop"] is False + assert by_name["A_ott1"].props["pop"] == 100.0 + + def test_duplicates_are_summarised_in_one_warning(self, tmp_path, caplog): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 2, "wikidata": 42, "raw_popularity": 100.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + with caplog.at_level(logging.WARNING): + popularity_add_prop(t) + + warnings = [r for r in caplog.records if r.levelno == logging.WARNING] + assert len(warnings) == 1 + assert "1 nodes share a wikidata Qid" in warnings[0].message + + +class TestPopularityAddProp: + def test_popularity_set_on_every_node_and_rounded(self, tmp_path): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "raw_popularity": 20.0}, + {"ott": 3, "raw_popularity": 30.0}, + ], + ) + + popularity_add_prop(t) + + by_name = {n.name: n for n in t.traverse()} + # Leaves: n_anc + n_desc == 1 -> sum of ancestor + descendant popsums. + # A: anc=10, desc=0 -> 10. B: anc=20, desc=0 -> 20. + assert by_name["A_ott1"].props["popularity"] == 10.0 + assert by_name["B_ott2"].props["popularity"] == 20.0 + # Root: (0 + 30) / log(2) — rounded to 2 dp. + assert by_name["Root_ott3"].props["popularity"] == round(30.0 / log(2), 2) + + def test_nodes_without_pop_get_zero_popularity(self, tmp_path): + # All zeros in -> popularity is 0 (special case applies at leaves). + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa(tmp_path, t, []) + + popularity_add_prop(t) + for n in t.traverse(): + assert n.props["popularity"] == 0 + + def test_exclude_taxa_passes_through(self, tmp_path): + # popularity_add_prop forwards exclude_taxa to sum_popularity_over_tree; + # the excluded node's own pop is zeroed so its rendered popularity + # reflects only descendants. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + _attach_taxa( + tmp_path, + t, + [ + {"ott": 1, "raw_popularity": 10.0}, + {"ott": 2, "raw_popularity": 20.0}, + {"ott": 3, "raw_popularity": 999.0}, + ], + ) + + popularity_add_prop(t, exclude_taxa=["Root_ott3"]) + root = next(n for n in t.traverse() if n.name == "Root_ott3") + # Root pop=0, descendants_popsum=30, n_desc=2 -> 30/log(2). + assert root.props["popularity"] == round(30.0 / log(2), 2) + + +class TestPopularityAddRank: + @staticmethod + def _set_leaf_pops(tree, pops): + # Manually attach popularity values to leaves (bypassing the full + # pipeline). popularity_add_rank only reads node.props["popularity"]. + for n in tree.traverse(): + if n.is_leaf and n.name in pops: + n.props["popularity"] = pops[n.name] + + def test_distinct_popularities_get_sequential_ranks(self): + # Higher popularity -> lower (better) rank, starting at 1. + t = ete4.Tree("(A,B,C)R;", parser=1) + self._set_leaf_pops(t, {"A": 30, "B": 20, "C": 10}) + + popularity_add_rank(t) + + by_name = {n.name: n for n in t.traverse()} + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 2 + assert by_name["C"].props["popularity_rank"] == 3 + + def test_ties_use_standard_competition_ranking(self): + # Two leaves tied at the top share rank 1; the next leaf gets + # rank 3 (not 2). Same for ties further down. + t = ete4.Tree("(A,B,C,D,E)R;", parser=1) + self._set_leaf_pops(t, {"A": 10, "B": 10, "C": 5, "D": 3, "E": 3}) + + popularity_add_rank(t) + + by_name = {n.name: n for n in t.traverse()} + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 1 + assert by_name["C"].props["popularity_rank"] == 3 + assert by_name["D"].props["popularity_rank"] == 4 + assert by_name["E"].props["popularity_rank"] == 4 + + def test_all_leaves_tied_share_rank_one(self): + t = ete4.Tree("(A,B,C)R;", parser=1) + self._set_leaf_pops(t, {"A": 7, "B": 7, "C": 7}) + + popularity_add_rank(t) + + for name in ("A", "B", "C"): + leaf = next(n for n in t.traverse() if n.name == name) + assert leaf.props["popularity_rank"] == 1 + + def test_single_leaf_gets_rank_one(self): + # The function ranks even the root if it is a leaf. + t = ete4.Tree("A;", parser=1) + t.props["popularity"] = 42 + popularity_add_rank(t) + assert t.props["popularity_rank"] == 1 + + def test_internal_nodes_do_not_get_a_rank(self): + # Only leaves are ranked; the internal node's popularity, even + # if set, is ignored both as a tie-breaker and as an output. + t = ete4.Tree("((A,B)I,C)R;", parser=1) + self._set_leaf_pops(t, {"A": 10, "B": 5, "C": 1}) + by_name = {n.name: n for n in t.traverse()} + by_name["I"].props["popularity"] = 999 # should not affect anything + by_name["R"].props["popularity"] = 999 + + popularity_add_rank(t) + + # Leaves ranked normally. + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 2 + assert by_name["C"].props["popularity_rank"] == 3 + # Internal nodes untouched by ranking. + assert "popularity_rank" not in by_name["I"].props + assert "popularity_rank" not in by_name["R"].props + + def test_internal_node_without_popularity_does_not_trigger_skip(self): + # The None-guard only inspects leaves. An internal node that + # has no popularity prop must not cause the early return. + t = ete4.Tree("((A,B)I,C)R;", parser=1) + self._set_leaf_pops(t, {"A": 10, "B": 5, "C": 1}) + # I and R deliberately have no popularity prop set. + + popularity_add_rank(t) + + by_name = {n.name: n for n in t.traverse()} + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 2 + assert by_name["C"].props["popularity_rank"] == 3 + + def test_rank_cumsum_with_mixed_group_sizes(self): + # 1 leaf at top, 3 tied below, 1 at the bottom. + # Expected ranks: top=1, tied group=2, bottom=5. + t = ete4.Tree("(A,B,C,D,E)R;", parser=1) + self._set_leaf_pops(t, {"A": 100, "B": 50, "C": 50, "D": 50, "E": 1}) + + popularity_add_rank(t) + + by_name = {n.name: n for n in t.traverse()} + assert by_name["A"].props["popularity_rank"] == 1 + assert by_name["B"].props["popularity_rank"] == 2 + assert by_name["C"].props["popularity_rank"] == 2 + assert by_name["D"].props["popularity_rank"] == 2 + assert by_name["E"].props["popularity_rank"] == 5 diff --git a/tests/test_tree_build_step_taxon.py b/tests/test_tree_build_step_taxon.py new file mode 100644 index 00000000..4c4de994 --- /dev/null +++ b/tests/test_tree_build_step_taxon.py @@ -0,0 +1,82 @@ +import ete4 + +from oz_tree_build.tree_build.step_taxon import taxon_add_prop + + +class TestTaxonAddProp: + def test_empty_taxon_map_assigns_empty_dict(self): + # Every node — leaf, internal, root — gets {} when the map is empty. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + taxon_add_prop(t, {}) + for n in t.traverse(): + assert n.props["taxon"] == {} + + def test_leaf_match_assigns_taxon_entry(self): + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + a_entry = {"ott": 1, "wikidata": 42, "raw_popularity": 1.5} + taxon_add_prop(t, {1: a_entry}) + + a = next(n for n in t.traverse() if n.name == "A_ott1") + b = next(n for n in t.traverse() if n.name == "B_ott2") + assert a.props["taxon"] == a_entry + assert b.props["taxon"] == {} + + def test_internal_node_match_assigns_taxon_entry(self): + # Internal nodes are also looked up via their ottN suffix. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + root_entry = {"ott": 3, "rank": "family"} + taxon_add_prop(t, {3: root_entry}) + + root = next(n for n in t.traverse() if n.name == "Root_ott3") + assert root.props["taxon"] == root_entry + + def test_node_without_ott_gets_empty_dict(self): + # "Foo" has no ottN suffix, so node_get_ott returns None and the + # entry must be {} regardless of what is in the taxon map. + t = ete4.Tree("(Foo,B_ott2)Root_ott3;", parser=1) + taxon_add_prop(t, {1: {"ott": 1}, 2: {"ott": 2}, 3: {"ott": 3}}) + + foo = next(n for n in t.traverse() if n.name == "Foo") + assert foo.props["taxon"] == {} + + def test_unnamed_node_gets_empty_dict(self): + # An unnamed internal node (e.g. result of resolve_polytomy) has no + # OTT and must end up with {}. + t = ete4.Tree("(A_ott1,B_ott2,C_ott4)Root_ott3;", parser=1) + t.resolve_polytomy() + + taxon_add_prop(t, {1: {"ott": 1}}) + + for n in t.traverse(): + if not n.name: + assert n.props["taxon"] == {} + + def test_unmatched_ott_gets_empty_dict(self): + # Node has an ottN but it's absent from the map. + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + taxon_add_prop(t, {99: {"ott": 99}}) + + for n in t.traverse(): + assert n.props["taxon"] == {} + + def test_space_separated_ott_in_name_is_matched(self): + # node_get_ott accepts both "_ottN" and " ottN" suffixes. + t = ete4.Tree("(A_ott1,B_ott2)Root;", parser=1) + # Rename a node to use the space-separated form. + a = next(n for n in t.traverse() if n.name == "A_ott1") + a.name = "Some name ott1" + entry = {"ott": 1} + taxon_add_prop(t, {1: entry}) + + assert a.props["taxon"] == entry + + def test_taxon_entry_is_assigned_by_reference(self): + # The function stores the same dict object on the node, so callers + # who mutate the taxon map afterwards see the change reflected on + # the tree (and vice versa). + t = ete4.Tree("(A_ott1,B_ott2)Root_ott3;", parser=1) + a_entry = {"ott": 1} + taxon_add_prop(t, {1: a_entry}) + + a = next(n for n in t.traverse() if n.name == "A_ott1") + assert a.props["taxon"] is a_entry diff --git a/tests/test_tree_build_step_tidy.py b/tests/test_tree_build_step_tidy.py new file mode 100644 index 00000000..4eaa473b --- /dev/null +++ b/tests/test_tree_build_step_tidy.py @@ -0,0 +1,371 @@ +import ete4 + +from oz_tree_build.tree_build.step_tidy import ( + POLYTOMY_COMB, + POLYTOMY_PROP, + POLYTOMY_RANDOM, + tidy_clear_conflicting_dates_topdown, + tidy_infill_dates_bottomup, + tidy_mark_resolved_polytomies, + tidy_prune_synthetic_leaves, + tidy_resolve_polytomies, +) + + +class TestTidyPruneSyntheticLeaves: + def test_childless_ot_mrca_node_is_pruned(self): + # An OT MRCA label left childless because both its children were + # extracted as separate subtrees. + t = ete4.Tree("(mrcaott118778ott781203,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 1 + assert [n.name for n in t.leaves()] == ["B"] + + def test_mrcaimp_and_mrcapoly_are_pruned(self): + t = ete4.Tree("(mrcaimp,mrcapoly,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 2 + assert [n.name for n in t.leaves()] == ["B"] + + def test_cascades_upwards(self): + # Root -> M (OT mrca) -> mrcaimp -> nothing. Pruning the mrcaimp + # empties M, which must then go too. + t = ete4.Tree("((mrcaimp)mrcaott90320ott145150,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 2 + assert [n.name for n in t.leaves()] == ["B"] + + def test_named_taxa_are_left_alone(self): + # A real taxon left childless is not ours to delete silently. + t = ete4.Tree("(Berycidae_ott118776,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 0 + assert sorted(n.name for n in t.leaves()) == ["B", "Berycidae_ott118776"] + + def test_synthetic_node_with_children_is_kept(self): + # Only *childless* synthetic nodes are debris; one still holding a + # subtree is doing its job as an ancestor. + t = ete4.Tree("((X:1,Y:1)mrcaimp,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 0 + assert sorted(n.name for n in t.leaves()) == ["B", "X", "Y"] + + def test_similar_names_are_not_matched(self): + # Guard the regex against eating real taxa that merely start "mrca". + t = ete4.Tree("(mrcaimposter,Mrcaimp,mrcaott12,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 0 + + def test_returns_zero_on_a_clean_tree(self): + t = ete4.Tree("(A:1,B:1)Root;", parser=1) + assert tidy_prune_synthetic_leaves(t) == 0 + + def test_root_is_never_detached(self): + # A tree that collapses entirely must not try to detach its own root. + t = ete4.Tree("(mrcaimp)mrcapoly;", parser=1) + tidy_prune_synthetic_leaves(t) + assert t.up is None + + +def _by_name(tree): + return {n.name: n for n in tree.traverse()} + + +class TestTidyResolvePolytomies: + def test_binary_tree_is_untouched(self): + t = ete4.Tree("((A:1,B:1)X:1,C:2)Root;", parser=1) + assert tidy_resolve_polytomies(t) == 0 + assert not any(n.props.get(POLYTOMY_PROP) for n in t.traverse()) + + def test_inserted_nodes_are_marked(self): + # A 4-way polytomy needs 2 extra nodes to become binary. + t = ete4.Tree("(A:1,B:1,C:1,D:1)Root;", parser=1) + assert tidy_resolve_polytomies(t) == 2 + marked = [n for n in t.traverse() if n.props.get(POLYTOMY_PROP)] + assert len(marked) == 2 + assert all(len(n.children) == 2 for n in t.traverse() if not n.is_leaf) + + def test_marked_as_comb_not_random(self): + # ete4's resolve_polytomy pairs children off in order rather than + # sampling a topology, so these are combs, not random draws. + t = ete4.Tree("(A:1,B:1,C:1)Root;", parser=1) + tidy_resolve_polytomies(t) + marked = [n for n in t.traverse() if n.props.get(POLYTOMY_PROP)] + assert [n.props[POLYTOMY_PROP] for n in marked] == [POLYTOMY_COMB] + + def test_resolution_is_deterministic_comb(self): + # Documents *why* the value is "comb": repeated runs give the same + # left-nested shape, so this is a systematic artefact, not a sample. + shapes = set() + for _ in range(5): + t = ete4.Tree("(A,B,C,D,E)Root;", parser=1) + tidy_resolve_polytomies(t) + shapes.add(t.write(parser=1, props=[])) + assert shapes == {"((((A,B):0,C):0,D):0,E);"} + + def test_pre_existing_nodes_are_not_marked(self): + t = ete4.Tree("(A:1,B:1,C:1)Root;", parser=1) + tidy_resolve_polytomies(t) + named = _by_name(t) + for name in ("Root", "A", "B", "C"): + assert not named[name].props.get(POLYTOMY_PROP) + + def test_marks_survive_zeroed_branch_lengths(self): + # The prop, not dist, is what identifies a resolution — clobbering + # branch lengths (as compute_branch_lengths later does) must not + # lose the marking. + t = ete4.Tree("(A:1,B:1,C:1)Root;", parser=1) + tidy_resolve_polytomies(t) + for n in t.traverse(): + n.dist = 5 + assert len([n for n in t.traverse() if n.props.get(POLYTOMY_PROP)]) == 1 + + +class TestTidyMarkResolvedPolytomies: + def test_marks_nodes_by_name_as_random(self): + t = ete4.Tree("((A:1,B:1)mrcapoly:1,C:2)Root;", parser=1) + assert tidy_mark_resolved_polytomies(t) == 1 + assert _by_name(t)["mrcapoly"].props[POLYTOMY_PROP] == POLYTOMY_RANDOM + + def test_both_kinds_are_truthy_for_consumers(self): + # step_output and step_jsnewick only ask "is this artificial?", so + # every kind has to survive a plain truth test. + assert POLYTOMY_COMB + assert POLYTOMY_RANDOM + assert POLYTOMY_COMB != POLYTOMY_RANDOM + + def test_leaves_other_nodes_alone(self): + t = ete4.Tree("((A:1,B:1)mrcapoly:1,C:2)Root;", parser=1) + tidy_mark_resolved_polytomies(t) + named = _by_name(t) + for name in ("Root", "A", "B", "C"): + assert not named[name].props.get(POLYTOMY_PROP) + + def test_no_matching_names_marks_nothing(self): + t = ete4.Tree("((A:1,B:1)X:0,C:2)Root;", parser=1) + assert tidy_mark_resolved_polytomies(t) == 0 + assert not any(n.props.get(POLYTOMY_PROP) for n in t.traverse()) + + def test_custom_name(self): + t = ete4.Tree("((A:1,B:1)poly:1,C:2)Root;", parser=1) + assert tidy_mark_resolved_polytomies(t, name="poly") == 1 + assert _by_name(t)["poly"].props[POLYTOMY_PROP] == POLYTOMY_RANDOM + + +class TestTidyInfillDatesBottomup: + def test_single_leaf_gets_date_zero(self): + t = ete4.Tree("A;", parser=1) + tidy_infill_dates_bottomup(t) + assert t.props["date"] == 0 + + def test_all_leaves_get_date_zero(self): + # Every leaf is reset to 0 regardless of any earlier value. + t = ete4.Tree("(A:1,B:2)Root;", parser=1) + nodes = _by_name(t) + nodes["A"].props["date"] = 99 # will be overwritten + tidy_infill_dates_bottomup(t) + assert nodes["A"].props["date"] == 0 + assert nodes["B"].props["date"] == 0 + + def test_parent_date_is_max_child_branch_length(self): + # Root date = max(child.date + child.dist) over leaves at date 0. + t = ete4.Tree("(A:1,B:2)Root;", parser=1) + tidy_infill_dates_bottomup(t) + assert t.props["date"] == 2 + + def test_multilevel_tree_accumulates_branch_lengths(self): + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + tidy_infill_dates_bottomup(t) + # Internal (parent of A,B) = max(0+1, 0+2) = 2 + # Root = max(2+3, 0+4) = 5 + internal = next(c for c in t.children if not c.is_leaf) + assert internal.props["date"] == 2 + assert t.props["date"] == 5 + + def test_existing_internal_date_preserved_when_larger(self): + # Pre-existing date on an internal node is kept if no child's + # accumulated date exceeds it. + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + internal = next(c for c in t.children if not c.is_leaf) + internal.props["date"] = 100 + tidy_infill_dates_bottomup(t) + assert internal.props["date"] == 100 + # Root sees the preserved internal date: max(100+3, 0+4) = 103 + assert t.props["date"] == 103 + + def test_existing_internal_date_overwritten_when_smaller(self): + # A child-derived date larger than the pre-existing one wins. + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + internal = next(c for c in t.children if not c.is_leaf) + internal.props["date"] = 0.5 + tidy_infill_dates_bottomup(t) + assert internal.props["date"] == 2 + assert t.props["date"] == 5 + + def test_child_without_dist_does_not_contribute(self): + # A child whose dist is None must not bump its parent's date. + t = ete4.Tree("(A,B:5)Root;", parser=1) + tidy_infill_dates_bottomup(t) + # Only B contributes: Root = 0 + 5 = 5 + assert t.props["date"] == 5 + + def test_internal_without_datable_descendants_has_no_date(self): + # If every child of an internal lacks dist, the internal stays + # without a date and cannot in turn propagate upward. + t = ete4.Tree("((A,B):4,C:1)Root;", parser=1) + tidy_infill_dates_bottomup(t) + internal = next(c for c in t.children if not c.is_leaf) + # Internal has no datable children (A and B both lack dist). + assert internal.props.get("date") is None + # Root sees only C contributing (internal has no date). + assert t.props["date"] == 1 + + def test_picks_oldest_subtree_when_branches_differ(self): + # Two subtrees with different total ages — root takes the older. + t = ete4.Tree("((A:1,B:1):10,(C:1,D:1):2)Root;", parser=1) + tidy_infill_dates_bottomup(t) + # left subtree internal = 1, +10 = 11; right subtree internal = 1, +2 = 3 + assert t.props["date"] == 11 + + +class TestTidyClearConflictingDatesTopdown: + def test_consistent_dates_are_all_kept(self): + # Descending dates: root oldest, leaves youngest. Nothing removed. + t = ete4.Tree("((A,B)I,C)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["I"].props["date"] = 5 + nodes["A"].props["date"] = 0 + nodes["B"].props["date"] = 2 + nodes["C"].props["date"] = 3 + tidy_clear_conflicting_dates_topdown(t) + assert nodes["Root"].props["date"] == 10 + assert nodes["I"].props["date"] == 5 + assert nodes["A"].props["date"] == 0 + assert nodes["B"].props["date"] == 2 + assert nodes["C"].props["date"] == 3 + + def test_child_older_than_parent_is_cleared(self): + # A child date that is older than its ancestor is a conflict + # and must be removed. + t = ete4.Tree("(A,B)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["A"].props["date"] = 20 # older than Root — conflict + nodes["B"].props["date"] = 3 + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] is None + assert nodes["B"].props["date"] == 3 + assert nodes["Root"].props["date"] == 10 + + def test_conflict_clears_only_that_node(self): + # Removing an intermediate conflicting date should not stop the + # descent — its descendants are still compared against the + # original ancestor's date. + t = ete4.Tree("((A,B)I,C)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["I"].props["date"] = 20 # conflict — older than Root + nodes["A"].props["date"] = 5 # consistent with Root (10) + nodes["B"].props["date"] = 50 # conflict — older than Root + tidy_clear_conflicting_dates_topdown(t) + assert nodes["Root"].props["date"] == 10 + assert nodes["I"].props["date"] is None + assert nodes["A"].props["date"] == 5 + assert nodes["B"].props["date"] is None + + def test_missing_ancestor_date_does_not_constrain_descendants(self): + # With no ancestor date set, the first node we meet on each path + # establishes the ceiling for everything below it. + t = ete4.Tree("(A,(B,C)I)Root;", parser=1) + nodes = _by_name(t) + # Root has no date. + nodes["A"].props["date"] = 100 # nothing above — kept + nodes["I"].props["date"] = 50 + nodes["B"].props["date"] = 1 + nodes["C"].props["date"] = 200 # conflict — older than I (50) + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] == 100 + assert nodes["I"].props["date"] == 50 + assert nodes["B"].props["date"] == 1 + assert nodes["C"].props["date"] is None + + def test_nodes_without_date_are_skipped_and_pass_mrad_through(self): + # An intermediate node without a date must not reset the ceiling; + # its descendants are still compared against the nearest ancestor + # that does have a date. + t = ete4.Tree("((A)I)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + # I has no date. + nodes["A"].props["date"] = 50 # still in conflict with Root via I + tidy_clear_conflicting_dates_topdown(t) + assert nodes["Root"].props["date"] == 10 + assert "date" not in nodes["I"].props + assert nodes["A"].props["date"] is None + + def test_small_overshoot_within_tolerance_is_kept(self): + # The function tolerates floating-point noise up to 1e-5. + t = ete4.Tree("(A)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["A"].props["date"] = 10 + 1e-6 # just within tolerance + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] == 10 + 1e-6 + + def test_overshoot_beyond_tolerance_is_cleared(self): + t = ete4.Tree("(A)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["A"].props["date"] = 10 + 1e-3 # outside tolerance + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] is None + + def test_equal_dates_are_kept(self): + # parent.date == ancestor.date is not a conflict. + t = ete4.Tree("(A)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 10 + nodes["A"].props["date"] = 10 + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] == 10 + + def test_mrad_tightens_as_we_descend(self): + # After we descend past a node with a smaller date, that smaller + # date becomes the new ceiling for everything below. + t = ete4.Tree("((A,B)I)Root;", parser=1) + nodes = _by_name(t) + nodes["Root"].props["date"] = 100 + nodes["I"].props["date"] = 5 # tighter than Root + nodes["A"].props["date"] = 3 # below I — fine + nodes["B"].props["date"] = 50 # would be fine vs Root but conflicts with I + tidy_clear_conflicting_dates_topdown(t) + assert nodes["A"].props["date"] == 3 + assert nodes["B"].props["date"] is None + + def test_runs_on_tree_with_no_dates_at_all(self): + # Nothing to do; should not raise. + t = ete4.Tree("((A,B),C)Root;", parser=1) + tidy_clear_conflicting_dates_topdown(t) + for n in t.traverse(): + assert n.props.get("date") is None + + +class TestTidyPipeline: + def test_bottomup_then_topdown_on_clean_tree_is_stable(self): + # End-to-end: branch-length-derived dates should already be + # self-consistent, so the topdown pass changes nothing. + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + tidy_infill_dates_bottomup(t) + before = {id(n): n.props.get("date") for n in t.traverse()} + tidy_clear_conflicting_dates_topdown(t) + after = {id(n): n.props.get("date") for n in t.traverse()} + assert before == after + + def test_bottomup_then_topdown_clears_oversized_pin(self): + # If a pre-pinned internal date is older than the root's + # inferred date, the topdown pass should strip it. + t = ete4.Tree("((A:1,B:2):3,C:4)Root;", parser=1) + internal = next(c for c in t.children if not c.is_leaf) + internal.props["date"] = 100 # pin older than root could ever be + tidy_infill_dates_bottomup(t) + # Root is forced up to 103 by the pin; that's fine. + # Now imagine an externally fixed Root date that's smaller: + t.props["date"] = 50 + tidy_clear_conflicting_dates_topdown(t) + assert t.props["date"] == 50 + assert internal.props["date"] is None diff --git a/tests/test_tree_build_step_treeprop.py b/tests/test_tree_build_step_treeprop.py new file mode 100644 index 00000000..ec2aecdc --- /dev/null +++ b/tests/test_tree_build_step_treeprop.py @@ -0,0 +1,79 @@ +import ete4 + +from oz_tree_build.tree_build.step_treeprop import ( + GEOLOGICAL_PERIODS, + treeprop_geological, +) + +######################################## +# treeprop_geological +######################################## + + +def set_dates_from_dist(tree): + """ + Postorder pass: leaves get date 0, interior nodes get max(child.date + child.dist). + If any child has an unknown date or dist, the parent's date becomes None. + """ + for node in tree.traverse("postorder"): + if node.is_leaf: + node.props["date"] = 0 + continue + parent_date = 0 + for c in node.children: + if c.props.get("date") is None or c.dist is None: + parent_date = None + break + new_date = c.props["date"] + c.dist + if new_date > parent_date: + parent_date = new_date + node.props["date"] = parent_date + + +def do_treeprop_geological(nwk, date_tree=True): + t = ete4.Tree(nwk, parser=1) + # Our tree needs to have the date prop set for this to work + if date_tree: + set_dates_from_dist(t) + assert treeprop_geological(t) == "geological" + + # Traverse tree, returning all periods + return [(n.name, n.props.get("date"), n.props["geological"]) for n in t.traverse("preorder")] + + +class TestTreepropGeological: + def test_undated_tree(self): + """Undated trees get 0 set""" + assert do_treeprop_geological("(A:10)B;", date_tree=False) == [ + ("B", None, 0), + ("A", None, 0), + ] + + def test_incomplete_date_tree(self): + """If not all dates set, we do what we can""" + assert do_treeprop_geological("((C:5,D:4)B)A:15;") == [ + ("A", None, 0), + ("B", 5.0, 4), + ("C", 0, 1), + ("D", 0, 1), + ] + + def test_complete_date_tree(self): + """If all dates set""" + assert do_treeprop_geological("((C:5,D:4)B:10)A:15;") == [ + ("A", 15.0, 5), + ("B", 5.0, 4), + ("C", 0, 1), + ("D", 0, 1), + ] + + def test_period_inclusive(self): + """Mya ranges are incclusive""" + + def get_period(x): + p = GEOLOGICAL_PERIODS[do_treeprop_geological(f"(B:{x})A;")[0][2]] + return (p["period"], p["epoch"], p["mya_start"]) + + assert get_period(520.99) == ("Cambrian", "Series 2", 521) + assert get_period(521) == ("Cambrian", "Series 2", 521) + assert get_period(521.01) == ("Cambrian", "Terreneuvian", 538.8) diff --git a/tests/test_versioned_outputs.py b/tests/test_versioned_outputs.py new file mode 100644 index 00000000..bd34ebde --- /dev/null +++ b/tests/test_versioned_outputs.py @@ -0,0 +1,169 @@ +""" +Unit tests for versioned_outputs.process +""" + +import gzip + +from oz_tree_build.versioned_outputs.versioned_outputs import process + + +def _write(path, content): + with open(path, "w") as f: + f.write(content) + + +def test_copies_with_version_appended(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + src = in_dir / "data.csv" + _write(src, "hello,world\n") + + process([str(src)], str(out_dir), 42) + + out_file = out_dir / "data_42.csv" + assert out_file.read_text() == "hello,world\n" + + +def test_replaces_existing_version_in_filename(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + src = in_dir / "data_99.csv" + _write(src, "row\n") + + process([str(src)], str(out_dir), 7) + + assert (out_dir / "data_7.csv").exists() + assert not (out_dir / "data_99_7.csv").exists() + + +def test_produces_gzip_alongside_plain_file(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + src = in_dir / "data.csv" + _write(src, "the quick brown fox\n") + + process([str(src)], str(out_dir), 3) + + gz_path = out_dir / "data_3.csv.gz" + assert gz_path.exists() + with gzip.open(gz_path, "rt") as f: + assert f.read() == "the quick brown fox\n" + + +def test_multiple_files_each_versioned(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + a = in_dir / "a.csv" + b = in_dir / "b.json" + _write(a, "A\n") + _write(b, "B\n") + + process([str(a), str(b)], str(out_dir), 11) + + assert (out_dir / "a_11.csv").read_text() == "A\n" + assert (out_dir / "b_11.json").read_text() == "B\n" + + +def test_import_sql_rewrites_filename_references(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + nodes = in_dir / "ordered_nodes.csv" + leaves = in_dir / "ordered_leaves.csv" + sql = in_dir / "import.sql" + _write(nodes, "n\n") + _write(leaves, "l\n") + _write( + sql, + "LOAD DATA INFILE 'ordered_nodes.csv' INTO TABLE ordered_nodes;\n" + "LOAD DATA INFILE 'ordered_leaves.csv' INTO TABLE ordered_leaves;\n", + ) + + process([str(nodes), str(leaves), str(sql)], str(out_dir), 55) + + out_sql = (out_dir / "import_55.sql").read_text() + assert "'ordered_nodes_55.csv'" in out_sql + assert "'ordered_leaves_55.csv'" in out_sql + assert "'ordered_nodes.csv'" not in out_sql + assert "'ordered_leaves.csv'" not in out_sql + + +def test_import_sql_appends_root_parent_update(tmp_path): + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + sql = in_dir / "import.sql" + _write(sql, "-- nothing to rewrite\n") + + process([str(sql)], str(out_dir), 123) + + out_sql = (out_dir / "import_123.sql").read_text() + assert out_sql.endswith("UPDATE ordered_nodes SET parent = -123 WHERE id = 1;\n") + assert "-- nothing to rewrite\n" in out_sql + + +def test_import_sql_only_replaces_quoted_names(tmp_path): + """Filename substring inside other identifiers (no quotes) should not be touched.""" + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + nodes = in_dir / "ordered_nodes.csv" + sql = in_dir / "import.sql" + _write(nodes, "n\n") + # Mention the bare filename (no quotes) in a comment; it must NOT be rewritten. + _write( + sql, + "-- see ordered_nodes.csv for schema\n" "LOAD DATA INFILE 'ordered_nodes.csv' INTO TABLE ordered_nodes;\n", + ) + + process([str(nodes), str(sql)], str(out_dir), 9) + + out_sql = (out_dir / "import_9.sql").read_text() + assert "-- see ordered_nodes.csv for schema\n" in out_sql + assert "'ordered_nodes_9.csv'" in out_sql + + +def test_input_file_outside_in_dir_uses_basename(tmp_path): + """Output is named from basename, regardless of input path.""" + nested = tmp_path / "deep" / "nested" / "dir" + nested.mkdir(parents=True) + out_dir = tmp_path / "out" + out_dir.mkdir() + src = nested / "thing.txt" + _write(src, "x\n") + + process([str(src)], str(out_dir), 1) + + assert (out_dir / "thing_1.txt").exists() + assert (out_dir / "thing_1.txt.gz").exists() + + +def test_overwrites_existing_output(tmp_path): + """gzip -f and shutil.copyfile both clobber prior outputs without error.""" + in_dir = tmp_path / "in" + out_dir = tmp_path / "out" + in_dir.mkdir() + out_dir.mkdir() + src = in_dir / "data.csv" + _write(src, "fresh\n") + + # Pre-existing stale outputs from an earlier run. + _write(out_dir / "data_5.csv", "stale\n") + _write(out_dir / "data_5.csv.gz", "not a real gzip") + + process([str(src)], str(out_dir), 5) + + assert (out_dir / "data_5.csv").read_text() == "fresh\n" + with gzip.open(out_dir / "data_5.csv.gz", "rt") as f: + assert f.read() == "fresh\n"