From 640b4ef8818d88fda669ef8980062263045a7177 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 05:04:28 +0000 Subject: [PATCH 01/14] Added per-asset and per-Dandiset delivery ratio summaries Computed a per-asset delivery ratio (total bytes delivered over the asset's true DANDI size) and emitted it as a new column in by_asset.tsv. Added a per-Dandiset delivery_ratio.tsv with the asset-weighted percentiles (p10, p25, p50, p75, p90) plus a volume-weighted ratio. Asset sizes are fetched from the DANDI API and cached locally in asset_sizes.json, reused across runs. Assets with missing or zero size are excluded from the percentile computation and the skipped count is logged. Co-Authored-By: Claude Code / Claude Opus 4.8 --- CHANGELOG.md | 1 + pyproject.toml | 3 +- .../summarize/_generate_dandiset_summaries.py | 223 ++++++++++++++++++ tests/test_dandi_summaries.py | 15 +- tests/test_delivery_ratio.py | 132 +++++++++++ 5 files changed, 369 insertions(+), 5 deletions(-) create mode 100644 tests/test_delivery_ratio.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f896b3f..cbf5bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### 🚀 Enhancement +- Added a per-asset `delivery_ratio` column to `by_asset.tsv` and a per-Dandiset `delivery_ratio.tsv` summary with `delivery_ratio_p10`, `delivery_ratio_p25`, `delivery_ratio_p50`, `delivery_ratio_p75`, `delivery_ratio_p90`, and `delivery_ratio_weighted`. The delivery ratio is the total bytes delivered over the asset's true DANDI size, signaling streaming versus download intensity. Asset sizes are fetched from the DANDI API and cached locally in `asset_sizes.json`. ([#86](https://github.com/dandi/dandi-s3-log-extraction/pull/86)) - Added `download` to `_dandi_extraction.awk` so extraction writes `download.txt` alongside the other per-request outputs. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) - Added `--cache-directory` to `dandis3logextraction extract` so remote extraction can use a custom cache directory. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) - Added `--inventory` to `dandis3logextraction extract --mode remote` so extraction can use a local S3 Inventory directory. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) diff --git a/pyproject.toml b/pyproject.toml index b06cc53..e7ad530 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ allow-direct-references = true [project] name = "dandi-s3-log-extraction" -version="1.0.2" +version="1.0.3" authors = [ { name="Cody Baker", email="cody.c.baker.phd@gmail.com" }, ] @@ -37,6 +37,7 @@ license = {file = "LICENSE.txt"} requires-python = ">=3.14" dependencies = [ "beartype", + "numpy", "pandas", "tqdm", "PyYAML", diff --git a/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py b/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py index 17023a3..4d8c9dd 100644 --- a/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py +++ b/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py @@ -4,8 +4,10 @@ import gzip import itertools import json +import logging import pathlib +import numpy import pandas import requests import s3_log_extraction @@ -14,11 +16,16 @@ from .._parallel._utils import _handle_max_workers +logger = logging.getLogger(name=__name__) + ASSET_TYPES_IN_ORDER = ("Neurophysiology", "Microscopy", "Video", "Miscellaneous") NEUROPHYSIOLOGY_SUFFIXES = {".nwb"} MICROSCOPY_SUFFIXES = {".nii", ".ome", ".tiff", ".tif", ".bvecs", ".bvals", ".trk"} VIDEO_SUFFIXES = {".mp4", ".mov", ".wmv", ".avi", ".mkv"} +# Percentile points of the per-asset delivery ratio distribution emitted per Dandiset +DELIVERY_RATIO_PERCENTILES = (10, 25, 50, 75, 90) + @beartype def generate_dandiset_summaries( @@ -87,6 +94,7 @@ def generate_dandiset_summaries( ) # Special key for no current association + # Undetermined content cannot be mapped back to a DANDI asset, so no asset sizes are available dandiset_id = "undetermined" _summarize_dandiset( dandiset_id=dandiset_id, @@ -94,6 +102,7 @@ def generate_dandiset_summaries( summary_directory=summary_directory, ip_to_region=ip_to_region, blob_id_to_asset_path=content_id_to_dandiset_path, + blob_id_to_size={}, ) else: dandiset_id_to_local_content_directories, content_id_to_dandiset_path = _get_determinable_dandi_asset_info( @@ -114,6 +123,14 @@ def generate_dandiset_summaries( else: dandiset_ids_to_summarize = [dandiset.identifier for dandiset in client.get_dandisets()] + # Resolve the true (DANDI metadata) byte size of each accessed asset, reusing a local cache across runs + blob_id_to_size = _load_asset_sizes( + client=client, + dandiset_ids=dandiset_ids_to_summarize, + dandiset_id_to_local_content_directories=dandiset_id_to_local_content_directories, + summary_directory=summary_directory, + ) + if max_workers == 1: for dandiset_id in tqdm.tqdm( iterable=dandiset_ids_to_summarize, @@ -133,6 +150,7 @@ def generate_dandiset_summaries( summary_directory=summary_directory, ip_to_region=ip_to_region, blob_id_to_asset_path=content_id_to_dandiset_path, + blob_id_to_size=blob_id_to_size, ) else: with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: @@ -144,6 +162,7 @@ def generate_dandiset_summaries( summary_directory=summary_directory, ip_to_region=ip_to_region, blob_id_to_asset_path=content_id_to_dandiset_path, + blob_id_to_size=blob_id_to_size, ) for dandiset_id in dandiset_ids_to_summarize ] @@ -257,6 +276,100 @@ def _get_undetermined_dandi_asset_info( return dandiset_id_to_local_content_directories, content_id_to_dandiset_path +def _load_asset_sizes( + *, + client, + dandiset_ids: list[str], + dandiset_id_to_local_content_directories: dict[str, list[pathlib.Path]], + summary_directory: pathlib.Path, +) -> dict[str, int | None]: + """ + Resolve the true DANDI metadata size (in bytes) of every accessed asset. + + Sizes are immutable per content (blob or Zarr) ID, so they are cached on disk keyed by content ID and reused + across runs to avoid slow per-asset API calls at archive scale. Each content ID is resolved at most once. A + content ID that the API cannot resolve is cached as ``None`` so it is not repeatedly re-fetched on later runs. + + Parameters + ---------- + client : dandi.dandiapi.DandiAPIClient + An open client used to list a Dandiset's assets when sizes are missing from the cache. + dandiset_ids : list of str + The Dandiset IDs that will be summarized. + dandiset_id_to_local_content_directories : dict + Mapping of Dandiset ID to the local per-content extraction directories (named by content ID). + summary_directory : pathlib.Path + Directory in which the persistent ``asset_sizes.json`` cache is stored. + + Returns + ------- + dict + Mapping of content ID to size in bytes, or ``None`` when the size could not be resolved. + """ + content_id_to_size = _load_asset_size_cache(summary_directory) + + cache_is_dirty = False + for dandiset_id in tqdm.tqdm( + iterable=dandiset_ids, + total=len(dandiset_ids), + desc="Resolving asset sizes", + unit="dandisets", + smoothing=0, + mininterval=5.0, + ): + blob_directories = dandiset_id_to_local_content_directories.get(dandiset_id, []) + requested_content_ids = [blob_directory.name for blob_directory in blob_directories] + missing_content_ids = [ + content_id for content_id in requested_content_ids if content_id not in content_id_to_size + ] + if len(missing_content_ids) == 0: + continue + + fetched_content_id_to_size = _fetch_dandiset_asset_sizes(client=client, dandiset_id=dandiset_id) + if fetched_content_id_to_size is None: + continue # Listing failed (for example, a transient network error); retry on a later run + + for content_id in missing_content_ids: + content_id_to_size[content_id] = fetched_content_id_to_size.get(content_id) + cache_is_dirty = True + + if cache_is_dirty: + _save_asset_size_cache(summary_directory=summary_directory, content_id_to_size=content_id_to_size) + + return content_id_to_size + + +def _load_asset_size_cache(summary_directory: pathlib.Path, /) -> dict[str, int | None]: + cache_file_path = summary_directory / "asset_sizes.json" + if not cache_file_path.exists(): + return {} + return json.loads(cache_file_path.read_text()) + + +def _save_asset_size_cache(*, summary_directory: pathlib.Path, content_id_to_size: dict[str, int | None]) -> None: + cache_file_path = summary_directory / "asset_sizes.json" + cache_file_path.write_text(json.dumps(content_id_to_size)) + + +def _fetch_dandiset_asset_sizes(*, client, dandiset_id: str) -> dict[str, int] | None: + """ + List a single Dandiset's assets and return a mapping of content ID to size in bytes. + + Returns ``None`` if the assets could not be listed at all so that the caller can retry on a later run rather + than caching the content IDs as permanently unresolvable. + """ + content_id_to_size: dict[str, int] = {} + try: + dandiset = client.get_dandiset(dandiset_id) + for asset in dandiset.get_assets(): + content_id = getattr(asset, "blob", None) or getattr(asset, "zarr", None) + if content_id is not None: + content_id_to_size[content_id] = int(asset.size) + except Exception: + return None + return content_id_to_size + + def _summarize_dandiset( *, dandiset_id: str, @@ -264,6 +377,7 @@ def _summarize_dandiset( summary_directory: pathlib.Path, ip_to_region: dict[str, str], blob_id_to_asset_path: dict[str, str], + blob_id_to_size: dict[str, int | None], ) -> None: _summarize_dandiset_by_day( blob_directories=blob_directories, summary_file_path=summary_directory / dandiset_id / "by_day.tsv" @@ -272,6 +386,12 @@ def _summarize_dandiset( blob_directories=blob_directories, summary_file_path=summary_directory / dandiset_id / "by_asset.tsv", blob_id_to_asset_path=blob_id_to_asset_path, + blob_id_to_size=blob_id_to_size, + ) + _summarize_dandiset_delivery_ratio( + blob_directories=blob_directories, + summary_file_path=summary_directory / dandiset_id / "delivery_ratio.tsv", + blob_id_to_size=blob_id_to_size, ) _summarize_dandiset_by_asset_per_week( blob_directories=blob_directories, @@ -525,11 +645,14 @@ def _summarize_dandiset_by_asset( blob_directories: list[pathlib.Path], summary_file_path: pathlib.Path, blob_id_to_asset_path: dict[str, str], + blob_id_to_size: dict[str, int | None], request_count_minimum: int = 50, ) -> None: summarized_activity_by_asset = collections.defaultdict(int) number_of_requests_by_asset = collections.defaultdict(int) number_of_downloads_by_asset = collections.defaultdict(int) + asset_size_by_asset = collections.defaultdict(int) + asset_size_known_by_asset: dict[str, bool] = collections.defaultdict(bool) for blob_directory in blob_directories: blob_id = blob_directory.name @@ -554,6 +677,12 @@ def _summarize_dandiset_by_asset( number_of_requests_by_asset[asset_path] += len(bytes_sent) number_of_downloads_by_asset[asset_path] += sum(downloads) + # Accumulate true asset size for the delivery ratio; sizes of 0 or missing do not contribute + asset_size = blob_id_to_size.get(blob_id) + if asset_size is not None and asset_size > 0: + asset_size_by_asset[asset_path] += asset_size + asset_size_known_by_asset[asset_path] = True + if len(summarized_activity_by_asset) == 0: return @@ -575,11 +704,105 @@ def _summarize_dandiset_by_asset( ) for path in all_asset_paths ], + "delivery_ratio": [ + ( + (summarized_activity_by_asset[path] / asset_size_by_asset[path]) + if asset_size_known_by_asset.get(path, False) + else float("nan") + ) + for path in all_asset_paths + ], } ) summary_table.to_csv(path_or_buf=summary_file_path, mode="w", sep="\t", header=True, index=False) +def _compute_delivery_ratio_percentiles(delivery_ratios: list[float], /) -> dict[str, float]: + """ + Compute the per-Dandiset delivery ratio percentiles from a list of per-asset delivery ratios. + + Uses linear interpolation between data points (the NumPy default). A single usable ratio yields that ratio for + all five percentiles, and an empty list yields ``NaN`` for all five percentiles. + + Parameters + ---------- + delivery_ratios : list of float + The per-asset delivery ratios for all usable assets in a Dandiset. + + Returns + ------- + dict + Mapping of percentile column name (for example ``"delivery_ratio_p10"``) to its float value. + """ + column_names = [f"delivery_ratio_p{percentile}" for percentile in DELIVERY_RATIO_PERCENTILES] + if len(delivery_ratios) == 0: + return {column_name: float("nan") for column_name in column_names} + + percentile_values = numpy.percentile(a=delivery_ratios, q=list(DELIVERY_RATIO_PERCENTILES), method="linear") + return {column_name: float(value) for column_name, value in zip(column_names, percentile_values)} + + +def _summarize_dandiset_delivery_ratio( + *, + blob_directories: list[pathlib.Path], + summary_file_path: pathlib.Path, + blob_id_to_size: dict[str, int | None], +) -> None: + """ + Write the per-Dandiset delivery ratio summary. + + The delivery ratio of an asset is the total bytes delivered across all logged GET requests divided by the + asset's true size in bytes. A ratio near 1 indicates download-dominated access while a ratio much greater than 1 + indicates streaming-dominated access. Assets with a missing or zero size are excluded from the percentile + computation and the count of skipped assets is logged. The asset-weighted percentiles plus a volume-weighted + ratio (total bytes delivered over total size) are written as a single row, always with all five percentile + columns present even when no asset is usable. + """ + delivery_ratios: list[float] = [] + total_bytes_delivered = 0 + total_asset_size = 0 + number_of_skipped_assets = 0 + number_of_accessed_assets = 0 + for blob_directory in blob_directories: + if not blob_directory.exists(): + continue # No extracted logs found (possible asset was never accessed); skip to next asset + number_of_accessed_assets += 1 + + blob_id = blob_directory.name + asset_size = blob_id_to_size.get(blob_id) + + bytes_sent_file_path = blob_directory / "bytes_sent.txt" + bytes_delivered = sum(int(value.strip()) for value in bytes_sent_file_path.read_text().splitlines()) + + if asset_size is None or asset_size == 0: + number_of_skipped_assets += 1 + continue + + delivery_ratios.append(bytes_delivered / asset_size) + total_bytes_delivered += bytes_delivered + total_asset_size += asset_size + + if number_of_accessed_assets == 0: + return # Dandiset was never accessed; match the other summaries by writing nothing + + if number_of_skipped_assets > 0: + logger.info( + "Skipped %d of %d accessed assets with missing or zero size while computing delivery ratios for %s", + number_of_skipped_assets, + number_of_accessed_assets, + summary_file_path.parent.name, + ) + + percentiles = _compute_delivery_ratio_percentiles(delivery_ratios) + weighted_ratio = (total_bytes_delivered / total_asset_size) if total_asset_size > 0 else float("nan") + + summary_file_path.parent.mkdir(parents=True, exist_ok=True) + data: dict[str, list[float]] = {column_name: [value] for column_name, value in percentiles.items()} + data["delivery_ratio_weighted"] = [weighted_ratio] + summary_table = pandas.DataFrame(data=data) + summary_table.to_csv(path_or_buf=summary_file_path, mode="w", sep="\t", header=True, index=False) + + def _summarize_dandiset_by_region( *, blob_directories: list[pathlib.Path], diff --git a/tests/test_dandi_summaries.py b/tests/test_dandi_summaries.py index ba56059..5ec6e8b 100644 --- a/tests/test_dandi_summaries.py +++ b/tests/test_dandi_summaries.py @@ -44,15 +44,18 @@ def test_dandiset_summaries(tmpdir: py.path.local): summary_file_path=test_summary_dir / "archive" / "requester_count.tsv", ) + # ``delivery_ratio.tsv`` and the ``delivery_ratio`` column depend on live DANDI asset sizes, so they are not + # part of the deterministic snapshot here; they are covered separately in ``test_delivery_ratio.py``. + skipped_file_names = {"requester_count.tsv", "delivery_ratio.tsv"} test_file_paths = { path.relative_to(test_summary_dir): path for path in test_summary_dir.rglob(pattern="*.tsv") - if path.name != "requester_count.tsv" + if path.name not in skipped_file_names } expected_file_paths = { path.relative_to(expected_summaries_dir): path for path in expected_summaries_dir.rglob(pattern="*.tsv") - if path.name != "requester_count.tsv" + if path.name not in skipped_file_names } assert set(test_file_paths.keys()) == set(expected_file_paths.keys()) @@ -60,8 +63,12 @@ def test_dandiset_summaries(tmpdir: py.path.local): relative_file_path = expected_file_path.relative_to(expected_summaries_dir) test_file_path = test_summary_dir / relative_file_path - test_mapped_log = pandas.read_table(filepath_or_buffer=test_file_path, index_col=0) - expected_mapped_log = pandas.read_table(filepath_or_buffer=expected_file_path, index_col=0) + test_mapped_log = pandas.read_table(filepath_or_buffer=test_file_path, index_col=0).drop( + columns=["delivery_ratio"], errors="ignore" + ) + expected_mapped_log = pandas.read_table(filepath_or_buffer=expected_file_path, index_col=0).drop( + columns=["delivery_ratio"], errors="ignore" + ) for column_name in ("number_of_requests", "number_of_downloads"): if column_name in expected_mapped_log.columns: expected_mapped_log[column_name] = expected_mapped_log[column_name].map( diff --git a/tests/test_delivery_ratio.py b/tests/test_delivery_ratio.py new file mode 100644 index 0000000..6745495 --- /dev/null +++ b/tests/test_delivery_ratio.py @@ -0,0 +1,132 @@ +import math +import pathlib + +import pandas +import pytest + +from dandi_s3_log_extraction.summarize._generate_dandiset_summaries import ( + _compute_delivery_ratio_percentiles, + _summarize_dandiset_delivery_ratio, +) + +PERCENTILE_COLUMN_NAMES = ( + "delivery_ratio_p10", + "delivery_ratio_p25", + "delivery_ratio_p50", + "delivery_ratio_p75", + "delivery_ratio_p90", +) + + +@pytest.mark.ai_generated +@pytest.mark.parametrize( + ("delivery_ratios", "expected_values"), + [ + # Several assets interpolate linearly between data points + ([1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0], (1.6, 2.5, 4.0, 7.5, 14.0)), + # Two assets + ([2.0, 3.0], (2.1, 2.25, 2.5, 2.75, 2.9)), + # A single usable asset yields that asset's ratio for all five percentiles + ([3.7], (3.7, 3.7, 3.7, 3.7, 3.7)), + ], +) +def test_compute_delivery_ratio_percentiles(delivery_ratios: list[float], expected_values: tuple[float, ...]) -> None: + percentiles = _compute_delivery_ratio_percentiles(delivery_ratios) + + assert tuple(percentiles.keys()) == PERCENTILE_COLUMN_NAMES + for column_name, expected_value in zip(PERCENTILE_COLUMN_NAMES, expected_values): + assert percentiles[column_name] == pytest.approx(expected_value) + + +@pytest.mark.ai_generated +def test_compute_delivery_ratio_percentiles_zero_assets() -> None: + percentiles = _compute_delivery_ratio_percentiles([]) + + assert tuple(percentiles.keys()) == PERCENTILE_COLUMN_NAMES + for column_name in PERCENTILE_COLUMN_NAMES: + assert math.isnan(percentiles[column_name]) + + +def _write_blob_directory(*, parent: pathlib.Path, blob_id: str, bytes_sent: list[int]) -> pathlib.Path: + blob_directory = parent / blob_id + blob_directory.mkdir(parents=True) + (blob_directory / "bytes_sent.txt").write_text("\n".join(str(value) for value in bytes_sent)) + return blob_directory + + +@pytest.mark.ai_generated +def test_summarize_dandiset_delivery_ratio_skips_unusable_assets(tmp_path: pathlib.Path) -> None: + extraction_directory = tmp_path / "extraction" + extraction_directory.mkdir() + + # Two usable assets (ratios 2.0 and 3.0), one with zero size, and one with a missing size; the latter two + # must be excluded from the percentile computation + blob_directories = [ + _write_blob_directory(parent=extraction_directory, blob_id="aaa", bytes_sent=[40, 60]), + _write_blob_directory(parent=extraction_directory, blob_id="bbb", bytes_sent=[100, 200]), + _write_blob_directory(parent=extraction_directory, blob_id="ccc", bytes_sent=[999]), + _write_blob_directory(parent=extraction_directory, blob_id="ddd", bytes_sent=[50]), + ] + blob_id_to_size = {"aaa": 50, "bbb": 100, "ccc": 0} # "ddd" is intentionally absent (missing size) + + summary_file_path = tmp_path / "summaries" / "000000" / "delivery_ratio.tsv" + _summarize_dandiset_delivery_ratio( + blob_directories=blob_directories, + summary_file_path=summary_file_path, + blob_id_to_size=blob_id_to_size, + ) + + summary_table = pandas.read_table(filepath_or_buffer=summary_file_path) + assert list(summary_table.columns) == [*PERCENTILE_COLUMN_NAMES, "delivery_ratio_weighted"] + assert len(summary_table) == 1 + + row = summary_table.iloc[0] + expected_percentiles = (2.1, 2.25, 2.5, 2.75, 2.9) + for column_name, expected_value in zip(PERCENTILE_COLUMN_NAMES, expected_percentiles): + assert row[column_name] == pytest.approx(expected_value) + # Volume-weighted ratio is the summed delivered bytes (100 + 300) over the summed usable sizes (50 + 100) + assert row["delivery_ratio_weighted"] == pytest.approx(400 / 150) + + +@pytest.mark.ai_generated +def test_summarize_dandiset_delivery_ratio_single_asset(tmp_path: pathlib.Path) -> None: + extraction_directory = tmp_path / "extraction" + extraction_directory.mkdir() + + blob_directories = [_write_blob_directory(parent=extraction_directory, blob_id="aaa", bytes_sent=[150])] + blob_id_to_size = {"aaa": 100} + + summary_file_path = tmp_path / "summaries" / "000000" / "delivery_ratio.tsv" + _summarize_dandiset_delivery_ratio( + blob_directories=blob_directories, + summary_file_path=summary_file_path, + blob_id_to_size=blob_id_to_size, + ) + + row = pandas.read_table(filepath_or_buffer=summary_file_path).iloc[0] + for column_name in PERCENTILE_COLUMN_NAMES: + assert row[column_name] == pytest.approx(1.5) + assert row["delivery_ratio_weighted"] == pytest.approx(1.5) + + +@pytest.mark.ai_generated +def test_summarize_dandiset_delivery_ratio_zero_usable_assets(tmp_path: pathlib.Path) -> None: + extraction_directory = tmp_path / "extraction" + extraction_directory.mkdir() + + # The asset was accessed but its size cannot be resolved, so the row is all NaN but still has every column + blob_directories = [_write_blob_directory(parent=extraction_directory, blob_id="aaa", bytes_sent=[150])] + blob_id_to_size: dict[str, int | None] = {} + + summary_file_path = tmp_path / "summaries" / "000000" / "delivery_ratio.tsv" + _summarize_dandiset_delivery_ratio( + blob_directories=blob_directories, + summary_file_path=summary_file_path, + blob_id_to_size=blob_id_to_size, + ) + + summary_table = pandas.read_table(filepath_or_buffer=summary_file_path) + assert list(summary_table.columns) == [*PERCENTILE_COLUMN_NAMES, "delivery_ratio_weighted"] + row = summary_table.iloc[0] + for column_name in [*PERCENTILE_COLUMN_NAMES, "delivery_ratio_weighted"]: + assert math.isnan(row[column_name]) From 12af7ed498f2c8f6706a102f3f3357779fcce609 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 17:12:14 +0000 Subject: [PATCH 02/14] Moved delivery ratio placement into DANDI totals and archive wrappers Reworked the experimental delivery ratio feature so the per-Dandiset and archive percentiles live in the totals and archive outputs rather than a standalone per-Dandiset file. Added DANDI-specific wrappers generate_dandiset_totals, generate_archive_totals, and generate_archive_summaries that call the upstream s3_log_extraction methods and then inject the asset-weighted delivery ratio percentiles plus the volume-weighted ratio, sourced from each by_asset.tsv delivery_ratio column. Per-Dandiset fields are written into totals.json, archive-wide fields into archive_totals.json, and an archive delivery_ratio.tsv. Added the update totals CLI command and routed update summaries --mode archive through the DANDI wrapper. Co-Authored-By: Claude Code / Claude Opus 4.8 --- CHANGELOG.md | 3 +- .../_command_line_interface/_cli.py | 45 ++- .../summarize/__init__.py | 10 +- .../summarize/_generate_dandiset_summaries.py | 266 ++++++++++++++---- tests/test_dandi_summaries.py | 9 +- tests/test_delivery_ratio.py | 214 +++++++++----- 6 files changed, 418 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbf5bf2..380d34b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ ### 🚀 Enhancement -- Added a per-asset `delivery_ratio` column to `by_asset.tsv` and a per-Dandiset `delivery_ratio.tsv` summary with `delivery_ratio_p10`, `delivery_ratio_p25`, `delivery_ratio_p50`, `delivery_ratio_p75`, `delivery_ratio_p90`, and `delivery_ratio_weighted`. The delivery ratio is the total bytes delivered over the asset's true DANDI size, signaling streaming versus download intensity. Asset sizes are fetched from the DANDI API and cached locally in `asset_sizes.json`. ([#86](https://github.com/dandi/dandi-s3-log-extraction/pull/86)) +- Added an experimental, DANDI-only delivery ratio feature. The delivery ratio is the total bytes delivered over the asset's true DANDI size, signaling streaming versus download intensity. A per-asset `delivery_ratio` column is added to `by_asset.tsv`. The asset-weighted percentiles `delivery_ratio_p10`, `delivery_ratio_p25`, `delivery_ratio_p50`, `delivery_ratio_p75`, `delivery_ratio_p90` plus the volume-weighted `delivery_ratio_weighted` are injected into per-Dandiset `totals.json`, into archive `archive_totals.json`, and into a new archive `delivery_ratio.tsv`, via DANDI wrappers around the upstream summary and totals methods. Asset sizes are fetched from the DANDI API and cached locally in `asset_sizes.json`. ([#86](https://github.com/dandi/dandi-s3-log-extraction/pull/86)) +- Added the `dandis3logextraction update totals` command, with `--mode archive` for archive-wide totals, wrapping the upstream totals generation to also emit the delivery ratio fields. ([#86](https://github.com/dandi/dandi-s3-log-extraction/pull/86)) - Added `download` to `_dandi_extraction.awk` so extraction writes `download.txt` alongside the other per-request outputs. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) - Added `--cache-directory` to `dandis3logextraction extract` so remote extraction can use a custom cache directory. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) - Added `--inventory` to `dandis3logextraction extract --mode remote` so extraction can use a local S3 Inventory directory. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) diff --git a/src/dandi_s3_log_extraction/_command_line_interface/_cli.py b/src/dandi_s3_log_extraction/_command_line_interface/_cli.py index b6c69f0..c3ba39e 100644 --- a/src/dandi_s3_log_extraction/_command_line_interface/_cli.py +++ b/src/dandi_s3_log_extraction/_command_line_interface/_cli.py @@ -8,7 +8,12 @@ import s3_log_extraction from ..extractors import DandiRemoteS3LogAccessExtractor -from ..summarize import generate_dandiset_summaries +from ..summarize import ( + generate_archive_summaries, + generate_archive_totals, + generate_dandiset_summaries, + generate_dandiset_totals, +) # dandis3logextraction @@ -231,7 +236,7 @@ def _update_summaries_cli( """Generate condensed summaries of activity.""" match mode: case "archive": - s3_log_extraction.summarize.generate_archive_summaries(cache_directory=cache_directory) + generate_archive_summaries(cache_directory=cache_directory) case _: pick_as_list = pick.split(",") if pick is not None else None skip_as_list = skip.split(",") if skip is not None else None @@ -244,3 +249,39 @@ def _update_summaries_cli( unassociated=unassociated, cache_directory=cache_directory, ) + + +# dandis3logextraction update totals +@_update_cli.command(name="totals") +@rich_click.option( + "--mode", + help=( + "Generate grand totals of activity across the extracted data. " + "Mode 'archive' aggregates over all Dandiset totals. " + "By default, per-Dandiset totals are generated." + ), + required=False, + type=rich_click.Choice(choices=["dandi", "archive"]), + default=None, +) +@rich_click.option( + "--cache", + "cache_directory", + help=( + "Path to the folder containing all previously extracted S3 access logs (`cache_directory`). " + "If not provided, the default cache directory from the configuration will be used." + ), + required=False, + type=rich_click.Path(file_okay=False, dir_okay=True), + default=None, +) +def _update_totals_cli( + mode: typing.Literal["dandi", "archive"] | None = None, + cache_directory: str | None = None, +) -> None: + """Generate grand totals of all extracted data, including experimental delivery ratio fields.""" + match mode: + case "archive": + generate_archive_totals(cache_directory=cache_directory) + case _: + generate_dandiset_totals(cache_directory=cache_directory) diff --git a/src/dandi_s3_log_extraction/summarize/__init__.py b/src/dandi_s3_log_extraction/summarize/__init__.py index 56980c7..ce07f85 100644 --- a/src/dandi_s3_log_extraction/summarize/__init__.py +++ b/src/dandi_s3_log_extraction/summarize/__init__.py @@ -1,5 +1,13 @@ -from ._generate_dandiset_summaries import generate_dandiset_summaries +from ._generate_dandiset_summaries import ( + generate_archive_summaries, + generate_archive_totals, + generate_dandiset_summaries, + generate_dandiset_totals, +) __all__ = [ + "generate_archive_summaries", + "generate_archive_totals", "generate_dandiset_summaries", + "generate_dandiset_totals", ] diff --git a/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py b/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py index 4d8c9dd..a7747c3 100644 --- a/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py +++ b/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py @@ -5,6 +5,7 @@ import itertools import json import logging +import math import pathlib import numpy @@ -388,11 +389,6 @@ def _summarize_dandiset( blob_id_to_asset_path=blob_id_to_asset_path, blob_id_to_size=blob_id_to_size, ) - _summarize_dandiset_delivery_ratio( - blob_directories=blob_directories, - summary_file_path=summary_directory / dandiset_id / "delivery_ratio.tsv", - blob_id_to_size=blob_id_to_size, - ) _summarize_dandiset_by_asset_per_week( blob_directories=blob_directories, summary_file_path=summary_directory / dandiset_id / "by_asset_per_week.tsv", @@ -742,65 +738,71 @@ def _compute_delivery_ratio_percentiles(delivery_ratios: list[float], /) -> dict return {column_name: float(value) for column_name, value in zip(column_names, percentile_values)} -def _summarize_dandiset_delivery_ratio( - *, - blob_directories: list[pathlib.Path], - summary_file_path: pathlib.Path, - blob_id_to_size: dict[str, int | None], -) -> None: - """ - Write the per-Dandiset delivery ratio summary. - - The delivery ratio of an asset is the total bytes delivered across all logged GET requests divided by the - asset's true size in bytes. A ratio near 1 indicates download-dominated access while a ratio much greater than 1 - indicates streaming-dominated access. Assets with a missing or zero size are excluded from the percentile - computation and the count of skipped assets is logged. The asset-weighted percentiles plus a volume-weighted - ratio (total bytes delivered over total size) are written as a single row, always with all five percentile - columns present even when no asset is usable. +DELIVERY_RATIO_FIELD_NAMES = ( + "delivery_ratio_p10", + "delivery_ratio_p25", + "delivery_ratio_p50", + "delivery_ratio_p75", + "delivery_ratio_p90", + "delivery_ratio_weighted", +) + + +def _read_by_asset_delivery_ratios(by_asset_file_path: pathlib.Path, /) -> tuple[list[float], list[int]]: """ - delivery_ratios: list[float] = [] - total_bytes_delivered = 0 - total_asset_size = 0 - number_of_skipped_assets = 0 - number_of_accessed_assets = 0 - for blob_directory in blob_directories: - if not blob_directory.exists(): - continue # No extracted logs found (possible asset was never accessed); skip to next asset - number_of_accessed_assets += 1 + Read the usable per-asset delivery ratios and delivered byte counts from a ``by_asset.tsv`` summary. - blob_id = blob_directory.name - asset_size = blob_id_to_size.get(blob_id) + Each row of ``by_asset.tsv`` that maps to a determined asset path corresponds to a single asset, so its + ``delivery_ratio`` is that asset's ratio. Rows whose ratio is missing (the asset's size could not be resolved) + and the catch-all ``"undetermined"`` row (which aggregates many blobs and is therefore not a single asset) are + excluded. - bytes_sent_file_path = blob_directory / "bytes_sent.txt" - bytes_delivered = sum(int(value.strip()) for value in bytes_sent_file_path.read_text().splitlines()) + Returns + ------- + tuple + A pair of equal-length lists holding the usable delivery ratios and their corresponding delivered bytes. + """ + if not by_asset_file_path.exists(): + return [], [] - if asset_size is None or asset_size == 0: - number_of_skipped_assets += 1 + table = pandas.read_table(filepath_or_buffer=by_asset_file_path) + if "delivery_ratio" not in table.columns: + return [], [] + + delivery_ratios: list[float] = [] + bytes_delivered: list[int] = [] + for asset_path, ratio, asset_bytes in zip(table["asset_path"], table["delivery_ratio"], table["bytes_sent"]): + if asset_path == "undetermined" or pandas.isna(ratio): continue + delivery_ratios.append(float(ratio)) + bytes_delivered.append(int(asset_bytes)) + return delivery_ratios, bytes_delivered + - delivery_ratios.append(bytes_delivered / asset_size) - total_bytes_delivered += bytes_delivered - total_asset_size += asset_size +def _compute_delivery_ratio_fields(*, delivery_ratios: list[float], bytes_delivered: list[int]) -> dict[str, float]: + """ + Compute the five asset-weighted percentile fields plus the volume-weighted delivery ratio. - if number_of_accessed_assets == 0: - return # Dandiset was never accessed; match the other summaries by writing nothing + The percentiles are asset-weighted (each asset contributes one ratio) while ``delivery_ratio_weighted`` is the + total delivered bytes over the total asset size. The deliberate gap between the weighted ratio and the median is + a heterogeneity signal, so both are reported. With no usable assets every field is ``NaN``. + """ + fields = _compute_delivery_ratio_percentiles(delivery_ratios) - if number_of_skipped_assets > 0: - logger.info( - "Skipped %d of %d accessed assets with missing or zero size while computing delivery ratios for %s", - number_of_skipped_assets, - number_of_accessed_assets, - summary_file_path.parent.name, - ) + total_bytes_delivered = sum(bytes_delivered) + # Recover each asset's size as delivered_bytes / ratio; ratios of 0 carry no size information and are skipped + total_asset_size = sum( + asset_bytes / ratio for asset_bytes, ratio in zip(bytes_delivered, delivery_ratios) if ratio > 0 + ) + fields["delivery_ratio_weighted"] = ( + (total_bytes_delivered / total_asset_size) if total_asset_size > 0 else float("nan") + ) + return fields - percentiles = _compute_delivery_ratio_percentiles(delivery_ratios) - weighted_ratio = (total_bytes_delivered / total_asset_size) if total_asset_size > 0 else float("nan") - summary_file_path.parent.mkdir(parents=True, exist_ok=True) - data: dict[str, list[float]] = {column_name: [value] for column_name, value in percentiles.items()} - data["delivery_ratio_weighted"] = [weighted_ratio] - summary_table = pandas.DataFrame(data=data) - summary_table.to_csv(path_or_buf=summary_file_path, mode="w", sep="\t", header=True, index=False) +def _jsonable_float(value: float, /) -> float | None: + """Return ``None`` in place of ``NaN`` so values serialize as valid JSON ``null``.""" + return None if math.isnan(value) else value def _summarize_dandiset_by_region( @@ -996,3 +998,159 @@ def _summarize_archive_unique_requester_count( rounded_count = _round_requester_count(count=len(unique_ips), modulo=modulo, minimum=minimum) summary_file_path.parent.mkdir(parents=True, exist_ok=True) summary_file_path.write_text(str(rounded_count)) + + +def _iter_dataset_by_asset_file_paths(summary_directory: pathlib.Path, /): + """Yield every per-Dandiset ``by_asset.tsv`` path, skipping the aggregated ``archive`` directory.""" + for by_asset_file_path in summary_directory.rglob(pattern="by_asset.tsv"): + if by_asset_file_path.parent.name == "archive": + continue + yield by_asset_file_path + + +def _pool_archive_delivery_ratios(summary_directory: pathlib.Path, /) -> tuple[list[float], list[int]]: + """Pool the usable per-asset delivery ratios and delivered bytes across every Dandiset in the archive.""" + pooled_delivery_ratios: list[float] = [] + pooled_bytes_delivered: list[int] = [] + for by_asset_file_path in _iter_dataset_by_asset_file_paths(summary_directory): + delivery_ratios, bytes_delivered = _read_by_asset_delivery_ratios(by_asset_file_path) + pooled_delivery_ratios.extend(delivery_ratios) + pooled_bytes_delivered.extend(bytes_delivered) + return pooled_delivery_ratios, pooled_bytes_delivered + + +def _resolve_summary_directory(*, cache_directory: str | pathlib.Path | None) -> pathlib.Path: + cache_directory = ( + pathlib.Path(cache_directory) if cache_directory is not None else s3_log_extraction.config.get_cache_directory() + ) + return cache_directory / "summaries" + + +@beartype +def generate_dandiset_totals( + *, + cache_directory: str | pathlib.Path | None = None, + privacy_threshold_minimum: int = 50, +) -> None: + """ + Generate per-Dandiset totals and augment them with experimental delivery ratio fields. + + This is a DANDI-specific wrapper around the generic ``s3_log_extraction`` per-dataset totals. After the upstream + ``totals.json`` is written, each Dandiset entry gains the asset-weighted delivery ratio percentiles + (``delivery_ratio_p10`` through ``delivery_ratio_p90``) and the volume-weighted ``delivery_ratio_weighted``, + derived from that Dandiset's ``by_asset.tsv``. Entries with no usable asset receive ``null`` for every field. + + Parameters + ---------- + cache_directory : path-like, optional + The top-level cache directory from which the summary directory is derived. + If not provided, the default cache directory is used. + privacy_threshold_minimum : int + Minimum disclosure threshold for privacy-rounded request and download totals. Default is ``50``. + """ + s3_log_extraction.summarize.generate_all_dataset_totals( + cache_directory=cache_directory, privacy_threshold_minimum=privacy_threshold_minimum + ) + + summary_directory = _resolve_summary_directory(cache_directory=cache_directory) + totals_file_path = summary_directory / "totals.json" + all_dataset_totals = json.loads(totals_file_path.read_text()) + + for dataset_id, dataset_totals in all_dataset_totals.items(): + delivery_ratios, bytes_delivered = _read_by_asset_delivery_ratios( + summary_directory / dataset_id / "by_asset.tsv" + ) + fields = _compute_delivery_ratio_fields(delivery_ratios=delivery_ratios, bytes_delivered=bytes_delivered) + for field_name in DELIVERY_RATIO_FIELD_NAMES: + dataset_totals[field_name] = _jsonable_float(fields[field_name]) + + with totals_file_path.open(mode="w") as io: + json.dump(obj=all_dataset_totals, fp=io, indent=2, sort_keys=True) + + +@beartype +def generate_archive_summaries( + *, + cache_directory: str | pathlib.Path | None = None, + asset_types_in_order: tuple[str, ...] | list[str] | None = None, + privacy_threshold_minimum: int = 50, +) -> None: + """ + Generate archive-wide summaries and add an experimental archive delivery ratio summary. + + This is a DANDI-specific wrapper around the generic ``s3_log_extraction`` archive summaries. After the upstream + archive summaries are written, an ``archive/delivery_ratio.tsv`` is written holding the asset-weighted delivery + ratio percentiles plus the volume-weighted ratio, pooled across every asset in every Dandiset. The single row + always contains all six columns, with ``NaN`` when the archive has no usable asset. + + Parameters + ---------- + cache_directory : path-like, optional + The top-level cache directory from which the summary directory is derived. + If not provided, the default cache directory is used. + asset_types_in_order : sequence of str, optional + Preferred output column ordering for known asset types in the archive ``by_asset_type_per_week.tsv``. + privacy_threshold_minimum : int + Minimum disclosure threshold for privacy-rounded request and download values. Default is ``50``. + """ + s3_log_extraction.summarize.generate_archive_summaries( + cache_directory=cache_directory, + asset_types_in_order=asset_types_in_order, + privacy_threshold_minimum=privacy_threshold_minimum, + ) + + summary_directory = _resolve_summary_directory(cache_directory=cache_directory) + pooled_delivery_ratios, pooled_bytes_delivered = _pool_archive_delivery_ratios(summary_directory) + fields = _compute_delivery_ratio_fields( + delivery_ratios=pooled_delivery_ratios, bytes_delivered=pooled_bytes_delivered + ) + + archive_directory = summary_directory / "archive" + archive_directory.mkdir(exist_ok=True) + summary_table = pandas.DataFrame( + data={field_name: [fields[field_name]] for field_name in DELIVERY_RATIO_FIELD_NAMES} + ) + summary_table.to_csv( + path_or_buf=archive_directory / "delivery_ratio.tsv", mode="w", sep="\t", header=True, index=False + ) + + +@beartype +def generate_archive_totals( + *, + cache_directory: str | pathlib.Path | None = None, + privacy_threshold_minimum: int = 50, +) -> None: + """ + Generate archive-wide totals and augment them with experimental delivery ratio fields. + + This is a DANDI-specific wrapper around the generic ``s3_log_extraction`` archive totals. After the upstream + ``archive_totals.json`` is written, it gains the asset-weighted delivery ratio percentiles plus the + volume-weighted ratio, pooled across every asset in every Dandiset. All fields are ``null`` when the archive has + no usable asset. + + Parameters + ---------- + cache_directory : path-like, optional + The top-level cache directory from which the summary directory is derived. + If not provided, the default cache directory is used. + privacy_threshold_minimum : int + Minimum disclosure threshold for privacy-rounded request and download totals. Default is ``50``. + """ + s3_log_extraction.summarize.generate_archive_totals( + cache_directory=cache_directory, privacy_threshold_minimum=privacy_threshold_minimum + ) + + summary_directory = _resolve_summary_directory(cache_directory=cache_directory) + pooled_delivery_ratios, pooled_bytes_delivered = _pool_archive_delivery_ratios(summary_directory) + fields = _compute_delivery_ratio_fields( + delivery_ratios=pooled_delivery_ratios, bytes_delivered=pooled_bytes_delivered + ) + + archive_totals_file_path = summary_directory / "archive_totals.json" + archive_totals = json.loads(archive_totals_file_path.read_text()) + for field_name in DELIVERY_RATIO_FIELD_NAMES: + archive_totals[field_name] = _jsonable_float(fields[field_name]) + + with archive_totals_file_path.open(mode="w") as io: + json.dump(obj=archive_totals, fp=io, indent=2, sort_keys=True) diff --git a/tests/test_dandi_summaries.py b/tests/test_dandi_summaries.py index 5ec6e8b..80f8fa7 100644 --- a/tests/test_dandi_summaries.py +++ b/tests/test_dandi_summaries.py @@ -44,18 +44,15 @@ def test_dandiset_summaries(tmpdir: py.path.local): summary_file_path=test_summary_dir / "archive" / "requester_count.tsv", ) - # ``delivery_ratio.tsv`` and the ``delivery_ratio`` column depend on live DANDI asset sizes, so they are not - # part of the deterministic snapshot here; they are covered separately in ``test_delivery_ratio.py``. - skipped_file_names = {"requester_count.tsv", "delivery_ratio.tsv"} test_file_paths = { path.relative_to(test_summary_dir): path for path in test_summary_dir.rglob(pattern="*.tsv") - if path.name not in skipped_file_names + if path.name != "requester_count.tsv" } expected_file_paths = { path.relative_to(expected_summaries_dir): path for path in expected_summaries_dir.rglob(pattern="*.tsv") - if path.name not in skipped_file_names + if path.name != "requester_count.tsv" } assert set(test_file_paths.keys()) == set(expected_file_paths.keys()) @@ -63,6 +60,8 @@ def test_dandiset_summaries(tmpdir: py.path.local): relative_file_path = expected_file_path.relative_to(expected_summaries_dir) test_file_path = test_summary_dir / relative_file_path + # The ``delivery_ratio`` column of ``by_asset.tsv`` depends on live DANDI asset sizes, so it is dropped from + # this deterministic snapshot and covered separately in ``test_delivery_ratio.py``. test_mapped_log = pandas.read_table(filepath_or_buffer=test_file_path, index_col=0).drop( columns=["delivery_ratio"], errors="ignore" ) diff --git a/tests/test_delivery_ratio.py b/tests/test_delivery_ratio.py index 6745495..787a42f 100644 --- a/tests/test_delivery_ratio.py +++ b/tests/test_delivery_ratio.py @@ -1,12 +1,16 @@ +import json import math import pathlib import pandas import pytest +import dandi_s3_log_extraction from dandi_s3_log_extraction.summarize._generate_dandiset_summaries import ( + _compute_delivery_ratio_fields, _compute_delivery_ratio_percentiles, - _summarize_dandiset_delivery_ratio, + _pool_archive_delivery_ratios, + _read_by_asset_delivery_ratios, ) PERCENTILE_COLUMN_NAMES = ( @@ -16,6 +20,7 @@ "delivery_ratio_p75", "delivery_ratio_p90", ) +DELIVERY_RATIO_FIELD_NAMES = (*PERCENTILE_COLUMN_NAMES, "delivery_ratio_weighted") @pytest.mark.ai_generated @@ -47,86 +52,163 @@ def test_compute_delivery_ratio_percentiles_zero_assets() -> None: assert math.isnan(percentiles[column_name]) -def _write_blob_directory(*, parent: pathlib.Path, blob_id: str, bytes_sent: list[int]) -> pathlib.Path: - blob_directory = parent / blob_id - blob_directory.mkdir(parents=True) - (blob_directory / "bytes_sent.txt").write_text("\n".join(str(value) for value in bytes_sent)) - return blob_directory +def _write_by_asset_tsv(*, directory: pathlib.Path, rows: list[tuple[str, int, float]]) -> pathlib.Path: + directory.mkdir(parents=True, exist_ok=True) + table = pandas.DataFrame( + data={ + "asset_path": [row[0] for row in rows], + "bytes_sent": [row[1] for row in rows], + "number_of_requests": ["<50"] * len(rows), + "number_of_downloads": ["<50"] * len(rows), + "delivery_ratio": [row[2] for row in rows], + } + ) + by_asset_file_path = directory / "by_asset.tsv" + table.to_csv(path_or_buf=by_asset_file_path, mode="w", sep="\t", header=True, index=False) + return by_asset_file_path @pytest.mark.ai_generated -def test_summarize_dandiset_delivery_ratio_skips_unusable_assets(tmp_path: pathlib.Path) -> None: - extraction_directory = tmp_path / "extraction" - extraction_directory.mkdir() - - # Two usable assets (ratios 2.0 and 3.0), one with zero size, and one with a missing size; the latter two - # must be excluded from the percentile computation - blob_directories = [ - _write_blob_directory(parent=extraction_directory, blob_id="aaa", bytes_sent=[40, 60]), - _write_blob_directory(parent=extraction_directory, blob_id="bbb", bytes_sent=[100, 200]), - _write_blob_directory(parent=extraction_directory, blob_id="ccc", bytes_sent=[999]), - _write_blob_directory(parent=extraction_directory, blob_id="ddd", bytes_sent=[50]), - ] - blob_id_to_size = {"aaa": 50, "bbb": 100, "ccc": 0} # "ddd" is intentionally absent (missing size) - - summary_file_path = tmp_path / "summaries" / "000000" / "delivery_ratio.tsv" - _summarize_dandiset_delivery_ratio( - blob_directories=blob_directories, - summary_file_path=summary_file_path, - blob_id_to_size=blob_id_to_size, +def test_read_by_asset_delivery_ratios_excludes_undetermined_and_missing(tmp_path: pathlib.Path) -> None: + by_asset_file_path = _write_by_asset_tsv( + directory=tmp_path / "000001", + rows=[ + ("sub-1/a.nwb", 100, 2.0), + ("sub-1/b.nwb", 300, 3.0), + ("sub-1/c.nwb", 7, float("nan")), # size could not be resolved + ("undetermined", 50, float("nan")), # aggregated bucket, not a single asset + ], ) - summary_table = pandas.read_table(filepath_or_buffer=summary_file_path) - assert list(summary_table.columns) == [*PERCENTILE_COLUMN_NAMES, "delivery_ratio_weighted"] - assert len(summary_table) == 1 + delivery_ratios, bytes_delivered = _read_by_asset_delivery_ratios(by_asset_file_path) + + assert delivery_ratios == [2.0, 3.0] + assert bytes_delivered == [100, 300] + + +@pytest.mark.ai_generated +def test_read_by_asset_delivery_ratios_missing_file() -> None: + delivery_ratios, bytes_delivered = _read_by_asset_delivery_ratios(pathlib.Path("does_not_exist.tsv")) + + assert delivery_ratios == [] + assert bytes_delivered == [] - row = summary_table.iloc[0] + +@pytest.mark.ai_generated +def test_compute_delivery_ratio_fields() -> None: + fields = _compute_delivery_ratio_fields(delivery_ratios=[2.0, 3.0], bytes_delivered=[100, 300]) + + assert tuple(fields.keys()) == DELIVERY_RATIO_FIELD_NAMES expected_percentiles = (2.1, 2.25, 2.5, 2.75, 2.9) for column_name, expected_value in zip(PERCENTILE_COLUMN_NAMES, expected_percentiles): - assert row[column_name] == pytest.approx(expected_value) - # Volume-weighted ratio is the summed delivered bytes (100 + 300) over the summed usable sizes (50 + 100) - assert row["delivery_ratio_weighted"] == pytest.approx(400 / 150) + assert fields[column_name] == pytest.approx(expected_value) + # Volume-weighted ratio is summed delivered bytes (100 + 300) over summed sizes (100/2 + 300/3) + assert fields["delivery_ratio_weighted"] == pytest.approx(400 / 150) @pytest.mark.ai_generated -def test_summarize_dandiset_delivery_ratio_single_asset(tmp_path: pathlib.Path) -> None: - extraction_directory = tmp_path / "extraction" - extraction_directory.mkdir() - - blob_directories = [_write_blob_directory(parent=extraction_directory, blob_id="aaa", bytes_sent=[150])] - blob_id_to_size = {"aaa": 100} - - summary_file_path = tmp_path / "summaries" / "000000" / "delivery_ratio.tsv" - _summarize_dandiset_delivery_ratio( - blob_directories=blob_directories, - summary_file_path=summary_file_path, - blob_id_to_size=blob_id_to_size, - ) +def test_compute_delivery_ratio_fields_single_asset() -> None: + fields = _compute_delivery_ratio_fields(delivery_ratios=[1.5], bytes_delivered=[150]) - row = pandas.read_table(filepath_or_buffer=summary_file_path).iloc[0] for column_name in PERCENTILE_COLUMN_NAMES: - assert row[column_name] == pytest.approx(1.5) - assert row["delivery_ratio_weighted"] == pytest.approx(1.5) + assert fields[column_name] == pytest.approx(1.5) + assert fields["delivery_ratio_weighted"] == pytest.approx(1.5) + + +@pytest.mark.ai_generated +def test_compute_delivery_ratio_fields_zero_usable_assets() -> None: + fields = _compute_delivery_ratio_fields(delivery_ratios=[], bytes_delivered=[]) + + for column_name in DELIVERY_RATIO_FIELD_NAMES: + assert math.isnan(fields[column_name]) + + +@pytest.mark.ai_generated +def test_pool_archive_delivery_ratios_skips_archive_directory(tmp_path: pathlib.Path) -> None: + _write_by_asset_tsv(directory=tmp_path / "000001", rows=[("sub-1/a.nwb", 100, 2.0)]) + _write_by_asset_tsv(directory=tmp_path / "000002", rows=[("sub-2/b.nwb", 300, 3.0)]) + # An archive rollup of by_asset.tsv must never be pooled back into the archive computation + _write_by_asset_tsv(directory=tmp_path / "archive", rows=[("sub-x/x.nwb", 999, 9.0)]) + + delivery_ratios, bytes_delivered = _pool_archive_delivery_ratios(tmp_path) + + assert sorted(delivery_ratios) == [2.0, 3.0] + assert sorted(bytes_delivered) == [100, 300] + + +def _write_minimal_dataset_summary(*, summary_directory: pathlib.Path, dataset_id: str, by_asset_rows) -> None: + dataset_directory = summary_directory / dataset_id + dataset_directory.mkdir(parents=True, exist_ok=True) + + pandas.DataFrame( + data={"region": ["US/east"], "bytes_sent": [123], "number_of_requests": [3], "number_of_downloads": [1]} + ).to_csv(path_or_buf=dataset_directory / "by_region.tsv", mode="w", sep="\t", header=True, index=False) + + pandas.DataFrame( + data={"date": ["2024-01-01"], "bytes_sent": [123], "number_of_requests": [3], "number_of_downloads": [1]} + ).to_csv(path_or_buf=dataset_directory / "by_day.tsv", mode="w", sep="\t", header=True, index=False) + + (dataset_directory / "requester_count.tsv").write_text("<50") + + _write_by_asset_tsv(directory=dataset_directory, rows=by_asset_rows) + + +@pytest.mark.ai_generated +def test_generate_dandiset_totals_adds_delivery_ratio_fields(tmp_path: pathlib.Path) -> None: + summary_directory = tmp_path / "summaries" + _write_minimal_dataset_summary( + summary_directory=summary_directory, + dataset_id="000001", + by_asset_rows=[("sub-1/a.nwb", 100, 2.0), ("sub-1/b.nwb", 300, 3.0), ("undetermined", 50, float("nan"))], + ) + # A Dandiset whose only accessed asset has an unresolved size yields no usable asset + _write_minimal_dataset_summary( + summary_directory=summary_directory, + dataset_id="000002", + by_asset_rows=[("sub-2/c.nwb", 10, float("nan"))], + ) + + dandi_s3_log_extraction.summarize.generate_dandiset_totals(cache_directory=tmp_path) + + all_dataset_totals = json.loads((summary_directory / "totals.json").read_text()) + + expected_percentiles = (2.1, 2.25, 2.5, 2.75, 2.9) + for column_name, expected_value in zip(PERCENTILE_COLUMN_NAMES, expected_percentiles): + assert all_dataset_totals["000001"][column_name] == pytest.approx(expected_value) + assert all_dataset_totals["000001"]["delivery_ratio_weighted"] == pytest.approx(400 / 150) + + # Zero usable assets means every delivery ratio field is null + for field_name in DELIVERY_RATIO_FIELD_NAMES: + assert all_dataset_totals["000002"][field_name] is None @pytest.mark.ai_generated -def test_summarize_dandiset_delivery_ratio_zero_usable_assets(tmp_path: pathlib.Path) -> None: - extraction_directory = tmp_path / "extraction" - extraction_directory.mkdir() - - # The asset was accessed but its size cannot be resolved, so the row is all NaN but still has every column - blob_directories = [_write_blob_directory(parent=extraction_directory, blob_id="aaa", bytes_sent=[150])] - blob_id_to_size: dict[str, int | None] = {} - - summary_file_path = tmp_path / "summaries" / "000000" / "delivery_ratio.tsv" - _summarize_dandiset_delivery_ratio( - blob_directories=blob_directories, - summary_file_path=summary_file_path, - blob_id_to_size=blob_id_to_size, +def test_generate_archive_summaries_and_totals_pool_across_dandisets(tmp_path: pathlib.Path) -> None: + summary_directory = tmp_path / "summaries" + _write_minimal_dataset_summary( + summary_directory=summary_directory, + dataset_id="000001", + by_asset_rows=[("sub-1/a.nwb", 100, 2.0), ("sub-1/b.nwb", 300, 3.0), ("undetermined", 50, float("nan"))], + ) + _write_minimal_dataset_summary( + summary_directory=summary_directory, + dataset_id="000002", + by_asset_rows=[("sub-2/c.nwb", 200, 4.0)], ) - summary_table = pandas.read_table(filepath_or_buffer=summary_file_path) - assert list(summary_table.columns) == [*PERCENTILE_COLUMN_NAMES, "delivery_ratio_weighted"] - row = summary_table.iloc[0] - for column_name in [*PERCENTILE_COLUMN_NAMES, "delivery_ratio_weighted"]: - assert math.isnan(row[column_name]) + dandi_s3_log_extraction.summarize.generate_archive_summaries(cache_directory=tmp_path) + dandi_s3_log_extraction.summarize.generate_archive_totals(cache_directory=tmp_path) + + expected_percentiles = (2.2, 2.5, 3.0, 3.5, 3.8) + + archive_delivery_ratio = pandas.read_table(filepath_or_buffer=summary_directory / "archive" / "delivery_ratio.tsv") + assert list(archive_delivery_ratio.columns) == list(DELIVERY_RATIO_FIELD_NAMES) + row = archive_delivery_ratio.iloc[0] + for column_name, expected_value in zip(PERCENTILE_COLUMN_NAMES, expected_percentiles): + assert row[column_name] == pytest.approx(expected_value) + assert row["delivery_ratio_weighted"] == pytest.approx(600 / 200) + + archive_totals = json.loads((summary_directory / "archive_totals.json").read_text()) + for column_name, expected_value in zip(PERCENTILE_COLUMN_NAMES, expected_percentiles): + assert archive_totals[column_name] == pytest.approx(expected_value) + assert archive_totals["delivery_ratio_weighted"] == pytest.approx(600 / 200) From 5b976a8c8f9b6311f16233e3abe24830668ca016 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 17:25:41 +0000 Subject: [PATCH 03/14] Documented the delivery ratio metric in the README Co-Authored-By: Claude Code / Claude Opus 4.8 --- CHANGELOG.md | 4 ++++ README.md | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 380d34b..eb3879b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ - Refactored `generate_dandiset_totals` to derive the summary directory from `cache_directory`. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) - Renamed the `--directory` CLI flag to `--cache` in the update commands. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) +### 📝 Documentation + +- Documented the experimental delivery ratio metric in the `README.md`. ([#86](https://github.com/dandi/dandi-s3-log-extraction/pull/86)) + ### 🔩 Dependency Updates - Updated compatibility for the latest `s3-log-extraction` release by pinning the lower bound to `>=1.9.2` and adapting extractor tests and summary columns. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) diff --git a/README.md b/README.md index 8d0c9c6..7ef2e7a 100644 --- a/README.md +++ b/README.md @@ -71,3 +71,21 @@ dandis3logextraction update totals dandis3logextraction update summaries --mode archive dandis3logextraction update totals --mode archive ``` + + + +## Delivery ratio (experimental) + +The delivery ratio is an experimental, DANDI-only signal of streaming versus download intensity. + +For each asset it is the total bytes delivered across all logged GET requests divided by the asset's true size in bytes. The true size comes from the DANDI API rather than the logs, and sizes are cached locally in `asset_sizes.json` so they are fetched only once. A ratio near 1 means access is download dominated. A ratio much greater than 1 means access is streaming dominated, since the same bytes are served many times. + +The metric appears in three places: + +- A per-asset `delivery_ratio` column in each Dandiset's `by_asset.tsv`. +- Per-Dandiset percentiles in `totals.json`, and archive wide percentiles in `archive_totals.json` and `archive/delivery_ratio.tsv`. The reported fields are `delivery_ratio_p10`, `delivery_ratio_p25`, `delivery_ratio_p50`, `delivery_ratio_p75`, `delivery_ratio_p90`, and `delivery_ratio_weighted`. + +The percentiles are asset weighted, where each asset contributes one ratio. The `delivery_ratio_weighted` field is volume weighted, computed as the total bytes delivered over the total asset size. The gap between the weighted value and the median is a deliberate heterogeneity signal, so both are reported. Assets with a missing or zero size are excluded from the computation. A Dandiset with no usable asset reports empty values for all six fields. + +These fields are produced by `dandis3logextraction update totals` and `dandis3logextraction update summaries --mode archive`, which wrap the generic totals and summary steps. + From 031e6926b218d86a8ed7d079bca9a7c3685e6640 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:25:54 +0000 Subject: [PATCH 04/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7ef2e7a..e24e6e0 100644 --- a/README.md +++ b/README.md @@ -88,4 +88,3 @@ The metric appears in three places: The percentiles are asset weighted, where each asset contributes one ratio. The `delivery_ratio_weighted` field is volume weighted, computed as the total bytes delivered over the total asset size. The gap between the weighted value and the median is a deliberate heterogeneity signal, so both are reported. Assets with a missing or zero size are excluded from the computation. A Dandiset with no usable asset reports empty values for all six fields. These fields are produced by `dandis3logextraction update totals` and `dandis3logextraction update summaries --mode archive`, which wrap the generic totals and summary steps. - From f3bbdf0e72deca73d2ef1b5d01e2e565d7fab23c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 17:31:48 +0000 Subject: [PATCH 05/14] Always do live asset size lookups and clarify ratio wording Removed the asset_sizes.json on-disk cache so asset sizes are looked up live from the DANDI API on every run; a persistent cache can be added later. Also reworded the README delivery ratio description to avoid implying the same byte ranges are re-served, since the logs do not distinguish ranges. Co-Authored-By: Claude Code / Claude Opus 4.8 --- CHANGELOG.md | 2 +- README.md | 2 +- .../summarize/_generate_dandiset_summaries.py | 51 +++++-------------- 3 files changed, 15 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb3879b..02a9e34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### 🚀 Enhancement -- Added an experimental, DANDI-only delivery ratio feature. The delivery ratio is the total bytes delivered over the asset's true DANDI size, signaling streaming versus download intensity. A per-asset `delivery_ratio` column is added to `by_asset.tsv`. The asset-weighted percentiles `delivery_ratio_p10`, `delivery_ratio_p25`, `delivery_ratio_p50`, `delivery_ratio_p75`, `delivery_ratio_p90` plus the volume-weighted `delivery_ratio_weighted` are injected into per-Dandiset `totals.json`, into archive `archive_totals.json`, and into a new archive `delivery_ratio.tsv`, via DANDI wrappers around the upstream summary and totals methods. Asset sizes are fetched from the DANDI API and cached locally in `asset_sizes.json`. ([#86](https://github.com/dandi/dandi-s3-log-extraction/pull/86)) +- Added an experimental, DANDI-only delivery ratio feature. The delivery ratio is the total bytes delivered over the asset's true DANDI size, signaling streaming versus download intensity. A per-asset `delivery_ratio` column is added to `by_asset.tsv`. The asset-weighted percentiles `delivery_ratio_p10`, `delivery_ratio_p25`, `delivery_ratio_p50`, `delivery_ratio_p75`, `delivery_ratio_p90` plus the volume-weighted `delivery_ratio_weighted` are injected into per-Dandiset `totals.json`, into archive `archive_totals.json`, and into a new archive `delivery_ratio.tsv`, via DANDI wrappers around the upstream summary and totals methods. Asset sizes are fetched live from the DANDI API on each run. ([#86](https://github.com/dandi/dandi-s3-log-extraction/pull/86)) - Added the `dandis3logextraction update totals` command, with `--mode archive` for archive-wide totals, wrapping the upstream totals generation to also emit the delivery ratio fields. ([#86](https://github.com/dandi/dandi-s3-log-extraction/pull/86)) - Added `download` to `_dandi_extraction.awk` so extraction writes `download.txt` alongside the other per-request outputs. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) - Added `--cache-directory` to `dandis3logextraction extract` so remote extraction can use a custom cache directory. ([#68](https://github.com/dandi/dandi-s3-log-extraction/pull/68)) diff --git a/README.md b/README.md index e24e6e0..ed77d91 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ dandis3logextraction update totals --mode archive The delivery ratio is an experimental, DANDI-only signal of streaming versus download intensity. -For each asset it is the total bytes delivered across all logged GET requests divided by the asset's true size in bytes. The true size comes from the DANDI API rather than the logs, and sizes are cached locally in `asset_sizes.json` so they are fetched only once. A ratio near 1 means access is download dominated. A ratio much greater than 1 means access is streaming dominated, since the same bytes are served many times. +For each asset it is the total bytes delivered across all logged GET requests divided by the asset's true size in bytes. The true size comes from the DANDI API rather than the logs, and is fetched with a live lookup on each run. A ratio near 1 means access is download dominated. A ratio much greater than 1 means access is streaming dominated, since the asset's size worth of bytes has been delivered many times over across requests. The metric appears in three places: diff --git a/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py b/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py index a7747c3..ba52c48 100644 --- a/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py +++ b/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py @@ -124,12 +124,11 @@ def generate_dandiset_summaries( else: dandiset_ids_to_summarize = [dandiset.identifier for dandiset in client.get_dandisets()] - # Resolve the true (DANDI metadata) byte size of each accessed asset, reusing a local cache across runs + # Resolve the true (DANDI metadata) byte size of each accessed asset via live API lookups blob_id_to_size = _load_asset_sizes( client=client, dandiset_ids=dandiset_ids_to_summarize, dandiset_id_to_local_content_directories=dandiset_id_to_local_content_directories, - summary_directory=summary_directory, ) if max_workers == 1: @@ -282,34 +281,29 @@ def _load_asset_sizes( client, dandiset_ids: list[str], dandiset_id_to_local_content_directories: dict[str, list[pathlib.Path]], - summary_directory: pathlib.Path, ) -> dict[str, int | None]: """ - Resolve the true DANDI metadata size (in bytes) of every accessed asset. + Resolve the true DANDI metadata size (in bytes) of every accessed asset via live API lookups. - Sizes are immutable per content (blob or Zarr) ID, so they are cached on disk keyed by content ID and reused - across runs to avoid slow per-asset API calls at archive scale. Each content ID is resolved at most once. A - content ID that the API cannot resolve is cached as ``None`` so it is not repeatedly re-fetched on later runs. + Each Dandiset's assets are listed once per run to build a content ID to size mapping. A content ID that the API + cannot resolve is recorded as ``None``. Sizes are not persisted between runs (a local cache may be added later), + so this performs live lookups on every invocation. Parameters ---------- client : dandi.dandiapi.DandiAPIClient - An open client used to list a Dandiset's assets when sizes are missing from the cache. + An open client used to list each Dandiset's assets. dandiset_ids : list of str The Dandiset IDs that will be summarized. dandiset_id_to_local_content_directories : dict Mapping of Dandiset ID to the local per-content extraction directories (named by content ID). - summary_directory : pathlib.Path - Directory in which the persistent ``asset_sizes.json`` cache is stored. Returns ------- dict Mapping of content ID to size in bytes, or ``None`` when the size could not be resolved. """ - content_id_to_size = _load_asset_size_cache(summary_directory) - - cache_is_dirty = False + content_id_to_size: dict[str, int | None] = {} for dandiset_id in tqdm.tqdm( iterable=dandiset_ids, total=len(dandiset_ids), @@ -319,45 +313,26 @@ def _load_asset_sizes( mininterval=5.0, ): blob_directories = dandiset_id_to_local_content_directories.get(dandiset_id, []) - requested_content_ids = [blob_directory.name for blob_directory in blob_directories] - missing_content_ids = [ - content_id for content_id in requested_content_ids if content_id not in content_id_to_size - ] - if len(missing_content_ids) == 0: + if len(blob_directories) == 0: continue fetched_content_id_to_size = _fetch_dandiset_asset_sizes(client=client, dandiset_id=dandiset_id) if fetched_content_id_to_size is None: - continue # Listing failed (for example, a transient network error); retry on a later run + continue # Listing failed (for example, a transient network error) - for content_id in missing_content_ids: + for blob_directory in blob_directories: + content_id = blob_directory.name content_id_to_size[content_id] = fetched_content_id_to_size.get(content_id) - cache_is_dirty = True - - if cache_is_dirty: - _save_asset_size_cache(summary_directory=summary_directory, content_id_to_size=content_id_to_size) return content_id_to_size -def _load_asset_size_cache(summary_directory: pathlib.Path, /) -> dict[str, int | None]: - cache_file_path = summary_directory / "asset_sizes.json" - if not cache_file_path.exists(): - return {} - return json.loads(cache_file_path.read_text()) - - -def _save_asset_size_cache(*, summary_directory: pathlib.Path, content_id_to_size: dict[str, int | None]) -> None: - cache_file_path = summary_directory / "asset_sizes.json" - cache_file_path.write_text(json.dumps(content_id_to_size)) - - def _fetch_dandiset_asset_sizes(*, client, dandiset_id: str) -> dict[str, int] | None: """ List a single Dandiset's assets and return a mapping of content ID to size in bytes. - Returns ``None`` if the assets could not be listed at all so that the caller can retry on a later run rather - than caching the content IDs as permanently unresolvable. + Returns ``None`` if the assets could not be listed at all so that the caller can treat every requested content + ID in that Dandiset as unresolved. """ content_id_to_size: dict[str, int] = {} try: From c0ee072c413d031f250370906da83fc94500464a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 17:32:36 +0000 Subject: [PATCH 06/14] Explained why delivery ratio is reported as percentiles Added a note to the README clarifying that per-asset delivery ratios are highly skewed, so the Dandiset and archive levels report distribution percentiles rather than a single average, while the per-asset values remain in by_asset.tsv. Co-Authored-By: Claude Code / Claude Opus 4.8 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ed77d91..09ab642 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ The metric appears in three places: - A per-asset `delivery_ratio` column in each Dandiset's `by_asset.tsv`. - Per-Dandiset percentiles in `totals.json`, and archive wide percentiles in `archive_totals.json` and `archive/delivery_ratio.tsv`. The reported fields are `delivery_ratio_p10`, `delivery_ratio_p25`, `delivery_ratio_p50`, `delivery_ratio_p75`, `delivery_ratio_p90`, and `delivery_ratio_weighted`. +Percentiles are reported at the Dandiset and archive level, rather than a single average, because per-asset delivery ratios are highly skewed. Within one Dandiset some assets are downloaded close to once while others are streamed many times over, so a mean would be dominated by a few heavily streamed assets and hide that spread. The five percentiles describe the shape of the distribution compactly, and the exact per-asset values are still available in `by_asset.tsv` for anyone who needs them. + The percentiles are asset weighted, where each asset contributes one ratio. The `delivery_ratio_weighted` field is volume weighted, computed as the total bytes delivered over the total asset size. The gap between the weighted value and the median is a deliberate heterogeneity signal, so both are reported. Assets with a missing or zero size are excluded from the computation. A Dandiset with no usable asset reports empty values for all six fields. These fields are produced by `dandis3logextraction update totals` and `dandis3logextraction update summaries --mode archive`, which wrap the generic totals and summary steps. From d3b14378264e6e2e0510f4b5ca799fb31a66b44b Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:33:30 -0400 Subject: [PATCH 07/14] Apply suggestion from @CodyCBakerPhD --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 09ab642..ae8d149 100644 --- a/README.md +++ b/README.md @@ -89,4 +89,3 @@ Percentiles are reported at the Dandiset and archive level, rather than a single The percentiles are asset weighted, where each asset contributes one ratio. The `delivery_ratio_weighted` field is volume weighted, computed as the total bytes delivered over the total asset size. The gap between the weighted value and the median is a deliberate heterogeneity signal, so both are reported. Assets with a missing or zero size are excluded from the computation. A Dandiset with no usable asset reports empty values for all six fields. -These fields are produced by `dandis3logextraction update totals` and `dandis3logextraction update summaries --mode archive`, which wrap the generic totals and summary steps. From 5a13b591845f343d1af95c9ff15ad817158d412a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 17:33:38 +0000 Subject: [PATCH 08/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index ae8d149..1e61e48 100644 --- a/README.md +++ b/README.md @@ -88,4 +88,3 @@ The metric appears in three places: Percentiles are reported at the Dandiset and archive level, rather than a single average, because per-asset delivery ratios are highly skewed. Within one Dandiset some assets are downloaded close to once while others are streamed many times over, so a mean would be dominated by a few heavily streamed assets and hide that spread. The five percentiles describe the shape of the distribution compactly, and the exact per-asset values are still available in `by_asset.tsv` for anyone who needs them. The percentiles are asset weighted, where each asset contributes one ratio. The `delivery_ratio_weighted` field is volume weighted, computed as the total bytes delivered over the total asset size. The gap between the weighted value and the median is a deliberate heterogeneity signal, so both are reported. Assets with a missing or zero size are excluded from the computation. A Dandiset with no usable asset reports empty values for all six fields. - From 4061a99a13af921d9e51006327f1d3c794818061 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:34:10 -0400 Subject: [PATCH 09/14] Apply suggestion from @CodyCBakerPhD --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e61e48..95c671c 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ dandis3logextraction update totals --mode archive The delivery ratio is an experimental, DANDI-only signal of streaming versus download intensity. -For each asset it is the total bytes delivered across all logged GET requests divided by the asset's true size in bytes. The true size comes from the DANDI API rather than the logs, and is fetched with a live lookup on each run. A ratio near 1 means access is download dominated. A ratio much greater than 1 means access is streaming dominated, since the asset's size worth of bytes has been delivered many times over across requests. +For each asset it is the total bytes delivered across all logged GET requests divided by the asset's true size in bytes. A ratio near 1 means access is download dominated. A ratio much greater than 1 means access is streaming dominated, since the asset's size worth of bytes has been delivered many times over across requests. The metric appears in three places: From a14a5df64c7015b77ca61277776f45d7cf295f49 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 17:35:27 +0000 Subject: [PATCH 10/14] Added LaTeX equations for the delivery ratio to the README Co-Authored-By: Claude Code / Claude Opus 4.8 --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 95c671c..792b5fe 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,13 @@ dandis3logextraction update totals --mode archive The delivery ratio is an experimental, DANDI-only signal of streaming versus download intensity. -For each asset it is the total bytes delivered across all logged GET requests divided by the asset's true size in bytes. A ratio near 1 means access is download dominated. A ratio much greater than 1 means access is streaming dominated, since the asset's size worth of bytes has been delivered many times over across requests. +For each asset it is the total bytes delivered across all logged GET requests divided by the asset's true size in bytes: + +$$r_a = \frac{\sum_{i \in G_a} b_i}{s_a}$$ + +where $G_a$ is the set of logged GET requests for asset $a$, $b_i$ is the bytes delivered in request $i$, and $s_a$ is the asset's true size in bytes. + +A ratio near 1 means access is download dominated. A ratio much greater than 1 means access is streaming dominated, since the asset's size worth of bytes has been delivered many times over across requests. The metric appears in three places: @@ -87,4 +93,8 @@ The metric appears in three places: Percentiles are reported at the Dandiset and archive level, rather than a single average, because per-asset delivery ratios are highly skewed. Within one Dandiset some assets are downloaded close to once while others are streamed many times over, so a mean would be dominated by a few heavily streamed assets and hide that spread. The five percentiles describe the shape of the distribution compactly, and the exact per-asset values are still available in `by_asset.tsv` for anyone who needs them. -The percentiles are asset weighted, where each asset contributes one ratio. The `delivery_ratio_weighted` field is volume weighted, computed as the total bytes delivered over the total asset size. The gap between the weighted value and the median is a deliberate heterogeneity signal, so both are reported. Assets with a missing or zero size are excluded from the computation. A Dandiset with no usable asset reports empty values for all six fields. +The percentiles are asset weighted, where each asset contributes one ratio $r_a$ regardless of its size. The `delivery_ratio_weighted` field is instead volume weighted, computed as the total bytes delivered over the total asset size across the usable assets $A$: + +$$r_{\text{vol}} = \frac{\sum_{a \in A} \sum_{i \in G_a} b_i}{\sum_{a \in A} s_a}$$ + +The gap between this volume-weighted value $r_{\text{vol}}$ and the median $p_{50}$ is a deliberate heterogeneity signal, so both are reported. Assets with a missing or zero size are excluded from the computation. A Dandiset with no usable asset reports empty values for all six fields. From 1fea4fbd9149bb488816ccc0f9e8d062348fc42e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 17:42:38 +0000 Subject: [PATCH 11/14] Wrote archive delivery ratio percentiles as one comma-separated TSV column The archive delivery_ratio.tsv now reports the five percentiles in a single "delivery_ratio(p10,p25,p50,p75,p90)" column with a comma-separated value rather than five separate columns, alongside the delivery_ratio_weighted scalar. The JSON totals keep individual keys. Co-Authored-By: Claude Code / Claude Opus 4.8 --- .../summarize/_generate_dandiset_summaries.py | 16 ++++++++++- tests/test_delivery_ratio.py | 28 +++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py b/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py index ba52c48..cde8ed9 100644 --- a/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py +++ b/src/dandi_s3_log_extraction/summarize/_generate_dandiset_summaries.py @@ -722,6 +722,17 @@ def _compute_delivery_ratio_percentiles(delivery_ratios: list[float], /) -> dict "delivery_ratio_weighted", ) +# Single TSV column holding the five percentiles as a comma-separated tuple (for example "0.1,0.2,0.3,0.4,0.5") +DELIVERY_RATIO_PERCENTILE_COLUMN = "delivery_ratio(" + ",".join(f"p{p}" for p in DELIVERY_RATIO_PERCENTILES) + ")" + + +def _format_delivery_ratio_percentiles(fields: dict[str, float], /) -> str: + """Format the five percentile fields as one comma-separated string, using an empty slot for ``NaN``.""" + return ",".join( + "" if math.isnan(value := fields[f"delivery_ratio_p{percentile}"]) else str(value) + for percentile in DELIVERY_RATIO_PERCENTILES + ) + def _read_by_asset_delivery_ratios(by_asset_file_path: pathlib.Path, /) -> tuple[list[float], list[int]]: """ @@ -1083,7 +1094,10 @@ def generate_archive_summaries( archive_directory = summary_directory / "archive" archive_directory.mkdir(exist_ok=True) summary_table = pandas.DataFrame( - data={field_name: [fields[field_name]] for field_name in DELIVERY_RATIO_FIELD_NAMES} + data={ + DELIVERY_RATIO_PERCENTILE_COLUMN: [_format_delivery_ratio_percentiles(fields)], + "delivery_ratio_weighted": [fields["delivery_ratio_weighted"]], + } ) summary_table.to_csv( path_or_buf=archive_directory / "delivery_ratio.tsv", mode="w", sep="\t", header=True, index=False diff --git a/tests/test_delivery_ratio.py b/tests/test_delivery_ratio.py index 787a42f..176e857 100644 --- a/tests/test_delivery_ratio.py +++ b/tests/test_delivery_ratio.py @@ -7,8 +7,10 @@ import dandi_s3_log_extraction from dandi_s3_log_extraction.summarize._generate_dandiset_summaries import ( + DELIVERY_RATIO_PERCENTILE_COLUMN, _compute_delivery_ratio_fields, _compute_delivery_ratio_percentiles, + _format_delivery_ratio_percentiles, _pool_archive_delivery_ratios, _read_by_asset_delivery_ratios, ) @@ -123,6 +125,26 @@ def test_compute_delivery_ratio_fields_zero_usable_assets() -> None: assert math.isnan(fields[column_name]) +@pytest.mark.ai_generated +def test_delivery_ratio_percentile_column_name() -> None: + assert DELIVERY_RATIO_PERCENTILE_COLUMN == "delivery_ratio(p10,p25,p50,p75,p90)" + + +@pytest.mark.ai_generated +def test_format_delivery_ratio_percentiles() -> None: + fields = _compute_delivery_ratio_fields(delivery_ratios=[2.0, 3.0], bytes_delivered=[100, 300]) + + assert _format_delivery_ratio_percentiles(fields) == "2.1,2.25,2.5,2.75,2.9" + + +@pytest.mark.ai_generated +def test_format_delivery_ratio_percentiles_zero_usable_assets() -> None: + fields = _compute_delivery_ratio_fields(delivery_ratios=[], bytes_delivered=[]) + + # Every percentile is empty so the comma-separated tuple keeps its five slots + assert _format_delivery_ratio_percentiles(fields) == ",,,," + + @pytest.mark.ai_generated def test_pool_archive_delivery_ratios_skips_archive_directory(tmp_path: pathlib.Path) -> None: _write_by_asset_tsv(directory=tmp_path / "000001", rows=[("sub-1/a.nwb", 100, 2.0)]) @@ -202,10 +224,10 @@ def test_generate_archive_summaries_and_totals_pool_across_dandisets(tmp_path: p expected_percentiles = (2.2, 2.5, 3.0, 3.5, 3.8) archive_delivery_ratio = pandas.read_table(filepath_or_buffer=summary_directory / "archive" / "delivery_ratio.tsv") - assert list(archive_delivery_ratio.columns) == list(DELIVERY_RATIO_FIELD_NAMES) + # The TSV reports the five percentiles as one comma-separated column plus the weighted scalar + assert list(archive_delivery_ratio.columns) == [DELIVERY_RATIO_PERCENTILE_COLUMN, "delivery_ratio_weighted"] row = archive_delivery_ratio.iloc[0] - for column_name, expected_value in zip(PERCENTILE_COLUMN_NAMES, expected_percentiles): - assert row[column_name] == pytest.approx(expected_value) + assert str(row[DELIVERY_RATIO_PERCENTILE_COLUMN]) == ",".join(str(value) for value in expected_percentiles) assert row["delivery_ratio_weighted"] == pytest.approx(600 / 200) archive_totals = json.loads((summary_directory / "archive_totals.json").read_text()) From 40a5feaa7f7f7dc18f6279b42b857d0309b55417 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 17:42:38 +0000 Subject: [PATCH 12/14] Improved delivery ratio equation rendering in the README Pulled 1/s_a out of the per-asset equation and forced the summation indices below the sigma with \limits so they render aligned. Co-Authored-By: Claude Code / Claude Opus 4.8 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 792b5fe..d68a8b5 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ The delivery ratio is an experimental, DANDI-only signal of streaming versus dow For each asset it is the total bytes delivered across all logged GET requests divided by the asset's true size in bytes: -$$r_a = \frac{\sum_{i \in G_a} b_i}{s_a}$$ +$$r_a = \frac{1}{s_a} \sum\limits_{i \in G_a} b_i$$ where $G_a$ is the set of logged GET requests for asset $a$, $b_i$ is the bytes delivered in request $i$, and $s_a$ is the asset's true size in bytes. @@ -95,6 +95,6 @@ Percentiles are reported at the Dandiset and archive level, rather than a single The percentiles are asset weighted, where each asset contributes one ratio $r_a$ regardless of its size. The `delivery_ratio_weighted` field is instead volume weighted, computed as the total bytes delivered over the total asset size across the usable assets $A$: -$$r_{\text{vol}} = \frac{\sum_{a \in A} \sum_{i \in G_a} b_i}{\sum_{a \in A} s_a}$$ +$$r_{\text{vol}} = \frac{\sum\limits_{a \in A} \sum\limits_{i \in G_a} b_i}{\sum\limits_{a \in A} s_a}$$ The gap between this volume-weighted value $r_{\text{vol}}$ and the median $p_{50}$ is a deliberate heterogeneity signal, so both are reported. Assets with a missing or zero size are excluded from the computation. A Dandiset with no usable asset reports empty values for all six fields. From cf2c0a0a476f4ab062869c9166d792d973040389 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:50:15 -0400 Subject: [PATCH 13/14] Apply suggestion from @CodyCBakerPhD --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d68a8b5..a5cd172 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,10 @@ For each asset it is the total bytes delivered across all logged GET requests di $$r_a = \frac{1}{s_a} \sum\limits_{i \in G_a} b_i$$ -where $G_a$ is the set of logged GET requests for asset $a$, $b_i$ is the bytes delivered in request $i$, and $s_a$ is the asset's true size in bytes. +where: +- $G_a$ is the set of logged GET requests for asset $a$ +- $b_i$ is the bytes delivered in request $i$ +- $s_a$ is the asset's true size in bytes. A ratio near 1 means access is download dominated. A ratio much greater than 1 means access is streaming dominated, since the asset's size worth of bytes has been delivered many times over across requests. From 6233473c7476c3cd5d350aa8271935b198152bb9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 22 Jun 2026 17:51:40 +0000 Subject: [PATCH 14/14] Used n as the delivery ratio request index variable Co-Authored-By: Claude Code / Claude Opus 4.8 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a5cd172..4b6ce9b 100644 --- a/README.md +++ b/README.md @@ -80,11 +80,11 @@ The delivery ratio is an experimental, DANDI-only signal of streaming versus dow For each asset it is the total bytes delivered across all logged GET requests divided by the asset's true size in bytes: -$$r_a = \frac{1}{s_a} \sum\limits_{i \in G_a} b_i$$ +$$r_a = \frac{1}{s_a} \sum\limits_{n \in G_a} b_n$$ where: - $G_a$ is the set of logged GET requests for asset $a$ -- $b_i$ is the bytes delivered in request $i$ +- $b_n$ is the bytes delivered in request $n$ - $s_a$ is the asset's true size in bytes. A ratio near 1 means access is download dominated. A ratio much greater than 1 means access is streaming dominated, since the asset's size worth of bytes has been delivered many times over across requests. @@ -98,6 +98,6 @@ Percentiles are reported at the Dandiset and archive level, rather than a single The percentiles are asset weighted, where each asset contributes one ratio $r_a$ regardless of its size. The `delivery_ratio_weighted` field is instead volume weighted, computed as the total bytes delivered over the total asset size across the usable assets $A$: -$$r_{\text{vol}} = \frac{\sum\limits_{a \in A} \sum\limits_{i \in G_a} b_i}{\sum\limits_{a \in A} s_a}$$ +$$r_{\text{vol}} = \frac{\sum\limits_{a \in A} \sum\limits_{n \in G_a} b_n}{\sum\limits_{a \in A} s_a}$$ The gap between this volume-weighted value $r_{\text{vol}}$ and the median $p_{50}$ is a deliberate heterogeneity signal, so both are reported. Assets with a missing or zero size are excluded from the computation. A Dandiset with no usable asset reports empty values for all six fields.