From ad3103bd4565f794fce66d19fcd7e31be94ddb33 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 16:52:33 +0000 Subject: [PATCH 1/2] ?_stream=1 fetches pages of max_returned_rows Streaming every row previously paginated using the default page size of 100, so a large export ran ten times more queries than it needed. It now uses the largest page size unless ?_size= is in the URL. Microbenchmark: /db/table.csv?_stream=1 for a 200,000 row, four column table (7MB of CSV), median of five runs against a local server: 100 row pages (before): 5.21s 1,000 row pages (after): 1.09s Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_016Bvfq9SBhxPegj4hvi5NF4 --- datasette/views/table.py | 13 ++++++++++++- docs/changelog.rst | 1 + docs/plugin_hooks.rst | 2 +- tests/test_renderer_streaming.py | 25 +++++++++++++++++++++++-- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/datasette/views/table.py b/datasette/views/table.py index 8fba1a05f7..7183ee7f23 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1953,6 +1953,11 @@ async def table_view_traced(datasette, request): context_for_html_hack = True default_labels = True + # ?_stream=1 for a format that can stream fetches the largest pages + stream = bool( + request.args.get("_stream") and datasette._renderer_can_stream.get(format_) + ) + start = time.perf_counter() view_data = await table_view_data( datasette, @@ -1962,6 +1967,7 @@ async def table_view_traced(datasette, request): context_for_html_hack=context_for_html_hack, default_labels=default_labels, renderer_extras=datasette._renderer_extras.get(format_), + stream=stream, ) if isinstance(view_data, Response): return view_data @@ -1984,6 +1990,7 @@ async def fetch_next_page(): default_labels=default_labels, _next=next_token, rows_only=True, + stream=stream, ) next_token = page_data["next"] return page_rows @@ -2340,6 +2347,7 @@ async def table_view_data( _next=None, renderer_extras=None, rows_only=False, + stream=False, ): extra_extras = extra_extras or set() # We have a table or view @@ -2469,7 +2477,10 @@ async def table_view_data( # Handle ?_size=500 # TODO: This was: # page_size = _size or request.args.get("_size") or table_metadata.get("size") - page_size = request.args.get("_size") or table_metadata.get("size") + page_size = request.args.get("_size") + if not page_size: + # Streaming every row is fastest using the largest possible pages + page_size = "max" if stream else table_metadata.get("size") if page_size: if page_size == "max": page_size = datasette.max_returned_rows diff --git a/docs/changelog.rst b/docs/changelog.rst index 2f13da3ae3..f31e294d29 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -34,6 +34,7 @@ Unreleased - The following ``datasette.utils`` functions and classes are now :ref:`documented ` and considered stable for use by plugins: ``escape_sqlite()``, ``escape_fts()``, ``to_css_class()``, ``modify_query_string()``, ``actor_matches_allow()``, ``add_cors_headers()``, ``StartupError``, ``BadMetadataError``, ``MultiParams``, ``PrefixedUrlString``, ``PaginatedResources`` and the ``sqlite3`` module. (:issue:`1630`) - **Backwards incompatible:** the ``format_bytes()`` function is no longer available in templates, and ``datasette.utils.format_bytes()`` has been removed. Custom templates that call ``{{ format_bytes(size) }}`` should use Jinja's built-in `filesizeformat `__ filter instead: ``{{ size|filesizeformat }}``. (:issue:`1841`) - File sizes shown on the database page and in the tooltips for binary values now use decimal units, where 1 MB is 1,000,000 bytes, matching the sizes already shown on the homepage. They previously used 1 MB to mean 1,048,576 bytes, so a 5,242,880 byte database that was displayed as ``5.0 MB`` is now displayed as ``5.2 MB``. +- ``?_stream=1`` now fetches rows in pages of :ref:`setting_max_returned_rows`, 1,000 by default, instead of the default page size of 100. This makes streaming every row of a large table several times faster. A ``?_size=`` in the URL is still respected. - New ``datasette.urls.modify(target, add=, replace=, remove=, format=)`` method for building links that change the query string of the current page or of any other path. It applies ``base_url`` for you and is available in templates as ``urls.modify()``. See :ref:`internals_datasette_urls_modify`. (:issue:`1342`) - Fixed a bug where sort, facet and filter-removal links on the table page dropped query string arguments with blank values, so sorting ``/db/table?name=`` lost the filter for rows where ``name`` is the empty string. diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index b9ae434fa4..0667f5ee70 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -769,7 +769,7 @@ A table can have far more rows than fit on one page. A ``"render"`` function tha ``row_stream`` yields the rows on the current page. If the URL includes ``?_stream=1`` it carries on past the end of that page, fetching one page at a time, until it has yielded every row in the table or view. Your function does not need to know which of those is happening. -Use ``async for rows in row_stream.pages()`` to receive a list of rows for each page instead of one row at a time. +Use ``async for rows in row_stream.pages()`` to receive a list of rows for each page instead of one row at a time. When streaming every row those pages are :ref:`setting_max_returned_rows` rows each, unless the URL sets a different ``?_size=``. Datasette's own :ref:`CSV and TSV export ` is implemented this way. Some things to be aware of: diff --git a/tests/test_renderer_streaming.py b/tests/test_renderer_streaming.py index 7499f6b822..bde23754d5 100644 --- a/tests/test_renderer_streaming.py +++ b/tests/test_renderer_streaming.py @@ -133,10 +133,14 @@ async def test_row_stream_respects_filters_and_sort(ds): async def test_row_stream_pages(ds): + # Streaming fetches max_returned_rows at a time, not default_page_size response = await ds.client.get("/data/numbers.pagesizes?_stream=1") - assert response.text.split() == ["10", "10", "10", "5"] + assert response.text.split() == ["35"] + # ... unless ?_size= asks for something else response = await ds.client.get("/data/numbers.pagesizes?_stream=1&_size=20") assert response.text.split() == ["20", "15"] + response = await ds.client.get("/data/numbers.pagesizes?_stream=1&_size=10") + assert response.text.split() == ["10", "10", "10", "5"] response = await ds.client.get("/data/numbers.pagesizes") assert response.text.split() == ["10"] @@ -229,7 +233,7 @@ async def broken_on_third_page(*args, **kwargs): monkeypatch.setattr(table, "table_view_data", broken_on_third_page) with pytest.raises(ValueError, match="database went away"): - await ds.client.get("/data/numbers.csv?_stream=1") + await ds.client.get("/data/numbers.csv?_stream=1&_size=10") assert calls == [None, "10", "20"] @@ -297,3 +301,20 @@ async def test_expanded_columns_extra(ds_client): await ds_client.get("/fixtures/facetable.json?_extra=expanded_columns&_size=1") ).json() assert data["expanded_columns"] == [] + + +@pytest.mark.asyncio +async def test_row_stream_pages_are_max_returned_rows(plugin): + ds = _make_datasette(max_returned_rows=15) + await _add_numbers(ds) + try: + response = await ds.client.get("/data/numbers.pagesizes?_stream=1") + assert response.text.split() == ["15", "15", "5"] + # A size set in the table configuration does not apply to streaming + ds.config = {"databases": {"data": {"tables": {"numbers": {"size": 5}}}}} + response = await ds.client.get("/data/numbers.pagesizes") + assert response.text.split() == ["5"] + response = await ds.client.get("/data/numbers.pagesizes?_stream=1") + assert response.text.split() == ["15", "15", "5"] + finally: + ds.close() From defca3282f66a58ed2d0bb9f4aec647c02b47632 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 16:57:54 +0000 Subject: [PATCH 2/2] Export dialog groups formats that can stream with TSV and CSV Plugin output formats that accept row_stream now share the form used by TSV and CSV, so the "Include all rows" and "Expand labels" options apply to them. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_016Bvfq9SBhxPegj4hvi5NF4 --- datasette/app.py | 2 + datasette/static/export-dialog.js | 20 ++++--- datasette/templates/_export_dialog.html | 47 ++++++++++----- docs/changelog.rst | 1 + docs/csv_export.rst | 4 +- docs/plugin_hooks.rst | 2 + tests/test_renderer_streaming.py | 78 +++++++++++++++++++++++++ 7 files changed, 131 insertions(+), 23 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index ad91b8f164..15e7a026db 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2668,6 +2668,8 @@ def _register_renderers(self): "description": renderer.get("description"), "is_binary": is_binary, "is_webpage": is_webpage, + # Formats that can stream are grouped with "Include all rows" + "can_stream": self._renderer_can_stream[extension], } async def render_template( diff --git a/datasette/static/export-dialog.js b/datasette/static/export-dialog.js index 5a880ece9e..7655e6e6d6 100644 --- a/datasette/static/export-dialog.js +++ b/datasette/static/export-dialog.js @@ -71,8 +71,9 @@ errorFor(el)?.remove(); } - // TSV and CSV build their URLs from the form they share, so its checkboxes - // apply. TSV's buttons send the form to its own path using formaction. + // TSV, CSV and plugin formats that can stream build their URLs from the + // form they share, so its checkboxes apply. All but CSV send the form to + // their own path using formaction. function exportUrl(el) { const row = el.closest(".export-option"); const form = el.closest(".export-csv-form"); @@ -82,8 +83,11 @@ const link = row.querySelector("a.export-action[href]"); return new URL(link.getAttribute("href"), location.href).href; } - const view = row.querySelector('[data-export-action="view"]'); - const path = view.getAttribute("formaction") || form.getAttribute("action"); + // Binary formats have no View button, so look at Download as well + const submit = row.querySelector("[formaction]"); + const path = submit + ? submit.getAttribute("formaction") + : form.getAttribute("action"); const url = new URL(path, location.href); // Row pages have no fields to send, so no "?" either url.search = new URLSearchParams(new FormData(form)); @@ -164,8 +168,8 @@ const rowsText = (n) => `${formatNumber(n)} row${n === 1 ? "" : "s"}`; // How many rows an export returns: JSON and plugin formats return the rows - // on this page, TSV and CSV ("csv" here) ask for _size=max starting from - // this page, and "Include all rows" streams every row from the first one. + // on this page, the formats that can stream - TSV, CSV and some plugin + // formats, "csv" here - ask for _size=max starting from this page, and "Include all rows" streams every row from the first one. // Mirrors the rows_label macro in _export_dialog.html. function rowsLabel(kind, rows, stream) { const { page, more, later, max, total } = rows; @@ -222,7 +226,7 @@ rows.total !== null && rows.total <= rows.max ) { - // TSV and CSV already include every row, so the option would do nothing + // Those formats already include every row, so the option would do nothing stream.checked = false; stream.disabled = true; stream.closest("label").hidden = true; @@ -328,7 +332,7 @@ const el = ev.target.closest("[data-export-action]"); const action = el && el.dataset.exportAction; if (action === "download" && el.closest(".export-csv-form")) { - // Download TSV or CSV through a link with a download attribute, like + // Download these formats through a link with a download attribute, like // the other formats, rather than submitting the form with _dl=on: // WebKit on Linux does not treat that response as a download ev.preventDefault(); diff --git a/datasette/templates/_export_dialog.html b/datasette/templates/_export_dialog.html index 8034a3c4d5..577b5cee07 100644 --- a/datasette/templates/_export_dialog.html +++ b/datasette/templates/_export_dialog.html @@ -12,15 +12,29 @@ that one row. Otherwise the dialog also offers more JSON shapes, using urls.modify(). Formats from plugins use renderer_options for their label, description - and which actions to offer. -#} + and which actions to offer. Those that can stream every row are grouped + with TSV and CSV, so that "Include all rows" applies to them. -#} {% set export_base = export_meta or "data" %} {% set export_later = export_rows.later if export_rows else false %} {#- "Include all rows" streams every row from the first one, so offer it on every later page, and on the first page when CSV would stop short -#} {% set csv_stream = export_rows and display_rows and (export_later or (next_url and not (export_rows.total is not none and export_rows.total <= export_rows.max))) and settings.default_allow_stream %} +{#- Formats from plugins that can stream, as (extension, path, options) -#} +{% set streaming = namespace(formats=[], names=(["TSV"] if url_tsv_path else []) + ["CSV"]) %} +{% if display_rows %} +{% for name, url in renderers.items() if name not in ("json", "csv") and not (name == "tsv" and url_tsv_path) %} + {% set options = (renderer_options or {}).get(name) or {} %} + {% if options.can_stream %} + {% set streaming.formats = streaming.formats + [(name, url.split("?")[0], options)] %} + {% set streaming.names = streaming.names + [options.label or name] %} + {% endif %} +{% endfor %} +{% endif %} +{% set streaming_extensions = streaming.formats|map("first")|list %} {% macro rows_text(n) %}{{ "{:,}".format(n) }} row{{ "" if n == 1 else "s" }}{% endmacro %} {#- How many rows an export returns. JSON and plugin formats return the rows - on this page; TSV and CSV ask for _size=max, starting from this page. + on this page; TSV, CSV and plugin formats that can stream ("csv" here) + ask for _size=max, starting from this page. Mirrored by rowsLabel() in export-dialog.js. -#} {% macro rows_label(kind, rows, id) %} {%- set n = rows.page -%} @@ -41,20 +55,22 @@ {%- else -%}First {{ rows_text(n) }}{%- endif -%} {%- endmacro %} -{#- TSV or CSV: submit buttons for the shared form, with TSV using formaction -#} -{% macro delimited_option(id, name, description, rows, formaction=none) %} -
+{#- TSV, CSV or a plugin format that can stream: submit buttons for the shared + form, with every format other than CSV using formaction. kind is as for + export_option below. -#} +{% macro delimited_option(id, name, description, rows, formaction=none, kind=none) %} +
{{ name }} {% if rows %}{{ rows_label("csv", rows, id) }}{% endif %}
- {{ description }} + {% if description %}{{ description }}{% endif %}
- - - + {% if not kind %}{% endif %} + {% if kind != "binary" %}{% endif %} + {% if kind != "webpage" %}{% endif %}
@@ -89,15 +105,18 @@