Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions datasette/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -2668,6 +2668,8 @@ def _register_renderers(self):
"description": renderer.get("description"),
"is_binary": is_binary,
"is_webpage": is_webpage,
# Formats that can stream are grouped with "Include all rows"
"can_stream": self._renderer_can_stream[extension],
}

async def render_template(
Expand Down
20 changes: 12 additions & 8 deletions datasette/static/export-dialog.js
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,9 @@
errorFor(el)?.remove();
}

// TSV and CSV build their URLs from the form they share, so its checkboxes
// apply. TSV's buttons send the form to its own path using formaction.
// TSV, CSV and plugin formats that can stream build their URLs from the
// form they share, so its checkboxes apply. All but CSV send the form to
// their own path using formaction.
function exportUrl(el) {
const row = el.closest(".export-option");
const form = el.closest(".export-csv-form");
Expand All @@ -82,8 +83,11 @@
const link = row.querySelector("a.export-action[href]");
return new URL(link.getAttribute("href"), location.href).href;
}
const view = row.querySelector('[data-export-action="view"]');
const path = view.getAttribute("formaction") || form.getAttribute("action");
// Binary formats have no View button, so look at Download as well
const submit = row.querySelector("[formaction]");
const path = submit
? submit.getAttribute("formaction")
: form.getAttribute("action");
const url = new URL(path, location.href);
// Row pages have no fields to send, so no "?" either
url.search = new URLSearchParams(new FormData(form));
Expand Down Expand Up @@ -164,8 +168,8 @@
const rowsText = (n) => `${formatNumber(n)} row${n === 1 ? "" : "s"}`;

// How many rows an export returns: JSON and plugin formats return the rows
// on this page, TSV and CSV ("csv" here) ask for _size=max starting from
// this page, and "Include all rows" streams every row from the first one.
// on this page, the formats that can stream - TSV, CSV and some plugin
// formats, "csv" here - ask for _size=max starting from this page, and "Include all rows" streams every row from the first one.
// Mirrors the rows_label macro in _export_dialog.html.
function rowsLabel(kind, rows, stream) {
const { page, more, later, max, total } = rows;
Expand Down Expand Up @@ -222,7 +226,7 @@
rows.total !== null &&
rows.total <= rows.max
) {
// TSV and CSV already include every row, so the option would do nothing
// Those formats already include every row, so the option would do nothing
stream.checked = false;
stream.disabled = true;
stream.closest("label").hidden = true;
Expand Down Expand Up @@ -328,7 +332,7 @@
const el = ev.target.closest("[data-export-action]");
const action = el && el.dataset.exportAction;
if (action === "download" && el.closest(".export-csv-form")) {
// Download TSV or CSV through a link with a download attribute, like
// Download these formats through a link with a download attribute, like
// the other formats, rather than submitting the form with _dl=on:
// WebKit on Linux does not treat that response as a download
ev.preventDefault();
Expand Down
47 changes: 33 additions & 14 deletions datasette/templates/_export_dialog.html
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,29 @@
that one row. Otherwise the dialog also offers more JSON shapes, using
urls.modify().
Formats from plugins use renderer_options for their label, description
and which actions to offer. -#}
and which actions to offer. Those that can stream every row are grouped
with TSV and CSV, so that "Include all rows" applies to them. -#}
{% set export_base = export_meta or "data" %}
{% set export_later = export_rows.later if export_rows else false %}
{#- "Include all rows" streams every row from the first one, so offer it on
every later page, and on the first page when CSV would stop short -#}
{% set csv_stream = export_rows and display_rows and (export_later or (next_url and not (export_rows.total is not none and export_rows.total <= export_rows.max))) and settings.default_allow_stream %}
{#- Formats from plugins that can stream, as (extension, path, options) -#}
{% set streaming = namespace(formats=[], names=(["TSV"] if url_tsv_path else []) + ["CSV"]) %}
{% if display_rows %}
{% for name, url in renderers.items() if name not in ("json", "csv") and not (name == "tsv" and url_tsv_path) %}
{% set options = (renderer_options or {}).get(name) or {} %}
{% if options.can_stream %}
{% set streaming.formats = streaming.formats + [(name, url.split("?")[0], options)] %}
{% set streaming.names = streaming.names + [options.label or name] %}
{% endif %}
{% endfor %}
{% endif %}
{% set streaming_extensions = streaming.formats|map("first")|list %}
{% macro rows_text(n) %}{{ "{:,}".format(n) }} row{{ "" if n == 1 else "s" }}{% endmacro %}
{#- How many rows an export returns. JSON and plugin formats return the rows
on this page; TSV and CSV ask for _size=max, starting from this page.
on this page; TSV, CSV and plugin formats that can stream ("csv" here)
ask for _size=max, starting from this page.
Mirrored by rowsLabel() in export-dialog.js. -#}
{% macro rows_label(kind, rows, id) %}
{%- set n = rows.page -%}
Expand All @@ -41,20 +55,22 @@
{%- else -%}First {{ rows_text(n) }}{%- endif -%}
</span>
{%- endmacro %}
{#- TSV or CSV: submit buttons for the shared form, with TSV using formaction -#}
{% macro delimited_option(id, name, description, rows, formaction=none) %}
<div class="export-option" data-export-format="{{ id }}">
{#- TSV, CSV or a plugin format that can stream: submit buttons for the shared
form, with every format other than CSV using formaction. kind is as for
export_option below. -#}
{% macro delimited_option(id, name, description, rows, formaction=none, kind=none) %}
<div class="export-option" data-export-format="{{ id }}"{% if kind %} data-export-kind="{{ kind }}"{% endif %}>
<div class="export-option-text">
<div class="export-option-heading">
<span class="export-option-name" id="export-{{ id }}-name">{{ name }}</span>
{% if rows %}{{ rows_label("csv", rows, id) }}{% endif %}
</div>
<span class="export-option-description">{{ description }}</span>
{% if description %}<span class="export-option-description">{{ description }}</span>{% endif %}
</div>
<div class="export-actions" role="group" aria-labelledby="export-{{ id }}-name"{% if rows %} aria-describedby="export-{{ id }}-rows"{% endif %}>
<button type="button" class="export-action" data-export-action="copy" hidden>Copy</button>
<button type="submit" class="export-action" data-export-action="view"{% if formaction %} formaction="{{ formaction }}"{% endif %}>View</button>
<button type="submit" class="export-action" data-export-action="download" name="_dl" value="on"{% if formaction %} formaction="{{ formaction }}"{% endif %}>Download</button>
{% if not kind %}<button type="button" class="export-action" data-export-action="copy" hidden>Copy</button>{% endif %}
{% if kind != "binary" %}<button type="submit" class="export-action" data-export-action="view"{% if formaction %} formaction="{{ formaction }}"{% endif %}>View</button>{% endif %}
{% if kind != "webpage" %}<button type="submit" class="export-action" data-export-action="download" name="_dl" value="on"{% if formaction %} formaction="{{ formaction }}"{% endif %}>Download</button>{% endif %}
<button type="button" class="export-action" data-export-action="copy-url" hidden>Copy URL</button>
</div>
</div>
Expand Down Expand Up @@ -89,15 +105,18 @@ <h2 class="modal-title" id="export-dialog-title">Export data</h2>
</div>
<div class="modal-body export-links">
{% if export_scope %}<p class="export-scope" id="export-scope" title="{{ export_scope }}{% if export_scope_description %} {{ export_scope_description }}{% endif %}"><strong>{{ export_scope }}</strong>{% if export_scope_description %} {{ export_scope_description }}{% endif %}</p>{% endif %}
{% if export_later %}<p class="export-page-note" id="export-page-note">These exports start from this page, not the first page.{% if csv_stream %} Check “Include all rows” to export every row as {{ "TSV or " if url_tsv_path }}CSV.{% endif %}</p>{% endif %}
{% if export_later %}<p class="export-page-note" id="export-page-note">These exports start from this page, not the first page.{% if csv_stream %} Check “Include all rows” to export every row as {{ streaming.names[:-1]|join(", ") }}{{ " or " if streaming.names|length > 1 }}{{ streaming.names[-1] }}.{% endif %}</p>{% endif %}
{% if display_rows %}
{#- TSV and CSV share a form, so the options under them apply to both.
The other formats are in a separate box below, as the options do
not apply to them. -#}
{#- TSV, CSV and plugin formats that can stream share a form, so the
options under them apply to them all. The other formats are in a
separate box below, as the options do not apply to them. -#}
<div class="export-options export-option-group">
<form class="export-csv-form" action="{{ url_csv_path }}" method="get">
{% if url_tsv_path %}{{ delimited_option("tsv", "TSV", "Tab-separated values, for pasting into Excel or Google Sheets", export_rows, formaction=url_tsv_path) }}{% endif %}
{{ delimited_option("csv", "CSV", "Comma-separated values, for spreadsheets and other tools", export_rows) }}
{% for name, path, options in streaming.formats %}
{{ delimited_option(name, options.label or name, options.description, export_rows, formaction=path, kind="binary" if options.is_binary else ("webpage" if options.is_webpage else none)) }}
{% endfor %}
{% if expandable_columns or csv_stream %}
<div class="export-csv-options">
{% if expandable_columns %}<label><input type="checkbox" name="_labels" checked> Expand labels</label>{% endif %}
Expand All @@ -121,7 +140,7 @@ <h2 class="modal-title" id="export-dialog-title">Export data</h2>
{{ export_option("json-object", "JSON object", "Rows keyed by their primary key", urls.modify(renderers['json'], add={'_shape': 'object'}), export_base ~ ".json", export_rows) }}
{% endif %}
{% endif %}
{% for name, url in renderers.items() if name not in ("json", "csv") and not (name == "tsv" and url_tsv_path) %}
{% for name, url in renderers.items() if name not in ("json", "csv") and not (name == "tsv" and url_tsv_path) and name not in streaming_extensions %}
{% set options = (renderer_options or {}).get(name) or {} %}
{{ export_option("format-" ~ loop.index, options.label or name, options.description, url, export_base ~ "." ~ name, export_rows, kind="binary" if options.is_binary else ("webpage" if options.is_webpage else none)) }}
{% endfor %}
Expand Down
13 changes: 12 additions & 1 deletion datasette/views/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -1953,6 +1953,11 @@ async def table_view_traced(datasette, request):
context_for_html_hack = True
default_labels = True

# ?_stream=1 for a format that can stream fetches the largest pages
stream = bool(
request.args.get("_stream") and datasette._renderer_can_stream.get(format_)
)

start = time.perf_counter()
view_data = await table_view_data(
datasette,
Expand All @@ -1962,6 +1967,7 @@ async def table_view_traced(datasette, request):
context_for_html_hack=context_for_html_hack,
default_labels=default_labels,
renderer_extras=datasette._renderer_extras.get(format_),
stream=stream,
)
if isinstance(view_data, Response):
return view_data
Expand All @@ -1984,6 +1990,7 @@ async def fetch_next_page():
default_labels=default_labels,
_next=next_token,
rows_only=True,
stream=stream,
)
next_token = page_data["next"]
return page_rows
Expand Down Expand Up @@ -2340,6 +2347,7 @@ async def table_view_data(
_next=None,
renderer_extras=None,
rows_only=False,
stream=False,
):
extra_extras = extra_extras or set()
# We have a table or view
Expand Down Expand Up @@ -2469,7 +2477,10 @@ async def table_view_data(
# Handle ?_size=500
# TODO: This was:
# page_size = _size or request.args.get("_size") or table_metadata.get("size")
page_size = request.args.get("_size") or table_metadata.get("size")
page_size = request.args.get("_size")
if not page_size:
# Streaming every row is fastest using the largest possible pages
page_size = "max" if stream else table_metadata.get("size")
if page_size:
if page_size == "max":
page_size = datasette.max_returned_rows
Expand Down
2 changes: 2 additions & 0 deletions docs/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ Unreleased
- The following ``datasette.utils`` functions and classes are now :ref:`documented <internals_utils>` and considered stable for use by plugins: ``escape_sqlite()``, ``escape_fts()``, ``to_css_class()``, ``modify_query_string()``, ``actor_matches_allow()``, ``add_cors_headers()``, ``StartupError``, ``BadMetadataError``, ``MultiParams``, ``PrefixedUrlString``, ``PaginatedResources`` and the ``sqlite3`` module. (:issue:`1630`)
- **Backwards incompatible:** the ``format_bytes()`` function is no longer available in templates, and ``datasette.utils.format_bytes()`` has been removed. Custom templates that call ``{{ format_bytes(size) }}`` should use Jinja's built-in `filesizeformat <https://jinja.palletsprojects.com/en/stable/templates/#jinja-filters.filesizeformat>`__ filter instead: ``{{ size|filesizeformat }}``. (:issue:`1841`)
- File sizes shown on the database page and in the tooltips for binary values now use decimal units, where 1 MB is 1,000,000 bytes, matching the sizes already shown on the homepage. They previously used 1 MB to mean 1,048,576 bytes, so a 5,242,880 byte database that was displayed as ``5.0 MB`` is now displayed as ``5.2 MB``.
- ``?_stream=1`` now fetches rows in pages of :ref:`setting_max_returned_rows`, 1,000 by default, instead of the default page size of 100. This makes streaming every row of a large table several times faster. A ``?_size=`` in the URL is still respected.
- The "Export data" dialog now groups plugin formats that can stream with TSV and CSV, so its "Include all rows" option applies to those formats too.

- New ``datasette.urls.modify(target, add=, replace=, remove=, format=)`` method for building links that change the query string of the current page or of any other path. It applies ``base_url`` for you and is available in templates as ``urls.modify()``. See :ref:`internals_datasette_urls_modify`. (:issue:`1342`)
- Fixed a bug where sort, facet and filter-removal links on the table page dropped query string arguments with blank values, so sorting ``/db/table?name=`` lost the filter for rows where ``name`` is the empty string.
Expand Down
4 changes: 3 additions & 1 deletion docs/csv_export.rst
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ Options
-------

TSV and CSV share the following options. These are only offered for tables
and views:
and views. Formats added by plugins that can
:ref:`stream every row <plugin_register_output_renderer_streaming>` are listed
alongside TSV and CSV, and these options apply to them as well:

* **expand labels** - if your table has any foreign key references this option
will cause the export to gain additional ``COLUMN_NAME_label`` columns with a
Expand Down
4 changes: 3 additions & 1 deletion docs/plugin_hooks.rst
Original file line number Diff line number Diff line change
Expand Up @@ -769,7 +769,9 @@ A table can have far more rows than fit on one page. A ``"render"`` function tha

``row_stream`` yields the rows on the current page. If the URL includes ``?_stream=1`` it carries on past the end of that page, fetching one page at a time, until it has yielded every row in the table or view. Your function does not need to know which of those is happening.

Use ``async for rows in row_stream.pages()`` to receive a list of rows for each page instead of one row at a time.
Use ``async for rows in row_stream.pages()`` to receive a list of rows for each page instead of one row at a time. When streaming every row those pages are :ref:`setting_max_returned_rows` rows each, unless the URL sets a different ``?_size=``.

The :ref:`Export data dialog <csv_export>` lists formats that accept ``row_stream`` in the same group as TSV and CSV, so that its "Include all rows" and "Expand labels" options apply to them. Like TSV and CSV, their links in the dialog use ``?_size=max``.

Datasette's own :ref:`CSV and TSV export <csv_export>` is implemented this way. Some things to be aware of:

Expand Down
Loading
Loading