Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions datasette/facets.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import time
import urllib

from datasette import hookimpl
Expand All @@ -12,6 +13,26 @@
sqlite3,
)

# Suggesting facets for a page can take at most this many multiples of
# the facet_suggest_time_limit_ms setting in total, see #862
SUGGEST_TOTAL_TIME_LIMIT_MULTIPLE = 10


def start_suggest_deadline(ds, facet_instances):
"""
Start the clock on the total time allowed for suggesting facets for a
page, sharing a single deadline across all of the facet instances.
"""
time_limit_ms = ds.setting("facet_suggest_time_limit_ms")
if not time_limit_ms:
# 0 means no per-query suggestion limit, so no total limit either
return
deadline = (
time.monotonic() + SUGGEST_TOTAL_TIME_LIMIT_MULTIPLE * time_limit_ms / 1000
)
for facet in facet_instances:
facet.suggest_deadline = deadline


def load_facet_configs(request, table_config):
# Given a request and the configuration for a table, return
Expand Down Expand Up @@ -86,6 +107,10 @@ class Facet:
type = None
# How many rows to consider when suggesting facets:
suggest_consider = 1000
# time.monotonic() value after which suggest() should stop considering
# further columns, or None for no deadline. Shared by every facet
# instance on a page, see start_suggest_deadline()
suggest_deadline = None

def __init__(
self,
Expand Down Expand Up @@ -152,6 +177,17 @@ def get_facet_size(self):
async def suggest(self):
return []

def suggest_time_exceeded(self):
"""
True if the page has used up its total time budget for suggesting
facets. suggest() implementations should check this before running
the query for each column and stop if it returns True.
"""
return (
self.suggest_deadline is not None
and time.monotonic() >= self.suggest_deadline
)

async def facet_results(self):
# returns ([results], [timed_out])
# TODO: Include "hideable" with each one somehow, which indicates if it was
Expand All @@ -177,6 +213,8 @@ async def suggest(self):
suggested_facets = []
already_enabled = [c["config"]["simple"] for c in self.get_configs()]
for column in columns:
if self.suggest_time_exceeded():
break
if column in already_enabled:
continue
suggested_facet_sql = f"""
Expand Down Expand Up @@ -334,6 +372,8 @@ async def suggest(self):
suggested_facets = []
already_enabled = [c["config"]["simple"] for c in self.get_configs()]
for column in columns:
if self.suggest_time_exceeded():
break
if column in already_enabled:
continue
# Is every value in this column either null or a JSON array?
Expand Down Expand Up @@ -492,6 +532,8 @@ async def suggest(self):
already_enabled = [c["config"]["simple"] for c in self.get_configs()]
suggested_facets = []
for column in columns:
if self.suggest_time_exceeded():
break
if column in already_enabled:
continue
# Does this column contain any dates in the first 100 rows?
Expand Down
10 changes: 6 additions & 4 deletions datasette/views/table_extras.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from datasette.column_types import SQLiteType
from datasette.database import QueryInterrupted
from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider
from datasette.facets import start_suggest_deadline
from datasette.plugins import pm
from datasette.resources import DatabaseResource, TableResource
from datasette.utils import (
Expand Down Expand Up @@ -291,10 +292,11 @@ async def resolve(self, context, facet_instances):
and not context.nofacet
and not context.nosuggest
):
facet_suggest_awaitables = [facet.suggest() for facet in facet_instances]
for suggest_result in await context.run_sequential(
*facet_suggest_awaitables
):
start_suggest_deadline(context.datasette, facet_instances)
for facet in facet_instances:
if facet.suggest_time_exceeded():
break
(suggest_result,) = await context.run_sequential(facet.suggest())
suggested_facets.extend(suggest_result)
return suggested_facets

Expand Down
1 change: 1 addition & 0 deletions docs/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Changelog
Unreleased
----------

- The total time spent calculating suggested facets for a page is now capped at ten times the :ref:`setting_facet_suggest_time_limit_ms` setting, so tables with a large number of columns can no longer spend several seconds suggesting facets. (:issue:`862`)
- The default :ref:`setting_max_post_body_bytes` limit has been increased from 2MB to 10MB (``10485760`` bytes), allowing larger JSON API requests.
- Fixed a bug where a database, table or saved query with a name such as ``count``, ``wrapper`` or ``tag`` could break the layout of its pages, because the class added to the ``<body>`` element for that name matched a class used by Datasette's own CSS and JavaScript.

Expand Down
3 changes: 3 additions & 0 deletions docs/plugin_hooks.rst
Original file line number Diff line number Diff line change
Expand Up @@ -847,6 +847,9 @@ Each Facet subclass implements a new type of facet operation. The class should l
async def suggest(self):
# Use self.sql and self.params to suggest some facets
suggested_facets = []
# If this runs a query per column, check
# self.suggest_time_exceeded() before each query and
# stop once it returns True
suggested_facets.append(
{
"name": column, # Or other unique name
Expand Down
2 changes: 2 additions & 0 deletions docs/settings.rst
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,8 @@ Default: ``50``

When Datasette calculates suggested facets it needs to run a SQL query for every column in your table. This time limit, in milliseconds, applies separately to each query. If the time limit is exceeded the column will not be suggested as a facet.

There is also a limit on the total time spent suggesting facets for a page, which is ten times this setting - 500ms by default. Once that total has been used up the remaining columns are not considered, so a table with a large number of columns may not have every possible facet suggested.

You can increase this time limit like so::

datasette mydatabase.db --setting facet_suggest_time_limit_ms 500
Expand Down
61 changes: 61 additions & 0 deletions tests/test_facets.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
from types import SimpleNamespace
from urllib.parse import parse_qsl, urlsplit

import pytest
Expand Down Expand Up @@ -975,3 +976,63 @@ async def test_facet_only_considers_first_x_rows():
assert data2["suggested_facets"] == []
finally:
Facet.suggest_consider = original_suggest_consider


# https://github.com/simonw/datasette/issues/862
@pytest.mark.asyncio
@pytest.mark.parametrize(
"time_limit_ms,expected_suggested",
(
# Each suggestion query takes 30ms on the fake clock, so the total
# budget of 10 * 10ms runs out after four: id, c0, c1 and c2
(10, ["c0", "c1", "c2"]),
# 0 disables the total limit
(0, [f"c{i}" for i in range(20)]),
),
)
async def test_facet_suggest_total_time_limit(
monkeypatch, time_limit_ms, expected_suggested
):
ds = Datasette(
[], memory=True, settings={"facet_suggest_time_limit_ms": time_limit_ms}
)
db = ds.add_database(Database(ds, memory_name="suggest_total_time_limit"))
columns = [f"c{i}" for i in range(20)]
await db.execute_write(
"create table items (id integer primary key, {})".format(
", ".join(f"{column} text" for column in columns)
)
)
for values in ("a", "a", "b"):
await db.execute_write(
"insert into items ({}) values ({})".format(
", ".join(columns), ", ".join("?" for _ in columns)
),
[values] * len(columns),
)

now = 1000.0
monkeypatch.setattr("datasette.facets.time", SimpleNamespace(monotonic=lambda: now))
execute = ds.execute
suggestion_queries = 0

async def slow_execute(*args, **kwargs):
nonlocal now, suggestion_queries
if "with limited as" in args[1]:
suggestion_queries += 1
now += 0.03
return await execute(*args, **kwargs)

monkeypatch.setattr(ds, "execute", slow_execute)
try:
response = await ds.client.get(
"/suggest_total_time_limit/items.json?_extra=suggested_facets"
)
assert response.status_code == 200
suggested = [facet["name"] for facet in response.json()["suggested_facets"]]
assert suggested == expected_suggested
if time_limit_ms:
# Remaining columns and facet types were never queried
assert suggestion_queries == 4
finally:
ds.close()
Loading