Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,22 @@ jobs:
- name: Run tests
run: |
python -m pytest
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.13"
cache: pip
cache-dependency-path: pyproject.toml
- name: Install dependencies
run: |
pip install . --group dev
- name: Run ruff check
run: |
ruff check .
- name: Run ruff format
run: |
ruff format --check .
18 changes: 9 additions & 9 deletions Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,25 @@
# Run linters
@lint:
echo "Linters..."
echo " Black"
pipenv run black . --check
echo " ruff"
pipenv run ruff .
echo " ruff check"
pipenv run ruff check .
echo " ruff format"
pipenv run ruff format --check .


# Serve live docs on localhost:8000
@docs:
rm -rf docs/_build
cd docs && pipenv run make livehtml

# Apply Black
@black:
pipenv run black .
# Apply ruff format
@format:
pipenv run ruff format .

# Run automatic fixes
@fix:
pipenv run ruff . --fix
pipenv run black .
pipenv run ruff check . --fix
pipenv run ruff format .

# Push commit if tests pass
@push: test lint
Expand Down
95 changes: 42 additions & 53 deletions datasette_enrichments/__init__.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,32 @@
from abc import ABC, abstractmethod
import asyncio
import datetime
from datasette import hookimpl
from datasette.permissions import Action, PermissionSQL
from datasette.resources import DatabaseResource
from datasette.utils import async_call_with_supported_arguments, tilde_encode, sqlite3
from datasette_secrets import Secret, get_secret
import json
import secrets
import sys
import time
import traceback
import urllib
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
from urllib.parse import quote

from datasette import hookimpl
from datasette.permissions import Action
from datasette.plugins import pm
from datasette.resources import DatabaseResource
from datasette.utils import (
async_call_with_supported_arguments,
await_me_maybe,
sqlite3,
tilde_encode,
)
from datasette_secrets import Secret, get_secret
from markupsafe import Markup, escape
from . import views
from wtforms import PasswordField
from wtforms.validators import DataRequired
from .utils import mark_job_complete, pks_for_rows
from urllib.parse import quote
from . import hookspecs

from datasette.utils import await_me_maybe

from typing import TYPE_CHECKING, List, Optional, Tuple, Union
from . import hookspecs, views
from .utils import mark_job_complete, pks_for_rows

if TYPE_CHECKING:
from datasette.app import Datasette
Expand All @@ -32,10 +35,10 @@
pm.add_hookspecs(hookspecs)


IdType = Union[int, str, Tuple[Union[int, str], ...]]
IdType = int | str | tuple[int | str, ...]

# Custom epoch to save space in the _enrichment_progress table
JAN_1_2025_EPOCH = int(datetime.datetime(2025, 1, 1).timestamp() * 1000)
JAN_1_2025_EPOCH = int(datetime.datetime(2025, 1, 1).timestamp() * 1000) # noqa: DTZ001


def ms_since_2025_to_datetime(ms_since_2025):
Expand Down Expand Up @@ -105,8 +108,8 @@ async def set_job_status(
db: "Database",
job_id: int,
status: str,
allowed_statuses: Optional[Tuple[str]] = None,
message: Optional[str] = None,
allowed_statuses: tuple[str] | None = None,
message: str | None = None,
):
if allowed_statuses:
# First check the current status
Expand Down Expand Up @@ -146,9 +149,7 @@ async def record_progress(db, job_id, success_count, error_count, message=""):
) values (
:job_id, :timestamp_ms_2025, :success_count, :error_count, {}
)
""".format(
":message" if message else "null"
),
""".format(":message" if message else "null"),
{
"job_id": job_id,
"timestamp_ms_2025": int(time.time() * 1000) - JAN_1_2025_EPOCH,
Expand All @@ -173,22 +174,22 @@ class SecretError(Exception):


class Enrichment(ABC):
_subclasses = []
_subclasses = [] # noqa: RUF012

batch_size: int = 100
# Cancel run after this many errors
default_max_errors: int = 5
log_traceback: bool = False

class Cancel(Exception):
def __init__(self, reason: Optional[str] = None):
def __init__(self, reason: str | None = None):
self.reason = reason

def __str__(self) -> str:
return self.reason or "Cancelled by enrichment"

class Pause(Exception):
def __init__(self, reason: Optional[str] = None):
def __init__(self, reason: str | None = None):
self.reason = reason

def __str__(self) -> str:
Expand All @@ -207,14 +208,14 @@ def name(self):
...

description: str = "" # Short description of this enrichment
secret: Optional[Secret] = None
secret: Secret | None = None

def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
cls._subclasses.append(cls)

def __repr__(self):
return "<Enrichment: {}>".format(self.slug)
return f"<Enrichment: {self.slug}>"

async def get_secret(self, datasette: "Datasette", config: dict):
if self.secret is None:
Expand All @@ -228,13 +229,11 @@ async def get_secret(self, datasette: "Datasette", config: dict):
stashed_keys = datasette._enrichments_stashed_secrets
stash_key = config.get("enrichment_secret")
if stash_key not in stashed_keys:
raise SecretError(
"No secret found in stash for {}".format(self.secret.name)
)
raise SecretError(f"No secret found in stash for {self.secret.name}")
return stashed_keys[stash_key]

async def log_error(
self, db: "Database", job_id: int, ids: List[IdType], error: str
self, db: "Database", job_id: int, ids: list[IdType], error: str
):
if self.log_traceback:
error += "\n\n" + traceback.format_exc()
Expand Down Expand Up @@ -324,8 +323,8 @@ async def enrich_batch(
pks: list,
config: dict,
job_id: int,
actor_id: str = None,
) -> Optional[int]:
actor_id: str | None = None,
) -> int | None:
raise NotImplementedError

async def increment_cost(
Expand All @@ -347,7 +346,7 @@ async def enqueue(
table: str,
filter_querystring: str,
config: dict,
actor_id: str = None,
actor_id: str | None = None,
) -> int:
# Enqueue a job
qs = filter_querystring
Expand Down Expand Up @@ -378,9 +377,7 @@ def _insert(conn):
:enrichment, 'pending', :database_name, :table_name, :filter_querystring, :config,
datetime('now'), :row_count, 0, 0, 0{}
)
""".format(
", :actor_id" if actor_id else ", null"
),
""".format(", :actor_id" if actor_id else ", null"),
{
"enrichment": self.slug,
"database_name": db.name,
Expand Down Expand Up @@ -435,8 +432,8 @@ async def run_enrichment():
)
qs = job["filter_querystring"]
if next_cursor:
qs += "&_next={}".format(next_cursor)
qs += "&_size={}&_shape=objects".format(self.batch_size)
qs += f"&_next={next_cursor}"
qs += f"&_size={self.batch_size}&_shape=objects"
response = await datasette.client.get(
table_path + "?" + qs, skip_permission_checks=True
)
Expand Down Expand Up @@ -466,7 +463,7 @@ async def run_enrichment():
except self.Pause as ex:
await set_job_status(db, job_id, "paused", message=str(ex))
return
except Exception as ex:
except Exception as ex: # noqa: BLE001
await self.log_error(db, job_id, pks_for_rows(rows, pks), str(ex))
# Update next_cursor
next_cursor = response.json()["next"]
Expand Down Expand Up @@ -553,7 +550,7 @@ async def inner():
database,
tilde_encode(table),
(
"?{}".format(request.query_string)
f"?{request.query_string}"
if request.query_string
else ""
),
Expand All @@ -575,9 +572,7 @@ async def inner():
items.append(
{
"href": datasette.urls.path(
"/-/enrich/{}/-/jobs?table={}".format(
database, quote(table)
),
f"/-/enrich/{database}/-/jobs?table={quote(table)}",
),
"label": "Enrichment jobs",
"description": "View and manage {} enrichment job{} for this table".format(
Expand Down Expand Up @@ -616,7 +611,7 @@ async def inner():
return [
{
"href": datasette.urls.path(
"/-/enrich/{}/-/jobs".format(database),
f"/-/enrich/{database}/-/jobs",
),
"label": "Enrichment jobs",
"description": "View and manage {} enrichment job{} for this database".format(
Expand Down Expand Up @@ -652,11 +647,7 @@ async def inner():
return [
{
"href": datasette.urls.path(
"/-/enrich/{}/{}?{}".format(
database,
tilde_encode(table),
query_string,
)
f"/-/enrich/{database}/{tilde_encode(table)}?{query_string}"
),
"label": "Enrich this row",
"description": "Run a data cleaning operation against this row",
Expand Down Expand Up @@ -958,12 +949,10 @@ async def _restart_running_jobs_task(datasette):

# Find jobs marked as 'running'
running_jobs = (
await db.execute(
"""
await db.execute("""
SELECT * FROM _enrichment_jobs
WHERE status = 'running'
"""
)
""")
).rows

# Grab all known enrichments
Expand All @@ -980,7 +969,7 @@ async def _restart_running_jobs_task(datasette):
# Resume from wherever it left off
await enrichment.start_enrichment_in_process(datasette, db, job_id)
else:
print("Unknown enrichment: {}".format(enrichment_slug), file=sys.stderr)
print(f"Unknown enrichment: {enrichment_slug}", file=sys.stderr)


async def restart_running_jobs(datasette):
Expand Down
11 changes: 5 additions & 6 deletions datasette_enrichments/utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import asyncio
import secrets
from typing import Optional, Union, TYPE_CHECKING
from typing import TYPE_CHECKING

if TYPE_CHECKING:
from datasette.app import Datasette
Expand All @@ -12,13 +11,13 @@ def __init__(self, job_id, msg):
self.msg = msg

def __repr__(self):
return "<WaitForJobException {}: {}>".format(self.job_id, self.msg)
return f"<WaitForJobException {self.job_id}: {self.msg}>"


async def wait_for_job(
datasette: "Datasette",
job_id: Union[int, str],
database: Optional[str] = None,
job_id: int | str,
database: str | None = None,
timeout=None,
):
"Returns when the job has completed, using an asyncio.Event"
Expand Down Expand Up @@ -49,7 +48,7 @@ async def wait_for_job(


async def mark_job_complete(
datasette: "Datasette", job_id: int, database: Optional[str] = None
datasette: "Datasette", job_id: int, database: str | None = None
):
_ensure_enrichment_properties(datasette)
if not database:
Expand Down
Loading
Loading