Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,42 @@ zeeker/{cli.py,commands/{assets.py,metadata.py,helpers.py}}
Sibling imports work: resource modules can `import helper` for helpers in `resources/` — import at module top level only (dir is appended to sys.path just for the load, lowest precedence, then removed; stdlib/site-packages always win name clashes)
`--setup-fts` is idempotent (safe on incremental `--sync-from-s3` builds)

## Status Contract (unambiguous build output)
**Skip with reason**: raise `Skip` from fetch_data instead of returning `[]` to say WHY:
```python
from zeeker import Skip # requires zeeker>=0.9.0; kinds: "up_to_date" (default), "blocked", "disabled"

def fetch_data(existing_table):
if not os.environ.get("TAILSCALE_PROXY"):
raise Skip("TAILSCALE_PROXY unset — proxy required", kind="blocked")
return [] # plain [] still works → skip with kind "up_to_date", no reason
```
Renders `[SKIP] name <reason> (kind) (1.2s)`; `skip_reason`/`skip_kind` in `--json`. Works in sync + async fetch_data and in fetch_fragments_data (skips just the fragments phase); safe with `fragments_on_skip` (fragments run with context `[]`). `zeeker build --fail-on-blocked` → exit 1 if any resource skipped with kind "blocked". "blocked"/"disabled" skips do NOT bump `_zeeker_updates.last_updated` (source wasn't checked — time-based incremental resources stay safe); "up_to_date" does. Caveats: on zeeker<0.9.0 the import is an ImportError → hard failure (shim with `try/except ImportError` if the repo must support both); don't raise Skip inside retry-wrapped code except zeeker-common's `sync_retry`/`async_retry`, which pass Skip through without retrying. Not honored in `transform_data` (recorded as transform failure).

**Enrichment counters** (`__zeeker_report__`): multi-phase resources that UPDATE rows (0 records inserted) report work via a module-level dict, read + cleared on every exit path (success, skip, even a fetch_data crash — work done before the crash stays visible) and re-read after the fragments phase (counters set in fetch_fragments_data merge in, summing on key collision):
```python
def fetch_data(existing_table):
global __zeeker_report__
__zeeker_report__ = {"updated": 50, "enriched": 25, "notes": "phase2 drained 25"}
return [] # renders "[SKIP] judgments no data returned; updated=50 enriched=25"
```
Int values only (others ignored, never fails the build); totals appear in the SUMMARY footer; `extra_counts`/`notes` in `--json`.

**Warnings are machine-readable**: per-resource warnings → `ResourceOutcome.warnings` (`WARN[resource]: ...` lines in plain mode), build-level warnings (e.g. S3 sync) → `BuildReport.build_warnings` (`WARN[build]: ...`); both in the `--json` payload.

**Resource logging** (zeeker-common): consistent per-resource log lines + optional JSONL sink (`ZEEKER_BUILDLOG_JSONL=/path/to/file.jsonl`):
```python
from zeeker_common import resource_logger

log = resource_logger("judgments")
log.info("discovery starting") # stdout: "judgments: discovery starting"
log.warn("proxy slow"); log.error("boom") # stderr
log.done(new=3, updated=50) # "judgments: done — 3 new, 50 updated"
log.aborted("circuit breaker", failed=5) # stderr: "judgments: ABORTED (circuit breaker) — 5 failed"
log.skipped("nothing new") # stderr: "judgments: SKIPPED (nothing new)"
```
Count grammar is noun-first (`3 new, 2 skipped, 0 failed`) — matches the data repos' Build Monitoring Guide regexes (`(\d+) new`); underscores in count names render as spaces.

## CRITICAL: Duplicate Handling
MUST filter existing IDs to avoid UNIQUE constraint errors:
```python
Expand Down
2 changes: 1 addition & 1 deletion packages/zeeker-common/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "zeeker-common"
version = "0.1.0"
version = "0.2.0"
description = "Common utilities for Zeeker data projects"
readme = "README.md"
requires-python = ">=3.12"
Expand Down
148 changes: 148 additions & 0 deletions packages/zeeker-common/tests/test_buildlog.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
"""Tests for zeeker_common.buildlog (resource_logger)."""

import json

import pytest

from zeeker_common import resource_logger
from zeeker_common.buildlog import ResourceLogger


def test_lazy_export_from_package_init():
import zeeker_common

assert zeeker_common.resource_logger is resource_logger
assert zeeker_common.ResourceLogger is ResourceLogger
with pytest.raises(AttributeError):
zeeker_common.does_not_exist


def test_info_goes_to_stdout_with_prefix(capsys):
log = resource_logger("judgments")
log.info("discovery starting")
captured = capsys.readouterr()
assert captured.out == "judgments: discovery starting\n"
assert captured.err == ""


def test_warn_and_error_go_to_stderr(capsys):
log = resource_logger("judgments")
log.warn("proxy slow")
log.error("boom")
captured = capsys.readouterr()
assert captured.out == ""
assert "judgments: proxy slow\n" in captured.err
assert "judgments: boom\n" in captured.err


def test_multiline_message_prefixes_every_line_preserving_whitespace(capsys):
log = resource_logger("res")
log.info("first\n indented\nlast")
captured = capsys.readouterr()
assert captured.out == "res: first\nres: indented\nres: last\n"


def test_done_with_counts_uses_noun_first_grammar(capsys):
"""Count grammar must match the data repos' adopted monitoring shape:
'done — 3 new, 2 skipped, 0 failed' (parsed by regexes like '(\\d+) new')."""
log = resource_logger("judgments")
log.done(new=3, updated=50)
captured = capsys.readouterr()
assert captured.out == "judgments: done — 3 new, 50 updated\n"
assert captured.err == ""


def test_done_counts_render_underscores_as_spaces(capsys):
log = resource_logger("pdpc")
log.done(ok=4, still_pending=2)
captured = capsys.readouterr()
assert captured.out == "pdpc: done — 4 ok, 2 still pending\n"


def test_done_without_counts(capsys):
log = resource_logger("judgments")
log.done()
captured = capsys.readouterr()
assert captured.out == "judgments: done\n"


def test_aborted_with_counts_goes_to_stderr(capsys):
log = resource_logger("pdpc")
log.aborted("circuit breaker", failed=5, ok=2)
captured = capsys.readouterr()
assert captured.out == ""
assert captured.err == "pdpc: ABORTED (circuit breaker) — 5 failed, 2 ok\n"


def test_aborted_without_counts(capsys):
log = resource_logger("pdpc")
log.aborted("circuit breaker")
captured = capsys.readouterr()
assert captured.err == "pdpc: ABORTED (circuit breaker)\n"


def test_skipped_goes_to_stderr(capsys):
log = resource_logger("pdpc")
log.skipped("TAILSCALE_PROXY unset")
captured = capsys.readouterr()
assert captured.out == ""
assert captured.err == "pdpc: SKIPPED (TAILSCALE_PROXY unset)\n"


def test_jsonl_sink_appends_records(tmp_path, monkeypatch, capsys):
jsonl = tmp_path / "build.jsonl"
monkeypatch.setenv("ZEEKER_BUILDLOG_JSONL", str(jsonl))

log = resource_logger("judgments")
log.info("starting")
log.done(updated=50)
log.skipped("nothing new")
capsys.readouterr() # drain stream output

lines = jsonl.read_text().strip().splitlines()
assert len(lines) == 3
records = [json.loads(line) for line in lines]

assert records[0]["resource"] == "judgments"
assert records[0]["level"] == "info"
assert records[0]["event"] == "info"
assert records[0]["message"] == "starting"
assert records[0]["counts"] == {}
assert "ts" in records[0]

assert records[1]["event"] == "done"
assert records[1]["counts"] == {"updated": 50}

assert records[2]["event"] == "skipped"
assert records[2]["level"] == "warn"


def test_jsonl_disabled_when_env_unset(tmp_path, monkeypatch, capsys):
monkeypatch.delenv("ZEEKER_BUILDLOG_JSONL", raising=False)
log = resource_logger("res")
log.info("hello")
capsys.readouterr()
assert list(tmp_path.iterdir()) == []


def test_jsonl_failure_never_raises(tmp_path, monkeypatch, capsys):
# Point the sink at a directory — open(..., "a") fails, but the logger
# must swallow it and still emit the stream line.
monkeypatch.setenv("ZEEKER_BUILDLOG_JSONL", str(tmp_path))
log = resource_logger("res")
log.info("still works")
log.done(x=1)
captured = capsys.readouterr()
assert "res: still works\n" in captured.out
assert "res: done — 1 x\n" in captured.out


def test_jsonl_handles_non_serializable_counts(tmp_path, monkeypatch, capsys):
jsonl = tmp_path / "log.jsonl"
monkeypatch.setenv("ZEEKER_BUILDLOG_JSONL", str(jsonl))
log = resource_logger("res")
log.done(when=object()) # default=str fallback
capsys.readouterr()
record = json.loads(jsonl.read_text().strip())
assert record["event"] == "done"
assert "when" in record["counts"]
54 changes: 54 additions & 0 deletions packages/zeeker-common/tests/test_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,3 +147,57 @@ def test_sync_with_delay(self):

assert elapsed >= 2.0
assert get_count() == 3


class _Skip(Exception):
"""Stand-in for zeeker.Skip (zeeker-common can't depend on zeeker).

The retry decorators exclude by class name, so any exception class named
``Skip`` anywhere in the MRO must pass through without retries.
"""


_Skip.__name__ = "Skip"


class TestSkipPassesThrough:
"""zeeker.Skip is control flow, not a transient failure — the decorators
must re-raise it immediately (no backoff, no RetryError wrapping)."""

def test_sync_retry_does_not_retry_skip(self):
call_count = 0

@sync_retry
def declares_skip():
nonlocal call_count
call_count += 1
raise _Skip("proxy required")

with pytest.raises(_Skip):
declares_skip()
assert call_count == 1

@pytest.mark.anyio
async def test_async_retry_does_not_retry_skip(self):
call_count = 0

@async_retry
async def declares_skip():
nonlocal call_count
call_count += 1
raise _Skip("proxy required")

with pytest.raises(_Skip):
await declares_skip()
assert call_count == 1

def test_skip_subclass_also_passes_through(self):
class SubSkip(_Skip):
pass

@sync_retry
def declares_skip():
raise SubSkip("blocked")

with pytest.raises(SubSkip):
declares_skip()
20 changes: 18 additions & 2 deletions packages/zeeker-common/zeeker_common/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,21 @@
from .jina import get_jina_reader_content
from .retry import async_retry, sync_retry

__version__ = "0.1.0"
__all__ = ["get_hash_id", "get_jina_reader_content", "async_retry", "sync_retry"]
__version__ = "0.2.0"
__all__ = [
"get_hash_id",
"get_jina_reader_content",
"async_retry",
"sync_retry",
"resource_logger",
"ResourceLogger",
]


def __getattr__(name):
# Lazy exports — buildlog is pure stdlib, but keep import-time work minimal.
if name in ("resource_logger", "ResourceLogger"):
from . import buildlog

return getattr(buildlog, name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
Loading
Loading