diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08f0ca9..676d81a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,8 @@ jobs: --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 with: go-version-file: go.mod - name: Build diff --git a/.gitignore b/.gitignore index 68ad742..44adcab 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ cookies.txt # Secrets .env + +# Local test-drive credentials (never commit) +/secrets/ +docker-compose.testdrive.yml diff --git a/AGENT.md b/AGENT.md index 9103cbb..7e0d44e 100644 --- a/AGENT.md +++ b/AGENT.md @@ -48,14 +48,15 @@ When using Amp with `make dev` running in another window: ### Manual Local Development 1. Start PostgreSQL: `docker run -d --name postgres -e POSTGRES_USER=user -e POSTGRES_PASSWORD=pass -e POSTGRES_DB=arker -p 5432:5432 postgres:15` 2. Install Playwright: `go install github.com/mxschmitt/playwright-go/cmd/playwright@latest && playwright install chromium` -3. Install yt-dlp: `pip install yt-dlp` +3. Install yt-dlp and gallery-dl: `pip install yt-dlp gallery-dl` 4. Run: `go run .` ### Dependencies -- **Go 1.24+** (using Go 1.24.5 toolchain) +- **Go 1.25.12+** - **PostgreSQL 15** - **Git** (for repository archiving) -- **Python 3 + yt-dlp** (for YouTube archiving) +- **Python 3 + yt-dlp** (for video archiving) +- **Python 3 + gallery-dl** (for photo posts and mixed photo/video carousels) - **Python 3 + itch-dl** (for itch.io game archiving) - **Playwright + Chromium** (for MHTML and screenshots) @@ -70,9 +71,11 @@ When using Amp with `make dev` running in another window: │ │ ├── mhtml.go # MHTML webpage archiving │ │ ├── screenshot.go # Full-page screenshot capture │ │ ├── git.go # Git repository cloning -│ │ ├── youtube.go # YouTube video downloading +│ │ ├── ytdlp.go # Video downloading via yt-dlp +│ │ ├── gallery_dl.go # Photo/carousel downloading via gallery-dl │ │ ├── itch.go # itch.io game archiving -│ │ └── browser_utils.go # Shared browser utilities +│ │ ├── pwbundle.go # Playwright browser/page lifecycle +│ │ └── utils.go # Shared browser utilities & page loading │ ├── handlers/ # HTTP handlers │ │ ├── admin.go # Admin interface endpoints │ │ ├── api.go # REST API endpoints @@ -80,16 +83,25 @@ When using Amp with `make dev` running in another window: │ │ ├── display.go # Archive display pages │ │ ├── git.go # Git HTTP backend │ │ ├── itch_serve.go # itch.io individual file serving +│ │ ├── gallery_dl_serve.go # gallery-dl manifest + per-file serving +│ │ ├── thumb.go # Thumbnail serving + placeholder │ │ └── serve.go # File serving with streaming │ ├── models/ # Database models & types │ │ └── models.go # User, ArchivedURL, Capture, ArchiveItem │ ├── storage/ # Storage interface & implementations -│ │ └── fs.go # Filesystem storage (S3-ready interface) +│ │ ├── fs.go # Filesystem storage +│ │ ├── s3.go # S3/R2 storage with presigned direct URLs +│ │ ├── direct.go # DirectURLStorage interface +│ │ └── memory_storage.go # In-memory storage (tests) +│ ├── thumbnail/ # Derived preview images +│ │ └── thumbnail.go # Crop/scale/encode helper │ ├── monitoring/ # Browser process monitoring │ ├── utils/ # Shared utilities │ └── workers/ # Async job processing │ ├── queue.go # Job queue management -│ └── worker.go # Background worker implementation +│ ├── archive_worker.go # Archive job processing +│ ├── thumbnail_worker.go # On-demand thumbnail backfill +│ └── cleanup_worker.go # Stuck-job reaper ├── templates/ # HTML templates for web interface └── Makefile # Development workflow commands ``` @@ -97,16 +109,22 @@ When using Amp with `make dev` running in another window: ## Core Interfaces & Architecture ### Key Interfaces -- **`Storage`** - Pluggable storage backend (filesystem now, S3-ready) +- **`Storage`** - Pluggable storage backend (filesystem or S3/R2) - Methods: `Writer(key)`, `Reader(key)`, `Exists(key)`, `Size(key)` - - Current: Filesystem storage with zstd compression + - `SeekableStorage` adds `SeekableReader` for range requests; `DirectURLStorage` + adds presigned redirects. There is no delete method: the production bucket is + locked, so objects are written once under a nonced key and never replaced. + - Objects are stored **uncompressed** — the bytes on disk are exactly what the + archiver wrote. - **`Archiver`** - Different archiving strategies - - Methods: `Archive(url, writer)`, content type detection - - Types: MHTML, Screenshot, Git, YouTube, Itch + - Method: `Archive(ctx, url, logWriter, db, itemID) (Result, error)` + - `Result` carries the artifact reader, extension, content type, the Playwright + bundle (browser archivers), and an optional derived thumbnail + - Types: MHTML, Screenshot, Git, yt-dlp, gallery-dl, Itch ### Performance Features - **Browser Instance Reuse**: Playwright browsers reused across jobs for efficiency -- **Streaming**: All file operations use streaming with zstd compression +- **Streaming**: All file operations use streaming - **Async Processing**: Queue-based job processing with configurable worker pools - **Concurrent Workers**: Default 5 workers (configurable via `MAX_WORKERS`) - **Browser Monitoring**: Tracks browser processes to prevent memory leaks @@ -135,6 +153,10 @@ When using Amp with `make dev` running in another window: - `GET /git/:shortid` - Git HTTP backend for cloning repositories - `GET /itch/:shortid/file/*filepath` - Stream individual files from itch.io game archives - `GET /itch/:shortid/list` - JSON list of files in itch.io game archive +- `GET /gallery/:shortid/list` - JSON post metadata + media file list for a gallery-dl archive +- `GET /gallery/:shortid/file/*filepath` - Stream one media file out of a gallery-dl archive +- `GET|HEAD /thumb/:shortid` - Preview image for a capture (480x270 JPEG); falls back to an SVG placeholder and queues generation +- `GET|HEAD /thumb/:shortid/:type` - Preview image for one archive type ### Admin Interface (Session Authentication) - `GET /login` - Admin login page @@ -152,7 +174,7 @@ When using Amp with `make dev` running in another window: ### Git Repository Access ```bash -git clone https://archive.selfhosted.hackclub.com/git/{shortid} +git clone https://archive.hackclub.com/git/{shortid} ``` ## Configuration @@ -170,6 +192,8 @@ git clone https://archive.selfhosted.hackclub.com/git/{shortid} - `YTDLP_COOKIES_B64` - Base64-encoded cookies.txt content, written to a temp file at startup (used when `YTDLP_COOKIES_FILE` is unset; convenient for Coolify secrets) - `YTDLP_PROXY` - Optional proxy URL (e.g. `http://user:pass@host:port`, `socks5://...`) applied to every yt-dlp call. Instagram aggressively rate-limits datacenter IP ranges; a residential/mobile proxy is the reliable way to archive Instagram from a server. yt-dlp itself must also be kept current (installed from the nightly `--pre` channel) since Instagram breaks the extractor frequently. - `YTDLP_IMPERSONATE` - Optional yt-dlp `--impersonate` target for Instagram/TikTok/Facebook video URLs. Docker images default to `chrome` and install `curl-cffi`; set empty to disable for manual installs without curl-cffi. +- `GALLERYDL_USER_AGENT` - Optional `--user-agent` override for gallery-dl. Leave unset: gallery-dl sets a per-site User-Agent already (Instagram gets a current Chrome UA because it serves lower-quality video to anything else), and this replaces those defaults everywhere. +- `GALLERYDL_SLEEP_REQUEST` - Optional `--sleep-request` override (`"1"`, `"0.5-1.5"`). Leave unset. gallery-dl ships per-site request intervals (Instagram waits a randomized 6-12s between API calls); because `--sleep-request` is a root config key it *replaces* those rather than acting as a floor, so any value below a site's own default makes throttling more likely, not less. Set it only to slow gallery-dl down further. - `LOGIN_TEXT` - Text to display under login form @@ -188,13 +212,18 @@ git clone https://archive.selfhosted.hackclub.com/git/{shortid} ## Testing ### Test Files -- `storage_test.go` - Storage interface tests -- `archiver_test.go` - Archiver interface tests -- `monitoring_test.go` - Browser monitoring tests -- `validation_test.go` - Input validation tests -- `login_text_test.go` - Login text handling tests -- `vimeo_test.go` - Vimeo video archiving tests +Most tests live beside the package they cover (`internal/*/..._test.go`); a few +integration-level ones sit at the repo root (`storage_test.go`, +`monitoring_test.go`, `validation_test.go`). Run `go test ./...` rather than +working from a list here — this section has drifted before. + +Two conventions worth knowing: +- DB-backed tests use in-memory SQLite (`gorm.io/driver/sqlite`) with + `AutoMigrate`, so they need no running Postgres. See `newWorkerTestDB` in + `internal/workers/archive_worker_test.go` for the pattern. +- Handler tests build a real `gin` engine and drive it with `httptest`, so route + registration and middleware are exercised too. See `internal/handlers/thumb_test.go`. ### Running Tests ```bash @@ -213,7 +242,8 @@ go test -run TestFSStorage # Run specific test ### Archive & Browser - **mxschmitt/playwright-go** v0.6100.0 - Browser automation - **go-git/go-git/v5** v5.8.1 - Git operations -- **klauspost/compress** v1.18.0 - zstd compression +- **HugoSmits86/nativewebp** v1.2.0 - WebP encoding (lossless only) +- **golang.org/x/image** v0.44.0 - WebP decoding + high-quality rescaling ### Utilities - **golang.org/x/crypto** v0.33.0 - Password hashing (bcrypt) @@ -231,9 +261,52 @@ go test -run TestFSStorage # Run specific test ### Adding New Archive Types 1. Implement `Archiver` interface in `internal/archivers/` -2. Add to `archiversMap` in `cmd/main.go` -3. Update content type detection in handlers -4. Add tests in `archiver_test.go` +2. Add the type constant to `internal/utils/archive_types.go` (`canonicalArchiveTypes`) +3. Add to `archiversMap` in `cmd/main.go` +4. Route URLs to it in `utils.GetArchiveTypes` +5. Give it a timeout case in `utils.TimeoutForJobType` (the default is only 2 minutes) +6. Update `contentTypeForArchive` in `internal/handlers/serve.go` +7. Add it to `defaultTypePreference`/`getDisplayName` in `internal/handlers/display.go` and to the content pane in `templates/display_type.html` +8. Add tests + +Type names are stable identifiers: they appear in stored rows, in permalinks +(`/{shortid}/{type}`), and in API requests. Renaming one means adding a +`legacyArchiveTypeAliases` entry so old names keep resolving, plus a rename in +`migrateLegacyArchiveTypes`. Never just change the string. + +An archiver returns an `archivers.Result` struct, not a list of values, so a new +derived artifact does not churn every archiver's signature. `Result.Thumbnail` +is optional and nil is not an error. + +### Thumbnails + +Thumbnails are a **derived artifact**, not an archive type — they have no tab, +no permalink type segment, and no entry in `canonicalArchiveTypes`. They live in +four columns on `archive_items` (`thumbnail_key/width/height/status`) and are +served from `/thumb/{shortid}[/{type}]`. + +- **Size/format**: 480x270 JPEG (`internal/thumbnail`). JPEG because the only + WebP encoder in the tree (`nativewebp`) is lossless-only — it produced a + ~490KB "thumbnail" in testing versus ~35KB for JPEG, and it panics on some + low-colour-count inputs. `x/image/webp` is decode-only and reads back what + `nativewebp` wrote. +- **Generation is two-path**. New captures: `ScreenshotArchiver` derives it from + the image it has already decoded, so it costs one downscale and no extra + browser work. Pre-existing archives: the `/thumb` handler enqueues a + `ThumbnailJobArgs` job on the `high_priority` queue the first time somebody + views one. Never generate inline in a request — a full-page screenshot reaches + 60 megapixels (~240MB decoded) and one dashboard render asks for hundreds. +- **`thumbnail_status` must be set to `unavailable` for permanent failures** + (unsupported type, oversized, undecodable). Without it the lazy path + re-enqueues the same impossible job on every page view. +- **A thumbnail failure must never fail an archive.** The archive is the + product; the preview is not. +- Keys carry an upload nonce like archive keys (`{shortid}/{type}-{nonce}-thumb.jpg`) + because the bucket forbids overwrites and deletes. Regenerating means writing a + new object and repointing the row. +- `/thumb` always returns an image, falling back to a generated SVG placeholder + with a short `max-age` so a refresh picks up the real one. Callers can render + a card unconditionally. ### Database Changes 1. Update models in `internal/models/models.go` @@ -245,18 +318,23 @@ go test -run TestFSStorage # Run specific test ### Common Issues - **Playwright fails**: Ensure Chromium is installed (`playwright install chromium`) - **yt-dlp not found**: Install with `pip install yt-dlp` +- **gallery-dl not found**: Install with `pip install gallery-dl`. Install it into the *same* Python environment as yt-dlp: gallery-dl's default Instagram video path hands DASH manifests to yt-dlp as an importable module, and silently falls back to lower-quality pre-merged MP4 otherwise. - **Database connection**: Check PostgreSQL is running and credentials match - **Permission errors**: Ensure storage/cache directories are writable - **Browser leaks**: Check `/status/browser` endpoint for monitoring data ### Production Debugging -- **SSH Access**: `ssh root@archive.selfhosted.hackclub.com` +- **SSH Access**: `ssh archive-hq-local.selfhosted.hackclub.com` (via cloudflared, see `~/.ssh/config`; use `sudo docker ...` on the host) - **Health Checks**: Monitor `/health` and `/status/browser` endpoints - **Logs**: Check Coolify dashboard or container logs - **Database**: Connect via environment variables in deployment +- **Container names rotate per deploy** — resolve the app container with + `sudo docker ps | grep ^hcmolbh8` rather than assuming a fixed name +- **Don't run one-off tools inside the app container**: a Coolify deploy will + kill it mid-run. Run them on the host instead. ### Health Monitoring -- Startup health checks verify yt-dlp and Playwright availability +- Startup health checks verify yt-dlp, gallery-dl, and Playwright availability - Browser process monitoring with leak detection - Automatic log cleanup (30 days for completed items) @@ -266,7 +344,7 @@ go test -run TestFSStorage # Run specific test - **Modular Design**: Clean separation of concerns with internal packages - **Interface-Driven**: Storage and Archiver interfaces for extensibility - **Resilient Processing**: Error handling with retries, timeouts, and status tracking -- **Memory Efficient**: Streaming operations for large files with compression +- **Memory Efficient**: Streaming operations for large files - **Production Ready**: Docker deployment with health checks and resource limits - **Git Integration**: Full Git HTTP backend for repository cloning - **API-First**: RESTful API with web interface as overlay diff --git a/Dockerfile b/Dockerfile index 91f9934..49a35e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ -FROM golang:1.24-bookworm +FROM golang:1.25.12-bookworm # Install system dependencies in a single layer with aggressive cleanup RUN apt-get update && apt-get install -y --no-install-recommends \ git \ + tini \ python3 \ python3-pip \ curl \ @@ -55,6 +56,20 @@ RUN pip3 install --break-system-packages --no-cache-dir --upgrade --pre "yt-dlp[ && yt-dlp --version > /etc/yt-dlp-version \ && rm -rf /root/.cache +# gallery-dl handles photo posts and mixed photo/video carousels, which yt-dlp +# rejects outright. Its extractors break on the same cadence as yt-dlp's, so the +# remote ADD cache-busts on the latest release for the same reason. +# +# It is installed into the SAME environment as yt-dlp on purpose: gallery-dl's +# default Instagram video path hands DASH manifests to yt-dlp as an importable +# Python module, and silently falls back to lower-quality pre-merged MP4 when it +# cannot import one. requests[socks] is what makes a socks5:// value in +# YTDLP_PROXY work rather than warn and bypass the proxy. +ADD https://api.github.com/repos/mikf/gallery-dl/releases/latest /tmp/gallery-dl-release.json +RUN pip3 install --break-system-packages --no-cache-dir --upgrade gallery-dl "requests[socks]" \ + && gallery-dl --version > /etc/gallery-dl-version \ + && rm -rf /root/.cache + # The Docker image includes curl-cffi, so use yt-dlp's browser impersonation by # default for Instagram/TikTok/Facebook anti-bot responses. Arker applies this # only to those URL families. Override to empty to disable, or to another target @@ -90,4 +105,9 @@ RUN mkdir -p /data /cache EXPOSE 8080 +# tini reaps the zombie headless_shell/browser children that arker's own PID-1 +# process never wait()s on. Without an init, prod accumulates ~1k defunct PIDs +# per day and eventually hits the container's pids cgroup limit, at which point +# archiving breaks with fork failures. +ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["./arker"] diff --git a/Dockerfile.dev b/Dockerfile.dev index dfe6e9f..38834ff 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,5 +1,5 @@ # Development Dockerfile with live reloading -FROM golang:1.24-bookworm +FROM golang:1.25.12-bookworm # Install required packages and Playwright dependencies (matching production) RUN apt-get update && apt-get install -y \ @@ -37,9 +37,11 @@ RUN apt-get update && apt-get install -y \ wget \ && rm -rf /var/lib/apt/lists/* -# Install yt-dlp and itch-dl. Match production: nightly yt-dlp plus curl-cffi -# for browser impersonation. -RUN pip3 install --break-system-packages --upgrade --pre "yt-dlp[default,curl-cffi]" itch-dl +# Install yt-dlp, gallery-dl, and itch-dl. Match production: nightly yt-dlp plus +# curl-cffi for browser impersonation, gallery-dl in the same environment so its +# Instagram video path can import yt-dlp, and requests[socks] for socks5 proxies. +RUN pip3 install --break-system-packages --upgrade --pre "yt-dlp[default,curl-cffi]" \ + && pip3 install --break-system-packages --upgrade gallery-dl "requests[socks]" itch-dl ENV YTDLP_IMPERSONATE=chrome # Set working directory diff --git a/README.md b/README.md index e4b6bf2..b8338cb 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A self-hostable minimalist version of . - Creates Chrome snapshots of URLs and serves them at nice short URLs like -- Also supports git clones, YouTube videos, itch.io games, and website screenshots +- Also supports git clones, videos (yt-dlp), photo posts and carousels (gallery-dl), itch.io games, and website screenshots - Comprehensive API - Stores everything compressed using [zstd](https://github.com/facebook/zstd) level 6 (seekable format for random access) @@ -40,8 +40,8 @@ The `.env` file is optional. If it doesn't exist, Arker will use environment var - `ADMIN_PASSWORD` - Admin login password (default: `admin`) - `LOGIN_TEXT` - Custom text to display under the login form. Useful for providing demo credentials (e.g., `LOGIN_TEXT="Demo: admin/admin"`). Supports basic HTML. - `GIN_MODE` - Gin framework mode (`debug` for development) -- `YTDLP_COOKIES_FILE` / `YTDLP_COOKIES_B64` - Optional Netscape cookies.txt for yt-dlp. Useful for Instagram and other sites that require login. -- `YTDLP_PROXY` - Optional proxy URL passed to yt-dlp. A residential/mobile proxy may be needed when Instagram rate-limits datacenter IPs. +- `YTDLP_COOKIES_FILE` / `YTDLP_COOKIES_B64` - Optional Netscape cookies.txt, shared by yt-dlp and gallery-dl. Required for Instagram and other sites that require login. +- `YTDLP_PROXY` - Optional proxy URL passed to yt-dlp and gallery-dl. A residential/mobile proxy may be needed when Instagram rate-limits datacenter IPs. `socks5://` needs PySocks (`pip install "requests[socks]"`, already in the Docker images). - `YTDLP_IMPERSONATE` - Optional yt-dlp `--impersonate` target for Instagram/TikTok/Facebook video URLs. Production/dev Docker images default this to `chrome` and install `curl-cffi`; set it empty to disable. ### Itch.io Game Archiving @@ -53,9 +53,9 @@ The `.env` file is optional. If it doesn't exist, Arker will use environment var - Python 3.10+ with `itch-dl` package: `pip install itch-dl` - itch.io API key: Generate at https://itch.io/user/settings → API Keys -### Video Archiving +### Video Archiving (yt-dlp) -Arker shells out to `yt-dlp` for YouTube/Vimeo/Instagram/TikTok/Facebook-style videos. The production Dockerfile installs yt-dlp from the nightly (`--pre`) channel with `curl-cffi` because Instagram extractor fixes often land before stable releases. The Docker build also cache-busts on the latest yt-dlp nightly release metadata so redeploys do not keep a stale yt-dlp layer. +Arker shells out to `yt-dlp` for YouTube/Vimeo/Instagram-reel/TikTok/Facebook-style videos. The production Dockerfile installs yt-dlp from the nightly (`--pre`) channel with `curl-cffi` because Instagram extractor fixes often land before stable releases. The Docker build also cache-busts on the latest yt-dlp nightly release metadata so redeploys do not keep a stale yt-dlp layer. For manual installs, prefer: @@ -64,6 +64,38 @@ pip3 install --upgrade --pre "yt-dlp[default,curl-cffi]" YTDLP_IMPERSONATE=chrome # used for Instagram/TikTok/Facebook video URLs ``` +### Photo and Carousel Archiving (gallery-dl) + +yt-dlp only downloads video. A URL whose media is photos — an Instagram feed +post, an X status, a Reddit gallery — makes it fail outright with *"There is no +video in this post"*. Those URLs go to `gallery-dl` instead, which fetches every +image and video in the post along with the caption, author, date, and like count. + +The result is a ZIP holding every downloaded file, gallery-dl's raw per-file +metadata sidecars, and a normalized `metadata.json` written by Arker. The viewer +renders it as the original post; `/gallery/:shortid/list` returns the same data +as JSON. + +Routed hosts (post-shaped URLs only, so a profile link never pulls a whole +account): Instagram `/p/` and `/tv/`, X/Twitter, Reddit, Tumblr, Bluesky, Flickr, +Imgur, DeviantArt, ArtStation, Pixiv, Pinterest, Newgrounds, VSCO. Adding one is +a single entry in `galleryDLSites` in `internal/utils/url_utils.go`. + +```bash +pip3 install --upgrade gallery-dl "requests[socks]" +``` + +Install gallery-dl into the **same** Python environment as yt-dlp: gallery-dl's +default Instagram video path hands DASH manifests to yt-dlp as an importable +module, and silently falls back to lower-quality pre-merged MP4 if it cannot +import one. + +Instagram archiving needs cookies (`YTDLP_COOKIES_FILE`) — logged out, every +request redirects to the login page. Do not add a `GALLERYDL_SLEEP_REQUEST` +override to "be safe": gallery-dl already waits a randomized 6-12 seconds between +Instagram API calls, and that setting replaces the per-site default rather than +acting as a floor. + ### Storage Configuration Arker supports both filesystem and S3-compatible storage backends. diff --git a/cmd/main.go b/cmd/main.go index eb670b5..f9d9659 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -72,12 +72,18 @@ type Config struct { ItchAPIKey string `envconfig:"ITCH_API_KEY"` ItchDlPath string `envconfig:"ITCH_DL_PATH" default:"itch-dl"` - // yt-dlp authentication: Instagram (and some other sites) refuse media - // requests from logged-out clients, so captures need session cookies. + // Media tool authentication: Instagram (and some other sites) refuse media + // requests from logged-out clients, so captures need session cookies. The + // same jar and proxy serve both yt-dlp and gallery-dl; the env vars keep + // their YTDLP_ names for compatibility with existing deployments. YtDlpCookiesFile string `envconfig:"YTDLP_COOKIES_FILE"` // Path to a Netscape-format cookies.txt YtDlpCookiesB64 string `envconfig:"YTDLP_COOKIES_B64"` // Base64-encoded cookies.txt content (used when no file path is set) - YtDlpProxy string `envconfig:"YTDLP_PROXY"` // Optional proxy (e.g. residential) for yt-dlp; Instagram throttles datacenter IPs + YtDlpProxy string `envconfig:"YTDLP_PROXY"` // Optional proxy (e.g. residential); Instagram throttles datacenter IPs YtDlpImpersonate string `envconfig:"YTDLP_IMPERSONATE"` // Optional yt-dlp --impersonate target (Docker defaults to chrome) + + // gallery-dl Configuration (photo posts and mixed photo/video carousels) + GalleryDlUserAgent string `envconfig:"GALLERYDL_USER_AGENT"` // Optional UA override; empty keeps gallery-dl's per-site defaults + GalleryDlSleepRequest string `envconfig:"GALLERYDL_SLEEP_REQUEST"` // Optional inter-request delay ("1", "0.5-1.5"); empty keeps per-site defaults } // CustomErrorHandler implements the River ErrorHandler interface and updates archive items. @@ -174,6 +180,51 @@ func getOrCreateConfigValue(db *gorm.DB, key string, defaultValue string) (strin return config.Value, nil } +// migrateLegacyArchiveTypes rewrites rows still carrying a retired type name +// ("youtube" -> "yt-dlp"). Requests and jobs using the old name are normalized +// on the way in, so this only has to run once; it is idempotent and a no-op on +// every boot after the first. +// +// archive_items and queued river_job arguments must move together, in one +// transaction. The cleanup worker pairs an item with its job by comparing +// archive_items.type to river_job.args->>'type'; if only the items were +// renamed, every in-flight item would look orphaned and get force-failed while +// its job was still queued and about to succeed. +func migrateLegacyArchiveTypes(db *gorm.DB) error { + return db.Transaction(func(tx *gorm.DB) error { + for legacy, canonical := range utils.LegacyArchiveTypeAliases() { + // UpdateColumn, not Update: rewriting updated_at on every historical + // row would corrupt "last updated" ordering and shift the cleanup + // worker's staleness window. Unscoped so soft-deleted rows are + // renamed too and cannot resurface under an unreachable type. + result := tx.Unscoped().Model(&models.ArchiveItem{}). + Where("type = ?", legacy). + UpdateColumn("type", canonical) + if result.Error != nil { + return fmt.Errorf("rename archive type %q to %q: %w", legacy, canonical, result.Error) + } + itemsRenamed := result.RowsAffected + + jobResult := tx.Exec(` + UPDATE river_job + SET args = jsonb_set(args::jsonb, '{type}', to_jsonb(?::text)) + WHERE args->>'type' = ? + AND state IN ('available', 'running', 'retryable', 'scheduled', 'pending') + `, canonical, legacy) + if jobResult.Error != nil { + return fmt.Errorf("rename queued job type %q to %q: %w", legacy, canonical, jobResult.Error) + } + + if itemsRenamed > 0 || jobResult.RowsAffected > 0 { + slog.Info("Renamed legacy archive type", + "from", legacy, "to", canonical, + "items", itemsRenamed, "queued_jobs", jobResult.RowsAffected) + } + } + return nil + }) +} + func populateFileSizes(db *gorm.DB, storage storage.Storage) { var items []models.ArchiveItem // Find completed archive items that don't have file size set @@ -361,22 +412,37 @@ func main() { // Populate file sizes for existing archives populateFileSizes(db, storageInstance) - // Configure yt-dlp cookies for sites that require authentication (e.g. Instagram) + // Configure cookies for sites that require authentication (e.g. Instagram). + // Shared by yt-dlp and gallery-dl. + // + // A bad cookie path must not be fatal: a mis-typed mount once took + // production down for minutes because an unreadable file crash-looped the + // app and the orchestrator gave up on it. Losing logged-in archiving is bad; + // losing the whole server, including every archive type that needs no + // cookies at all, is much worse. cookiesPath, err := utils.InitYtDlpCookies(cfg.YtDlpCookiesFile, cfg.YtDlpCookiesB64, os.TempDir()) - if err != nil { - log.Fatalf("Failed to configure yt-dlp cookies: %v", err) - } - if cookiesPath != "" { - slog.Info("yt-dlp cookies configured", "path", cookiesPath) - } else { - slog.Warn("No yt-dlp cookies configured; videos on sites requiring login (e.g. Instagram) will fail to archive") + switch { + case err != nil: + slog.Error("Failed to configure media cookies; continuing without them. "+ + "Logged-in-only sites (e.g. Instagram) will fail to archive until this is fixed.", + "error", err) + case cookiesPath != "": + slog.Info("Media cookies configured", "path", cookiesPath) + default: + slog.Warn("No media cookies configured; posts on sites requiring login (e.g. Instagram) will fail to archive") } if proxy := utils.InitYtDlpProxy(cfg.YtDlpProxy); proxy != "" { - slog.Info("yt-dlp proxy configured") + slog.Info("Media proxy configured") } if impersonate := utils.InitYtDlpImpersonate(cfg.YtDlpImpersonate); impersonate != "" { slog.Info("yt-dlp browser impersonation configured", "target", impersonate) } + if userAgent := utils.InitGalleryDlUserAgent(cfg.GalleryDlUserAgent); userAgent != "" { + slog.Info("gallery-dl user agent override configured", "user_agent", userAgent) + } + if sleepRequest := utils.InitGalleryDlSleepRequest(cfg.GalleryDlSleepRequest); sleepRequest != "" { + slog.Info("gallery-dl request interval override configured", "sleep_request", sleepRequest) + } // Perform health checks on startup log.Println("Performing startup health checks...") @@ -402,11 +468,12 @@ func main() { // No shared browser manager - each job creates its own browser instance archiversMap := map[string]archivers.Archiver{ - "mhtml": &archivers.MHTMLArchiver{}, - "screenshot": &archivers.ScreenshotArchiver{}, - "git": &archivers.GitArchiver{}, - "youtube": &archivers.YTArchiver{}, - "itch": &archivers.ItchArchiver{ItchDlPath: cfg.ItchDlPath, APIKey: cfg.ItchAPIKey}, + utils.ArchiveTypeMHTML: &archivers.MHTMLArchiver{}, + utils.ArchiveTypeScreenshot: &archivers.ScreenshotArchiver{}, + utils.ArchiveTypeGit: &archivers.GitArchiver{}, + utils.ArchiveTypeYtDlp: &archivers.YtDlpArchiver{}, + utils.ArchiveTypeGalleryDl: &archivers.GalleryDLArchiver{}, + utils.ArchiveTypeItch: &archivers.ItchArchiver{ItchDlPath: cfg.ItchDlPath, APIKey: cfg.ItchAPIKey}, } os.MkdirAll(cfg.CachePath, 0755) @@ -428,10 +495,21 @@ func main() { } slog.Info("River migrations completed successfully") + // Runs here, not next to AutoMigrate, because it has to rewrite queued + // River job arguments too and river_job does not exist until the migration + // above has run. It still completes before riverClient.Start below, so no + // worker can observe a half-renamed queue. + if err := migrateLegacyArchiveTypes(db); err != nil { + slog.Error("Archive type rename migration failed", "error", err) + } + // Create worker registry riverWorkers := river.NewWorkers() archiveWorker := workers.NewArchiveWorker(storageInstance, db, archiversMap) river.AddWorker(riverWorkers, archiveWorker) + // Backfills thumbnails for archives captured before the feature existed. + // New captures produce theirs inline and never enqueue this. + river.AddWorker(riverWorkers, workers.NewThumbnailWorker(storageInstance, db)) // Create River client with configuration errorHandler := &CustomErrorHandler{db: db} @@ -543,7 +621,10 @@ func main() { admin.POST("/api-keys/:id/toggle", func(c *gin.Context) { handlers.ApiKeysToggle(c, db) }) admin.DELETE("/api-keys/:id", func(c *gin.Context) { handlers.ApiKeysDelete(c, db) }) admin.POST("/retry-failed", func(c *gin.Context) { handlers.RetryAllFailedJobs(c, db, riverClient) }) - admin.POST("/backfill-videos", func(c *gin.Context) { handlers.BackfillMissingVideoItems(c, db, riverClient) }) + admin.POST("/backfill-media", func(c *gin.Context) { handlers.BackfillMissingMediaItems(c, db, riverClient) }) + // Retained: the previous name for the endpoint above, kept working for + // existing operator scripts and runbooks. + admin.POST("/backfill-videos", func(c *gin.Context) { handlers.BackfillMissingMediaItems(c, db, riverClient) }) admin.POST("/url/:id/capture", func(c *gin.Context) { handlers.RequestCapture(c, db, riverClient) }) admin.POST("/archive", func(c *gin.Context) { handlers.AdminArchive(c, db, riverClient) }) admin.GET("/item/:id/log", func(c *gin.Context) { handlers.GetItemLog(c, db) }) @@ -569,11 +650,24 @@ func main() { r.HEAD("/archive/:shortid/:type", func(c *gin.Context) { handlers.ServeArchive(c, storageInstance, db) }) r.GET("/archive/:shortid/mhtml/html", func(c *gin.Context) { handlers.ServeMHTMLAsHTML(c, storageInstance, db) }) + // Thumbnail routes - MUST come before /:shortid/:type catch-all. + // HEAD is registered alongside GET, matching /archive/:shortid/:type: + // caches and link-preview crawlers probe with HEAD before fetching. + thumbHandler := func(c *gin.Context) { handlers.ServeThumbnail(c, storageInstance, db, riverClient) } + r.GET("/thumb/:shortid", thumbHandler) + r.HEAD("/thumb/:shortid", thumbHandler) + r.GET("/thumb/:shortid/*type", thumbHandler) + r.HEAD("/thumb/:shortid/*type", thumbHandler) + // Itch routes - MUST come before /:shortid/:type catch-all r.GET("/itch/health", handlers.ServeItchHealth) r.GET("/itch/:shortid/file/*filepath", func(c *gin.Context) { handlers.ServeItchFile(c, storageInstance, db) }) r.GET("/itch/:shortid/list", func(c *gin.Context) { handlers.ServeItchGameList(c, storageInstance, db) }) + // Gallery routes - MUST come before /:shortid/:type catch-all + r.GET("/gallery/:shortid/list", func(c *gin.Context) { handlers.ServeGalleryManifest(c, storageInstance, db) }) + r.GET("/gallery/:shortid/file/*filepath", func(c *gin.Context) { handlers.ServeGalleryFile(c, storageInstance, db) }) + r.Any("/git/*path", func(c *gin.Context) { handlers.GitHandler(c, storageInstance, db, cfg.CachePath) }) // Catch-all routes - MUST come last diff --git a/go.mod b/go.mod index e58866d..76b9e77 100644 --- a/go.mod +++ b/go.mod @@ -1,28 +1,27 @@ module arker -go 1.24.4 - -toolchain go1.24.5 +go 1.25.12 require ( github.com/HugoSmits86/nativewebp v1.2.0 - github.com/aws/aws-sdk-go-v2 v1.39.0 - github.com/aws/aws-sdk-go-v2/config v1.31.8 - github.com/aws/aws-sdk-go-v2/credentials v1.18.12 - github.com/aws/aws-sdk-go-v2/service/s3 v1.88.1 + github.com/aws/aws-sdk-go-v2 v1.42.1 + github.com/aws/aws-sdk-go-v2/config v1.32.30 + github.com/aws/aws-sdk-go-v2/credentials v1.19.29 + github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1 github.com/gin-contrib/sessions v0.0.5 github.com/gin-gonic/gin v1.9.1 - github.com/go-git/go-git/v5 v5.8.1 + github.com/go-git/go-git/v5 v5.19.2 github.com/go-playground/validator/v10 v10.27.0 - github.com/jackc/pgx/v5 v5.7.5 + github.com/jackc/pgx/v5 v5.10.0 github.com/joho/godotenv v1.5.1 github.com/kelseyhightower/envconfig v1.4.0 github.com/mxschmitt/playwright-go v0.6100.0 github.com/riverqueue/river v0.23.1 github.com/riverqueue/river/riverdriver/riverpgxv5 v0.23.1 github.com/riverqueue/river/rivertype v0.23.1 - golang.org/x/crypto v0.38.0 - golang.org/x/net v0.39.0 + golang.org/x/crypto v0.54.0 + golang.org/x/image v0.44.0 + golang.org/x/net v0.57.0 gorm.io/driver/postgres v1.5.2 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.30.0 @@ -31,39 +30,39 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect - github.com/acomagu/bufpipe v1.0.4 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.29.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.38.4 // indirect - github.com/aws/smithy-go v1.23.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.4.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.32.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.44.1 // indirect + github.com/aws/smithy-go v1.27.3 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.6.4 // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set/v2 v2.8.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.4.1 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-jose/go-jose/v3 v3.0.5 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/securecookie v1.1.1 // indirect github.com/gorilla/sessions v1.2.1 // indirect @@ -76,21 +75,21 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.44 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/riverqueue/apiframe v0.0.0-20250408034821-b206bbbd0fb4 // indirect github.com/riverqueue/river/riverdriver v0.23.1 // indirect github.com/riverqueue/river/rivershared v0.23.1 // indirect - github.com/sergi/go-diff v1.1.0 // indirect - github.com/skeema/knownhosts v1.2.0 // indirect - github.com/stretchr/testify v1.10.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/stretchr/testify v1.11.1 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -100,13 +99,10 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect go.uber.org/goleak v1.3.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/image v0.24.0 // indirect - golang.org/x/mod v0.24.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect - golang.org/x/tools v0.32.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index fc2fa66..3d3043c 100644 --- a/go.sum +++ b/go.sum @@ -3,62 +3,60 @@ dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/HugoSmits86/nativewebp v1.2.0 h1:XJtXeTg7FsOi9VB1elQYZy3n6VjYLqofSr3gGRLUOp4= github.com/HugoSmits86/nativewebp v1.2.0/go.mod h1:YNQuWenlVmSUUASVNhTDwf4d7FwYQGbGhklC8p72Vr8= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs= -github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go-v2 v1.39.0 h1:xm5WV/2L4emMRmMjHFykqiA4M/ra0DJVSWUkDyBjbg4= -github.com/aws/aws-sdk-go-v2 v1.39.0/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00= -github.com/aws/aws-sdk-go-v2/config v1.31.8 h1:kQjtOLlTU4m4A64TsRcqwNChhGCwaPBt+zCQt/oWsHU= -github.com/aws/aws-sdk-go-v2/config v1.31.8/go.mod h1:QPpc7IgljrKwH0+E6/KolCgr4WPLerURiU592AYzfSY= -github.com/aws/aws-sdk-go-v2/credentials v1.18.12 h1:zmc9e1q90wMn8wQbjryy8IwA6Q4XlaL9Bx2zIqdNNbk= -github.com/aws/aws-sdk-go-v2/credentials v1.18.12/go.mod h1:3VzdRDR5u3sSJRI4kYcOSIBbeYsgtVk7dG5R/U6qLWY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 h1:Is2tPmieqGS2edBnmOJIbdvOA6Op+rRpaYR60iBAwXM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7/go.mod h1:F1i5V5421EGci570yABvpIXgRIBPb5JM+lSkHF6Dq5w= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 h1:UCxq0X9O3xrlENdKf1r9eRJoKz/b0AfGkpp3a7FPlhg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7/go.mod h1:rHRoJUNUASj5Z/0eqI4w32vKvC7atoWR0jC+IkmVH8k= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 h1:Y6DTZUn7ZUC4th9FMBbo8LVE+1fyq3ofw+tRwkUd3PY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7/go.mod h1:x3XE6vMnU9QvHN/Wrx2s44kwzV2o2g5x/siw4ZUJ9g8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7 h1:BszAktdUo2xlzmYHjWMq70DqJ7cROM8iBd3f6hrpuMQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7/go.mod h1:XJ1yHki/P7ZPuG4fd3f0Pg/dSGA2cTQBCLw82MH2H48= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7 h1:zmZ8qvtE9chfhBPuKB2aQFxW5F/rpwXUgmcVCgQzqRw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7/go.mod h1:vVYfbpd2l+pKqlSIDIOgouxNsGu5il9uDp0ooWb0jys= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 h1:mLgc5QIgOy26qyh5bvW+nDoAppxgn3J2WV3m9ewq7+8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7/go.mod h1:wXb/eQnqt8mDQIQTTmcw58B5mYGxzLGZGK8PWNFZ0BA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7 h1:u3VbDKUCWarWiU+aIUK4gjTr/wQFXV17y3hgNno9fcA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7/go.mod h1:/OuMQwhSyRapYxq6ZNpPer8juGNrB4P5Oz8bZ2cgjQE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.88.1 h1:+RpGuaQ72qnU83qBKVwxkznewEdAGhIWo/PQCmkhhog= -github.com/aws/aws-sdk-go-v2/service/s3 v1.88.1/go.mod h1:xajPTguLoeQMAOE44AAP2RQoUhF8ey1g5IFHARv71po= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.3 h1:7PKX3VYsZ8LUWceVRuv0+PU+E7OtQb1lgmi5vmUE9CM= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.3/go.mod h1:Ql6jE9kyyWI5JHn+61UT/Y5Z0oyVJGmgmJbZD5g4unY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.4 h1:e0XBRn3AptQotkyBFrHAxFB8mDhAIOfsG+7KyJ0dg98= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.4/go.mod h1:XclEty74bsGBCr1s0VSaA11hQ4ZidK4viWK7rRfO88I= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.4 h1:PR00NXRYgY4FWHqOGx3fC3lhVKjsp1GdloDv2ynMSd8= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.4/go.mod h1:Z+Gd23v97pX9zK97+tX4ppAgqCt3Z2dIXB02CtBncK8= -github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= -github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= -github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek= +github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= +github.com/aws/aws-sdk-go-v2/config v1.32.30 h1:XwsEzpTJfQYJbFicz/QMLwAZdyeNVVoOEkbF7R3gPJk= +github.com/aws/aws-sdk-go-v2/config v1.32.30/go.mod h1:Ud32SuMc+/9BGxfpSVld7HrE2o05JwKmXY4M3jOQNZU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29 h1:WHZGssHH887cO0ox07SIQZsFx3MKD4ps6w0xUEmnKYQ= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29/go.mod h1:Mhl0xR6zjguiuj00XRx2wMx22sAltk7oya39sT7fdg8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1 h1:LkBKxAOE5WXjlFuFZqPG1rREnl6I6QCMElcXFDEidos= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.1/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do= +github.com/aws/aws-sdk-go-v2/service/signin v1.4.1 h1:V7ZZ300WPXGjvkyore5DGe0ljVPOxCXie/thWdtSBXE= +github.com/aws/aws-sdk-go-v2/service/signin v1.4.1/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg= +github.com/aws/aws-sdk-go-v2/service/sso v1.32.1 h1:gYFYh4iLLcAOJRLNPY2aD2g9DIhKn4eof8UkIrr1rTk= +github.com/aws/aws-sdk-go-v2/service/sso v1.32.1/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1 h1:arjT9Cm3/WYbGmD5TUZHk4UQn4Lle1fUNZs5FC6CtF0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84= +github.com/aws/aws-sdk-go-v2/service/sts v1.44.1 h1:RvfHDg+xvAeZ+5741vUEjpOVtYSIm93W2zhx10Xtydw= +github.com/aws/aws-sdk-go-v2/service/sts v1.44.1/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= -github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U= +github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -66,8 +64,8 @@ github.com/deckarep/golang-set/v2 v2.8.0 h1:swm0rlPCmdWn9mESxKOjWk8hXSqoxOp+Zlfu github.com/deckarep/golang-set/v2 v2.8.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= -github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= @@ -78,16 +76,16 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= -github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= -github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= -github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= -github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A= -github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQlVQ= github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -102,13 +100,11 @@ github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -124,8 +120,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= -github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -143,10 +139,9 @@ github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -157,8 +152,6 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8= @@ -172,11 +165,12 @@ github.com/mxschmitt/playwright-go v0.6100.0 h1:HYNnbGZsTHz8veJyDGe4fU1iPxfvXqzm github.com/mxschmitt/playwright-go v0.6100.0/go.mod h1:A7VtrS3j/c8ToGnSVUaOfNtQQVxi6JotUS0jeuus6r4= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= -github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -203,11 +197,11 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM= -github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -219,8 +213,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -246,34 +240,28 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= -golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= -golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ= -golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= +golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -281,48 +269,37 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= -golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -330,7 +307,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/archivers/archiver.go b/internal/archivers/archiver.go index 3e2cc80..c607da7 100644 --- a/internal/archivers/archiver.go +++ b/internal/archivers/archiver.go @@ -2,11 +2,47 @@ package archivers import ( "context" - "gorm.io/gorm" "io" + + "gorm.io/gorm" ) -// Archiver interface +// Thumbnail is an encoded preview image produced alongside the main artifact. +// +// It carries bytes rather than an io.Reader on purpose. Thumbnails are small +// (tens of KB), and the main artifact's reader is already a live process or a +// goroutine writing into an io.Pipe whose close semantics are delicate. A +// second reader with the same lifetime rules would be a second way to leak a +// blocked goroutine for no benefit. +type Thumbnail struct { + Data []byte + Width int + Height int +} + +// Result is what an archiver produces for one item. +// +// This is a struct rather than a list of return values so that adding a derived +// artifact -- a thumbnail today, extracted metadata or a transcript later -- does +// not churn the signature of every archiver again. +type Result struct { + // Data is the archived content. The worker owns closing it if it is also an + // io.Closer; see saveArchiveData. + Data io.Reader + Extension string + // ContentType is advisory. Serving derives its own content type from the + // archive type and extension, since that is what survives a restart. + ContentType string + // Bundle is the Playwright bundle for browser-based archivers. It must be + // returned even on the error paths so the worker can always clean it up. + Bundle *PWBundle + // Thumbnail is optional. A nil thumbnail is not an error: most archive + // types cannot produce one cheaply, and no caller may treat its absence as + // a failure. + Thumbnail *Thumbnail +} + +// Archiver captures a URL into a single stored artifact. type Archiver interface { - Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (data io.Reader, extension string, contentType string, bundle *PWBundle, err error) + Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (Result, error) } diff --git a/internal/archivers/gallery_dl.go b/internal/archivers/gallery_dl.go new file mode 100644 index 0000000..ff93af4 --- /dev/null +++ b/internal/archivers/gallery_dl.go @@ -0,0 +1,673 @@ +package archivers + +import ( + "archive/zip" + "context" + "encoding/json" + "fmt" + "io" + "mime" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "gorm.io/gorm" + + "arker/internal/utils" +) + +// GalleryDLArchiver downloads every media file behind a "post" URL using +// gallery-dl, the image-world counterpart to yt-dlp. It covers Instagram, +// X/Twitter, Reddit, Tumblr, Bluesky, Flickr, Imgur, Pixiv and ~300 other +// sites, and unlike yt-dlp it handles photo posts and mixed photo/video +// carousels instead of failing with "There is no video in this post". +// +// The output is a ZIP containing every downloaded file, gallery-dl's raw +// per-file metadata sidecars, and a normalized metadata.json written by Arker. +type GalleryDLArchiver struct{} + +// GalleryMetadata is Arker's normalized view of a gallery-dl capture. The +// raw, site-specific metadata is preserved alongside it in the ZIP, so this +// only needs to carry the fields a viewer wants: who posted, what they said, +// when, and what files came back. +type GalleryMetadata struct { + SourceURL string `json:"source_url"` + Extractor string `json:"extractor,omitempty"` + Subcategory string `json:"subcategory,omitempty"` + PostID string `json:"post_id,omitempty"` + PostURL string `json:"post_url,omitempty"` + Author string `json:"author,omitempty"` + AuthorName string `json:"author_name,omitempty"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + Date string `json:"date,omitempty"` + Likes *int64 `json:"likes,omitempty"` + Tags []string `json:"tags,omitempty"` + FileCount int `json:"file_count"` + Files []GalleryFile `json:"files"` + ToolVersion string `json:"gallery_dl_version,omitempty"` + ArchivedAt string `json:"archived_at"` +} + +// GalleryFile describes one downloaded media file inside the ZIP. +type GalleryFile struct { + Name string `json:"name"` + Size int64 `json:"size"` + ContentType string `json:"content_type,omitempty"` + IsVideo bool `json:"is_video"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + MetadataFile string `json:"metadata_file,omitempty"` +} + +// galleryMetadataFilename is the normalized metadata Arker adds at the ZIP +// root. gallery-dl is configured to emit numeric filenames (001.jpg), so this +// name can never collide with a downloaded media file. +const galleryMetadataFilename = "metadata.json" + +func (a *GalleryDLArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (Result, error) { + fmt.Fprintf(logWriter, "Starting gallery archive for: %s\n", url) + + select { + case <-ctx.Done(): + return Result{}, ctx.Err() + default: + } + + // gallery-dl only reads the jar (there is a separate --cookies-export for + // writing), but reuse the per-run private copy anyway: it keeps a + // read-only mounted secret safe and matches how yt-dlp is invoked. + cookieArgs, cleanupCookies, err := utils.MediaCookieArgsForRun() + if err != nil { + fmt.Fprintf(logWriter, "Failed to prepare gallery-dl cookies: %v\n", err) + return Result{}, err + } + defer cleanupCookies() + if len(cookieArgs) == 0 { + fmt.Fprintf(logWriter, "No cookies configured; logged-in-only sites (e.g. Instagram) will fail\n") + } + + version, versionErr := utils.GalleryDlVersion(ctx) + if versionErr == nil { + fmt.Fprintf(logWriter, "gallery-dl version: %s\n", version) + } else { + fmt.Fprintf(logWriter, "Could not determine gallery-dl version: %v\n", versionErr) + } + + tmpDir, err := os.MkdirTemp("", "arker-gallery-*") + if err != nil { + return Result{}, fmt.Errorf("failed to create temp directory: %w", err) + } + // Cleanup is handed to the ZIP goroutine on the success path; every early + // return below removes the directory itself. + cleanupTmp := func() { _ = os.RemoveAll(tmpDir) } + success := false + defer func() { + if !success { + cleanupTmp() + } + }() + + redactedLog := utils.NewRedactingWriter(logWriter, utils.MediaProxyRedactionSecrets()) + + args := galleryDlDownloadArgs(tmpDir) + args = append(args, cookieArgs...) + args = append(args, utils.MediaProxyArgs()...) + args = append(args, utils.GalleryDlUserAgentArgs()...) + args = append(args, utils.GalleryDlSleepArgs()...) + args = append(args, url) + + cmd := exec.CommandContext(ctx, "gallery-dl") + cmd.Args = append(cmd.Args, args...) + cmd.Stdout = redactedLog + cmd.Stderr = redactedLog + // Own process group so a timeout kills the whole tree, not just the parent. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + fmt.Fprintf(logWriter, "Starting gallery-dl download process...\n") + if err := cmd.Start(); err != nil { + fmt.Fprintf(logWriter, "Failed to start gallery-dl: %v\n", err) + return Result{}, err + } + + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-ctx.Done(): + if cmd.Process != nil { + fmt.Fprintf(logWriter, "Context cancelled, killing gallery-dl process group\n") + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } + case <-done: + } + }() + + runErr := cmd.Wait() + + media, sidecars, err := collectGalleryFiles(tmpDir) + if err != nil { + fmt.Fprintf(logWriter, "Failed to inspect gallery-dl output: %v\n", err) + return Result{}, err + } + + // gallery-dl exits non-zero for partial failures too, so a run that still + // produced media is worth keeping. Only fail when nothing came back. + if len(media) == 0 { + if runErr != nil { + fmt.Fprintf(logWriter, "gallery-dl failed: %v (%s)\n", runErr, describeGalleryDlExit(runErr)) + return Result{}, fmt.Errorf("gallery-dl failed: %w (%s)", runErr, describeGalleryDlExit(runErr)) + } + fmt.Fprintf(logWriter, "gallery-dl downloaded no files for %s\n", url) + return Result{}, fmt.Errorf("gallery-dl downloaded no files for %s", url) + } + if runErr != nil { + fmt.Fprintf(logWriter, "gallery-dl exited with %v (%s) but produced %d file(s); keeping partial archive\n", + runErr, describeGalleryDlExit(runErr), len(media)) + } + + metadata := buildGalleryMetadata(tmpDir, url, version, media, sidecars, logWriter) + metadataJSON, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + fmt.Fprintf(logWriter, "Failed to encode gallery metadata: %v\n", err) + return Result{}, fmt.Errorf("failed to encode gallery metadata: %w", err) + } + + fmt.Fprintf(logWriter, "Downloaded %d file(s) from %s\n", len(media), metadata.Extractor) + if metadata.Author != "" { + fmt.Fprintf(logWriter, "Author: %s\n", metadata.Author) + } + if metadata.Description != "" { + fmt.Fprintf(logWriter, "Caption: %s\n", utils.TruncateForLog(metadata.Description, 300)) + } + + // Stream the ZIP so a large carousel never has to sit in memory. + pipeReader, pipeWriter := io.Pipe() + success = true + go func() { + defer pipeWriter.Close() + defer cleanupTmp() + + zipWriter := zip.NewWriter(pipeWriter) + if err := writeGalleryZip(zipWriter, tmpDir, metadataJSON, logWriter); err != nil { + fmt.Fprintf(logWriter, "Error building gallery ZIP: %v\n", err) + _ = zipWriter.Close() + pipeWriter.CloseWithError(err) + return + } + if err := zipWriter.Close(); err != nil { + fmt.Fprintf(logWriter, "Error finalizing gallery ZIP: %v\n", err) + pipeWriter.CloseWithError(err) + return + } + fmt.Fprintf(logWriter, "Successfully created gallery ZIP archive\n") + }() + + return Result{Data: pipeReader, Extension: ".zip", ContentType: "application/zip"}, nil +} + +// galleryDlDownloadArgs builds the invariant part of the gallery-dl command +// line. Output is forced flat with zero-padded numeric names so the ZIP layout +// is identical for every site and can never collide with metadata.json. +func galleryDlDownloadArgs(destDir string) []string { + return []string{ + // Never pick up a gallery-dl.conf from the host: the archiver's + // behavior must depend only on what Arker passes. + "--config-ignore", + // -D (not -d) sets the destination AND clears the per-extractor + // subdirectory template, giving one flat directory per job. + "-D", destDir, + "-f", "{num:>03}.{extension}", + "--write-metadata", + // gallery-dl rewrites its --cookies file on exit by default, dumping + // every cookie it picked up along the way (CDN hosts included) into + // the jar. The run gets a private copy so the configured secret is + // never touched, but there is no reason to pay for the rewrite. + "-o", "cookies-update=false", + "--no-part", + "-R", "3", + "--http-timeout", "30", + // Deliberately not --verbose. Default output already names every file + // written and prints a one-line "[site][error] ..." on failure, which + // is what the archive log and the failure pane need; verbose adds + // urllib3 chatter and a full Python traceback around it. + } +} + +// galleryDlExitReasons maps gallery-dl's exit bits to human-readable causes. +// +// The exit status is a bitmask OR'd across every URL and extractor in the +// invocation, not an enum, so a single run can report several of these at once. +// The values are not documented upstream; they come from reading +// gallery_dl/exception.py and were confirmed against live runs. +var galleryDlExitReasons = []struct { + bit int + reason string +}{ + {1, "unexpected error"}, + {4, "extraction or download failed (404, redirect to login, rate limit)"}, + {8, "anti-bot challenge (e.g. Cloudflare)"}, + {16, "authentication required or rejected"}, + {32, "bad input (format string, filter, or input file)"}, + {64, "no gallery-dl extractor supports this URL"}, + {128, "local filesystem error"}, +} + +// describeGalleryDlExit turns a gallery-dl exit status into something a human +// reading archive logs can act on. +func describeGalleryDlExit(err error) string { + exitErr, ok := err.(*exec.ExitError) + if !ok { + return "gallery-dl did not run" + } + + code := exitErr.ExitCode() + var reasons []string + for _, entry := range galleryDlExitReasons { + if code&entry.bit != 0 { + reasons = append(reasons, entry.reason) + } + } + if len(reasons) == 0 { + return fmt.Sprintf("gallery-dl exit code %d", code) + } + return fmt.Sprintf("%s (exit code %d)", strings.Join(reasons, "; "), code) +} + +// collectGalleryFiles splits gallery-dl's flat output directory into media +// files and their JSON sidecars. +func collectGalleryFiles(dir string) (media []string, sidecars map[string]string, err error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, nil, err + } + + sidecars = make(map[string]string) + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if strings.HasSuffix(name, ".json") { + // gallery-dl names sidecars ".json". + sidecars[strings.TrimSuffix(name, ".json")] = name + continue + } + media = append(media, name) + } + + sort.Strings(media) + return media, sidecars, nil +} + +// buildGalleryMetadata normalizes gallery-dl's site-specific metadata into the +// handful of fields a viewer actually needs. Every site names things +// differently, so each field is resolved from a priority list of known keys +// and simply omitted when nothing matches. +func buildGalleryMetadata(dir, sourceURL, version string, media []string, sidecars map[string]string, logWriter io.Writer) *GalleryMetadata { + meta := &GalleryMetadata{ + SourceURL: sourceURL, + FileCount: len(media), + ToolVersion: version, + ArchivedAt: time.Now().UTC().Format(time.RFC3339), + } + + // Read the post record off the first media file's sidecar. gallery-dl + // merges the post-level metadata into every file's dict, so slide 1 + // carries the caption, author, and date for the whole post. + var raw map[string]interface{} + if len(media) > 0 { + if sidecar, ok := sidecars[media[0]]; ok { + raw = readGalleryJSON(filepath.Join(dir, sidecar), logWriter) + } + } + + if raw != nil { + meta.Extractor = galleryString(raw, "category") + meta.Subcategory = galleryString(raw, "subcategory") + + // "id" and "url" are ambiguous: at the top level they name the + // individual file (Imgur's image id and its CDN link), while the + // enclosing album/post object holds the ones a viewer wants. Prefer an + // explicit post-level key, then the container, and only then fall back. + meta.PostID = firstNonEmpty( + galleryString(raw, "post_id", "tweet_id", "post_shortcode", "shortcode"), + galleryNested(raw, "id"), + galleryString(raw, "id"), + ) + meta.PostURL = firstNonEmpty( + galleryString(raw, "post_url", "webpage_url"), + galleryNested(raw, "url"), + ) + + meta.Author, meta.AuthorName = resolveGalleryAuthor(raw) + + meta.Title = firstNonEmpty(galleryString(raw, "title"), galleryNested(raw, "title")) + + // Caption key order matters. Bluesky uses "text" for the post body and + // "description" for an image's alt text, so checking description first + // would surface alt text as the caption. Instagram has no "text" key, + // so it still resolves to "description". + meta.Description = firstNonEmpty( + galleryString(raw, "text", "content", "caption"), + galleryString(raw, "description"), + galleryNested(raw, "description"), + ) + + meta.Date = galleryString(raw, "post_date", "date", "created_at", "taken_at") + // Every site counts approval differently: likes, hearts, upvotes, + // favorites. Treat them as one number rather than modelling each. + // + // Order is by closeness to "likes", not convenience: a site can expose + // several of these at once and the first match wins. Imgur carries both + // upvote_count (366) and favorite_count (0) on the same album, so + // checking favourites first would report zero likes on a popular post. + meta.Likes = galleryInt(raw, + "likes", "like_count", "likeCount", + "upvote_count", "point_count", + "favorite_count", "favorites", "score") + meta.Tags = galleryStrings(raw, "tags", "hashtags") + } + + for _, name := range media { + file := GalleryFile{ + Name: name, + ContentType: galleryContentType(name), + MetadataFile: sidecars[name], + } + file.IsVideo = strings.HasPrefix(file.ContentType, "video/") + if info, err := os.Stat(filepath.Join(dir, name)); err == nil { + file.Size = info.Size() + } + if sidecar, ok := sidecars[name]; ok { + if perFile := readGalleryJSON(filepath.Join(dir, sidecar), logWriter); perFile != nil { + if w := galleryInt(perFile, "width"); w != nil { + file.Width = int(*w) + } + if h := galleryInt(perFile, "height"); h != nil { + file.Height = int(*h) + } + } + } + meta.Files = append(meta.Files, file) + } + + return meta +} + +func readGalleryJSON(path string, logWriter io.Writer) map[string]interface{} { + file, err := os.Open(path) + if err != nil { + return nil + } + defer file.Close() + + // UseNumber keeps large integers exact. Post IDs on X and Instagram exceed + // 2^53, so decoding them as float64 would silently corrupt the value + // recorded in metadata.json (1234567890123456789 -> ...456768). + decoder := json.NewDecoder(file) + decoder.UseNumber() + + var parsed map[string]interface{} + if err := decoder.Decode(&parsed); err != nil { + fmt.Fprintf(logWriter, "Could not parse %s: %v\n", filepath.Base(path), err) + return nil + } + return parsed +} + +// galleryContainers are objects extractors nest post-level metadata under. +// gallery-dl merges the post record into every file's dict, but some sites keep +// it in a sub-object instead: Imgur puts the album's title, URL and vote counts +// under "album" while the top level describes just the one image. +var galleryContainers = []string{"album", "post", "gallery", "tweet", "submission", "record"} + +// galleryPersonKeys are the objects that hold author details when an extractor +// models the poster as a nested record rather than flat fields. +var galleryPersonKeys = []string{"author", "user", "account", "owner", "uploader", "artist"} + +// galleryString returns the first key holding a non-empty string, searching the +// top level only. +func galleryString(raw map[string]interface{}, keys ...string) string { + for _, key := range keys { + switch value := raw[key].(type) { + case string: + if trimmed := strings.TrimSpace(value); trimmed != "" { + return trimmed + } + case json.Number: + // Numeric IDs keep their exact source digits. + return value.String() + } + } + return "" +} + +// galleryNested runs the same lookup against the known container objects only, +// so a caller can decide whether the container or the top level should win. +func galleryNested(raw map[string]interface{}, keys ...string) string { + for _, container := range galleryContainers { + nested, ok := raw[container].(map[string]interface{}) + if !ok { + continue + } + if found := galleryString(nested, keys...); found != "" { + return found + } + } + return "" +} + +// galleryObject returns a nested object by key, looking at the top level first +// and then inside the container objects. +func galleryObject(raw map[string]interface{}, key string) map[string]interface{} { + if object, ok := raw[key].(map[string]interface{}); ok { + return object + } + for _, container := range galleryContainers { + nested, ok := raw[container].(map[string]interface{}) + if !ok { + continue + } + if object, ok := nested[key].(map[string]interface{}); ok { + return object + } + } + return nil +} + +// resolveGalleryAuthor pulls a handle and a display name out of whichever shape +// the extractor uses: flat fields (Instagram's username/fullname) or a nested +// person object (Bluesky's author{handle,displayName}, Imgur's album.account). +func resolveGalleryAuthor(raw map[string]interface{}) (handle, display string) { + handle = galleryString(raw, "username", "uploader", "artist", "owner") + display = galleryString(raw, "fullname", "full_name", "author_name", "displayName", "display_name") + + for _, key := range galleryPersonKeys { + if handle != "" && display != "" { + break + } + person := galleryObject(raw, key) + if person == nil { + // Some extractors store the author as a bare string. + if handle == "" { + handle = galleryString(raw, key) + } + continue + } + if handle == "" { + handle = galleryString(person, "username", "handle", "name", "login", "nick") + } + if display == "" { + display = galleryString(person, "displayName", "display_name", "fullname", "full_name", "name") + } + } + + // A single name is a handle, not a display name. + if handle == "" && display != "" { + handle, display = display, "" + } + // Don't render "someone (someone)". + if handle == display { + display = "" + } + return handle, display +} + +// galleryInt returns the first key holding an integer, searching the top level +// then the container objects. +func galleryInt(raw map[string]interface{}, keys ...string) *int64 { + if found := galleryIntIn(raw, keys...); found != nil { + return found + } + for _, container := range galleryContainers { + nested, ok := raw[container].(map[string]interface{}) + if !ok { + continue + } + if found := galleryIntIn(nested, keys...); found != nil { + return found + } + } + return nil +} + +func galleryIntIn(raw map[string]interface{}, keys ...string) *int64 { + for _, key := range keys { + number, ok := raw[key].(json.Number) + if !ok { + continue + } + result, err := number.Int64() + if err != nil { + continue + } + return &result + } + return nil +} + +// firstNonEmpty returns the first non-empty candidate. +func firstNonEmpty(candidates ...string) string { + for _, candidate := range candidates { + if candidate != "" { + return candidate + } + } + return "" +} + +func galleryStrings(raw map[string]interface{}, keys ...string) []string { + for _, key := range keys { + values, ok := raw[key].([]interface{}) + if !ok { + continue + } + var result []string + for _, value := range values { + if str, ok := value.(string); ok && strings.TrimSpace(str) != "" { + result = append(result, strings.TrimSpace(str)) + } + } + if len(result) > 0 { + return result + } + } + return nil +} + +// galleryContentType maps a downloaded filename to a MIME type. mime's table +// misses a few formats social sites actually serve, so those are pinned here. +func galleryContentType(name string) string { + switch strings.ToLower(filepath.Ext(name)) { + case ".jpg", ".jpeg": + return "image/jpeg" + case ".png": + return "image/png" + case ".gif": + return "image/gif" + case ".webp": + return "image/webp" + case ".avif": + return "image/avif" + case ".mp4", ".m4v": + return "video/mp4" + case ".webm": + return "video/webm" + case ".mov": + return "video/quicktime" + case ".mkv": + return "video/x-matroska" + } + if byExt := mime.TypeByExtension(filepath.Ext(name)); byExt != "" { + return byExt + } + return "application/octet-stream" +} + +// writeGalleryZip stores Arker's metadata.json first so a reader can stream +// the header without buffering the whole archive, then every gallery-dl file. +// Media is stored uncompressed: JPEG/MP4 payloads do not deflate meaningfully +// and skipping it keeps large carousels cheap to write. +func writeGalleryZip(zipWriter *zip.Writer, dir string, metadataJSON []byte, logWriter io.Writer) error { + header := &zip.FileHeader{Name: galleryMetadataFilename, Method: zip.Deflate} + header.SetModTime(time.Now()) + writer, err := zipWriter.CreateHeader(header) + if err != nil { + return fmt.Errorf("create %s: %w", galleryMetadataFilename, err) + } + if _, err := writer.Write(metadataJSON); err != nil { + return fmt.Errorf("write %s: %w", galleryMetadataFilename, err) + } + + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + names = append(names, entry.Name()) + } + } + sort.Strings(names) + + for _, name := range names { + method := zip.Deflate + if !strings.HasSuffix(name, ".json") { + method = zip.Store + } + if err := addGalleryFileToZip(zipWriter, dir, name, method); err != nil { + fmt.Fprintf(logWriter, "Failed to add %s to ZIP: %v\n", name, err) + return fmt.Errorf("add file %s: %w", name, err) + } + } + return nil +} + +func addGalleryFileToZip(zipWriter *zip.Writer, dir, name string, method uint16) error { + file, err := os.Open(filepath.Join(dir, name)) + if err != nil { + return err + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return err + } + + header := &zip.FileHeader{Name: name, Method: method} + header.SetModTime(info.ModTime()) + + writer, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + _, err = io.Copy(writer, file) + return err +} diff --git a/internal/archivers/gallery_dl_test.go b/internal/archivers/gallery_dl_test.go new file mode 100644 index 0000000..f1be632 --- /dev/null +++ b/internal/archivers/gallery_dl_test.go @@ -0,0 +1,535 @@ +package archivers + +import ( + "archive/zip" + "bytes" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// writeGalleryFixture lays out a directory the way gallery-dl does with the +// flags the archiver passes: flat, numeric filenames, one JSON sidecar per file. +func writeGalleryFixture(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for name, contents := range files { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + return dir +} + +const instagramCarouselSidecar = `{ + "category": "instagram", + "subcategory": "post", + "post_id": "3955281333542808561", + "post_shortcode": "DbktPO1Eopi", + "post_url": "https://www.instagram.com/p/DbktPO1Eopi/", + "username": "agentverseinsta", + "fullname": "Agent Verse", + "description": "the news community is buzzing", + "post_date": "2026-08-04 18:12:03", + "likes": 42, + "tags": ["coding", "devops"], + "sidecar_media_id": "3955281333542808561", + "width": 1080, + "height": 1350, + "num": 1, + "count": 3 +}` + +func TestCollectGalleryFilesSeparatesMediaFromSidecars(t *testing.T) { + dir := writeGalleryFixture(t, map[string]string{ + "001.jpg": "a", + "001.jpg.json": "{}", + "002.mp4": "b", + "002.mp4.json": "{}", + "003.jpg": "c", + }) + + media, sidecars, err := collectGalleryFiles(dir) + if err != nil { + t.Fatalf("collectGalleryFiles: %v", err) + } + + want := []string{"001.jpg", "002.mp4", "003.jpg"} + if len(media) != len(want) { + t.Fatalf("media = %v, want %v", media, want) + } + for i := range want { + if media[i] != want[i] { + t.Fatalf("media = %v, want %v (must be sorted so slide order is stable)", media, want) + } + } + if sidecars["001.jpg"] != "001.jpg.json" { + t.Errorf("sidecars[001.jpg] = %q, want 001.jpg.json", sidecars["001.jpg"]) + } + // A media file with no sidecar must still be listed, not dropped. + if _, ok := sidecars["003.jpg"]; ok { + t.Error("003.jpg should have no sidecar") + } +} + +func TestBuildGalleryMetadataFromInstagramCarousel(t *testing.T) { + dir := writeGalleryFixture(t, map[string]string{ + "001.jpg": "aaaa", + "001.jpg.json": instagramCarouselSidecar, + "002.mp4": "bb", + "002.mp4.json": `{"width": 720, "height": 1280}`, + }) + + media, sidecars, err := collectGalleryFiles(dir) + if err != nil { + t.Fatalf("collectGalleryFiles: %v", err) + } + meta := buildGalleryMetadata(dir, "https://www.instagram.com/p/DbktPO1Eopi/", "1.32.9", media, sidecars, io.Discard) + + if meta.Extractor != "instagram" { + t.Errorf("Extractor = %q, want instagram", meta.Extractor) + } + if meta.Author != "agentverseinsta" { + t.Errorf("Author = %q, want agentverseinsta", meta.Author) + } + if meta.AuthorName != "Agent Verse" { + t.Errorf("AuthorName = %q, want Agent Verse", meta.AuthorName) + } + if meta.Description != "the news community is buzzing" { + t.Errorf("Description = %q, want the caption text", meta.Description) + } + if meta.PostID != "3955281333542808561" { + t.Errorf("PostID = %q, want 3955281333542808561", meta.PostID) + } + if meta.Date != "2026-08-04 18:12:03" { + t.Errorf("Date = %q, want the post date", meta.Date) + } + if meta.Likes == nil || *meta.Likes != 42 { + t.Errorf("Likes = %v, want 42", meta.Likes) + } + if len(meta.Tags) != 2 { + t.Errorf("Tags = %v, want 2 entries", meta.Tags) + } + if meta.FileCount != 2 || len(meta.Files) != 2 { + t.Fatalf("FileCount = %d / Files = %d, want 2 each", meta.FileCount, len(meta.Files)) + } + + image := meta.Files[0] + if image.ContentType != "image/jpeg" || image.IsVideo { + t.Errorf("file[0] = %+v, want a non-video image/jpeg", image) + } + if image.Size != 4 { + t.Errorf("file[0].Size = %d, want 4", image.Size) + } + if image.Width != 1080 || image.Height != 1350 { + t.Errorf("file[0] dimensions = %dx%d, want 1080x1350", image.Width, image.Height) + } + + // A carousel can mix stills and video; the video slide must be marked so + // the viewer renders a player instead of a broken image. + video := meta.Files[1] + if video.ContentType != "video/mp4" || !video.IsVideo { + t.Errorf("file[1] = %+v, want a video/mp4 marked IsVideo", video) + } +} + +func TestBuildGalleryMetadataToleratesMissingAndMalformedSidecars(t *testing.T) { + dir := writeGalleryFixture(t, map[string]string{ + "001.jpg": "a", + "001.jpg.json": "{not json", + "002.png": "b", + }) + + media, sidecars, err := collectGalleryFiles(dir) + if err != nil { + t.Fatalf("collectGalleryFiles: %v", err) + } + meta := buildGalleryMetadata(dir, "https://example.com/post/1", "1.32.9", media, sidecars, io.Discard) + + // Unparseable metadata must degrade to "no metadata", never lose the media. + if meta.FileCount != 2 { + t.Errorf("FileCount = %d, want 2", meta.FileCount) + } + if meta.SourceURL != "https://example.com/post/1" { + t.Errorf("SourceURL = %q, want the source URL", meta.SourceURL) + } + if meta.Author != "" || meta.Description != "" { + t.Errorf("expected empty post fields, got author=%q description=%q", meta.Author, meta.Description) + } +} + +// Some extractors nest the author in an object rather than a flat username. +// Unwrapping that is resolveGalleryAuthor's job, not galleryString's: +// galleryString stays a plain top-level string lookup so callers can control +// whether the top level or a container object wins. +func TestNestedAuthorObjectsAreResolvedByAuthorResolver(t *testing.T) { + raw := map[string]interface{}{ + "user": map[string]interface{}{"name": "someone"}, + } + if got := galleryString(raw, "user"); got != "" { + t.Errorf("galleryString on an object = %q, want empty", got) + } + if handle, _ := resolveGalleryAuthor(raw); handle != "someone" { + t.Errorf("resolveGalleryAuthor = %q, want someone", handle) + } +} + +func TestWriteGalleryZipContainsMetadataAndMedia(t *testing.T) { + dir := writeGalleryFixture(t, map[string]string{ + "001.jpg": "image-bytes", + "001.jpg.json": instagramCarouselSidecar, + }) + + var buf bytes.Buffer + zipWriter := zip.NewWriter(&buf) + if err := writeGalleryZip(zipWriter, dir, []byte(`{"source_url":"x"}`), io.Discard); err != nil { + t.Fatalf("writeGalleryZip: %v", err) + } + if err := zipWriter.Close(); err != nil { + t.Fatalf("close zip: %v", err) + } + + reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) + if err != nil { + t.Fatalf("read zip: %v", err) + } + + got := map[string]string{} + for _, file := range reader.File { + contents, err := file.Open() + if err != nil { + t.Fatalf("open %s: %v", file.Name, err) + } + data, err := io.ReadAll(contents) + contents.Close() + if err != nil { + t.Fatalf("read %s: %v", file.Name, err) + } + got[file.Name] = string(data) + } + + // Arker's metadata.json must be present alongside gallery-dl's raw sidecar, + // so the archive is self-describing without re-running gallery-dl. + for _, name := range []string{galleryMetadataFilename, "001.jpg", "001.jpg.json"} { + if _, ok := got[name]; !ok { + t.Errorf("zip is missing %s (has %v)", name, keys(got)) + } + } + if got["001.jpg"] != "image-bytes" { + t.Errorf("001.jpg = %q, want image-bytes", got["001.jpg"]) + } + var metadata map[string]interface{} + if err := json.Unmarshal([]byte(got[galleryMetadataFilename]), &metadata); err != nil { + t.Errorf("metadata.json is not valid JSON: %v", err) + } +} + +// A file that cannot be read must fail the whole archive rather than being +// silently skipped, or a partial ZIP gets stored as completed. +func TestWriteGalleryZipPropagatesFileError(t *testing.T) { + dir := writeGalleryFixture(t, map[string]string{"001.jpg": "a"}) + if err := os.Chmod(filepath.Join(dir, "001.jpg"), 0o000); err != nil { + t.Skipf("cannot chmod in this environment: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(filepath.Join(dir, "001.jpg"), 0o600) }) + + if os.Geteuid() == 0 { + t.Skip("running as root; permission bits do not restrict reads") + } + + zipWriter := zip.NewWriter(io.Discard) + if err := writeGalleryZip(zipWriter, dir, []byte("{}"), io.Discard); err == nil { + t.Fatal("expected writeGalleryZip to fail on an unreadable file, got nil") + } +} + +func TestGalleryContentType(t *testing.T) { + tests := map[string]struct { + contentType string + isVideo bool + }{ + "001.jpg": {"image/jpeg", false}, + "001.JPEG": {"image/jpeg", false}, + "002.png": {"image/png", false}, + "003.webp": {"image/webp", false}, + "004.mp4": {"video/mp4", true}, + "005.webm": {"video/webm", true}, + "006.mov": {"video/quicktime", true}, + "007.bin": {"application/octet-stream", false}, + } + + for name, want := range tests { + got := galleryContentType(name) + if got != want.contentType { + t.Errorf("galleryContentType(%q) = %q, want %q", name, got, want.contentType) + } + if strings.HasPrefix(got, "video/") != want.isVideo { + t.Errorf("galleryContentType(%q) video detection = %v, want %v", name, !want.isVideo, want.isVideo) + } + } +} + +// gallery-dl's exit status is a bitmask, so a single run can report several +// causes at once. Decoding it as an enum would mislabel most real failures. +func TestDescribeGalleryDlExitDecodesBitmask(t *testing.T) { + tests := []struct { + code int + contains []string + }{ + {64, []string{"no gallery-dl extractor"}}, + {4, []string{"extraction or download failed"}}, + {16, []string{"authentication"}}, + {8, []string{"anti-bot challenge"}}, + {20, []string{"extraction or download failed", "authentication"}}, + {68, []string{"extraction or download failed", "no gallery-dl extractor"}}, + } + + for _, tt := range tests { + err := exitErrorWithCode(t, tt.code) + got := describeGalleryDlExit(err) + for _, want := range tt.contains { + if !strings.Contains(got, want) { + t.Errorf("describeGalleryDlExit(%d) = %q, want it to mention %q", tt.code, got, want) + } + } + } + + if got := describeGalleryDlExit(errors.New("boom")); got != "gallery-dl did not run" { + t.Errorf("describeGalleryDlExit(non-exit error) = %q, want %q", got, "gallery-dl did not run") + } +} + +// The archiver must never let a gallery-dl config file on the host change what +// it does, and must not let gallery-dl rewrite the shared cookie jar. +func TestGalleryDlDownloadArgsAreHermetic(t *testing.T) { + args := galleryDlDownloadArgs("/tmp/out") + joined := strings.Join(args, " ") + + for _, want := range []string{ + "--config-ignore", + "-D /tmp/out", + "cookies-update=false", + "--write-metadata", + "--no-part", + } { + if !strings.Contains(joined, want) { + t.Errorf("gallery-dl args %v missing %q", args, want) + } + } + + // -d keeps gallery-dl's per-site subdirectory tree; only -D flattens it, + // and a flat layout is what the ZIP and the viewer assume. + for i, arg := range args { + if arg == "-d" { + t.Errorf("args[%d] uses -d, which preserves subdirectories; want -D", i) + } + } +} + +func exitErrorWithCode(t *testing.T, code int) error { + t.Helper() + err := exec.Command("sh", "-c", "exit "+itoa(code)).Run() + if err == nil { + t.Fatalf("expected a non-zero exit for code %d", code) + } + return err +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + var digits []byte + for n > 0 { + digits = append([]byte{byte('0' + n%10)}, digits...) + n /= 10 + } + return string(digits) +} + +func keys(m map[string]string) []string { + result := make([]string, 0, len(m)) + for key := range m { + result = append(result, key) + } + return result +} + +// JSON numbers decode to float64 by default, which silently rounds integers +// above 2^53. Real X and Instagram post IDs are in that range, so the ID +// recorded in metadata.json would not match the ID on the site. +func TestBuildGalleryMetadataPreservesLargeNumericIDs(t *testing.T) { + dir := writeGalleryFixture(t, map[string]string{ + "001.jpg": "a", + "001.jpg.json": `{ + "category": "twitter", + "tweet_id": 1234567890123456789, + "likes": 9007199254740995, + "width": 1200, + "height": 675 + }`, + }) + + media, sidecars, err := collectGalleryFiles(dir) + if err != nil { + t.Fatalf("collectGalleryFiles: %v", err) + } + meta := buildGalleryMetadata(dir, "https://x.com/a/status/1234567890123456789", "1.32.9", media, sidecars, io.Discard) + + if meta.PostID != "1234567890123456789" { + t.Errorf("PostID = %q, want 1234567890123456789 exactly (float64 would give ...456768)", meta.PostID) + } + if meta.Likes == nil || *meta.Likes != 9007199254740995 { + t.Errorf("Likes = %v, want 9007199254740995 exactly", meta.Likes) + } + if meta.Files[0].Width != 1200 || meta.Files[0].Height != 675 { + t.Errorf("dimensions = %dx%d, want 1200x675", meta.Files[0].Width, meta.Files[0].Height) + } +} + +// Bluesky nests the poster in author{handle,displayName}, counts approval as +// likeCount, and — critically — uses "text" for the post body while +// "description" holds the image's alt text. Reading description as the caption +// put alt text on the page where the post should be. +const blueskySidecar = `{ + "category": "bluesky", + "subcategory": "post", + "post_id": "3mqafridzgk2e", + "author": {"did": "did:plc:z72", "handle": "bsky.app", "displayName": "Bluesky"}, + "likeCount": 3485, + "repostCount": 582, + "text": "v1.127 is live! We're rolling out improvements to search", + "description": "A rendering of the new Filters button", + "date": "2026-07-09 19:50:16", + "width": 2140, + "height": 2000 +}` + +// Imgur describes the individual image at the top level and keeps the album's +// title, URL, uploader and vote counts in a nested object. +const imgurAlbumSidecar = `{ + "category": "imgur", + "subcategory": "album", + "id": "kKu3U5P", + "url": "https://i.imgur.com/kKu3U5P.jpg", + "title": "", + "description": "", + "width": 1537, + "height": 2048, + "date": "2026-08-07 14:32:03", + "album": { + "id": "zJjxIyO", + "title": "The Baroness", + "url": "https://imgur.com/a/zJjxIyO", + "upvote_count": 366, + "point_count": 342, + "favorite_count": 0, + "score": 0, + "image_count": 5, + "account": {"id": 27958845, "username": "somebody"} + } +}` + +func metaFrom(t *testing.T, sidecar string) *GalleryMetadata { + t.Helper() + dir := writeGalleryFixture(t, map[string]string{ + "001.jpg": "x", + "001.jpg.json": sidecar, + }) + media, sidecars, err := collectGalleryFiles(dir) + if err != nil { + t.Fatalf("collectGalleryFiles: %v", err) + } + return buildGalleryMetadata(dir, "https://example.com/post", "1.32.9", media, sidecars, io.Discard) +} + +func TestBuildGalleryMetadataFromBlueskyPost(t *testing.T) { + meta := metaFrom(t, blueskySidecar) + + if meta.Author != "bsky.app" { + t.Errorf("Author = %q, want bsky.app (from the nested author object)", meta.Author) + } + if meta.AuthorName != "Bluesky" { + t.Errorf("AuthorName = %q, want Bluesky", meta.AuthorName) + } + if meta.Likes == nil || *meta.Likes != 3485 { + t.Errorf("Likes = %v, want 3485 (likeCount)", meta.Likes) + } + // The regression that motivated this: alt text must not become the caption. + if meta.Description != "v1.127 is live! We're rolling out improvements to search" { + t.Errorf("Description = %q, want the post text, not the image alt text", meta.Description) + } +} + +func TestBuildGalleryMetadataFromImgurAlbum(t *testing.T) { + meta := metaFrom(t, imgurAlbumSidecar) + + if meta.Title != "The Baroness" { + t.Errorf("Title = %q, want The Baroness (from album.title)", meta.Title) + } + if meta.Author != "somebody" { + t.Errorf("Author = %q, want somebody (from album.account.username)", meta.Author) + } + // The album carries upvote_count 366 and favorite_count 0 side by side; + // a key order that checked favourites first reported 0 likes on a post + // with hundreds. + if meta.Likes == nil || *meta.Likes != 366 { + t.Errorf("Likes = %v, want 366 (album.upvote_count, not favorite_count 0)", meta.Likes) + } + // The album identifies the post; the top-level id/url describe one image. + if meta.PostID != "zJjxIyO" { + t.Errorf("PostID = %q, want the album id zJjxIyO, not the image id", meta.PostID) + } + if meta.PostURL != "https://imgur.com/a/zJjxIyO" { + t.Errorf("PostURL = %q, want the album URL, not the image CDN URL", meta.PostURL) + } +} + +// Instagram must keep working: it has no "text" key, flat username/fullname, +// and its caption genuinely lives in "description". +func TestBuildGalleryMetadataInstagramStillResolves(t *testing.T) { + meta := metaFrom(t, instagramCarouselSidecar) + + if meta.Author != "agentverseinsta" || meta.AuthorName != "Agent Verse" { + t.Errorf("author = %q / %q, want agentverseinsta / Agent Verse", meta.Author, meta.AuthorName) + } + if meta.Description != "the news community is buzzing" { + t.Errorf("Description = %q, want the caption from description", meta.Description) + } + if meta.Likes == nil || *meta.Likes != 42 { + t.Errorf("Likes = %v, want 42", meta.Likes) + } + if meta.PostID != "3955281333542808561" { + t.Errorf("PostID = %q, want the post_id", meta.PostID) + } +} + +func TestResolveGalleryAuthorEdgeCases(t *testing.T) { + // A bare string author. + h, d := resolveGalleryAuthor(map[string]interface{}{"author": "someone"}) + if h != "someone" || d != "" { + t.Errorf("bare string author = %q/%q, want someone/''", h, d) + } + // Only a display name available: it becomes the handle rather than + // rendering an empty handle with a parenthesised name. + h, d = resolveGalleryAuthor(map[string]interface{}{"fullname": "Some One"}) + if h != "Some One" || d != "" { + t.Errorf("display-only = %q/%q, want Some One/''", h, d) + } + // Identical handle and display name must not render as "x (x)". + h, d = resolveGalleryAuthor(map[string]interface{}{ + "author": map[string]interface{}{"handle": "x", "displayName": "x"}, + }) + if h != "x" || d != "" { + t.Errorf("duplicate name = %q/%q, want x/''", h, d) + } + // Nothing at all. + if h, d = resolveGalleryAuthor(map[string]interface{}{}); h != "" || d != "" { + t.Errorf("empty = %q/%q, want empty", h, d) + } +} diff --git a/internal/archivers/git.go b/internal/archivers/git.go index acbe19c..ea16fbf 100644 --- a/internal/archivers/git.go +++ b/internal/archivers/git.go @@ -59,13 +59,13 @@ func installGitProtocols() { }) } -func (a *GitArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (io.Reader, string, string, *PWBundle, error) { +func (a *GitArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (Result, error) { fmt.Fprintf(logWriter, "Starting git archive for: %s\n", url) // Check context before starting select { case <-ctx.Done(): - return nil, "", "", nil, ctx.Err() + return Result{}, ctx.Err() default: } @@ -81,7 +81,7 @@ func (a *GitArchiver) Archive(ctx context.Context, url string, logWriter io.Writ tempDir, err := os.MkdirTemp("", "git-archive-") if err != nil { fmt.Fprintf(logWriter, "Failed to create temp directory: %v\n", err) - return nil, "", "", nil, err + return Result{}, err } cleanup := func() { os.RemoveAll(tempDir) } @@ -93,7 +93,7 @@ func (a *GitArchiver) Archive(ctx context.Context, url string, logWriter io.Writ if err != nil { fmt.Fprintf(logWriter, "Failed to clone repository: %v\n", err) cleanup() - return nil, "", "", nil, err + return Result{}, err } fmt.Fprintf(logWriter, "Repository cloned successfully\n") @@ -139,7 +139,7 @@ func (a *GitArchiver) Archive(ctx context.Context, url string, logWriter io.Writ } }() - return pr, ".tar", "application/x-tar", nil, nil + return Result{Data: pr, Extension: ".tar", ContentType: "application/x-tar"}, nil } // extractGitRepoURL extracts the repository URL from GitHub URLs with extra paths and fragments diff --git a/internal/archivers/git_cleanup_test.go b/internal/archivers/git_cleanup_test.go index 8ed0506..ee9f9e8 100644 --- a/internal/archivers/git_cleanup_test.go +++ b/internal/archivers/git_cleanup_test.go @@ -31,7 +31,8 @@ func TestGitArchiveCleansTempDir(t *testing.T) { before := countTempCloneDirs(t) a := &GitArchiver{} - r, _, _, _, err := a.Archive(context.Background(), "file://"+repo, io.Discard, nil, 0) + res, err := a.Archive(context.Background(), "file://"+repo, io.Discard, nil, 0) + r := res.Data if err != nil { t.Fatalf("archive: %v", err) } diff --git a/internal/archivers/itch.go b/internal/archivers/itch.go index 5b58110..bea02e6 100644 --- a/internal/archivers/itch.go +++ b/internal/archivers/itch.go @@ -47,25 +47,25 @@ type GameFile struct { Size int64 `json:"size"` } -func (a *ItchArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (io.Reader, string, string, *PWBundle, error) { +func (a *ItchArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (Result, error) { fmt.Fprintf(logWriter, "Starting itch archive for: %s\n", url) // Check context before starting select { case <-ctx.Done(): - return nil, "", "", nil, ctx.Err() + return Result{}, ctx.Err() default: } // Check if API key is available if a.APIKey == "" { - return nil, "", "", nil, fmt.Errorf("itch.io API key not configured") + return Result{}, fmt.Errorf("itch.io API key not configured") } // Create temporary directory for itch-dl output tmpDir, err := os.MkdirTemp("", "itch-archive-*") if err != nil { - return nil, "", "", nil, fmt.Errorf("failed to create temp directory: %w", err) + return Result{}, fmt.Errorf("failed to create temp directory: %w", err) } // Note: Don't defer cleanup here - it will happen before ZIP creation completes @@ -81,7 +81,7 @@ func (a *ItchArchiver) Archive(ctx context.Context, url string, logWriter io.Wri output, err := cmd.CombinedOutput() if err != nil { fmt.Fprintf(logWriter, "itch-dl error: %v\nOutput: %s\n", err, string(output)) - return nil, "", "", nil, fmt.Errorf("itch-dl failed: %w", err) + return Result{}, fmt.Errorf("itch-dl failed: %w", err) } fmt.Fprintf(logWriter, "itch-dl completed successfully\n") @@ -90,7 +90,7 @@ func (a *ItchArchiver) Archive(ctx context.Context, url string, logWriter io.Wri // Find the downloaded game directory gameDir, err := findGameDirectory(tmpDir) if err != nil { - return nil, "", "", nil, fmt.Errorf("failed to find game directory: %w", err) + return Result{}, fmt.Errorf("failed to find game directory: %w", err) } fmt.Fprintf(logWriter, "Found game directory: %s\n", gameDir) @@ -98,7 +98,7 @@ func (a *ItchArchiver) Archive(ctx context.Context, url string, logWriter io.Wri // Parse metadata metadata, err := parseItchMetadata(gameDir, logWriter) if err != nil { - return nil, "", "", nil, fmt.Errorf("failed to parse metadata: %w", err) + return Result{}, fmt.Errorf("failed to parse metadata: %w", err) } fmt.Fprintf(logWriter, "Parsed metadata: %s\n", metadata.Title) @@ -123,7 +123,7 @@ func (a *ItchArchiver) Archive(ctx context.Context, url string, logWriter io.Wri fmt.Fprintf(logWriter, "Successfully created ZIP archive\n") }() - return pipeReader, ".zip", "application/zip", nil, nil + return Result{Data: pipeReader, Extension: ".zip", ContentType: "application/zip"}, nil } // findGameDirectory locates the downloaded game directory diff --git a/internal/archivers/mhtml.go b/internal/archivers/mhtml.go index 38c4067..80bb3c1 100644 --- a/internal/archivers/mhtml.go +++ b/internal/archivers/mhtml.go @@ -13,37 +13,37 @@ import ( type MHTMLArchiver struct { } -func (a *MHTMLArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (io.Reader, string, string, *PWBundle, error) { +func (a *MHTMLArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (Result, error) { fmt.Fprintf(logWriter, "Starting MHTML archive for: %s\n", url) bundle, page, err := setupBrowserForArchiving(logWriter) if err != nil { // If bundle is not nil, it means the browser was created and must be cleaned up by the worker. - return nil, "", "", bundle, err + return Result{Bundle: bundle}, err } // Note: PWBundle cleanup is deferred in the main worker loop. if err = PerformCompletePageLoadWithContext(ctx, page, url, logWriter, true); err != nil { - return nil, "", "", bundle, err + return Result{Bundle: bundle}, err } return a.ArchiveWithPageContext(ctx, page, url, logWriter, bundle) } -func (a *MHTMLArchiver) ArchiveWithPageContext(ctx context.Context, page playwright.Page, url string, logWriter io.Writer, bundle *PWBundle) (io.Reader, string, string, *PWBundle, error) { +func (a *MHTMLArchiver) ArchiveWithPageContext(ctx context.Context, page playwright.Page, url string, logWriter io.Writer, bundle *PWBundle) (Result, error) { fmt.Fprintf(logWriter, "Creating CDP session for MHTML capture...\n") // Check context before creating CDP session select { case <-ctx.Done(): - return nil, "", "", bundle, ctx.Err() + return Result{Bundle: bundle}, ctx.Err() default: } session, err := page.Context().NewCDPSession(page) if err != nil { fmt.Fprintf(logWriter, "Failed to create CDP session: %v\n", err) - return nil, "", "", bundle, err + return Result{Bundle: bundle}, err } fmt.Fprintf(logWriter, "Capturing MHTML snapshot with context awareness...\n") @@ -51,7 +51,7 @@ func (a *MHTMLArchiver) ArchiveWithPageContext(ctx context.Context, page playwri // Check context before MHTML capture select { case <-ctx.Done(): - return nil, "", "", bundle, ctx.Err() + return Result{Bundle: bundle}, ctx.Err() default: } @@ -72,7 +72,7 @@ func (a *MHTMLArchiver) ArchiveWithPageContext(ctx context.Context, page playwri select { case <-ctx.Done(): fmt.Fprintf(logWriter, "Context cancelled during MHTML capture\n") - return nil, "", "", bundle, ctx.Err() + return Result{Bundle: bundle}, ctx.Err() case cdpRes := <-resultChan: result = cdpRes.result err = cdpRes.err @@ -80,16 +80,16 @@ func (a *MHTMLArchiver) ArchiveWithPageContext(ctx context.Context, page playwri if err != nil { fmt.Fprintf(logWriter, "Failed to capture MHTML snapshot: %v\n", err) - return nil, "", "", bundle, err + return Result{Bundle: bundle}, err } dataStr, err := parseMHTMLSnapshot(result) if err != nil { fmt.Fprintf(logWriter, "Failed to parse MHTML snapshot result: %v\n", err) - return nil, "", "", bundle, err + return Result{Bundle: bundle}, err } fmt.Fprintf(logWriter, "MHTML archive completed successfully, size: %d bytes\n", len(dataStr)) - return strings.NewReader(dataStr), ".mhtml", "application/x-mhtml", bundle, nil + return Result{Data: strings.NewReader(dataStr), Extension: ".mhtml", ContentType: "application/x-mhtml", Bundle: bundle}, nil } // parseMHTMLSnapshot extracts the MHTML payload from a Page.captureSnapshot CDP diff --git a/internal/archivers/screenshot.go b/internal/archivers/screenshot.go index 36f5fdc..95fbcaa 100644 --- a/internal/archivers/screenshot.go +++ b/internal/archivers/screenshot.go @@ -11,13 +11,15 @@ import ( "image/jpeg" "image/png" "io" + + "arker/internal/thumbnail" ) // ScreenshotArchiver type ScreenshotArchiver struct { } -func (a *ScreenshotArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (io.Reader, string, string, *PWBundle, error) { +func (a *ScreenshotArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (Result, error) { fmt.Fprintf(logWriter, "Starting screenshot archive for: %s\n", url) pageOpts := playwright.BrowserNewPageOptions{ @@ -30,22 +32,22 @@ func (a *ScreenshotArchiver) Archive(ctx context.Context, url string, logWriter bundle, page, err := setupBrowserForArchiving(logWriter, pageOpts) if err != nil { - return nil, "", "", bundle, err + return Result{Bundle: bundle}, err } // Note: PWBundle cleanup is deferred in the main worker loop. if err = PerformCompletePageLoadWithContext(ctx, page, url, logWriter, true); err != nil { - return nil, "", "", bundle, err + return Result{Bundle: bundle}, err } return a.ArchiveWithPageContext(ctx, page, url, logWriter, bundle) } -func (a *ScreenshotArchiver) ArchiveWithPageContext(ctx context.Context, page playwright.Page, url string, logWriter io.Writer, bundle *PWBundle) (io.Reader, string, string, *PWBundle, error) { +func (a *ScreenshotArchiver) ArchiveWithPageContext(ctx context.Context, page playwright.Page, url string, logWriter io.Writer, bundle *PWBundle) (Result, error) { // Check context before screenshot operations select { case <-ctx.Done(): - return nil, "", "", bundle, ctx.Err() + return Result{Bundle: bundle}, ctx.Err() default: } @@ -67,7 +69,7 @@ func (a *ScreenshotArchiver) ArchiveWithPageContext(ctx context.Context, page pl // Check context before taking screenshot select { case <-ctx.Done(): - return nil, "", "", bundle, ctx.Err() + return Result{Bundle: bundle}, ctx.Err() default: } @@ -78,7 +80,7 @@ func (a *ScreenshotArchiver) ArchiveWithPageContext(ctx context.Context, page pl }) if err != nil { fmt.Fprintf(logWriter, "Failed to take screenshot: %v\n", err) - return nil, "", "", bundle, err + return Result{Bundle: bundle}, err } // Decode PNG and select optimal format @@ -87,11 +89,23 @@ func (a *ScreenshotArchiver) ArchiveWithPageContext(ctx context.Context, page pl img, err := png.Decode(bytes.NewReader(data)) if err != nil { fmt.Fprintf(logWriter, "Failed to decode PNG: %v\n", err) - return nil, "", "", bundle, err + return Result{Bundle: bundle}, err } fmt.Fprintf(logWriter, "Image decoded, bounds: %v\n", img.Bounds()) + // Derive the thumbnail here, from the image we have already decoded. + // + // This is why the screenshot archiver is the thumbnail source: the decode + // above is work we do regardless, so the thumbnail costs one downscale and + // no second browser round-trip. Deriving it later from the stored artifact + // would mean re-decoding a full-page screenshot that can reach 60 + // megapixels. + // + // A thumbnail failure is never an archive failure: the screenshot itself is + // fine, and a missing preview is a cosmetic loss. + thumb := deriveThumbnail(img, logWriter) + // Select format based on image dimensions extension, mimeType, format := selectImageFormat(img, logWriter) @@ -138,7 +152,25 @@ func (a *ScreenshotArchiver) ArchiveWithPageContext(ctx context.Context, page pl } }() - return pipeReader, extension, mimeType, bundle, nil + return Result{ + Data: pipeReader, + Extension: extension, + ContentType: mimeType, + Bundle: bundle, + Thumbnail: thumb, + }, nil +} + +// deriveThumbnail builds the preview image for a screenshot, returning nil (and +// logging) rather than an error on any failure. +func deriveThumbnail(img image.Image, logWriter io.Writer) *Thumbnail { + t, err := thumbnail.FromImage(img) + if err != nil { + fmt.Fprintf(logWriter, "Thumbnail generation skipped: %v\n", err) + return nil + } + fmt.Fprintf(logWriter, "Thumbnail generated: %dx%d, %d bytes\n", t.Width, t.Height, len(t.Data)) + return &Thumbnail{Data: t.Data, Width: t.Width, Height: t.Height} } // selectImageFormat determines the best format based on image dimensions diff --git a/internal/archivers/youtube.go b/internal/archivers/ytdlp.go similarity index 84% rename from internal/archivers/youtube.go rename to internal/archivers/ytdlp.go index cd3e9f3..49167d1 100644 --- a/internal/archivers/youtube.go +++ b/internal/archivers/ytdlp.go @@ -27,16 +27,19 @@ func (r *tempVideoReader) Close() error { return err2 } -// YTArchiver downloads videos from YouTube, Vimeo, and other platforms (streams directly from yt-dlp stdout) -type YTArchiver struct{} +// YtDlpArchiver downloads videos from YouTube, Vimeo, Instagram reels, TikTok +// and other platforms via yt-dlp. It handles video only; a URL whose media is +// photos (or a mixed photo/video carousel) belongs to GalleryDLArchiver, which +// yt-dlp rejects with "There is no video in this post". +type YtDlpArchiver struct{} -func (a *YTArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (io.Reader, string, string, *PWBundle, error) { +func (a *YtDlpArchiver) Archive(ctx context.Context, url string, logWriter io.Writer, db *gorm.DB, itemID uint) (Result, error) { fmt.Fprintf(logWriter, "Starting video archive for: %s\n", url) // Check context before starting select { case <-ctx.Done(): - return nil, "", "", nil, ctx.Err() + return Result{}, ctx.Err() default: } @@ -46,7 +49,7 @@ func (a *YTArchiver) Archive(ctx context.Context, url string, logWriter io.Write cookieArgs, cleanupCookies, err := utils.YtDlpCookieArgsForRun() if err != nil { fmt.Fprintf(logWriter, "Failed to prepare yt-dlp cookies: %v\n", err) - return nil, "", "", nil, err + return Result{}, err } defer cleanupCookies() @@ -72,21 +75,21 @@ func (a *YTArchiver) Archive(ctx context.Context, url string, logWriter io.Write testOutput, err := testCmd.CombinedOutput() if err != nil { fmt.Fprintf(redactedLog, "yt-dlp test failed: %v\nOutput: %s\n", err, string(testOutput)) - return nil, "", "", nil, fmt.Errorf("yt-dlp cannot access video: %v", err) + return Result{}, fmt.Errorf("yt-dlp cannot access video: %v", err) } fmt.Fprintf(redactedLog, "Video info:\n%s\n", string(testOutput)) // Check context before main download select { case <-ctx.Done(): - return nil, "", "", nil, ctx.Err() + return Result{}, ctx.Err() default: } tempBase, err := createTempVideoBase() if err != nil { fmt.Fprintf(logWriter, "Failed to create temp video path: %v\n", err) - return nil, "", "", nil, err + return Result{}, err } keepTempFile := "" defer func() { @@ -109,7 +112,7 @@ func (a *YTArchiver) Archive(ctx context.Context, url string, logWriter io.Write fmt.Fprintf(logWriter, "Starting yt-dlp download process...\n") if err = cmd.Start(); err != nil { fmt.Fprintf(logWriter, "Failed to start yt-dlp: %v\n", err) - return nil, "", "", nil, err + return Result{}, err } // Kill the whole process group when the context times out @@ -128,25 +131,25 @@ func (a *YTArchiver) Archive(ctx context.Context, url string, logWriter io.Write if err = cmd.Wait(); err != nil { fmt.Fprintf(logWriter, "yt-dlp download failed: %v\n", err) - return nil, "", "", nil, fmt.Errorf("yt-dlp download failed: %w", err) + return Result{}, fmt.Errorf("yt-dlp download failed: %w", err) } outputPath, err := findDownloadedMP4(tempBase) if err != nil { fmt.Fprintf(logWriter, "Failed to find downloaded MP4: %v\n", err) - return nil, "", "", nil, err + return Result{}, err } file, err := os.Open(outputPath) if err != nil { fmt.Fprintf(logWriter, "Failed to open downloaded MP4: %v\n", err) - return nil, "", "", nil, err + return Result{}, err } keepTempFile = outputPath fmt.Fprintf(logWriter, "Video download completed successfully\n") - return &tempVideoReader{File: file, path: outputPath}, ".mp4", "video/mp4", nil, nil + return Result{Data: &tempVideoReader{File: file, path: outputPath}, Extension: ".mp4", ContentType: "video/mp4"}, nil } func ytDlpDownloadArgs(outputTemplate string) []string { diff --git a/internal/archivers/youtube_test.go b/internal/archivers/ytdlp_test.go similarity index 100% rename from internal/archivers/youtube_test.go rename to internal/archivers/ytdlp_test.go diff --git a/internal/handlers/admin.go b/internal/handlers/admin.go index ebef59a..d282dc8 100644 --- a/internal/handlers/admin.go +++ b/internal/handlers/admin.go @@ -139,11 +139,13 @@ func GetItemLog(c *gin.Context, db *gorm.DB) { } // RetryAllFailedJobs directly retries all failed archive items. -// Pass ?type=youtube to retry only one archive type. +// Pass ?type=yt-dlp (or any archive type) to retry only one archive type. func RetryAllFailedJobs(c *gin.Context, db *gorm.DB, riverClient *river.Client[pgx.Tx]) { query := db.Where("status = 'failed'") if typ := c.Query("type"); typ != "" { - query = query.Where("type = ?", typ) + // Normalize so a runbook that still says ?type=youtube keeps matching + // rows rather than silently retrying nothing. + query = query.Where("type = ?", utils.NormalizeArchiveType(typ)) } // Get all failed items @@ -199,73 +201,135 @@ func RetryAllFailedJobs(c *gin.Context, db *gorm.DB, riverClient *river.Client[p }) } -// BackfillMissingVideoItems creates and enqueues youtube archive items for -// captures of video URLs that never had one (e.g. TikTok short links archived -// before short-link detection existed). Failed youtube items are re-run via -// RetryAllFailedJobs instead. Pass ?dry_run=true to preview without queueing. -func BackfillMissingVideoItems(c *gin.Context, db *gorm.DB, riverClient *river.Client[pgx.Tx]) { - dryRun := c.Query("dry_run") == "true" +// mediaBackfillURLPattern pre-filters candidate URLs in SQL for each media +// archive type. Loading every capture and filtering in Go blows Postgres's +// 65535-parameter limit at production scale (~100k captures); the Go predicate +// named alongside each pattern stays the exact filter. +var mediaBackfillURLPattern = map[string]struct { + sqlLike string + matches func(string) bool +}{ + utils.ArchiveTypeYtDlp: { + sqlLike: "(LOWER(archived_urls.original) LIKE '%youtube.com%' OR LOWER(archived_urls.original) LIKE '%youtu.be%' OR LOWER(archived_urls.original) LIKE '%vimeo.com%' OR LOWER(archived_urls.original) LIKE '%instagram.com%' OR LOWER(archived_urls.original) LIKE '%tiktok.com%' OR LOWER(archived_urls.original) LIKE '%facebook.com%' OR LOWER(archived_urls.original) LIKE '%fb.watch%')", + matches: utils.IsVideoURL, + }, + utils.ArchiveTypeGalleryDl: { + sqlLike: "(LOWER(archived_urls.original) LIKE '%instagram.com%' OR LOWER(archived_urls.original) LIKE '%twitter.com%' OR LOWER(archived_urls.original) LIKE '%x.com%' OR LOWER(archived_urls.original) LIKE '%reddit.com%' OR LOWER(archived_urls.original) LIKE '%redd.it%' OR LOWER(archived_urls.original) LIKE '%tumblr.com%' OR LOWER(archived_urls.original) LIKE '%bsky.app%' OR LOWER(archived_urls.original) LIKE '%flickr.com%' OR LOWER(archived_urls.original) LIKE '%imgur.com%' OR LOWER(archived_urls.original) LIKE '%deviantart.com%' OR LOWER(archived_urls.original) LIKE '%artstation.com%' OR LOWER(archived_urls.original) LIKE '%pixiv.net%' OR LOWER(archived_urls.original) LIKE '%pinterest.com%' OR LOWER(archived_urls.original) LIKE '%newgrounds.com%' OR LOWER(archived_urls.original) LIKE '%vsco.co%')", + matches: utils.IsGalleryDLURL, + }, +} - // Pre-filter video URLs in SQL: loading every youtube-less capture and - // filtering in Go blows Postgres's 65535-parameter limit at production - // scale (~100k captures). IsVideoURL below stays the exact filter. - videoURLPattern := "(LOWER(archived_urls.original) LIKE '%youtube.com%' OR LOWER(archived_urls.original) LIKE '%youtu.be%' OR LOWER(archived_urls.original) LIKE '%vimeo.com%' OR LOWER(archived_urls.original) LIKE '%instagram.com%' OR LOWER(archived_urls.original) LIKE '%tiktok.com%' OR LOWER(archived_urls.original) LIKE '%facebook.com%' OR LOWER(archived_urls.original) LIKE '%fb.watch%')" +// BackfillMissingMediaItems creates and enqueues missing media archive items +// for captures whose URL should have one. +// +// Two things produce these gaps: a URL family that gained support after the +// capture was taken (TikTok short links, or every Instagram photo post taken +// before gallery-dl existed), and detection rules that changed underneath +// existing rows. Failed items are re-run via RetryAllFailedJobs instead — this +// only creates items that are absent entirely. +// +// Pass ?type=gallery-dl or ?type=yt-dlp to backfill one type (default: both), +// ?dry_run=true to preview without queueing, and ?limit=N to bound a run. +// Bounding matters for Instagram, which has soft-blocked this account for hours +// in response to bulk traffic. +func BackfillMissingMediaItems(c *gin.Context, db *gorm.DB, riverClient *river.Client[pgx.Tx]) { + dryRun := c.Query("dry_run") == "true" - var candidates []struct { - ID uint - ShortID string - Original string - } - if err := db.Table("captures"). - Select("captures.id, captures.short_id, archived_urls.original"). - Joins("JOIN archived_urls ON archived_urls.id = captures.archived_url_id"). - Where("captures.deleted_at IS NULL AND archived_urls.deleted_at IS NULL"). - Where("NOT EXISTS (SELECT 1 FROM archive_items WHERE archive_items.capture_id = captures.id AND archive_items.type = 'youtube' AND archive_items.deleted_at IS NULL)"). - Where(videoURLPattern). - Scan(&candidates).Error; err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to query captures"}) - return + requestedTypes := []string{utils.ArchiveTypeGalleryDl, utils.ArchiveTypeYtDlp} + if requested := c.Query("type"); requested != "" { + canonical := utils.NormalizeArchiveType(requested) + if _, ok := mediaBackfillURLPattern[canonical]; !ok { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("unsupported backfill type: %s", requested)}) + return + } + requestedTypes = []string{canonical} } - backfilled := []string{} - for _, capture := range candidates { - if !utils.IsVideoURL(capture.Original) { - continue - } - if dryRun { - backfilled = append(backfilled, capture.ShortID) - continue + limit := 0 + if rawLimit := c.Query("limit"); rawLimit != "" { + parsed, err := strconv.Atoi(rawLimit) + if err != nil || parsed < 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "limit must be a non-negative integer"}) + return } + limit = parsed + } - item := models.ArchiveItem{CaptureID: capture.ID, Type: "youtube", Status: "pending"} - if err := db.Create(&item).Error; err != nil { - continue - } + backfilled := map[string][]string{} + total := 0 - args := workers.ArchiveJobArgs{ - ShortID: capture.ShortID, - Type: "youtube", - URL: capture.Original, + for _, archiveType := range requestedTypes { + filter := mediaBackfillURLPattern[archiveType] + // The limit is per type, not a shared budget: with both types selected + // a shared budget would be spent entirely on whichever runs first and + // silently do none of the other. + typeCount := 0 + + var candidates []struct { + ID uint + ShortID string + Original string } - opts := &river.InsertOpts{ - MaxAttempts: 3, - Tags: []string{"archive", "youtube", "backfill"}, - UniqueOpts: river.UniqueOpts{ - ByArgs: true, - ByPeriod: 1 * time.Minute, - }, + if err := db.Table("captures"). + Select("captures.id, captures.short_id, archived_urls.original"). + Joins("JOIN archived_urls ON archived_urls.id = captures.archived_url_id"). + Where("captures.deleted_at IS NULL AND archived_urls.deleted_at IS NULL"). + Where("NOT EXISTS (SELECT 1 FROM archive_items WHERE archive_items.capture_id = captures.id AND archive_items.type = ? AND archive_items.deleted_at IS NULL)", archiveType). + Where(filter.sqlLike). + // Deterministic order so a bounded dry run previews the same rows + // the real run will take. + Order("captures.id"). + Scan(&candidates).Error; err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to query captures"}) + return } - if _, err := riverClient.Insert(c.Request.Context(), args, opts); err != nil { - continue + + for _, capture := range candidates { + if limit > 0 && typeCount >= limit { + break + } + if !filter.matches(capture.Original) { + continue + } + if dryRun { + backfilled[archiveType] = append(backfilled[archiveType], capture.ShortID) + typeCount++ + total++ + continue + } + + item := models.ArchiveItem{CaptureID: capture.ID, Type: archiveType, Status: "pending"} + if err := db.Create(&item).Error; err != nil { + continue + } + + args := workers.ArchiveJobArgs{ + ShortID: capture.ShortID, + Type: archiveType, + URL: capture.Original, + } + opts := &river.InsertOpts{ + MaxAttempts: 3, + Tags: []string{"archive", archiveType, "backfill"}, + UniqueOpts: river.UniqueOpts{ + ByArgs: true, + ByPeriod: 1 * time.Minute, + }, + } + if _, err := riverClient.Insert(c.Request.Context(), args, opts); err != nil { + continue + } + backfilled[archiveType] = append(backfilled[archiveType], capture.ShortID) + typeCount++ + total++ } - backfilled = append(backfilled, capture.ShortID) } - message := fmt.Sprintf("Backfilled %d captures with video archive jobs", len(backfilled)) + message := fmt.Sprintf("Backfilled %d captures with media archive jobs", total) if dryRun { - message = fmt.Sprintf("Dry run: %d captures would be backfilled with video archive jobs", len(backfilled)) + message = fmt.Sprintf("Dry run: %d captures would be backfilled with media archive jobs", total) } - c.JSON(http.StatusOK, gin.H{"message": message, "short_ids": backfilled}) + c.JSON(http.StatusOK, gin.H{"message": message, "count": total, "short_ids": backfilled}) } func AdminArchive(c *gin.Context, db *gorm.DB, riverClient *river.Client[pgx.Tx]) { diff --git a/internal/handlers/api.go b/internal/handlers/api.go index 6b0c225..2f67939 100644 --- a/internal/handlers/api.go +++ b/internal/handlers/api.go @@ -18,6 +18,10 @@ import ( type PastArchiveResponse struct { ShortID string `json:"short_id"` Timestamp time.Time `json:"timestamp"` + // ThumbnailURL always points at a servable image. The endpoint falls back + // to a placeholder, so consumers can render a card unconditionally instead + // of branching on whether a preview happens to exist yet. + ThumbnailURL string `json:"thumbnail_url"` } // getPastArchives is the shared logic for retrieving past archives. @@ -49,8 +53,9 @@ func getPastArchives(c *gin.Context, db *gorm.DB) { response := make([]PastArchiveResponse, len(captures)) for i, capture := range captures { response[i] = PastArchiveResponse{ - ShortID: capture.ShortID, - Timestamp: capture.Timestamp, + ShortID: capture.ShortID, + Timestamp: capture.Timestamp, + ThumbnailURL: ThumbnailURL(c, capture.ShortID), } } diff --git a/internal/handlers/display.go b/internal/handlers/display.go index 6af9a4d..1d502f1 100644 --- a/internal/handlers/display.go +++ b/internal/handlers/display.go @@ -24,32 +24,140 @@ func calculateQueuePosition(db *gorm.DB, item *models.ArchiveItem) int { return int(count) + 1 // Add 1 because position is 1-based } -// URL type mapping: user-facing URLs use "web" instead of "mhtml" +// URL type mapping: user-facing URLs use "web" instead of "mhtml". +// +// This also resolves retired type names, so permalinks handed out before the +// yt-dlp rename (/{shortid}/youtube) keep working forever. func urlTypeToInternalType(urlType string) string { if urlType == "web" { - return "mhtml" + return utils.ArchiveTypeMHTML } - return urlType + return utils.NormalizeArchiveType(urlType) } +// internalTypeToURLType maps a stored type to the segment used in links. It +// canonicalizes, so a row still holding a retired type name links to (and +// highlights as) its current name rather than producing a tab that 404s. func internalTypeToURLType(internalType string) string { - if internalType == "mhtml" { + canonical := utils.NormalizeArchiveType(internalType) + if canonical == utils.ArchiveTypeMHTML { return "web" } - return internalType + return canonical } func getDisplayName(internalType string) string { - switch internalType { - case "mhtml": + switch utils.NormalizeArchiveType(internalType) { + case utils.ArchiveTypeMHTML: return "Web" - case "itch": + case utils.ArchiveTypeItch: return "Itch" + case utils.ArchiveTypeYtDlp: + return "Video" + case utils.ArchiveTypeGalleryDl: + return "Media" default: return internalType } } +// defaultTypePreference returns the archive types to land on, best first, for +// the kind of page this URL is. +func defaultTypePreference(originalURL string) []string { + switch { + case utils.IsItchURL(originalURL): + return []string{utils.ArchiveTypeItch, utils.ArchiveTypeMHTML, utils.ArchiveTypeScreenshot, utils.ArchiveTypeYtDlp, utils.ArchiveTypeGit} + case utils.IsGitURL(originalURL): + return []string{utils.ArchiveTypeGit, utils.ArchiveTypeMHTML, utils.ArchiveTypeScreenshot, utils.ArchiveTypeYtDlp} + case utils.IsGalleryDLURL(originalURL): + return []string{utils.ArchiveTypeGalleryDl, utils.ArchiveTypeMHTML, utils.ArchiveTypeScreenshot, utils.ArchiveTypeYtDlp, utils.ArchiveTypeGit} + case utils.IsVideoURL(originalURL): + return []string{utils.ArchiveTypeYtDlp, utils.ArchiveTypeMHTML, utils.ArchiveTypeScreenshot, utils.ArchiveTypeGit} + default: + return []string{utils.ArchiveTypeMHTML, utils.ArchiveTypeScreenshot, utils.ArchiveTypeGit, utils.ArchiveTypeYtDlp} + } +} + +// archiveTab is one rendered tab in the viewer. +type archiveTab struct { + URLType string + DisplayName string + Status string + IsActive bool +} + +// buildTabs orders a capture's archive items the way the viewer should show +// them: the types this kind of URL cares about most first, then anything else +// in creation order. Building this server-side keeps the template to a single +// loop instead of one near-identical copy per URL kind. +func buildTabs(items []models.ArchiveItem, preference []string, currentURLType string) []archiveTab { + tabs := make([]archiveTab, 0, len(items)) + used := make(map[string]bool, len(items)) + + appendTab := func(item models.ArchiveItem) { + urlType := internalTypeToURLType(item.Type) + tabs = append(tabs, archiveTab{ + URLType: urlType, + DisplayName: getDisplayName(item.Type), + Status: item.Status, + IsActive: urlType == currentURLType, + }) + } + + for _, preferredType := range preference { + for _, item := range items { + canonical := utils.NormalizeArchiveType(item.Type) + if canonical == preferredType && !used[canonical] { + used[canonical] = true + appendTab(item) + } + } + } + for _, item := range items { + canonical := utils.NormalizeArchiveType(item.Type) + if !used[canonical] { + used[canonical] = true + appendTab(item) + } + } + return tabs +} + +// selectDefaultType picks which tab to open. +// +// Preference order decides, except that a failed archive is skipped while any +// non-failed one exists: landing a visitor on a red "Archive Failed" page while +// a perfectly good screenshot sits one tab over is the worst of the available +// options. Pending and processing archives are deliberately still eligible — a +// freshly queued capture should open on the tab that is working, with its live +// log and auto-reload, not on whichever fast archive happened to finish first. +func selectDefaultType(items []models.ArchiveItem, preference []string) string { + byType := make(map[string]string, len(items)) + for _, item := range items { + byType[utils.NormalizeArchiveType(item.Type)] = item.Status + } + + for _, preferredType := range preference { + if status, ok := byType[preferredType]; ok && status != "failed" { + return preferredType + } + } + for _, preferredType := range preference { + if _, ok := byType[preferredType]; ok { + return preferredType + } + } + for _, item := range items { + if item.Status != "failed" { + return utils.NormalizeArchiveType(item.Type) + } + } + if len(items) > 0 { + return utils.NormalizeArchiveType(items[0].Type) + } + return "" +} + // DisplayDefault serves the default archive type view directly (no redirect) func DisplayDefault(c *gin.Context, db *gorm.DB) { shortID := c.Param("shortid") @@ -65,68 +173,9 @@ func DisplayDefault(c *gin.Context, db *gorm.DB) { // Determine the default archive type based on URL type isGit := utils.IsGitURL(archivedURL.Original) - isVideo := utils.IsVideoURL(archivedURL.Original) - isItch := utils.IsItchURL(archivedURL.Original) - var defaultType string - - if isItch { - // For itch.io URLs, prefer itch -> mhtml -> screenshot -> youtube -> git - for _, preferredType := range []string{"itch", "mhtml", "screenshot", "youtube", "git"} { - for _, item := range capture.ArchiveItems { - if item.Type == preferredType { - defaultType = preferredType - break - } - } - if defaultType != "" { - break - } - } - } else if isGit { - // For git repositories, prefer git -> mhtml -> screenshot -> youtube - for _, preferredType := range []string{"git", "mhtml", "screenshot", "youtube"} { - for _, item := range capture.ArchiveItems { - if item.Type == preferredType { - defaultType = preferredType - break - } - } - if defaultType != "" { - break - } - } - } else if isVideo { - // For video URLs (YouTube, Vimeo, etc.), prefer youtube -> mhtml -> screenshot -> git - for _, preferredType := range []string{"youtube", "mhtml", "screenshot", "git"} { - for _, item := range capture.ArchiveItems { - if item.Type == preferredType { - defaultType = preferredType - break - } - } - if defaultType != "" { - break - } - } - } else { - // For websites, prefer mhtml -> screenshot -> git -> youtube - for _, preferredType := range []string{"mhtml", "screenshot", "git", "youtube"} { - for _, item := range capture.ArchiveItems { - if item.Type == preferredType { - defaultType = preferredType - break - } - } - if defaultType != "" { - break - } - } - } - // If no preferred type found, use the first available - if defaultType == "" && len(capture.ArchiveItems) > 0 { - defaultType = capture.ArchiveItems[0].Type - } + preference := defaultTypePreference(archivedURL.Original) + defaultType := selectDefaultType(capture.ArchiveItems, preference) if defaultType == "" { c.Status(http.StatusNotFound) @@ -136,7 +185,7 @@ func DisplayDefault(c *gin.Context, db *gorm.DB) { // Find the specific archive item var targetItem *models.ArchiveItem for i := range capture.ArchiveItems { - if capture.ArchiveItems[i].Type == defaultType { + if utils.ArchiveTypesEqual(capture.ArchiveItems[i].Type, defaultType) { targetItem = &capture.ArchiveItems[i] break } @@ -166,15 +215,12 @@ func DisplayDefault(c *gin.Context, db *gorm.DB) { c.HTML(http.StatusOK, "display_type.html", gin.H{ "date": capture.Timestamp.Format(time.RFC1123), "timestamp": capture.Timestamp.Format(time.RFC3339), // For JavaScript parsing - "archives": capture.ArchiveItems, + "tabs": buildTabs(capture.ArchiveItems, preference, internalTypeToURLType(defaultType)), "current_item": targetItem, "current_type": internalTypeToURLType(defaultType), // Convert to URL type for display "short_id": shortID, "host": c.Request.Host, "original_url": archivedURL.Original, - "is_git": isGit, - "is_video": isVideo, - "is_itch": isItch, "git_repo_name": gitRepoName, "download_filename": filename, "queue_position": queuePosition, @@ -199,10 +245,11 @@ func DisplayType(c *gin.Context, db *gorm.DB) { var archivedURL models.ArchivedURL db.First(&archivedURL, capture.ArchivedURLID) - // Find the specific archive item using internal type + // Find the specific archive item using internal type. Compares canonically + // so a row still holding a retired type name stays reachable. var targetItem *models.ArchiveItem for i := range capture.ArchiveItems { - if capture.ArchiveItems[i].Type == internalType { + if utils.ArchiveTypesEqual(capture.ArchiveItems[i].Type, internalType) { targetItem = &capture.ArchiveItems[i] break } @@ -214,11 +261,8 @@ func DisplayType(c *gin.Context, db *gorm.DB) { } // Check if this is a git repository and generate clone info - isGit := utils.IsGitURL(archivedURL.Original) - isVideo := utils.IsVideoURL(archivedURL.Original) - isItch := utils.IsItchURL(archivedURL.Original) var gitRepoName string - if isGit { + if utils.IsGitURL(archivedURL.Original) { gitRepoName = utils.ExtractRepoName(archivedURL.Original) } @@ -232,17 +276,16 @@ func DisplayType(c *gin.Context, db *gorm.DB) { } c.HTML(http.StatusOK, "display_type.html", gin.H{ - "date": capture.Timestamp.Format(time.RFC1123), - "timestamp": capture.Timestamp.Format(time.RFC3339), // For JavaScript parsing - "archives": capture.ArchiveItems, - "current_item": targetItem, - "current_type": urlType, // Use the URL type for display + "date": capture.Timestamp.Format(time.RFC1123), + "timestamp": capture.Timestamp.Format(time.RFC3339), // For JavaScript parsing + "tabs": buildTabs(capture.ArchiveItems, defaultTypePreference(archivedURL.Original), internalTypeToURLType(internalType)), + "current_item": targetItem, + // Canonicalize rather than echoing urlType: a legacy /{id}/youtube + // permalink must still match the tab links, which are canonical. + "current_type": internalTypeToURLType(internalType), "short_id": shortID, "host": c.Request.Host, "original_url": archivedURL.Original, - "is_git": isGit, - "is_video": isVideo, - "is_itch": isItch, "git_repo_name": gitRepoName, "download_filename": filename, "queue_position": queuePosition, @@ -258,7 +301,7 @@ func GetLogs(c *gin.Context, db *gorm.DB) { var item models.ArchiveItem if err := db.Joins("JOIN captures ON captures.id = archive_items.capture_id"). - Where("captures.short_id = ? AND archive_items.type = ?", shortID, internalType). + Where("captures.short_id = ? AND archive_items.type IN ?", shortID, utils.ArchiveTypeMatchValues(internalType)). First(&item).Error; err != nil { c.JSON(http.StatusNotFound, gin.H{"error": "Not found"}) return diff --git a/internal/handlers/display_tabs_test.go b/internal/handlers/display_tabs_test.go new file mode 100644 index 0000000..81e5c1d --- /dev/null +++ b/internal/handlers/display_tabs_test.go @@ -0,0 +1,148 @@ +package handlers + +import ( + "testing" + + "arker/internal/models" + "arker/internal/utils" +) + +func items(pairs ...string) []models.ArchiveItem { + result := make([]models.ArchiveItem, 0, len(pairs)/2) + for i := 0; i+1 < len(pairs); i += 2 { + result = append(result, models.ArchiveItem{Type: pairs[i], Status: pairs[i+1]}) + } + return result +} + +// The bug this fixes: an Instagram photo post got a yt-dlp item that always +// failed, and the viewer opened that tab by preference, so every visitor landed +// on a red "Archive Failed" page while a good screenshot sat one tab over. +func TestSelectDefaultTypePrefersCompletedOverPreferred(t *testing.T) { + tests := []struct { + name string + items []models.ArchiveItem + preference []string + want string + }{ + { + name: "skips a failed preferred type for a completed one", + items: items("yt-dlp", "failed", "mhtml", "completed", "screenshot", "completed"), + preference: []string{utils.ArchiveTypeYtDlp, utils.ArchiveTypeMHTML, utils.ArchiveTypeScreenshot}, + want: utils.ArchiveTypeMHTML, + }, + { + name: "uses the preferred type when it completed", + items: items("gallery-dl", "completed", "mhtml", "completed"), + preference: []string{utils.ArchiveTypeGalleryDl, utils.ArchiveTypeMHTML}, + want: utils.ArchiveTypeGalleryDl, + }, + { + name: "falls back to preference order when nothing completed", + items: items("mhtml", "failed", "gallery-dl", "pending"), + preference: []string{utils.ArchiveTypeGalleryDl, utils.ArchiveTypeMHTML}, + want: utils.ArchiveTypeGalleryDl, + }, + { + name: "falls back to any completed item outside the preference list", + items: items("itch", "completed"), + preference: []string{utils.ArchiveTypeGalleryDl, utils.ArchiveTypeMHTML}, + want: utils.ArchiveTypeItch, + }, + { + name: "returns empty for a capture with no items", + items: nil, + preference: []string{utils.ArchiveTypeMHTML}, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := selectDefaultType(tt.items, tt.preference); got != tt.want { + t.Errorf("selectDefaultType = %q, want %q", got, tt.want) + } + }) + } +} + +func TestDefaultTypePreferenceRoutesByURLKind(t *testing.T) { + tests := []struct { + name string + url string + want string + }{ + {"instagram post leads with gallery-dl", "https://www.instagram.com/p/ABC123/", utils.ArchiveTypeGalleryDl}, + {"instagram reel leads with yt-dlp", "https://www.instagram.com/reel/ABC123/", utils.ArchiveTypeYtDlp}, + {"youtube leads with yt-dlp", "https://www.youtube.com/watch?v=123", utils.ArchiveTypeYtDlp}, + {"git repo leads with git", "https://github.com/user/repo", utils.ArchiveTypeGit}, + {"itch leads with itch", "https://someone.itch.io/game", utils.ArchiveTypeItch}, + {"plain site leads with mhtml", "https://example.com", utils.ArchiveTypeMHTML}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + preference := defaultTypePreference(tt.url) + if len(preference) == 0 || preference[0] != tt.want { + t.Errorf("defaultTypePreference(%q) = %v, want it to lead with %q", tt.url, preference, tt.want) + } + }) + } +} + +func TestBuildTabsOrdersByPreferenceAndMarksActive(t *testing.T) { + archiveItems := items( + "mhtml", "completed", + "screenshot", "completed", + "gallery-dl", "failed", + ) + preference := defaultTypePreference("https://www.instagram.com/p/ABC123/") + + tabs := buildTabs(archiveItems, preference, "screenshot") + + if len(tabs) != 3 { + t.Fatalf("got %d tabs, want 3", len(tabs)) + } + if tabs[0].URLType != utils.ArchiveTypeGalleryDl { + t.Errorf("tabs[0] = %q, want gallery-dl first for an Instagram post", tabs[0].URLType) + } + if tabs[0].DisplayName != "Media" { + t.Errorf("gallery-dl display name = %q, want Media", tabs[0].DisplayName) + } + // mhtml is stored as "mhtml" but linked as "web". + if tabs[1].URLType != "web" || tabs[1].DisplayName != "Web" { + t.Errorf("tabs[1] = %+v, want the mhtml item exposed as web/Web", tabs[1]) + } + if !tabs[2].IsActive { + t.Errorf("tabs[2] = %+v, want the screenshot tab marked active", tabs[2]) + } + if tabs[0].IsActive || tabs[1].IsActive { + t.Error("only the current tab may be marked active") + } +} + +// A capture may hold a type the preference list does not mention; it must still +// get a tab rather than disappearing from the viewer. +func TestBuildTabsIncludesUnlistedTypes(t *testing.T) { + tabs := buildTabs(items("git", "completed"), []string{utils.ArchiveTypeMHTML}, "git") + if len(tabs) != 1 || tabs[0].URLType != utils.ArchiveTypeGit { + t.Fatalf("tabs = %+v, want a single git tab", tabs) + } +} + +// Permalinks handed out before the rename use /{shortid}/youtube and must keep +// resolving to the yt-dlp item forever. +func TestURLTypeMappingResolvesLegacyNames(t *testing.T) { + if got := urlTypeToInternalType("youtube"); got != utils.ArchiveTypeYtDlp { + t.Errorf("urlTypeToInternalType(youtube) = %q, want yt-dlp", got) + } + if got := urlTypeToInternalType("web"); got != utils.ArchiveTypeMHTML { + t.Errorf("urlTypeToInternalType(web) = %q, want mhtml", got) + } + if got := urlTypeToInternalType("gallery-dl"); got != utils.ArchiveTypeGalleryDl { + t.Errorf("urlTypeToInternalType(gallery-dl) = %q, want gallery-dl", got) + } + if got := internalTypeToURLType(utils.ArchiveTypeMHTML); got != "web" { + t.Errorf("internalTypeToURLType(mhtml) = %q, want web", got) + } +} diff --git a/internal/handlers/gallery_dl_serve.go b/internal/handlers/gallery_dl_serve.go new file mode 100644 index 0000000..75c504c --- /dev/null +++ b/internal/handlers/gallery_dl_serve.go @@ -0,0 +1,284 @@ +package handlers + +import ( + "archive/zip" + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "sync" + "time" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" + + "arker/internal/models" + "arker/internal/storage" + "arker/internal/utils" +) + +// galleryMetadataFilename mirrors the name the archiver writes at the ZIP root. +const galleryMetadataFilename = "metadata.json" + +// maxGalleryBufferedSize caps how large a gallery ZIP may be before it is +// refused when the storage backend cannot seek. Seekable backends (S3 ranged +// GETs, local files) stream and are not subject to this. +const maxGalleryBufferedSize = 200 * 1024 * 1024 + +// maxGalleryEntrySize caps a single media file served inline. One entry is one +// image or one video from a post, so this is generous; anything larger is only +// available through the full-archive download, which streams. +const maxGalleryEntrySize = 256 * 1024 * 1024 + +// seekerReaderAt adapts a ReadSeekCloser to io.ReaderAt for archive/zip. The +// mutex is what makes it safe: ReaderAt is documented as safe for concurrent +// use, but a seek-based implementation shares one cursor. +type seekerReaderAt struct { + mu sync.Mutex + seeker storage.ReadSeekCloser +} + +func (r *seekerReaderAt) ReadAt(p []byte, off int64) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + if _, err := r.seeker.Seek(off, io.SeekStart); err != nil { + return 0, err + } + n, err := io.ReadFull(r.seeker, p) + // io.ReaderAt's contract is to report a short read at end-of-input as + // io.EOF; io.ReadFull reports it as io.ErrUnexpectedEOF, which callers + // like archive/zip treat as a hard failure rather than end-of-file. + if err == io.ErrUnexpectedEOF { + err = io.EOF + } + return n, err +} + +// bytesReaderAtCloser adapts an in-memory buffer to the same interface. +type bytesReaderAtCloser struct{ data []byte } + +func (r *bytesReaderAtCloser) ReadAt(p []byte, off int64) (int, error) { + if off < 0 || off >= int64(len(r.data)) { + return 0, io.EOF + } + n := copy(p, r.data[off:]) + if n < len(p) { + return n, io.EOF + } + return n, nil +} + +// openGalleryZip locates a capture's gallery-dl archive and opens it for +// random access, preferring ranged reads over buffering the whole ZIP. +func openGalleryZip(c *gin.Context, storageInstance storage.Storage, db *gorm.DB, shortID string) (*zip.Reader, func(), bool) { + var item models.ArchiveItem + if err := db.Joins("JOIN captures ON captures.id = archive_items.capture_id"). + Where("captures.short_id = ? AND archive_items.type = ?", shortID, utils.ArchiveTypeGalleryDl). + First(&item).Error; err != nil { + c.Status(http.StatusNotFound) + return nil, nil, false + } + if item.Status != "completed" { + c.Status(http.StatusNotFound) + return nil, nil, false + } + + size, err := storageInstance.Size(item.StorageKey) + if err != nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Archive temporarily unavailable"}) + return nil, nil, false + } + + if seekable, ok := storageInstance.(storage.SeekableStorage); ok { + reader, err := seekable.SeekableReader(item.StorageKey) + if err == nil { + zipReader, err := zip.NewReader(&seekerReaderAt{seeker: reader}, size) + if err != nil { + reader.Close() + c.JSON(http.StatusInternalServerError, gin.H{"error": "Archive is not a readable ZIP"}) + return nil, nil, false + } + return zipReader, func() { reader.Close() }, true + } + // Fall through to buffering when the backend cannot seek this object. + } + + if size > maxGalleryBufferedSize { + c.JSON(http.StatusServiceUnavailable, gin.H{ + "error": "Archive too large to browse; download the full archive instead", + }) + return nil, nil, false + } + + reader, err := storageInstance.Reader(item.StorageKey) + if err != nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Archive temporarily unavailable"}) + return nil, nil, false + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Failed to read archive"}) + return nil, nil, false + } + + zipReader, err := zip.NewReader(&bytesReaderAtCloser{data: data}, int64(len(data))) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Archive is not a readable ZIP"}) + return nil, nil, false + } + return zipReader, func() {}, true +} + +// ServeGalleryManifest returns the normalized post metadata plus the list of +// media files in the archive. The viewer calls this to render a post. +func ServeGalleryManifest(c *gin.Context, storageInstance storage.Storage, db *gorm.DB) { + shortID := c.Param("shortid") + + zipReader, cleanup, ok := openGalleryZip(c, storageInstance, db, shortID) + if !ok { + return + } + defer cleanup() + + manifest := gin.H{"short_id": shortID} + + for _, file := range zipReader.File { + if file.Name != galleryMetadataFilename { + continue + } + contents, err := file.Open() + if err != nil { + break + } + raw, err := io.ReadAll(io.LimitReader(contents, 4*1024*1024)) + contents.Close() + if err != nil { + break + } + var metadata map[string]interface{} + if err := json.Unmarshal(raw, &metadata); err == nil { + manifest["metadata"] = metadata + } + break + } + + // Derive the file list from the ZIP itself rather than trusting + // metadata.json, so the viewer can never link to an entry that is not there. + files := make([]gin.H, 0, len(zipReader.File)) + for _, file := range zipReader.File { + if file.Name == galleryMetadataFilename || strings.HasSuffix(file.Name, ".json") { + continue + } + files = append(files, gin.H{ + "name": file.Name, + "size": file.UncompressedSize64, + "content_type": galleryFileContentType(file.Name), + "url": fmt.Sprintf("/gallery/%s/file/%s", shortID, url.PathEscape(file.Name)), + }) + } + manifest["files"] = files + + c.JSON(http.StatusOK, manifest) +} + +// ServeGalleryFile serves a single media file out of the gallery-dl ZIP. +func ServeGalleryFile(c *gin.Context, storageInstance storage.Storage, db *gorm.DB) { + shortID := c.Param("shortid") + + requestedPath := c.Param("filepath") + if decoded, err := url.QueryUnescape(requestedPath); err == nil { + requestedPath = decoded + } + requestedPath = path.Clean(strings.TrimPrefix(requestedPath, "/")) + // The archiver writes a flat ZIP, so any traversal or nesting is bogus. + if requestedPath == "" || requestedPath == "." || strings.Contains(requestedPath, "/") { + c.Status(http.StatusNotFound) + return + } + + zipReader, cleanup, ok := openGalleryZip(c, storageInstance, db, shortID) + if !ok { + return + } + defer cleanup() + + var target *zip.File + for _, file := range zipReader.File { + if file.Name == requestedPath { + target = file + break + } + } + if target == nil { + c.Status(http.StatusNotFound) + return + } + + contents, err := target.Open() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to read file from archive"}) + return + } + defer contents.Close() + + // Buffer the entry so http.ServeContent can answer Range requests. Video + // slides need this: Safari and iOS open a video with "Range: bytes=0-1" and + // refuse to play against a 200, and seeking is broken everywhere without it. + // Entries are single post media, bounded by maxGalleryEntrySize. + if target.UncompressedSize64 > maxGalleryEntrySize { + c.JSON(http.StatusServiceUnavailable, gin.H{ + "error": "File too large to serve individually; download the full archive instead", + }) + return + } + data, err := io.ReadAll(io.LimitReader(contents, int64(maxGalleryEntrySize))) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to read file from archive"}) + return + } + + c.Header("Content-Type", galleryFileContentType(target.Name)) + c.Header("X-Content-Type-Options", "nosniff") + c.Header("ETag", fmt.Sprintf("\"%s-%d-%x\"", shortID, target.UncompressedSize64, target.CRC32)) + // ServeContent sets Content-Length/Content-Range and handles Range and + // If-None-Match. The zero modtime keeps it from emitting Last-Modified. + http.ServeContent(c.Writer, c.Request, target.Name, time.Time{}, bytes.NewReader(data)) +} + +// galleryFileContentType maps a ZIP entry name to a MIME type, restricted to +// formats media sites actually serve. Anything unrecognized is served as an +// opaque download rather than something a browser might try to execute. +func galleryFileContentType(name string) string { + switch strings.ToLower(path.Ext(name)) { + case ".jpg", ".jpeg": + return "image/jpeg" + case ".png": + return "image/png" + case ".gif": + return "image/gif" + case ".webp": + return "image/webp" + case ".avif": + return "image/avif" + case ".mp4", ".m4v": + return "video/mp4" + case ".webm": + return "video/webm" + case ".mov": + return "video/quicktime" + case ".mp3": + return "audio/mpeg" + case ".m4a": + return "audio/mp4" + case ".json": + return "application/json" + default: + return "application/octet-stream" + } +} diff --git a/internal/handlers/gallery_dl_serve_test.go b/internal/handlers/gallery_dl_serve_test.go new file mode 100644 index 0000000..1b01ce8 --- /dev/null +++ b/internal/handlers/gallery_dl_serve_test.go @@ -0,0 +1,254 @@ +package handlers + +import ( + "archive/zip" + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" + + "arker/internal/models" + "arker/internal/storage" + "arker/internal/utils" +) + +// buildGalleryArchive produces a ZIP shaped exactly like the one +// GalleryDLArchiver stores: Arker's metadata.json, the media files, and +// gallery-dl's raw per-file sidecars. +func buildGalleryArchive(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + + entries := []struct{ name, body string }{ + {"metadata.json", `{"source_url":"https://www.instagram.com/p/ABC123/","extractor":"instagram","author":"someone","author_name":"Some One","description":"a caption","file_count":2}`}, + {"001.jpg", "jpeg-bytes"}, + {"001.jpg.json", `{"width":1080,"height":1350}`}, + {"002.mp4", "mp4-bytes"}, + {"002.mp4.json", `{"width":720,"height":1280}`}, + } + for _, entry := range entries { + w, err := zw.Create(entry.name) + if err != nil { + t.Fatalf("create %s: %v", entry.name, err) + } + if _, err := w.Write([]byte(entry.body)); err != nil { + t.Fatalf("write %s: %v", entry.name, err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("close zip: %v", err) + } + return buf.Bytes() +} + +func seedGalleryCapture(t *testing.T, db *gorm.DB, storageInstance storage.Storage, shortID, status string) { + t.Helper() + + key := "archive/" + shortID + "/gallery-dl.zip" + if status == "completed" { + writer, err := storageInstance.Writer(key) + if err != nil { + t.Fatalf("storage writer: %v", err) + } + if _, err := writer.Write(buildGalleryArchive(t)); err != nil { + t.Fatalf("storage write: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("storage close: %v", err) + } + } + + createVideoCapture(t, db, shortID, "https://www.instagram.com/p/"+shortID+"/", nil) + + var capture models.Capture + if err := db.Where("short_id = ?", shortID).First(&capture).Error; err != nil { + t.Fatalf("find capture: %v", err) + } + item := models.ArchiveItem{ + CaptureID: capture.ID, + Type: utils.ArchiveTypeGalleryDl, + Status: status, + StorageKey: key, + Extension: ".zip", + } + if err := db.Create(&item).Error; err != nil { + t.Fatalf("create item: %v", err) + } +} + +func newGalleryRouter(db *gorm.DB, storageInstance storage.Storage) *gin.Engine { + gin.SetMode(gin.TestMode) + r := gin.New() + r.GET("/gallery/:shortid/list", func(c *gin.Context) { ServeGalleryManifest(c, storageInstance, db) }) + r.GET("/gallery/:shortid/file/*filepath", func(c *gin.Context) { ServeGalleryFile(c, storageInstance, db) }) + return r +} + +func TestServeGalleryManifestReturnsMetadataAndMediaOnly(t *testing.T) { + db := newHandlerLogTestDB(t) + storageInstance := storage.NewMemoryStorage() + seedGalleryCapture(t, db, storageInstance, "7fbf9", "completed") + + req := httptest.NewRequest(http.MethodGet, "/gallery/7fbf9/list", nil) + rec := httptest.NewRecorder() + newGalleryRouter(db, storageInstance).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + + var body struct { + Metadata map[string]interface{} `json:"metadata"` + Files []struct { + Name string `json:"name"` + Size int64 `json:"size"` + ContentType string `json:"content_type"` + URL string `json:"url"` + } `json:"files"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + + if body.Metadata["author"] != "someone" { + t.Errorf("metadata.author = %v, want someone", body.Metadata["author"]) + } + if body.Metadata["description"] != "a caption" { + t.Errorf("metadata.description = %v, want the caption", body.Metadata["description"]) + } + + // JSON sidecars and metadata.json must not be offered as gallery media. + if len(body.Files) != 2 { + t.Fatalf("files = %+v, want exactly the 2 media files", body.Files) + } + if body.Files[0].Name != "001.jpg" || body.Files[0].ContentType != "image/jpeg" { + t.Errorf("files[0] = %+v, want 001.jpg as image/jpeg", body.Files[0]) + } + if body.Files[1].ContentType != "video/mp4" { + t.Errorf("files[1] content type = %q, want video/mp4", body.Files[1].ContentType) + } + if body.Files[0].URL != "/gallery/7fbf9/file/001.jpg" { + t.Errorf("files[0].URL = %q, want /gallery/7fbf9/file/001.jpg", body.Files[0].URL) + } + if body.Files[0].Size != int64(len("jpeg-bytes")) { + t.Errorf("files[0].Size = %d, want %d", body.Files[0].Size, len("jpeg-bytes")) + } +} + +func TestServeGalleryFileServesMediaWithCorrectType(t *testing.T) { + db := newHandlerLogTestDB(t) + storageInstance := storage.NewMemoryStorage() + seedGalleryCapture(t, db, storageInstance, "7fbf9", "completed") + router := newGalleryRouter(db, storageInstance) + + req := httptest.NewRequest(http.MethodGet, "/gallery/7fbf9/file/001.jpg", nil) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "jpeg-bytes" { + t.Errorf("body = %q, want jpeg-bytes", got) + } + if got := rec.Header().Get("Content-Type"); got != "image/jpeg" { + t.Errorf("Content-Type = %q, want image/jpeg", got) + } + // Archived media is attacker-controlled, so the browser must not be + // allowed to sniff a different (possibly executable) type. + if got := rec.Header().Get("X-Content-Type-Options"); got != "nosniff" { + t.Errorf("X-Content-Type-Options = %q, want nosniff", got) + } +} + +func TestServeGalleryFileRejectsTraversalAndMissingFiles(t *testing.T) { + db := newHandlerLogTestDB(t) + storageInstance := storage.NewMemoryStorage() + seedGalleryCapture(t, db, storageInstance, "7fbf9", "completed") + router := newGalleryRouter(db, storageInstance) + + for _, path := range []string{ + "/gallery/7fbf9/file/../../etc/passwd", + "/gallery/7fbf9/file/nested/001.jpg", + "/gallery/7fbf9/file/999.jpg", + "/gallery/7fbf9/file/", + } { + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusNotFound { + t.Errorf("GET %s = %d, want 404", path, rec.Code) + } + } +} + +// An archive that has not finished (or does not exist) must not be browsable. +func TestServeGalleryRejectsIncompleteAndUnknownCaptures(t *testing.T) { + db := newHandlerLogTestDB(t) + storageInstance := storage.NewMemoryStorage() + seedGalleryCapture(t, db, storageInstance, "pend1", "pending") + router := newGalleryRouter(db, storageInstance) + + for _, path := range []string{ + "/gallery/pend1/list", + "/gallery/pend1/file/001.jpg", + "/gallery/nope1/list", + } { + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + if rec.Code != http.StatusNotFound { + t.Errorf("GET %s = %d, want 404", path, rec.Code) + } + } +} + +// Safari and iOS open a video with "Range: bytes=0-1" and refuse to play +// against a 200, and seeking is broken everywhere without range support. +func TestServeGalleryFileSupportsRangeRequests(t *testing.T) { + db := newHandlerLogTestDB(t) + storageInstance := storage.NewMemoryStorage() + seedGalleryCapture(t, db, storageInstance, "7fbf9", "completed") + router := newGalleryRouter(db, storageInstance) + + req := httptest.NewRequest(http.MethodGet, "/gallery/7fbf9/file/002.mp4", nil) + req.Header.Set("Range", "bytes=0-2") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusPartialContent { + t.Fatalf("status = %d, want 206 Partial Content", rec.Code) + } + if got := rec.Body.String(); got != "mp4" { + t.Errorf("body = %q, want the first 3 bytes (%q)", got, "mp4") + } + if got := rec.Header().Get("Content-Range"); got != "bytes 0-2/9" { + t.Errorf("Content-Range = %q, want bytes 0-2/9", got) + } + if got := rec.Header().Get("Content-Type"); got != "video/mp4" { + t.Errorf("Content-Type = %q, want video/mp4", got) + } +} + +func TestServeGalleryFileAdvertisesRangeSupport(t *testing.T) { + db := newHandlerLogTestDB(t) + storageInstance := storage.NewMemoryStorage() + seedGalleryCapture(t, db, storageInstance, "7fbf9", "completed") + router := newGalleryRouter(db, storageInstance) + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/gallery/7fbf9/file/001.jpg", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + if got := rec.Header().Get("Accept-Ranges"); got != "bytes" { + t.Errorf("Accept-Ranges = %q, want bytes", got) + } + if got := rec.Body.String(); got != "jpeg-bytes" { + t.Errorf("body = %q, want the full file when no Range is sent", got) + } +} diff --git a/internal/handlers/serve.go b/internal/handlers/serve.go index bc5b240..09d02f3 100644 --- a/internal/handlers/serve.go +++ b/internal/handlers/serve.go @@ -17,12 +17,15 @@ import ( func ServeArchive(c *gin.Context, storageInstance storage.Storage, db *gorm.DB) { shortID := c.Param("shortid") - typ := c.Param("type") + // Match on both the canonical name and any retired alias: pre-rename pages + // embed /archive/{id}/youtube in every