Bump github.com/go-git/go-git/v5 from 5.19.1 to 5.19.2

.github/workflows/ci.yml

@@ -23,8 +23,8 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v7
         with:
           go-version-file: go.mod
       - name: Build

.gitignore

@@ -33,3 +33,7 @@ cookies.txt
 
 # Secrets
 .env
+
+# Local test-drive credentials (never commit)
+/secrets/
+docker-compose.testdrive.yml

AGENT.md

@@ -48,14 +48,15 @@ When using Amp with `make dev` running in another window:
 ### Manual Local Development
 1. Start PostgreSQL: `docker run -d --name postgres -e POSTGRES_USER=user -e POSTGRES_PASSWORD=pass -e POSTGRES_DB=arker -p 5432:5432 postgres:15`
 2. Install Playwright: `go install github.com/mxschmitt/playwright-go/cmd/playwright@latest && playwright install chromium`
-3. Install yt-dlp: `pip install yt-dlp`
+3. Install yt-dlp and gallery-dl: `pip install yt-dlp gallery-dl`
 4. Run: `go run .`
 
 ### Dependencies
-- **Go 1.24+** (using Go 1.24.5 toolchain)
+- **Go 1.25.12+**
 - **PostgreSQL 15**
 - **Git** (for repository archiving)
-- **Python 3 + yt-dlp** (for YouTube archiving)
+- **Python 3 + yt-dlp** (for video archiving)
+- **Python 3 + gallery-dl** (for photo posts and mixed photo/video carousels)
 - **Python 3 + itch-dl** (for itch.io game archiving)
 - **Playwright + Chromium** (for MHTML and screenshots)
 
@@ -70,43 +71,60 @@ When using Amp with `make dev` running in another window:
 │   │   ├── mhtml.go        # MHTML webpage archiving
 │   │   ├── screenshot.go   # Full-page screenshot capture
 │   │   ├── git.go          # Git repository cloning
-│   │   ├── youtube.go      # YouTube video downloading
+│   │   ├── ytdlp.go        # Video downloading via yt-dlp
+│   │   ├── gallery_dl.go   # Photo/carousel downloading via gallery-dl
 │   │   ├── itch.go         # itch.io game archiving
-│   │   └── browser_utils.go # Shared browser utilities
+│   │   ├── pwbundle.go     # Playwright browser/page lifecycle
+│   │   └── utils.go        # Shared browser utilities & page loading
 │   ├── handlers/           # HTTP handlers
 │   │   ├── admin.go        # Admin interface endpoints
 │   │   ├── api.go          # REST API endpoints
 │   │   ├── auth.go         # Authentication handlers
 │   │   ├── display.go      # Archive display pages
 │   │   ├── git.go          # Git HTTP backend
 │   │   ├── itch_serve.go   # itch.io individual file serving
+│   │   ├── gallery_dl_serve.go # gallery-dl manifest + per-file serving
+│   │   ├── thumb.go        # Thumbnail serving + placeholder
 │   │   └── serve.go        # File serving with streaming
 │   ├── models/             # Database models & types
 │   │   └── models.go       # User, ArchivedURL, Capture, ArchiveItem
 │   ├── storage/            # Storage interface & implementations
-│   │   └── fs.go           # Filesystem storage (S3-ready interface)
+│   │   ├── fs.go           # Filesystem storage
+│   │   ├── s3.go           # S3/R2 storage with presigned direct URLs
+│   │   ├── direct.go       # DirectURLStorage interface
+│   │   └── memory_storage.go # In-memory storage (tests)
+│   ├── thumbnail/          # Derived preview images
+│   │   └── thumbnail.go    # Crop/scale/encode helper
 │   ├── monitoring/         # Browser process monitoring
 │   ├── utils/              # Shared utilities
 │   └── workers/            # Async job processing
 │       ├── queue.go        # Job queue management
-│       └── worker.go       # Background worker implementation
+│       ├── archive_worker.go   # Archive job processing
+│       ├── thumbnail_worker.go # On-demand thumbnail backfill
+│       └── cleanup_worker.go   # Stuck-job reaper
 ├── templates/              # HTML templates for web interface
 └── Makefile               # Development workflow commands
 ```
 
 ## Core Interfaces & Architecture
 
 ### Key Interfaces
-- **`Storage`** - Pluggable storage backend (filesystem now, S3-ready)
+- **`Storage`** - Pluggable storage backend (filesystem or S3/R2)
   - Methods: `Writer(key)`, `Reader(key)`, `Exists(key)`, `Size(key)`
-  - Current: Filesystem storage with zstd compression
+  - `SeekableStorage` adds `SeekableReader` for range requests; `DirectURLStorage`
+    adds presigned redirects. There is no delete method: the production bucket is
+    locked, so objects are written once under a nonced key and never replaced.
+  - Objects are stored **uncompressed** — the bytes on disk are exactly what the
+    archiver wrote.
 - **`Archiver`** - Different archiving strategies
-  - Methods: `Archive(url, writer)`, content type detection
-  - Types: MHTML, Screenshot, Git, YouTube, Itch
+  - Method: `Archive(ctx, url, logWriter, db, itemID) (Result, error)`
+  - `Result` carries the artifact reader, extension, content type, the Playwright
+    bundle (browser archivers), and an optional derived thumbnail
+  - Types: MHTML, Screenshot, Git, yt-dlp, gallery-dl, Itch
 
 ### Performance Features
 - **Browser Instance Reuse**: Playwright browsers reused across jobs for efficiency
-- **Streaming**: All file operations use streaming with zstd compression
+- **Streaming**: All file operations use streaming
 - **Async Processing**: Queue-based job processing with configurable worker pools
 - **Concurrent Workers**: Default 5 workers (configurable via `MAX_WORKERS`)
 - **Browser Monitoring**: Tracks browser processes to prevent memory leaks
@@ -135,6 +153,10 @@ When using Amp with `make dev` running in another window:
 - `GET /git/:shortid` - Git HTTP backend for cloning repositories
 - `GET /itch/:shortid/file/*filepath` - Stream individual files from itch.io game archives
 - `GET /itch/:shortid/list` - JSON list of files in itch.io game archive
+- `GET /gallery/:shortid/list` - JSON post metadata + media file list for a gallery-dl archive
+- `GET /gallery/:shortid/file/*filepath` - Stream one media file out of a gallery-dl archive
+- `GET|HEAD /thumb/:shortid` - Preview image for a capture (480x270 JPEG); falls back to an SVG placeholder and queues generation
+- `GET|HEAD /thumb/:shortid/:type` - Preview image for one archive type
 
 ### Admin Interface (Session Authentication)
 - `GET /login` - Admin login page
@@ -152,7 +174,7 @@ When using Amp with `make dev` running in another window:
 
 ### Git Repository Access
 ```bash
-git clone https://archive.selfhosted.hackclub.com/git/{shortid}
+git clone https://archive.hackclub.com/git/{shortid}
 ```
 
 ## Configuration
@@ -170,6 +192,8 @@ git clone https://archive.selfhosted.hackclub.com/git/{shortid}
 - `YTDLP_COOKIES_B64` - Base64-encoded cookies.txt content, written to a temp file at startup (used when `YTDLP_COOKIES_FILE` is unset; convenient for Coolify secrets)
 - `YTDLP_PROXY` - Optional proxy URL (e.g. `http://user:pass@host:port`, `socks5://...`) applied to every yt-dlp call. Instagram aggressively rate-limits datacenter IP ranges; a residential/mobile proxy is the reliable way to archive Instagram from a server. yt-dlp itself must also be kept current (installed from the nightly `--pre` channel) since Instagram breaks the extractor frequently.
 - `YTDLP_IMPERSONATE` - Optional yt-dlp `--impersonate` target for Instagram/TikTok/Facebook video URLs. Docker images default to `chrome` and install `curl-cffi`; set empty to disable for manual installs without curl-cffi.
+- `GALLERYDL_USER_AGENT` - Optional `--user-agent` override for gallery-dl. Leave unset: gallery-dl sets a per-site User-Agent already (Instagram gets a current Chrome UA because it serves lower-quality video to anything else), and this replaces those defaults everywhere.
+- `GALLERYDL_SLEEP_REQUEST` - Optional `--sleep-request` override (`"1"`, `"0.5-1.5"`). Leave unset. gallery-dl ships per-site request intervals (Instagram waits a randomized 6-12s between API calls); because `--sleep-request` is a root config key it *replaces* those rather than acting as a floor, so any value below a site's own default makes throttling more likely, not less. Set it only to slow gallery-dl down further.
 
 - `LOGIN_TEXT` - Text to display under login form
 
@@ -188,13 +212,18 @@ git clone https://archive.selfhosted.hackclub.com/git/{shortid}
 ## Testing
 
 ### Test Files
-- `storage_test.go` - Storage interface tests
-- `archiver_test.go` - Archiver interface tests
-- `monitoring_test.go` - Browser monitoring tests
-- `validation_test.go` - Input validation tests
-- `login_text_test.go` - Login text handling tests
 
-- `vimeo_test.go` - Vimeo video archiving tests
+Most tests live beside the package they cover (`internal/*/..._test.go`); a few
+integration-level ones sit at the repo root (`storage_test.go`,
+`monitoring_test.go`, `validation_test.go`). Run `go test ./...` rather than
+working from a list here — this section has drifted before.
+
+Two conventions worth knowing:
+- DB-backed tests use in-memory SQLite (`gorm.io/driver/sqlite`) with
+  `AutoMigrate`, so they need no running Postgres. See `newWorkerTestDB` in
+  `internal/workers/archive_worker_test.go` for the pattern.
+- Handler tests build a real `gin` engine and drive it with `httptest`, so route
+  registration and middleware are exercised too. See `internal/handlers/thumb_test.go`.
 
 ### Running Tests
 ```bash
@@ -213,7 +242,8 @@ go test -run TestFSStorage    # Run specific test
 ### Archive & Browser
 - **mxschmitt/playwright-go** v0.6100.0 - Browser automation
 - **go-git/go-git/v5** v5.8.1 - Git operations
-- **klauspost/compress** v1.18.0 - zstd compression
+- **HugoSmits86/nativewebp** v1.2.0 - WebP encoding (lossless only)
+- **golang.org/x/image** v0.44.0 - WebP decoding + high-quality rescaling
 
 ### Utilities
 - **golang.org/x/crypto** v0.33.0 - Password hashing (bcrypt)
@@ -231,9 +261,52 @@ go test -run TestFSStorage    # Run specific test
 
 ### Adding New Archive Types
 1. Implement `Archiver` interface in `internal/archivers/`
-2. Add to `archiversMap` in `cmd/main.go`
-3. Update content type detection in handlers
-4. Add tests in `archiver_test.go`
+2. Add the type constant to `internal/utils/archive_types.go` (`canonicalArchiveTypes`)
+3. Add to `archiversMap` in `cmd/main.go`
+4. Route URLs to it in `utils.GetArchiveTypes`
+5. Give it a timeout case in `utils.TimeoutForJobType` (the default is only 2 minutes)
+6. Update `contentTypeForArchive` in `internal/handlers/serve.go`
+7. Add it to `defaultTypePreference`/`getDisplayName` in `internal/handlers/display.go` and to the content pane in `templates/display_type.html`
+8. Add tests
+
+Type names are stable identifiers: they appear in stored rows, in permalinks
+(`/{shortid}/{type}`), and in API requests. Renaming one means adding a
+`legacyArchiveTypeAliases` entry so old names keep resolving, plus a rename in
+`migrateLegacyArchiveTypes`. Never just change the string.
+
+An archiver returns an `archivers.Result` struct, not a list of values, so a new
+derived artifact does not churn every archiver's signature. `Result.Thumbnail`
+is optional and nil is not an error.
+
+### Thumbnails
+
+Thumbnails are a **derived artifact**, not an archive type — they have no tab,
+no permalink type segment, and no entry in `canonicalArchiveTypes`. They live in
+four columns on `archive_items` (`thumbnail_key/width/height/status`) and are
+served from `/thumb/{shortid}[/{type}]`.
+
+- **Size/format**: 480x270 JPEG (`internal/thumbnail`). JPEG because the only
+  WebP encoder in the tree (`nativewebp`) is lossless-only — it produced a
+  ~490KB "thumbnail" in testing versus ~35KB for JPEG, and it panics on some
+  low-colour-count inputs. `x/image/webp` is decode-only and reads back what
+  `nativewebp` wrote.
+- **Generation is two-path**. New captures: `ScreenshotArchiver` derives it from
+  the image it has already decoded, so it costs one downscale and no extra
+  browser work. Pre-existing archives: the `/thumb` handler enqueues a
+  `ThumbnailJobArgs` job on the `high_priority` queue the first time somebody
+  views one. Never generate inline in a request — a full-page screenshot reaches
+  60 megapixels (~240MB decoded) and one dashboard render asks for hundreds.
+- **`thumbnail_status` must be set to `unavailable` for permanent failures**
+  (unsupported type, oversized, undecodable). Without it the lazy path
+  re-enqueues the same impossible job on every page view.
+- **A thumbnail failure must never fail an archive.** The archive is the
+  product; the preview is not.
+- Keys carry an upload nonce like archive keys (`{shortid}/{type}-{nonce}-thumb.jpg`)
+  because the bucket forbids overwrites and deletes. Regenerating means writing a
+  new object and repointing the row.
+- `/thumb` always returns an image, falling back to a generated SVG placeholder
+  with a short `max-age` so a refresh picks up the real one. Callers can render
+  a card unconditionally.
 
 ### Database Changes
 1. Update models in `internal/models/models.go`
@@ -245,18 +318,23 @@ go test -run TestFSStorage    # Run specific test
 ### Common Issues
 - **Playwright fails**: Ensure Chromium is installed (`playwright install chromium`)
 - **yt-dlp not found**: Install with `pip install yt-dlp`
+- **gallery-dl not found**: Install with `pip install gallery-dl`. Install it into the *same* Python environment as yt-dlp: gallery-dl's default Instagram video path hands DASH manifests to yt-dlp as an importable module, and silently falls back to lower-quality pre-merged MP4 otherwise.
 - **Database connection**: Check PostgreSQL is running and credentials match
 - **Permission errors**: Ensure storage/cache directories are writable
 - **Browser leaks**: Check `/status/browser` endpoint for monitoring data
 
 ### Production Debugging
-- **SSH Access**: `ssh root@archive.selfhosted.hackclub.com`
+- **SSH Access**: `ssh archive-hq-local.selfhosted.hackclub.com` (via cloudflared, see `~/.ssh/config`; use `sudo docker ...` on the host)
 - **Health Checks**: Monitor `/health` and `/status/browser` endpoints
 - **Logs**: Check Coolify dashboard or container logs
 - **Database**: Connect via environment variables in deployment
+- **Container names rotate per deploy** — resolve the app container with
+  `sudo docker ps | grep ^hcmolbh8` rather than assuming a fixed name
+- **Don't run one-off tools inside the app container**: a Coolify deploy will
+  kill it mid-run. Run them on the host instead.
 
 ### Health Monitoring
-- Startup health checks verify yt-dlp and Playwright availability
+- Startup health checks verify yt-dlp, gallery-dl, and Playwright availability
 - Browser process monitoring with leak detection
 
 - Automatic log cleanup (30 days for completed items)
@@ -266,7 +344,7 @@ go test -run TestFSStorage    # Run specific test
 - **Modular Design**: Clean separation of concerns with internal packages
 - **Interface-Driven**: Storage and Archiver interfaces for extensibility
 - **Resilient Processing**: Error handling with retries, timeouts, and status tracking
-- **Memory Efficient**: Streaming operations for large files with compression
+- **Memory Efficient**: Streaming operations for large files
 - **Production Ready**: Docker deployment with health checks and resource limits
 - **Git Integration**: Full Git HTTP backend for repository cloning
 - **API-First**: RESTful API with web interface as overlay

Dockerfile

@@ -1,8 +1,9 @@
-FROM golang:1.24-bookworm
+FROM golang:1.25.12-bookworm
 
 # Install system dependencies in a single layer with aggressive cleanup
 RUN apt-get update && apt-get install -y --no-install-recommends \
     git \
+    tini \
     python3 \
     python3-pip \
     curl \
@@ -55,6 +56,20 @@ RUN pip3 install --break-system-packages --no-cache-dir --upgrade --pre "yt-dlp[
     && yt-dlp --version > /etc/yt-dlp-version \
     && rm -rf /root/.cache
 
+# gallery-dl handles photo posts and mixed photo/video carousels, which yt-dlp
+# rejects outright. Its extractors break on the same cadence as yt-dlp's, so the
+# remote ADD cache-busts on the latest release for the same reason.
+#
+# It is installed into the SAME environment as yt-dlp on purpose: gallery-dl's
+# default Instagram video path hands DASH manifests to yt-dlp as an importable
+# Python module, and silently falls back to lower-quality pre-merged MP4 when it
+# cannot import one. requests[socks] is what makes a socks5:// value in
+# YTDLP_PROXY work rather than warn and bypass the proxy.
+ADD https://api.github.com/repos/mikf/gallery-dl/releases/latest /tmp/gallery-dl-release.json
+RUN pip3 install --break-system-packages --no-cache-dir --upgrade gallery-dl "requests[socks]" \
+    && gallery-dl --version > /etc/gallery-dl-version \
+    && rm -rf /root/.cache
+
 # The Docker image includes curl-cffi, so use yt-dlp's browser impersonation by
 # default for Instagram/TikTok/Facebook anti-bot responses. Arker applies this
 # only to those URL families. Override to empty to disable, or to another target
@@ -90,4 +105,9 @@ RUN mkdir -p /data /cache
 
 EXPOSE 8080
 
+# tini reaps the zombie headless_shell/browser children that arker's own PID-1
+# process never wait()s on. Without an init, prod accumulates ~1k defunct PIDs
+# per day and eventually hits the container's pids cgroup limit, at which point
+# archiving breaks with fork failures.
+ENTRYPOINT ["/usr/bin/tini", "--"]
 CMD ["./arker"]

Dockerfile.dev

@@ -1,5 +1,5 @@
 # Development Dockerfile with live reloading
-FROM golang:1.24-bookworm
+FROM golang:1.25.12-bookworm
 
 # Install required packages and Playwright dependencies (matching production)
 RUN apt-get update && apt-get install -y \
@@ -37,9 +37,11 @@ RUN apt-get update && apt-get install -y \
     wget \
     && rm -rf /var/lib/apt/lists/*
 
-# Install yt-dlp and itch-dl. Match production: nightly yt-dlp plus curl-cffi
-# for browser impersonation.
-RUN pip3 install --break-system-packages --upgrade --pre "yt-dlp[default,curl-cffi]" itch-dl
+# Install yt-dlp, gallery-dl, and itch-dl. Match production: nightly yt-dlp plus
+# curl-cffi for browser impersonation, gallery-dl in the same environment so its
+# Instagram video path can import yt-dlp, and requests[socks] for socks5 proxies.
+RUN pip3 install --break-system-packages --upgrade --pre "yt-dlp[default,curl-cffi]" \
+    && pip3 install --break-system-packages --upgrade gallery-dl "requests[socks]" itch-dl
 ENV YTDLP_IMPERSONATE=chrome
 
 # Set working directory