Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file removed .DS_Store
Binary file not shown.
22 changes: 19 additions & 3 deletions .env.sample
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
MOTHER_DUCK_TOKEN=
MOTHER_DUCK_CONN=
RAPID_API_KEY=
# Copy to .env to override defaults. Every variable here is optional — the
# stack (Airflow + dbt + Streamlit on local DuckDB) runs with the defaults and
# needs no cloud services or secrets.
#
# Consumed by docker-compose.yaml:

# Host path used for the container volume mounts (dags, logs, dbt, data...).
AIRFLOW_PROJ_DIR=.

# UID that owns files inside the containers. On Linux set to your host UID
# (run: id -u) to avoid root-owned files in mounted volumes.
AIRFLOW_UID=50000

# Airflow web login (defaults: airflow / airflow).
_AIRFLOW_WWW_USER_USERNAME=airflow
_AIRFLOW_WWW_USER_PASSWORD=airflow

# Extra pip packages to install into the Airflow containers at startup.
_PIP_ADDITIONAL_REQUIREMENTS=
38 changes: 38 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: ci

on:
push:
branches: [main]
pull_request:

jobs:
pipeline:
name: dbt build + dashboard smoke test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip

- name: Install dbt and dashboard dependencies
run: |
python -m pip install --upgrade pip
pip install "dbt-duckdb==1.9.4" "duckdb==1.3.1"
pip install -r dashboard/requirements.txt

- name: Seed raw_receipts landing table (mirrors the Airflow ingestion)
run: python scripts/build_local_db.py

- name: Install dbt packages
working-directory: dbt/receipts_analytics
run: dbt deps

- name: dbt build (runs every model and data test)
working-directory: dbt/receipts_analytics
run: dbt build --target dev --profiles-dir .

- name: Dashboard smoke test
run: python scripts/smoke_test_dashboard.py
25 changes: 24 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -164,4 +164,27 @@ cython_debug/

#Minio data
include/data/minio/
upgrade-test*
upgrade-test*
# ---- Project-specific (added for showcase cleanup) ----
# OS cruft
.DS_Store
**/.DS_Store

# DuckDB is a build artifact produced by the pipeline, not source
*.duckdb
*.duckdb.wal

# dbt build artifacts / local state
dbt_packages/
**/dbt_packages/
**/target/
**/logs/
.user.yml
**/.user.yml

# Local secrets
.env

# Airflow runtime
logs/
plugins/
1 change: 1 addition & 0 deletions .python-version
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3.12.13
19 changes: 15 additions & 4 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,19 @@
FROM apache/airflow:latest
# Pinned for reproducible builds (Airflow 3 SDK is used by the DAGs).
FROM apache/airflow:3.0.6
WORKDIR /opt/airflow/

# Install Python deps first (dbt-duckdb + Cosmos) for better layer caching.
# dbt is installed into the image here; there is no separate venv to copy.
COPY requirements.txt requirements.txt
COPY dbt/ dbt/
RUN pip install --no-cache-dir -r requirements.txt

COPY config/airflow.cfg config/airflow.cfg
COPY .venv/ .venv/
COPY dags/ dags/
RUN pip install --no-cache-dir -r requirements.txt
# --chown so the airflow user can write dbt_packages/, target/ and logs/.
COPY --chown=airflow:0 dbt/ dbt/

# Install dbt packages and pre-compile the manifest so Cosmos can render the
# DAG from the manifest at parse time (fast, no dbt shell-out on every parse).
RUN cd /opt/airflow/dbt/receipts_analytics \
&& dbt deps \
&& dbt parse --profiles-dir . --target prod
17 changes: 11 additions & 6 deletions Dockerfile_streamlit
Original file line number Diff line number Diff line change
@@ -1,18 +1,23 @@
FROM python:3.12-slim
# Pinned to Debian bookworm for reproducible builds (the floating slim tag
# moved to trixie, which dropped some package names).
FROM python:3.12-slim-bookworm
LABEL authors="jacktoke"

WORKDIR /opt/airflow

RUN apt-get update && apt-get install -y \
build-essential \
# Only curl is needed at runtime, for the Streamlit healthcheck in
# docker-compose.yaml. The dashboard dependencies all install from wheels,
# so no compiler/build tooling is required.
RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
software-properties-common \
&& rm -rf /var/lib/apt/lists/*

RUN pip install --upgrade pip

COPY ./dashboard ./dashboard

COPY ./dbt/airbnb/receipts.duckdb dbt/airbnb/receipts.duckdb
RUN pip install -r dashboard/requirements.txt

RUN pip install -r dashboard/requirements.txt
# The DuckDB database is NOT baked into the image. It is the pipeline's output
# and is mounted at runtime (see the dc-streamlit volume in docker-compose.yaml),
# so the dashboard always reflects the latest dbt build.
Loading
Loading