Mitigate issues with read/write concurrency

.github/workflows/continuous-integration.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - 'release_v0.9.*'
   pull_request:
 
 env:
@@ -13,43 +14,29 @@ env:
   PIP_BREAK_SYSTEM_PACKAGES: 1
 
 jobs:
-  changes:
+  # This is the 0.9 maintenance line, whose Dockerfiles have diverged from main's.
+  # Base images are always built from the branch under test and tagged "0.9-<branch>":
+  # reading main's tag would test against 0.10 images, writing it would replace them.
+  images:
     runs-on: ubuntu-latest
-    permissions:
-      pull-requests: read
     outputs:
       pgdocker: ${{ steps.check.outputs.pgtag }}
-      buildpgdocker: ${{ steps.check.outputs.buildpg }}
       pyrustdocker: ${{ steps.check.outputs.pytag }}
-      buildpyrustdocker: ${{ steps.check.outputs.buildpy }}
-
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2
-        id: filter
-        with:
-          filters: |
-            pgstac:
-              - 'docker/pgstac/**'
-            pypgstac:
-              - 'docker/pypgstac/**'
       - id: check
         run: |
-          buildpg=false;
-          ref=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | tr / _);
-          [[ "${{ steps.filter.outputs.pgstac }}" == "true" ]] && buildpg=true || ref=main;
-          echo "pgtag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-postgres:$ref" >>$GITHUB_OUTPUT;
-          echo "buildpg=$buildpg" >>$GITHUB_OUTPUT;
-          buildpy=false;
-          [[ "${{ steps.filter.outputs.pypgstac }}" == "true" ]] && buildpy=true || ref=main;
-          echo "pytag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-pyrust:$ref" >>$GITHUB_OUTPUT;
-          echo "buildpy=$buildpy" >>$GITHUB_OUTPUT;
+          ref=$(echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | tr / _)
+          echo "pgtag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-postgres:0.9-$ref" >>$GITHUB_OUTPUT
+          echo "pytag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-pyrust:0.9-$ref" >>$GITHUB_OUTPUT
 
   # This builds a base postgres image that has everything installed to be able to run pgstac. This image does not have pgstac itself installed.
   buildpg:
     name: Build and push base postgres image
     runs-on: ubuntu-latest
-    needs: [changes]
+    needs: [images]
+    permissions:
+      contents: read
+      packages: write
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
@@ -60,22 +47,24 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and Push Base Postgres
-        if: ${{ needs.changes.outputs.buildpgdocker == 'true' }}
         uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4
         with:
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           context: .
           target: pgstacbase
           file: docker/pgstac/Dockerfile
-          tags: ${{ needs.changes.outputs.pgdocker }}
+          tags: ${{ needs.images.outputs.pgdocker }}
           push: true
           cache-from: type=gha
           cache-to: type=gha, mode=max
 
   buildpyrust:
     name: Build and push base pyrust
     runs-on: ubuntu-latest
-    needs: [changes]
+    needs: [images]
+    permissions:
+      contents: read
+      packages: write
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
@@ -86,29 +75,28 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and Push Base pyrust
-        if: ${{ needs.changes.outputs.buildpyrustdocker == 'true' }}
         uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4
         with:
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           context: .
           target: pyrustbase
           file: docker/pypgstac/Dockerfile
-          tags: ${{ needs.changes.outputs.pyrustdocker }}
+          tags: ${{ needs.images.outputs.pyrustdocker }}
           push: true
           cache-from: type=gha
           cache-to: type=gha, mode=max
 
   test:
     name: test
-    needs: [changes, buildpg, buildpyrust]
+    needs: [images, buildpg, buildpyrust]
     runs-on: ubuntu-latest
     strategy:
       matrix:
         flags:
           - ""
           - "--resolution lowest-direct"
     container:
-      image: ${{ needs.changes.outputs.pyrustdocker }}
+      image: ${{ needs.images.outputs.pyrustdocker }}
       options: --user root
       env:
         PGPASSWORD: postgres
@@ -119,7 +107,7 @@ jobs:
       postgres:
         env:
           POSTGRES_PASSWORD: postgres
-        image: ${{ needs.changes.outputs.pgdocker }}
+        image: ${{ needs.images.outputs.pgdocker }}
         options: >-
           --health-cmd pg_isready
           --health-interval 10s

CHANGELOG.md

@@ -6,6 +6,47 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 
+## [v0.9.12]
+
+### Fixed
+- Ingest and search contending on the partition materialized views, which every write
+  refreshed and every search read.
+- Deadlocks between concurrent writers, and between writers and collection deletes.
+- Searches serializing on the statistics cache whenever `context` is enabled.
+- Privilege handling across the `SECURITY DEFINER` functions. Only the operations that
+  need ownership of pgstac's own objects are still elevated; those are no longer
+  executable by `PUBLIC`, no longer run SQL supplied by their caller, and act only on
+  partitions of `items`.
+- `delete_collection` failing for `pgstac_ingest`.
+- Partitions widened through `items_staging` losing the CHECK constraints used for
+  partition pruning.
+- `update_collection_extents()` overwriting a valid extent, and deriving it from sampled
+  statistics rather than from the data.
+
+### Changed
+- Partition metadata is tracked on `partition_stats` instead of being derived by walking
+  the partition tree on the write path. `partition_steps` is removed and `partitions` is
+  now a plain view.
+- `update_partition_stats` does only the work its caller will read and no longer runs
+  `ANALYZE`; planner statistics come from autovacuum or `analyze_items()`.
+- `pypgstac` updates partition statistics after the load transaction commits rather than
+  while it still holds the load's lock.
+- `maintain_index()` takes the identity of the index to build rather than the statement
+  to run.
+- PostGIS must be installed in the `public` schema; the install now fails clearly if it
+  is not.
+
+### Added
+- Concurrency tests covering both ingest paths — the SQL staging tables and the
+  `pypgstac` loader — asserting on the server's deadlock counter rather than on raised
+  exceptions, which the loader's retries would otherwise hide.
+- Security tests asserting which functions are elevated, who may execute them, and that
+  they act only on partitions of `items`.
+- Tests for the partition metadata search relies on, and for the CHECK constraints that
+  make partition pruning work.
+- A migration test that populates a database on the previous release, migrates it, and
+  verifies nothing was lost and that ingest and search still behave.
+
 ## [v0.9.11]
 
 ### Fixed
@@ -616,6 +657,7 @@ _TODO_
 
 - Fixed issue with pypgstac loads which caused some writes to fail ([#18](https://github.com/stac-utils/pgstac/pull/18))
 
+[v0.9.12]: https://github.com/stac-utils/pgstac/compare/v0.9.11...v0.9.12
 [v0.9.11]: https://github.com/stac-utils/pgstac/compare/v0.9.10...v0.9.11
 [v0.9.10]: https://github.com/stac-utils/pgstac/compare/v0.9.9...v0.9.10
 [v0.9.9]: https://github.com/stac-utils/pgstac/compare/v0.9.8...v0.9.9

docker/pypgstac/bin/test

@@ -241,6 +241,46 @@ EOSQL
     echo "pg_dump/pg_restore test PASSED!"
 }
 
+function test_migrations_with_data(){
+    # test_migrations starts from an empty 0.3.0 database, so nothing there
+    # exercises migrating a database that already holds collections, items and
+    # partitions -- which is what every real upgrade is.
+    local _prev_pgdatabase="${PGDATABASE:-}"
+    psql -X -q -v ON_ERROR_STOP=1 <<EOSQL
+DROP DATABASE IF EXISTS pgstac_test_migration_data WITH (force);
+CREATE DATABASE pgstac_test_migration_data;
+ALTER DATABASE pgstac_test_migration_data SET search_path to pgstac, public;
+ALTER DATABASE pgstac_test_migration_data SET client_min_messages to $CLIENTMESSAGES;
+EOSQL
+    export PGDATABASE=pgstac_test_migration_data
+
+    local PREVIOUS
+    PREVIOUS=$(find $SRCDIR/pgstac/migrations -name "pgstac.*.sql" \
+        | sed -En 's/^.*pgstac\.([0-9]+\.[0-9]+\.[0-9]+)\.sql$/\1/p' \
+        | sort -Vr | head -1)
+    echo "Populating a $PREVIOUS database, then migrating it."
+
+    psql -X -q -v ON_ERROR_STOP=1 -f $SRCDIR/pgstac/migrations/pgstac.$PREVIOUS.sql >/dev/null
+    psql -X -q -v ON_ERROR_STOP=1 -f $SRCDIR/pgstac/tests/migration/snapshot.sql
+
+    cd $SRCDIR/pypgstac
+    source venv/bin/activate
+    pypgstac migrate
+    pypgstac --version
+
+    echo "Verifying nothing was lost."
+    psql -X -q -v ON_ERROR_STOP=1 -f $SRCDIR/pgstac/tests/migration/verify.sql
+    local _rc=$?
+
+    psql -X -q -c "DROP DATABASE IF EXISTS pgstac_test_migration_data WITH (force);" postgres
+    export PGDATABASE="${_prev_pgdatabase}"
+    if [ $_rc -ne 0 ]; then
+        echo "***MIGRATION WITH DATA FAILED***"
+        return 1
+    fi
+    echo "Migration with data PASSED!"
+}
+
 function test_migrations(){
     psql -X -q -v ON_ERROR_STOP=1 <<EOSQL
 DROP DATABASE IF EXISTS pgstac_test_migration WITH (force);
@@ -383,6 +423,7 @@ fi
 [ $BASICSQL -eq 1 ] && test_basicsql
 [ $PYPGSTAC -eq 1 ] && test_pypgstac
 [ $MIGRATIONS -eq 1 ] && test_migrations
+[ $MIGRATIONS -eq 1 ] && test_migrations_with_data
 [ $PGDUMP -eq 1 ] && test_pgdump
 
 exit 0

docs/src/pgstac.md

@@ -28,6 +28,14 @@ To grant pgstac permissions to a current postgresql user:
 GRANT pgstac_read TO <user>;
 ```
 
+#### PostGIS must be installed in `public`
+
+PgSTAC references PostGIS functions without schema qualification and pins its own
+`search_path` to `pgstac, public`, so the postgis extension has to be reachable from
+there. Installing PostGIS into its own schema (`CREATE EXTENSION postgis SCHEMA postgis`)
+is not supported. Installing PgSTAC against such a database now fails immediately with
+a clear error rather than producing a partially created schema.
+
 #### PgSTAC Search Path
 The search_path can be set at the database level or role level or by setting within the current session. The search_path is already set if you are directly using one of the pgstac users. If you are not logging in directly as one of the pgstac users, you will need to set the search_path by adding it to the search_path of the user you are using:
 ```sql
@@ -188,6 +196,26 @@ SELECT cron.schedule('0 * * * *', 'CALL validate_constraints();');
 SELECT cron.schedule('10, * * * *', 'CALL analyze_items();');
 ```
 
+#### Migrating a large catalog
+
+The last step of every install and migration recalculates statistics and CHECK
+constraints for every partition. On a catalog with many partitions this is the most
+expensive part of the migration and it holds locks while it runs.
+
+To keep that work off the migration itself, migrate with the queue enabled and drain it
+afterwards:
+
+```bash
+pypgstac --usequeue migrate
+pypgstac runqueue   # repeat until it reports nothing left to do
+```
+
+`--usequeue` is a global flag, so it goes before the subcommand. Search is correct as
+soon as the migration commits — partition visibility is always written synchronously —
+but until the queue is drained the observed datetime ranges and the CHECK constraints
+used for partition pruning are stale. `check_pgstac_settings()` warns while anything is
+still queued, and `runqueue` must be run as a role with `pgstac_admin`.
+
 ### System Checks
 
 #### System and pgSTAC Settings
@@ -260,7 +288,7 @@ ALTER DATABASE target_database SET search_path TO pgstac, public;
 - Always use `--schema=pgstac` on `pg_dump` to capture only the pgstac schema.
 - Do **not** use `pg_restore` directly — use `pgstac_restore` instead to handle the search_path issue.
 - After restoring, you may want to run `ANALYZE` on the restored database to update planner statistics.
-- Materialized views (`partitions`, `partition_steps`) are included in the dump. If you need to refresh them after restore, run `REFRESH MATERIALIZED VIEW pgstac.partitions; REFRESH MATERIALIZED VIEW pgstac.partition_steps;`.
+- Partition metadata lives in the `partition_stats` table and is included in the dump. If partitions and `partition_stats` ever get out of sync, `SELECT pgstac.sync_partition_stats();` reconciles them against the partition tree.
 
 ### Notification Triggers