diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 5a0c18ea..784cb627 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - 'release_v0.9.*' pull_request: env: @@ -13,43 +14,29 @@ env: PIP_BREAK_SYSTEM_PACKAGES: 1 jobs: - changes: + # This is the 0.9 maintenance line, whose Dockerfiles have diverged from main's. + # Base images are always built from the branch under test and tagged "0.9-": + # reading main's tag would test against 0.10 images, writing it would replace them. + images: runs-on: ubuntu-latest - permissions: - pull-requests: read outputs: pgdocker: ${{ steps.check.outputs.pgtag }} - buildpgdocker: ${{ steps.check.outputs.buildpg }} pyrustdocker: ${{ steps.check.outputs.pytag }} - buildpyrustdocker: ${{ steps.check.outputs.buildpy }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2 - id: filter - with: - filters: | - pgstac: - - 'docker/pgstac/**' - pypgstac: - - 'docker/pypgstac/**' - id: check run: | - buildpg=false; - ref=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | tr / _); - [[ "${{ steps.filter.outputs.pgstac }}" == "true" ]] && buildpg=true || ref=main; - echo "pgtag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-postgres:$ref" >>$GITHUB_OUTPUT; - echo "buildpg=$buildpg" >>$GITHUB_OUTPUT; - buildpy=false; - [[ "${{ steps.filter.outputs.pypgstac }}" == "true" ]] && buildpy=true || ref=main; - echo "pytag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-pyrust:$ref" >>$GITHUB_OUTPUT; - echo "buildpy=$buildpy" >>$GITHUB_OUTPUT; + ref=$(echo "${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | tr / _) + echo "pgtag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-postgres:0.9-$ref" >>$GITHUB_OUTPUT + echo "pytag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-pyrust:0.9-$ref" >>$GITHUB_OUTPUT # This builds a base postgres image that has everything installed to be able to run pgstac. This image does not have pgstac itself installed. buildpg: name: Build and push base postgres image runs-on: ubuntu-latest - needs: [changes] + needs: [images] + permissions: + contents: read + packages: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 @@ -60,14 +47,13 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Push Base Postgres - if: ${{ needs.changes.outputs.buildpgdocker == 'true' }} uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4 with: - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 context: . target: pgstacbase file: docker/pgstac/Dockerfile - tags: ${{ needs.changes.outputs.pgdocker }} + tags: ${{ needs.images.outputs.pgdocker }} push: true cache-from: type=gha cache-to: type=gha, mode=max @@ -75,7 +61,10 @@ jobs: buildpyrust: name: Build and push base pyrust runs-on: ubuntu-latest - needs: [changes] + needs: [images] + permissions: + contents: read + packages: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 @@ -86,21 +75,20 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Push Base pyrust - if: ${{ needs.changes.outputs.buildpyrustdocker == 'true' }} uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4 with: - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 context: . target: pyrustbase file: docker/pypgstac/Dockerfile - tags: ${{ needs.changes.outputs.pyrustdocker }} + tags: ${{ needs.images.outputs.pyrustdocker }} push: true cache-from: type=gha cache-to: type=gha, mode=max test: name: test - needs: [changes, buildpg, buildpyrust] + needs: [images, buildpg, buildpyrust] runs-on: ubuntu-latest strategy: matrix: @@ -108,7 +96,7 @@ jobs: - "" - "--resolution lowest-direct" container: - image: ${{ needs.changes.outputs.pyrustdocker }} + image: ${{ needs.images.outputs.pyrustdocker }} options: --user root env: PGPASSWORD: postgres @@ -119,7 +107,7 @@ jobs: postgres: env: POSTGRES_PASSWORD: postgres - image: ${{ needs.changes.outputs.pgdocker }} + image: ${{ needs.images.outputs.pgdocker }} options: >- --health-cmd pg_isready --health-interval 10s diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c440b79..e7c9a596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,47 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [v0.9.12] + +### Fixed +- Ingest and search contending on the partition materialized views, which every write + refreshed and every search read. +- Deadlocks between concurrent writers, and between writers and collection deletes. +- Searches serializing on the statistics cache whenever `context` is enabled. +- Privilege handling across the `SECURITY DEFINER` functions. Only the operations that + need ownership of pgstac's own objects are still elevated; those are no longer + executable by `PUBLIC`, no longer run SQL supplied by their caller, and act only on + partitions of `items`. +- `delete_collection` failing for `pgstac_ingest`. +- Partitions widened through `items_staging` losing the CHECK constraints used for + partition pruning. +- `update_collection_extents()` overwriting a valid extent, and deriving it from sampled + statistics rather than from the data. + +### Changed +- Partition metadata is tracked on `partition_stats` instead of being derived by walking + the partition tree on the write path. `partition_steps` is removed and `partitions` is + now a plain view. +- `update_partition_stats` does only the work its caller will read and no longer runs + `ANALYZE`; planner statistics come from autovacuum or `analyze_items()`. +- `pypgstac` updates partition statistics after the load transaction commits rather than + while it still holds the load's lock. +- `maintain_index()` takes the identity of the index to build rather than the statement + to run. +- PostGIS must be installed in the `public` schema; the install now fails clearly if it + is not. + +### Added +- Concurrency tests covering both ingest paths — the SQL staging tables and the + `pypgstac` loader — asserting on the server's deadlock counter rather than on raised + exceptions, which the loader's retries would otherwise hide. +- Security tests asserting which functions are elevated, who may execute them, and that + they act only on partitions of `items`. +- Tests for the partition metadata search relies on, and for the CHECK constraints that + make partition pruning work. +- A migration test that populates a database on the previous release, migrates it, and + verifies nothing was lost and that ingest and search still behave. + ## [v0.9.11] ### Fixed @@ -616,6 +657,7 @@ _TODO_ - Fixed issue with pypgstac loads which caused some writes to fail ([#18](https://github.com/stac-utils/pgstac/pull/18)) +[v0.9.12]: https://github.com/stac-utils/pgstac/compare/v0.9.11...v0.9.12 [v0.9.11]: https://github.com/stac-utils/pgstac/compare/v0.9.10...v0.9.11 [v0.9.10]: https://github.com/stac-utils/pgstac/compare/v0.9.9...v0.9.10 [v0.9.9]: https://github.com/stac-utils/pgstac/compare/v0.9.8...v0.9.9 diff --git a/docker/pypgstac/bin/test b/docker/pypgstac/bin/test index 3d2c5b56..0409c39e 100755 --- a/docker/pypgstac/bin/test +++ b/docker/pypgstac/bin/test @@ -241,6 +241,46 @@ EOSQL echo "pg_dump/pg_restore test PASSED!" } +function test_migrations_with_data(){ + # test_migrations starts from an empty 0.3.0 database, so nothing there + # exercises migrating a database that already holds collections, items and + # partitions -- which is what every real upgrade is. + local _prev_pgdatabase="${PGDATABASE:-}" + psql -X -q -v ON_ERROR_STOP=1 </dev/null + psql -X -q -v ON_ERROR_STOP=1 -f $SRCDIR/pgstac/tests/migration/snapshot.sql + + cd $SRCDIR/pypgstac + source venv/bin/activate + pypgstac migrate + pypgstac --version + + echo "Verifying nothing was lost." + psql -X -q -v ON_ERROR_STOP=1 -f $SRCDIR/pgstac/tests/migration/verify.sql + local _rc=$? + + psql -X -q -c "DROP DATABASE IF EXISTS pgstac_test_migration_data WITH (force);" postgres + export PGDATABASE="${_prev_pgdatabase}" + if [ $_rc -ne 0 ]; then + echo "***MIGRATION WITH DATA FAILED***" + return 1 + fi + echo "Migration with data PASSED!" +} + function test_migrations(){ psql -X -q -v ON_ERROR_STOP=1 <; ``` +#### PostGIS must be installed in `public` + +PgSTAC references PostGIS functions without schema qualification and pins its own +`search_path` to `pgstac, public`, so the postgis extension has to be reachable from +there. Installing PostGIS into its own schema (`CREATE EXTENSION postgis SCHEMA postgis`) +is not supported. Installing PgSTAC against such a database now fails immediately with +a clear error rather than producing a partially created schema. + #### PgSTAC Search Path The search_path can be set at the database level or role level or by setting within the current session. The search_path is already set if you are directly using one of the pgstac users. If you are not logging in directly as one of the pgstac users, you will need to set the search_path by adding it to the search_path of the user you are using: ```sql @@ -188,6 +196,26 @@ SELECT cron.schedule('0 * * * *', 'CALL validate_constraints();'); SELECT cron.schedule('10, * * * *', 'CALL analyze_items();'); ``` +#### Migrating a large catalog + +The last step of every install and migration recalculates statistics and CHECK +constraints for every partition. On a catalog with many partitions this is the most +expensive part of the migration and it holds locks while it runs. + +To keep that work off the migration itself, migrate with the queue enabled and drain it +afterwards: + +```bash +pypgstac --usequeue migrate +pypgstac runqueue # repeat until it reports nothing left to do +``` + +`--usequeue` is a global flag, so it goes before the subcommand. Search is correct as +soon as the migration commits — partition visibility is always written synchronously — +but until the queue is drained the observed datetime ranges and the CHECK constraints +used for partition pruning are stale. `check_pgstac_settings()` warns while anything is +still queued, and `runqueue` must be run as a role with `pgstac_admin`. + ### System Checks #### System and pgSTAC Settings @@ -260,7 +288,7 @@ ALTER DATABASE target_database SET search_path TO pgstac, public; - Always use `--schema=pgstac` on `pg_dump` to capture only the pgstac schema. - Do **not** use `pg_restore` directly — use `pgstac_restore` instead to handle the search_path issue. - After restoring, you may want to run `ANALYZE` on the restored database to update planner statistics. -- Materialized views (`partitions`, `partition_steps`) are included in the dump. If you need to refresh them after restore, run `REFRESH MATERIALIZED VIEW pgstac.partitions; REFRESH MATERIALIZED VIEW pgstac.partition_steps;`. +- Partition metadata lives in the `partition_stats` table and is included in the dump. If partitions and `partition_stats` ever get out of sync, `SELECT pgstac.sync_partition_stats();` reconciles them against the partition tree. ### Notification Triggers diff --git a/src/pgstac/migrations/pgstac.0.9.11-0.9.12.sql b/src/pgstac/migrations/pgstac.0.9.11-0.9.12.sql new file mode 100644 index 00000000..9fb50871 --- /dev/null +++ b/src/pgstac/migrations/pgstac.0.9.11-0.9.12.sql @@ -0,0 +1,1873 @@ +SET client_min_messages TO WARNING; +SET SEARCH_PATH to pgstac, public; +RESET ROLE; +DO $$ +DECLARE +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN + CREATE EXTENSION IF NOT EXISTS postgis; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN + CREATE EXTENSION IF NOT EXISTS btree_gist; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN + CREATE EXTENSION IF NOT EXISTS unaccent; + END IF; +END; +$$ LANGUAGE PLPGSQL; + +DO $$ + BEGIN + CREATE ROLE pgstac_admin; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_read; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_ingest; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + + +GRANT pgstac_admin TO current_user; + +-- Function to make sure pgstac_admin is the owner of items +CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ +DECLARE + f RECORD; +BEGIN + FOR f IN ( + SELECT + concat( + oid::regproc::text, + '(', + coalesce(pg_get_function_identity_arguments(oid),''), + ')' + ) AS name, + CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ + FROM pg_proc + WHERE + pronamespace=to_regnamespace('pgstac') + AND proowner != to_regrole('pgstac_admin') + AND proname NOT LIKE 'pg_stat%' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + FOR f IN ( + SELECT + oid::regclass::text as name, + CASE relkind + WHEN 'i' THEN 'INDEX' + WHEN 'I' THEN 'INDEX' + WHEN 'p' THEN 'TABLE' + WHEN 'r' THEN 'TABLE' + WHEN 'v' THEN 'VIEW' + WHEN 'S' THEN 'SEQUENCE' + ELSE NULL + END as typ + FROM pg_class + WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + RETURN; +END; +$$ LANGUAGE PLPGSQL; +SELECT pgstac_admin_owns(); + +CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; + +GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; + +ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; + +GRANT pgstac_read TO pgstac_ingest; +GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; + +SET ROLE pgstac_admin; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; +RESET ROLE; + +SET ROLE pgstac_ingest; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; +RESET ROLE; + +SET SEARCH_PATH TO pgstac, public; + +-- PgSTAC references PostGIS without schema qualification and runs with +-- search_path pinned to "pgstac, public", so PostGIS has to be reachable from +-- there. Fail here rather than part way through creating the schema. +DO $$ + BEGIN + IF to_regtype('geometry') IS NULL THEN + RAISE EXCEPTION 'PostGIS is not reachable from search_path "pgstac, public"' + USING HINT = 'PgSTAC requires the postgis extension in the public schema.'; + END IF; + END +$$; + +SET ROLE pgstac_admin; + +DO $$ + BEGIN + DROP FUNCTION IF EXISTS analyze_items; + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; +DO $$ + BEGIN + DROP FUNCTION IF EXISTS validate_constraints; + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; +-- Drain the queue here, where the functions its entries name still have the +-- signatures they were queued against and pgstac_admin owns the objects any +-- queued DDL touches. Statistics updates are discarded instead: +-- 998_idempotent_post recalculates every partition regardless. +DO $$ + BEGIN + DELETE FROM query_queue WHERE query LIKE 'SELECT update_partition_stats(%'; + PERFORM run_queued_queries_intransaction(); + EXCEPTION WHEN undefined_table OR undefined_function THEN + RAISE NOTICE 'No query queue to drain.'; + END +$$; + +-- Return type or argument list differs from an earlier release, which +-- CREATE OR REPLACE cannot change. +DROP FUNCTION IF EXISTS run_or_queue(text); +DROP FUNCTION IF EXISTS update_partition_stats_q(text, boolean); +DROP FUNCTION IF EXISTS update_partition_stats(text, boolean); +DROP FUNCTION IF EXISTS maintain_index(text, text, boolean, boolean, boolean); +DROP FUNCTION IF EXISTS queryable_indexes(text, boolean); + +-- Install these idempotently as migrations do not put them before trying to modify the collections table + + +CREATE OR REPLACE FUNCTION collection_geom(content jsonb) +RETURNS geometry AS $$ + WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) + SELECT + st_makeenvelope( + (box->>0)::float, + (box->>1)::float, + (box->>2)::float, + (box->>3)::float, + 4326 + ) + FROM box; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) +RETURNS timestamptz AS $$ + SELECT + CASE + WHEN + (content->'extent'->'temporal'->'interval'->0->>0) IS NULL + THEN '-infinity'::timestamptz + ELSE + (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz + END + ; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) +RETURNS timestamptz AS $$ + SELECT + CASE + WHEN + (content->'extent'->'temporal'->'interval'->0->>1) IS NULL + THEN 'infinity'::timestamptz + ELSE + (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz + END + ; +$$ LANGUAGE SQL IMMUTABLE STRICT; +-- BEGIN migra calculated SQL +drop index if exists "pgstac"."partitions_partition_idx"; + +drop function if exists "pgstac"."maintain_index"(indexname text, queryable_idx text, dropindexes boolean, rebuildindexes boolean, idxconcurrently boolean); + +drop materialized view if exists "pgstac"."partition_steps"; + +drop function if exists "pgstac"."queryable_indexes"(treeroot text, changes boolean, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text); + +drop function if exists "pgstac"."update_partition_stats"(_partition text, istrigger boolean); + +drop materialized view if exists "pgstac"."partitions"; + +drop view if exists "pgstac"."partitions_view"; + +drop index if exists "pgstac"."partitions_range_idx"; + +alter table "pgstac"."partition_stats" add column "collection" text; + +alter table "pgstac"."partition_stats" add column "partition_dtrange" tstzrange; + +CREATE INDEX partition_stats_collection_idx ON pgstac.partition_stats USING btree (collection); + +set check_function_bodies = off; + +CREATE OR REPLACE FUNCTION pgstac.maintain_index(_partition text, _indexname text, _queryable_id bigint, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) + RETURNS void + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path TO 'pgstac', 'public' +AS $function$ +DECLARE + _queryable_idx text; +BEGIN + -- Runs elevated, so it may only touch partitions of items. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(_partition)) THEN + RETURN; + END IF; + IF _queryable_id IS NOT NULL AND _partition IS NOT NULL THEN + SELECT format(indexdef(q), _partition) INTO _queryable_idx + FROM queryables q WHERE q.id = _queryable_id; + END IF; + IF _indexname IS NOT NULL THEN + IF dropindexes OR _queryable_idx IS NOT NULL THEN + EXECUTE format('DROP INDEX IF EXISTS %I;', _indexname); + ELSIF rebuildindexes THEN + IF idxconcurrently THEN + EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', _indexname); + ELSE + EXECUTE format('REINDEX INDEX %I;', _indexname); + END IF; + END IF; + END IF; + IF _queryable_idx IS NOT NULL THEN + IF idxconcurrently THEN + EXECUTE replace(_queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); + ELSE EXECUTE _queryable_idx; + END IF; + END IF; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.partition_catalog_meta(_partition text) + RETURNS TABLE(collection text, partition_dtrange tstzrange, constraint_dtrange tstzrange, constraint_edtrange tstzrange) + LANGUAGE plpgsql + STABLE +AS $function$ +DECLARE + _oid oid; + _parent oid; + _expr text; + _inf tstzrange := tstzrange('-infinity', 'infinity', '[]'); + _dtrange tstzrange; +BEGIN + _oid := partition_oid(_partition); + -- Leaves only, matching partitions_view: an intermediate partition has no + -- meaningful range of its own. + IF _oid IS NULL OR EXISTS (SELECT 1 FROM pg_inherits WHERE inhparent = _oid) THEN + RETURN; + END IF; + + SELECT inhparent INTO _parent FROM pg_inherits WHERE inhrelid = _oid; + + -- Partitions of items only. The SECURITY DEFINER functions below use this + -- to decide what they may alter, so any other relation must return nothing. + IF _parent IS NULL + OR ( + _parent <> 'pgstac.items'::regclass + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits + WHERE inhrelid = _parent AND inhparent = 'pgstac.items'::regclass + ) + ) + THEN + RETURN; + END IF; + + -- A partition of a sub-partitioned collection carries a datetime range + -- bound; its collection is on the parent. A direct partition of items + -- carries the collection itself. + IF _parent = 'pgstac.items'::regclass THEN + SELECT pg_get_expr(relpartbound, oid) INTO _expr FROM pg_class WHERE oid = _oid; + ELSE + SELECT pg_get_expr(relpartbound, oid) INTO _expr FROM pg_class WHERE oid = _parent; + END IF; + + SELECT COALESCE( + constraint_tstzrange(pg_get_expr(relpartbound, oid)), + _inf + ) INTO _dtrange FROM pg_class WHERE oid = _oid; + + RETURN QUERY SELECT + replace(replace(_expr, 'FOR VALUES IN (''', ''), ''')', ''), + _dtrange, + COALESCE(get_tstz_constraint(_oid, 'datetime'), _dtrange, _inf), + COALESCE(get_tstz_constraint(_oid, 'end_datetime'), _inf); +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.partition_oid(_partition text) + RETURNS oid + LANGUAGE sql + STABLE STRICT +AS $function$ + SELECT CASE + WHEN cardinality(parts) > 1 AND parts[cardinality(parts) - 1] <> 'pgstac' THEN NULL + ELSE to_regclass(format('pgstac.%I', parts[cardinality(parts)])) + END + FROM parse_ident(_partition) AS parts; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.queryable_indexes(treeroot text DEFAULT 'items'::text, changes boolean DEFAULT false, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text, OUT queryable_id bigint) + RETURNS SETOF record + LANGUAGE sql +AS $function$ +WITH p AS ( + SELECT + relid::text as partition, + replace(replace( + CASE + WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) + ELSE pg_get_expr(parent.relpartbound, parent.oid) + END, + 'FOR VALUES IN (''',''), ''')', + '' + ) AS collection + FROM pg_partition_tree(treeroot) + JOIN pg_class c ON (relid::regclass = c.oid) + JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) + ), i AS ( + SELECT + partition, + indexname, + regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, + COALESCE( + (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], + (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], + CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END + ) AS field + FROM + pg_indexes + JOIN p ON (tablename=partition) + ), q AS ( + SELECT + name AS field, + collection, + partition, + format(indexdef(queryables), partition) as qidx, + queryables.id as qid + FROM queryables, unnest_collection(queryables.collection_ids) collection + JOIN p USING (collection) + WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') + ) + SELECT + collection, + partition, + field, + indexname, + iidx as existing_idx, + qidx as queryable_idx, + qid as queryable_id + FROM i FULL JOIN q USING (field, partition) + WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; +; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.sync_partition_stats() + RETURNS void + LANGUAGE plpgsql + SET search_path TO 'pgstac', 'public' +AS $function$ +BEGIN + -- Ordered by partition, as every other writer of these rows is. + INSERT INTO partition_stats (partition, collection, partition_dtrange) + SELECT partition, collection, partition_dtrange FROM partitions_view + ORDER BY partition + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + + DELETE FROM partition_stats ps + WHERE ps.partition IN ( + SELECT partition FROM partition_stats stale + WHERE NOT EXISTS ( + SELECT 1 FROM partitions_view pv WHERE pv.partition = stale.partition + ) + ORDER BY partition + FOR UPDATE + ); +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.update_partition_stats(_partition text, istrigger boolean DEFAULT false, _extent boolean DEFAULT NULL::boolean) + RETURNS void + LANGUAGE plpgsql + SET search_path TO 'pgstac', 'public' +AS $function$ +DECLARE + dtrange tstzrange; + edtrange tstzrange; + cdtrange tstzrange; + cedtrange tstzrange; + extent geometry; + collection text; + pdtrange tstzrange; + auto_extent boolean := get_setting_bool('update_collection_extent'); + do_extent boolean := COALESCE(_extent, auto_extent); +BEGIN + -- Cannot be STRICT: _extent is three valued, and STRICT would skip the + -- body whenever it is NULL. + IF _partition IS NULL OR istrigger IS NULL THEN + RETURN; + END IF; + RAISE NOTICE 'Updating stats for %.', _partition; + + SELECT m.collection, m.partition_dtrange, m.constraint_dtrange, m.constraint_edtrange + INTO collection, pdtrange, cdtrange, cedtrange + FROM partition_catalog_meta(_partition) m; + + -- A queued update can outlive the partition it names. + IF NOT FOUND THEN + RAISE NOTICE 'Partition % no longer exists, skipping stats update.', _partition; + RETURN; + END IF; + + -- Taken before the partition is read. The constraint rebuild below needs + -- ACCESS EXCLUSIVE, and escalating to that mid-transaction deadlocks + -- against another session doing the same. SHARE UPDATE EXCLUSIVE conflicts + -- with itself but not with readers. + IF NOT istrigger THEN + EXECUTE format('LOCK TABLE %I IN SHARE UPDATE EXCLUSIVE MODE', _partition); + END IF; + + -- The observed ranges feed the constraint tightening below and collection + -- extents. When neither will read them, only the partition's identity is + -- written, which is what keeps it visible to search. + IF NOT istrigger OR do_extent THEN + -- st_extent visits every geometry, so it is only run when wanted. + IF do_extent THEN + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]'), + st_extent(geometry)::geometry + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange, extent; + RAISE DEBUG 'Extent: %', extent; + ELSE + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]') + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange; + END IF; + + INSERT INTO partition_stats + (partition, collection, partition_dtrange, dtrange, edtrange, spatial, last_updated) + VALUES (_partition, collection, pdtrange, dtrange, edtrange, extent, now()) + ON CONFLICT (partition) DO + UPDATE SET + collection=EXCLUDED.collection, + partition_dtrange=EXCLUDED.partition_dtrange, + dtrange=EXCLUDED.dtrange, + edtrange=EXCLUDED.edtrange, + spatial=COALESCE(EXCLUDED.spatial, partition_stats.spatial), + last_updated=EXCLUDED.last_updated + ; + ELSE + INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES (_partition, collection, pdtrange) + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + END IF; + + RAISE NOTICE 'Checking if we need to modify constraints...'; + RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; + IF + (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) + AND NOT istrigger + THEN + RAISE NOTICE 'Modifying Constraints'; + RAISE NOTICE 'Existing % %', cdtrange, cedtrange; + RAISE NOTICE 'New % %', dtrange, edtrange; + PERFORM drop_table_constraints(_partition); + PERFORM create_table_constraints(_partition, dtrange, edtrange); + END IF; + -- auto_extent, not do_extent: a caller that passed _extent aggregates the + -- extent itself, and update_collection_extents would then be updating + -- collections from inside its own UPDATE of collections. + RAISE NOTICE 'Checking if we need to update collection extents.'; + IF auto_extent THEN + RAISE NOTICE 'updating collection extent for %', collection; + PERFORM run_or_queue(format($q$ + UPDATE collections + SET content = jsonb_set_lax( + content, + '{extent}'::text[], + collection_extent(%L, FALSE), + true, + 'use_json_null' + ) WHERE id=%L + ; + $q$, collection, collection)); + ELSE + RAISE NOTICE 'Not updating collection extent for %', collection; + END IF; + +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.check_partition(_collection text, _dtrange tstzrange, _edtrange tstzrange) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path TO 'pgstac', 'public' +AS $function$ +DECLARE + c RECORD; + pm RECORD; + _partition_name text; + _partition_dtrange tstzrange; + _constraint_dtrange tstzrange; + _constraint_edtrange tstzrange; + q text; + err_context text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + + IF c.partition_trunc IS NOT NULL THEN + _partition_dtrange := tstzrange( + date_trunc(c.partition_trunc, lower(_dtrange)), + date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, + '[)' + ); + ELSE + _partition_dtrange := '[-infinity, infinity]'::tstzrange; + END IF; + + IF NOT _partition_dtrange @> _dtrange THEN + RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; + END IF; + + + IF c.partition_trunc = 'year' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); + ELSE + _partition_name := format('_items_%s', c.key); + END IF; + + -- Constraint ranges are maintained asynchronously, so they come from the + -- catalog rather than partition_stats. + SELECT ps.partition, m.constraint_dtrange, m.constraint_edtrange + INTO pm + FROM partition_stats ps + JOIN LATERAL partition_catalog_meta(ps.partition) m ON TRUE + WHERE ps.collection = _collection AND ps.partition_dtrange @> _dtrange + LIMIT 1; + IF FOUND THEN + RAISE NOTICE '% % %', _edtrange, _dtrange, pm; + _constraint_edtrange := + tstzrange( + least( + lower(_edtrange), + nullif(lower(pm.constraint_edtrange), '-infinity') + ), + greatest( + upper(_edtrange), + nullif(upper(pm.constraint_edtrange), 'infinity') + ), + '[]' + ); + _constraint_dtrange := + tstzrange( + least( + lower(_dtrange), + nullif(lower(pm.constraint_dtrange), '-infinity') + ), + greatest( + upper(_dtrange), + nullif(upper(pm.constraint_dtrange), 'infinity') + ), + '[]' + ); + + IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN + RETURN pm.partition; + ELSE + PERFORM drop_table_constraints(_partition_name); + END IF; + ELSE + _constraint_edtrange := _edtrange; + _constraint_dtrange := _dtrange; + END IF; + RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; + RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; + IF c.partition_trunc IS NULL THEN + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + GRANT ALL ON %I to pgstac_ingest; + $q$, + _partition_name, + _collection, + concat(_partition_name,'_pk'), + _partition_name, + _partition_name + ); + ELSE + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); + CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + GRANT ALL ON %I TO pgstac_ingest; + $q$, + format('_items_%s', c.key), + _collection, + _partition_name, + format('_items_%s', c.key), + lower(_partition_dtrange), + upper(_partition_dtrange), + format('%s_pk', _partition_name), + _partition_name, + _partition_name + ); + END IF; + + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', _partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + -- The _constraint_ ranges are the union of the existing constraint and the + -- incoming batch, so they hold for rows already present as well as the ones + -- about to be added. Queueable: rebuilding validates the partition under an + -- ACCESS EXCLUSIVE lock. + PERFORM run_or_queue(format( + 'SELECT create_table_constraints(%L, %L, %L);', + _partition_name, + _constraint_dtrange, + _constraint_edtrange + )); + PERFORM maintain_partitions(_partition_name); + -- Search finds partitions through partition_stats, so the row has to exist + -- before this transaction commits. A queued stats update has not written it. + IF NOT update_partition_stats_q(_partition_name, true) THEN + INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES (_partition_name, _collection, _partition_dtrange) + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + END IF; + RETURN _partition_name; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.check_pgstac_settings(_sysmem text DEFAULT NULL::text) + RETURNS void + LANGUAGE plpgsql + SET search_path TO 'pgstac', 'public' + SET client_min_messages TO 'notice' +AS $function$ +DECLARE + settingval text; + sysmem bigint := pg_size_bytes(_sysmem); + effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); + shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); + work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); + max_connections int := current_setting('max_connections', TRUE); + maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); + seq_page_cost float := current_setting('seq_page_cost', TRUE); + random_page_cost float := current_setting('random_page_cost', TRUE); + temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); + r record; +BEGIN + IF _sysmem IS NULL THEN + RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; + ELSE + IF effective_cache_size < (sysmem * 0.5) THEN + RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); + ELSIF effective_cache_size > (sysmem * 0.75) THEN + RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); + ELSE + RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); + END IF; + + IF shared_buffers < (sysmem * 0.2) THEN + RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); + ELSIF shared_buffers > (sysmem * 0.3) THEN + RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); + ELSE + RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); + END IF; + shared_buffers = sysmem * 0.3; + IF maintenance_work_mem < (sysmem * 0.2) THEN + RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); + ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN + RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); + ELSE + RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); + END IF; + + IF work_mem * max_connections > shared_buffers THEN + RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); + ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN + RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); + ELSE + RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); + END IF; + + IF random_page_cost / seq_page_cost != 1.1 THEN + RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; + ELSE + RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; + END IF; + + IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN + RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); + END IF; + END IF; + + RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; + RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; + + FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP + RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; + END LOOP; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; + ELSE + RAISE NOTICE 'pg_cron % is installed', settingval; + END IF; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; + ELSE + RAISE NOTICE 'pgstattuple % is installed', settingval; + END IF; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; + ELSE + RAISE NOTICE 'pg_stat_statements % is installed', settingval; + IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN + RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; + END IF; + END IF; + + -- Undrained queue means stale statistics and constraints, silently. + SELECT count(*) INTO settingval FROM query_queue; + IF settingval::bigint > 0 THEN + RAISE WARNING '% queries are waiting in query_queue. Run "CALL run_queued_queries();" (pypgstac runqueue) to drain it -- until then partition statistics and constraints are stale.', settingval; + END IF; + +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.chunker(_where text, OUT s timestamp with time zone, OUT e timestamp with time zone) + RETURNS SETOF record + LANGUAGE plpgsql +AS $function$ +DECLARE + explain jsonb; +BEGIN + IF _where IS NULL THEN + _where := ' TRUE '; + END IF; + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) + INTO explain; + RAISE DEBUG 'EXPLAIN: %', explain; + + RETURN QUERY + WITH t AS ( + SELECT j->>0 as p FROM + jsonb_path_query( + explain, + 'strict $.**."Relation Name" ? (@ != null)' + ) j + ), + parts AS ( + -- = ANY(array) uses the partition_stats primary key; the planner + -- cannot estimate jsonb_path_query, so a join plans as a full scan. + SELECT + date_trunc('month', lower(partition_dtrange)) as sdate, + date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate + FROM partition_stats + WHERE + partition = ANY (ARRAY(SELECT p FROM t)) + AND partition_dtrange IS NOT NULL + AND partition_dtrange != 'empty'::tstzrange + ), + times AS ( + SELECT sdate FROM parts + UNION + SELECT edate FROM parts + ), + uniq AS ( + SELECT DISTINCT sdate FROM times ORDER BY sdate + ), + last AS ( + SELECT sdate, lead(sdate, 1) over () as edate FROM uniq + ) + SELECT sdate, edate FROM last WHERE edate IS NOT NULL; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.collection_delete_trigger_func() + RETURNS trigger + LANGUAGE plpgsql +AS $function$ +DECLARE + collection_base_partition text := concat('_items_', OLD.key); +BEGIN + -- Tables before rows: check_partition takes these locks in the same order, + -- and the reverse deadlocks against a concurrent partition create. + EXECUTE format($q$ + DROP TABLE IF EXISTS %I CASCADE; + DELETE FROM partition_stats WHERE collection=%L; + $q$, + collection_base_partition, + OLD.id + ); + RETURN OLD; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.collection_extent(_collection text, runupdate boolean DEFAULT false) + RETURNS jsonb + LANGUAGE plpgsql +AS $function$ +DECLARE + geom_extent geometry; + mind timestamptz; + maxd timestamptz; + extent jsonb; + _partition text; +BEGIN + IF runupdate THEN + -- Not queued: the aggregate below reads what this writes. Ordered by + -- partition, as every other multi-partition writer is. + FOR _partition IN + SELECT partition FROM partition_stats + WHERE collection=_collection + ORDER BY partition + LOOP + PERFORM update_partition_stats(_partition, false, true); + END LOOP; + END IF; + SELECT + min(lower(dtrange)), + max(upper(edtrange)), + st_extent(spatial) + INTO + mind, + maxd, + geom_extent + FROM partition_stats + WHERE collection=_collection; + + IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN + extent := jsonb_build_object( + 'spatial', jsonb_build_object( + 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) + ), + 'temporal', jsonb_build_object( + 'interval', to_jsonb(array[array[mind, maxd]]) + ) + ); + RETURN extent; + END IF; + RETURN NULL; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path TO 'pgstac', 'public' +AS $function$ +DECLARE + q text; + _oid oid := partition_oid(t); +BEGIN + IF _oid IS NULL THEN + RETURN NULL; + END IF; + -- Only partitions of items. This runs elevated, so without the check it + -- would alter any table pgstac_admin owns that the caller names. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(t)) THEN + RETURN NULL; + END IF; + -- Reduce to the bare name so the ALTER statements below quote it correctly + -- even when the caller passed a schema qualified name. + t := get_partition_name(_oid); + RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; + IF _dtrange = 'empty' AND _edtrange = 'empty' THEN + q :=format( + $q$ + DO $block$ + BEGIN + ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + + + + EXCEPTION WHEN others THEN + RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; + END; + $block$; + $q$, + t, + format('%s_dt', t), + t, + format('%s_dt', t), + t, + format('%s_dt', t), + t + ); + ELSE + q :=format( + $q$ + DO $block$ + BEGIN + + ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK ( + (datetime >= %L) + AND (datetime <= %L) + AND (end_datetime >= %L) + AND (end_datetime <= %L) + ) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + + + + EXCEPTION WHEN others THEN + RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; + END; + $block$; + $q$, + t, + format('%s_dt', t), + t, + format('%s_dt', t), + lower(_dtrange), + upper(_dtrange), + lower(_edtrange), + upper(_edtrange), + t, + format('%s_dt', t), + t + ); + END IF; + -- Run, not queued: the queue runner is not a definer, so queued DDL + -- executes as whoever drains it. Defer by queueing a call to this function. + EXECUTE q; + RETURN t; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.delete_collection(_id text) + RETURNS void + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path TO 'pgstac', 'public' +AS $function$ +BEGIN + DELETE FROM collections WHERE id = _id; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _id USING ERRCODE = 'no_data_found'; + END IF; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.drop_table_constraints(t text) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path TO 'pgstac', 'public' +AS $function$ +DECLARE + q text; + _oid oid := partition_oid(t); +BEGIN + IF _oid IS NULL THEN + RETURN NULL; + END IF; + -- Only partitions of items. This runs elevated, so without the check it + -- would alter any table pgstac_admin owns that the caller names. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(t)) THEN + RETURN NULL; + END IF; + -- Reduce to the bare name so the ALTER statements below quote it correctly + -- even when the caller passed a schema qualified name. + t := get_partition_name(_oid); + FOR q IN SELECT FORMAT( + $q$ + ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; + $q$, + t, + conname + ) FROM pg_constraint + WHERE conrelid=_oid AND contype='c' + LOOP + EXECUTE q; + END LOOP; + RETURN t; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.format_item(_item items, _fields jsonb DEFAULT '{}'::jsonb, _hydrated boolean DEFAULT true) + RETURNS jsonb + LANGUAGE plpgsql +AS $function$ +DECLARE + cache bool := get_setting_bool('format_cache'); + _output jsonb := null; + t timestamptz := clock_timestamp(); +BEGIN + IF cache THEN + SELECT output INTO _output FROM format_item_cache + WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; + END IF; + IF _output IS NULL THEN + IF _hydrated THEN + _output := content_hydrate(_item, _fields); + ELSE + _output := content_nonhydrated(_item, _fields); + END IF; + END IF; + IF cache THEN + INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) + VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) + ON CONFLICT(collection, id, fields, hydrated) DO + UPDATE + SET lastused=now(), usecount = format_item_cache.usecount + 1 + ; + END IF; + RETURN _output; + +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.items_staging_triggerfunc() + RETURNS trigger + LANGUAGE plpgsql +AS $function$ +DECLARE + part text; + ts timestamptz := clock_timestamp(); + nrows int; +BEGIN + RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; + + FOR part IN WITH t AS ( + SELECT + n.content->>'collection' as collection, + stac_daterange(n.content->'properties') as dtr, + partition_trunc + FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) + ), p AS ( + SELECT + collection, + COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, + tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, + tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange + FROM t + GROUP BY 1,2 + -- Ordered: check_partition holds DDL and row locks until commit. + ) SELECT check_partition(collection, dtrange, edtrange) FROM ( + SELECT * FROM p ORDER BY collection, d + ) ordered LOOP + RAISE NOTICE 'Partition %', part; + END LOOP; + + RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; + DROP TABLE IF EXISTS tmpdata; + CREATE TEMP TABLE tmpdata ON COMMIT DROP AS + SELECT + (content_dehydrate(content)).* + FROM newdata; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; + + RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; + IF TG_TABLE_NAME = 'items_staging' THEN + INSERT INTO items + SELECT * FROM tmpdata; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; + ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN + INSERT INTO items + SELECT * FROM tmpdata + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; + ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN + -- Locked in a fixed order first, so concurrent upserts over an + -- overlapping id set cannot deadlock. A bare DELETE gives no ordering; + -- ORDER BY ... FOR UPDATE does, because LockRows sits above the sort. + WITH locked AS ( + SELECT o.collection, o.id + FROM tmpdata s + JOIN items o ON (o.id = s.id AND o.collection = s.collection) + WHERE o IS DISTINCT FROM s + ORDER BY o.collection, o.id + FOR UPDATE OF o + ) + DELETE FROM items i + USING locked l + WHERE i.collection = l.collection AND i.id = l.id + ; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; + INSERT INTO items AS t + SELECT * FROM tmpdata + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; + END IF; + + RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; + EXECUTE format('DELETE FROM %I', TG_TABLE_NAME); + RAISE NOTICE 'Done. %', clock_timestamp() - ts; + + RETURN NULL; + +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) + RETURNS SETOF text + LANGUAGE plpgsql +AS $function$ +DECLARE + rec record; + q text; +BEGIN + FOR rec IN ( + SELECT * FROM queryable_indexes(part,true) + ) LOOP + q := format( + 'SELECT maintain_index( + %L,%L,%L,%L,%L,%L + );', + rec.partition, + rec.indexname, + rec.queryable_id, + dropindexes, + rebuildindexes, + idxconcurrently + ); + RAISE NOTICE 'Q: %', q; + RETURN NEXT q; + END LOOP; + RETURN; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.partition_after_triggerfunc() + RETURNS trigger + LANGUAGE plpgsql + SET search_path TO 'pgstac', 'public' +AS $function$ +DECLARE + p text; + t timestamptz := clock_timestamp(); +BEGIN + RAISE NOTICE 'Updating partition stats %', t; + -- Ordered: each iteration holds a partition_stats row lock until commit. + FOR p IN SELECT DISTINCT partition + FROM newdata n JOIN partition_stats p + ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) + ORDER BY 1 + LOOP + PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); + END LOOP; + IF TG_OP IN ('DELETE','UPDATE') THEN + DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; + END IF; + RAISE NOTICE 't: % %', t, clock_timestamp() - t; + RETURN NULL; +END; +$function$ +; + +create or replace view "pgstac"."partition_sys_meta" as SELECT partition.partition, + replace(replace( + CASE + WHEN (pg_partition_tree.level = 1) THEN partition_expr.partition_expr + ELSE parent_partition_expr.parent_partition_expr + END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, + pg_partition_tree.level, + c.reltuples, + c.relhastriggers, + partition_dtrange.partition_dtrange, + COALESCE(get_tstz_constraint(c.oid, 'datetime'::text), partition_dtrange.partition_dtrange, inf_range.inf_range) AS constraint_dtrange, + COALESCE(get_tstz_constraint(c.oid, 'end_datetime'::text), inf_range.inf_range) AS constraint_edtrange + FROM (((((((pg_partition_tree('items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) + JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) + JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) + JOIN LATERAL get_partition_name(pg_partition_tree.relid) partition(partition) ON (true)) + JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) partition_expr(partition_expr) ON (true)) + JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) parent_partition_expr(parent_partition_expr) ON (true)) + JOIN LATERAL tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text) inf_range(inf_range) ON (true)) + JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range.inf_range) partition_dtrange(partition_dtrange) ON (true)) + WHERE pg_partition_tree.isleaf; + + +create or replace view "pgstac"."partitions_view" as SELECT sm.partition, + sm.collection, + sm.level, + sm.reltuples, + sm.relhastriggers, + sm.partition_dtrange, + sm.constraint_dtrange, + sm.constraint_edtrange, + ps.dtrange, + ps.edtrange, + ps.spatial, + ps.last_updated + FROM (partition_sys_meta sm + LEFT JOIN partition_stats ps USING (partition)); + + +CREATE OR REPLACE FUNCTION pgstac.repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT false) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path TO 'pgstac', 'public' +AS $function$ +DECLARE + c RECORD; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + IF triggered THEN + RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; + ELSE + RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; + IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN + RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; + RETURN _collection; + END IF; + END IF; + + IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN + EXECUTE format( + $q$ + CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; + DROP TABLE IF EXISTS %I CASCADE; + DELETE FROM partition_stats WHERE collection = %L; + WITH p AS ( + SELECT + collection, + CASE + WHEN %L IS NULL THEN '-infinity'::timestamptz + ELSE date_trunc(%L, datetime) + END as d, + tstzrange(min(datetime),max(datetime),'[]') as dtrange, + tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange + FROM changepartitionstaging + GROUP BY 1,2 + ) SELECT check_partition(collection, dtrange, edtrange) FROM p; + INSERT INTO items SELECT * FROM changepartitionstaging; + DROP TABLE changepartitionstaging; + $q$, + concat('_items_', c.key), + concat('_items_', c.key), + _collection, + c.partition_trunc, + c.partition_trunc + ); + END IF; + RETURN _collection; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.run_or_queue(query text) + RETURNS boolean + LANGUAGE plpgsql +AS $function$ +DECLARE + use_queue boolean := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; +BEGIN + IF get_setting_bool('debug') THEN + RAISE NOTICE '%', query; + END IF; + IF use_queue THEN + INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; + RETURN FALSE; + END IF; + EXECUTE query; + RETURN TRUE; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) + RETURNS searches + LANGUAGE plpgsql +AS $function$ +DECLARE + search searches%ROWTYPE; + cached_search searches%ROWTYPE; + pexplain jsonb; + t timestamptz; + i interval; + doupdate boolean := FALSE; + insertfound boolean := FALSE; + ro boolean := pgstac.readonly(); + found_search text; +BEGIN + RAISE NOTICE 'SEARCH: %', _search; + -- Calculate hash, where clause, and order by statement + search.search := _search; + search.metadata := _metadata; + search.hash := search_hash(_search, _metadata); + search._where := stac_search_to_where(_search); + search.orderby := sort_sqlorderby(_search); + search.lastused := now(); + search.usecount := 1; + + -- If we are in read only mode, directly return search + IF ro THEN + RETURN search; + END IF; + + RAISE NOTICE 'Updating Statistics for search: %s', search; + -- Update statistics for times used and and when last used + -- If the entry is locked, rather than waiting, skip updating the stats + INSERT INTO searches (search, lastused, usecount, metadata) + VALUES (search.search, now(), 1, search.metadata) + ON CONFLICT DO NOTHING + RETURNING * INTO cached_search + ; + + IF NOT FOUND OR cached_search IS NULL THEN + UPDATE searches SET + lastused = now(), + usecount = searches.usecount + 1 + WHERE hash = ( + SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED + ) + RETURNING * INTO cached_search + ; + END IF; + + IF cached_search IS NOT NULL THEN + cached_search._where = search._where; + cached_search.orderby = search.orderby; + RETURN cached_search; + END IF; + RETURN search; + +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.update_collection_extents() + RETURNS void + LANGUAGE sql +AS $function$ +UPDATE collections + SET content = jsonb_set_lax( + content, + '{extent}'::text[], + collection_extent(id, TRUE), + true, + 'return_target' + ) +; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.update_partition_stats_q(_partition text, istrigger boolean DEFAULT false) + RETURNS boolean + LANGUAGE plpgsql +AS $function$ +DECLARE +BEGIN + RETURN run_or_queue( + format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) + ); +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) + RETURNS search_wheres + LANGUAGE plpgsql +AS $function$ +DECLARE + t timestamptz; + i interval; + explain_json jsonb; + sw search_wheres%ROWTYPE; + inwhere_hash text := md5(inwhere); + _context text := lower(context(conf)); + _stats_ttl interval := context_stats_ttl(conf); + _estimated_cost_threshold float := context_estimated_cost(conf); + _estimated_count_threshold int := context_estimated_count(conf); + ro bool := pgstac.readonly(conf); +BEGIN + -- If updatestats is true then set ttl to 0 + IF updatestats THEN + RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; + _stats_ttl := '0'::interval; + END IF; + + -- If we don't need to calculate context, just return + IF _context = 'off' THEN + sw._where = inwhere; + RETURN sw; + END IF; + + -- Unlocked read. A fresh hit only bumps bookkeeping counters, and that is + -- the common case when identical searches run concurrently. + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; + + -- Within ttl: bump usage counters and return. The bump skips locked rows so + -- identical searches do not serialize; a missed increment is harmless. + -- sw.id, not "sw IS NOT NULL": a composite is only IS NOT NULL when every + -- field is, and search_wheres.partitions is never populated. + IF + sw.id IS NOT NULL + AND sw.statslastupdated IS NOT NULL + AND sw.total_count IS NOT NULL + AND now() - sw.statslastupdated <= _stats_ttl + THEN + RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; + IF NOT ro THEN + UPDATE search_wheres SET + lastused = now(), + usecount = search_wheres.usecount + 1 + WHERE id = ( + SELECT id FROM search_wheres + WHERE md5(_where) = inwhere_hash + FOR UPDATE SKIP LOCKED + ); + END IF; + RAISE DEBUG 'Returning cached counts. %', sw; + RETURN sw; + END IF; + + -- Missing or stale, so lock the row to compute once, then re-check + -- freshness in case another session finished while we waited. + IF NOT ro THEN + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; + IF + sw.statslastupdated IS NOT NULL + AND sw.total_count IS NOT NULL + AND now() - sw.statslastupdated <= _stats_ttl + THEN + RAISE DEBUG 'Another process refreshed stats while we waited: %', sw; + UPDATE search_wheres SET + lastused = now(), + usecount = search_wheres.usecount + 1 + WHERE md5(_where) = inwhere_hash + RETURNING * INTO sw; + RETURN sw; + END IF; + END IF; + + -- Calculate estimated cost and rows + -- Use explain to get estimated count/cost + IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN + RAISE DEBUG 'Calculating estimated stats'; + t := clock_timestamp(); + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) + INTO explain_json; + RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; + i := clock_timestamp() - t; + + sw.estimated_count := (explain_json->0->'Plan'->>'Plan Rows')::bigint; + sw.estimated_cost := (explain_json->0->'Plan'->>'Total Cost')::float; + sw.time_to_estimate := extract(epoch from i); + END IF; + + RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; + RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; + + -- If context is set to auto and the costs are within the threshold return the estimated costs + IF + _context = 'auto' + AND sw.estimated_count >= _estimated_count_threshold + AND sw.estimated_cost >= _estimated_cost_threshold + THEN + IF NOT ro THEN + INSERT INTO search_wheres ( + _where, + lastused, + usecount, + statslastupdated, + estimated_count, + estimated_cost, + time_to_estimate, + total_count, + time_to_count + ) VALUES ( + inwhere, + now(), + 1, + now(), + sw.estimated_count, + sw.estimated_cost, + sw.time_to_estimate, + null, + null + ) ON CONFLICT ((md5(_where))) + DO UPDATE SET + lastused = EXCLUDED.lastused, + usecount = search_wheres.usecount + 1, + statslastupdated = EXCLUDED.statslastupdated, + estimated_count = EXCLUDED.estimated_count, + estimated_cost = EXCLUDED.estimated_cost, + time_to_estimate = EXCLUDED.time_to_estimate, + total_count = EXCLUDED.total_count, + time_to_count = EXCLUDED.time_to_count + RETURNING * INTO sw; + END IF; + RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; + RETURN sw; + END IF; + + -- Calculate Actual Count + t := clock_timestamp(); + RAISE NOTICE 'Calculating actual count...'; + EXECUTE format( + 'SELECT count(*) FROM items WHERE %s', + inwhere + ) INTO sw.total_count; + i := clock_timestamp() - t; + RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; + sw.time_to_count := extract(epoch FROM i); + + IF NOT ro THEN + INSERT INTO search_wheres ( + _where, + lastused, + usecount, + statslastupdated, + estimated_count, + estimated_cost, + time_to_estimate, + total_count, + time_to_count + ) VALUES ( + inwhere, + now(), + 1, + now(), + sw.estimated_count, + sw.estimated_cost, + sw.time_to_estimate, + sw.total_count, + sw.time_to_count + ) ON CONFLICT ((md5(_where))) + DO UPDATE SET + lastused = EXCLUDED.lastused, + usecount = search_wheres.usecount + 1, + statslastupdated = EXCLUDED.statslastupdated, + estimated_count = EXCLUDED.estimated_count, + estimated_cost = EXCLUDED.estimated_cost, + time_to_estimate = EXCLUDED.time_to_estimate, + total_count = EXCLUDED.total_count, + time_to_count = EXCLUDED.time_to_count + RETURNING * INTO sw; + END IF; + RAISE DEBUG 'Returning with actual count. %', sw; + RETURN sw; +END; +$function$ +; + +create or replace view "pgstac"."partitions" as SELECT partition, + collection, + level, + reltuples, + relhastriggers, + partition_dtrange, + constraint_dtrange, + constraint_edtrange, + dtrange, + edtrange, + spatial, + last_updated + FROM partitions_view; + + + +-- END migra calculated SQL +DO $$ + BEGIN + INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES + ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); + EXCEPTION WHEN unique_violation THEN + RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES + ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); + EXCEPTION WHEN unique_violation THEN + RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES + ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); + EXCEPTION WHEN unique_violation THEN + RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DELETE FROM queryables a USING queryables b + WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; + + +INSERT INTO pgstac_settings (name, value) VALUES + ('context', 'off'), + ('context_estimated_count', '100000'), + ('context_estimated_cost', '100000'), + ('context_stats_ttl', '1 day'), + ('default_filter_lang', 'cql2-json'), + ('additional_properties', 'true'), + ('use_queue', 'false'), + ('queue_timeout', '10 minutes'), + ('update_collection_extent', 'false'), + ('format_cache', 'false'), + ('readonly', 'false') +ON CONFLICT DO NOTHING +; + + +INSERT INTO cql2_ops (op, template, types) VALUES + ('eq', '%s = %s', NULL), + ('neq', '%s != %s', NULL), + ('ne', '%s != %s', NULL), + ('!=', '%s != %s', NULL), + ('<>', '%s != %s', NULL), + ('lt', '%s < %s', NULL), + ('lte', '%s <= %s', NULL), + ('gt', '%s > %s', NULL), + ('gte', '%s >= %s', NULL), + ('le', '%s <= %s', NULL), + ('ge', '%s >= %s', NULL), + ('=', '%s = %s', NULL), + ('<', '%s < %s', NULL), + ('<=', '%s <= %s', NULL), + ('>', '%s > %s', NULL), + ('>=', '%s >= %s', NULL), + ('like', '%s LIKE %s', NULL), + ('ilike', '%s ILIKE %s', NULL), + ('+', '%s + %s', NULL), + ('-', '%s - %s', NULL), + ('*', '%s * %s', NULL), + ('/', '%s / %s', NULL), + ('not', 'NOT (%s)', NULL), + ('between', '%s BETWEEN %s AND %s', NULL), + ('isnull', '%s IS NULL', NULL), + ('upper', 'upper(%s)', NULL), + ('lower', 'lower(%s)', NULL), + ('casei', 'upper(%s)', NULL), + ('accenti', 'unaccent(%s)', NULL) +ON CONFLICT (op) DO UPDATE + SET + template = EXCLUDED.template +; + + +ALTER FUNCTION to_text COST 5000; +ALTER FUNCTION to_float COST 5000; +ALTER FUNCTION to_int COST 5000; +ALTER FUNCTION to_tstz COST 5000; +ALTER FUNCTION to_text_array COST 5000; + +ALTER FUNCTION drop_table_constraints SECURITY DEFINER; +ALTER FUNCTION create_table_constraints SECURITY DEFINER; +ALTER FUNCTION check_partition SECURITY DEFINER; +ALTER FUNCTION repartition SECURITY DEFINER; +ALTER FUNCTION maintain_index SECURITY DEFINER; + +-- Elevated only where ownership of pgstac_admin's objects is required. +-- Everything else relies on table privileges, which any role inheriting +-- pgstac_ingest or pgstac_read already has. +ALTER FUNCTION where_stats SECURITY INVOKER; +ALTER FUNCTION search_query SECURITY INVOKER; +ALTER FUNCTION format_item SECURITY INVOKER; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +GRANT ALL ON SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON SCHEMA pgstac to pgstac_admin; + +-- pgstac_read role limited to using function apis +GRANT EXECUTE ON FUNCTION search TO pgstac_read; +GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; +GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; +GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; +GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; + +-- The caches maintained by where_stats, search_query and format_item. +GRANT SELECT, INSERT, UPDATE ON search_wheres TO pgstac_read; +GRANT SELECT, INSERT, UPDATE ON searches TO pgstac_read; +GRANT SELECT, INSERT, UPDATE ON format_item_cache TO pgstac_read; + + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; + +REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; +GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; + +REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; +GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; + +-- PostgreSQL grants EXECUTE to PUBLIC on every new function, so each definer +-- is revoked and granted back to the roles that need it. Keep in step with the +-- ALTER FUNCTION ... SECURITY DEFINER statements above; the pgtap suite fails +-- if a definer is left PUBLIC executable. +REVOKE ALL PRIVILEGES ON FUNCTION + drop_table_constraints, + create_table_constraints, + check_partition, + repartition, + maintain_index, + delete_collection +FROM public; + +GRANT EXECUTE ON FUNCTION + drop_table_constraints, + create_table_constraints, + check_partition, + repartition, + maintain_index, + delete_collection +TO pgstac_ingest; + +RESET ROLE; + +SET ROLE pgstac_ingest; + +-- Search finds partitions through partition_stats, so this must be synchronous +-- rather than queued. +SELECT sync_partition_stats(); + +-- Repairs observed ranges and CHECK constraints for every partition, ordered +-- by partition as every other writer of these rows is. The most expensive part +-- of an install on a large catalog: run with pgstac.use_queue on (pypgstac +-- --usequeue) and drain with pypgstac runqueue to keep it off the migration. +SELECT update_partition_stats_q(partition) FROM partitions_view ORDER BY partition; +SELECT set_version('0.9.12'); diff --git a/src/pgstac/migrations/pgstac.0.9.12.sql b/src/pgstac/migrations/pgstac.0.9.12.sql new file mode 100644 index 00000000..6d71b2b4 --- /dev/null +++ b/src/pgstac/migrations/pgstac.0.9.12.sql @@ -0,0 +1,4979 @@ +RESET ROLE; +DO $$ +DECLARE +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN + CREATE EXTENSION IF NOT EXISTS postgis; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN + CREATE EXTENSION IF NOT EXISTS btree_gist; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN + CREATE EXTENSION IF NOT EXISTS unaccent; + END IF; +END; +$$ LANGUAGE PLPGSQL; + +DO $$ + BEGIN + CREATE ROLE pgstac_admin; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_read; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_ingest; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + + +GRANT pgstac_admin TO current_user; + +-- Function to make sure pgstac_admin is the owner of items +CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ +DECLARE + f RECORD; +BEGIN + FOR f IN ( + SELECT + concat( + oid::regproc::text, + '(', + coalesce(pg_get_function_identity_arguments(oid),''), + ')' + ) AS name, + CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ + FROM pg_proc + WHERE + pronamespace=to_regnamespace('pgstac') + AND proowner != to_regrole('pgstac_admin') + AND proname NOT LIKE 'pg_stat%' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + FOR f IN ( + SELECT + oid::regclass::text as name, + CASE relkind + WHEN 'i' THEN 'INDEX' + WHEN 'I' THEN 'INDEX' + WHEN 'p' THEN 'TABLE' + WHEN 'r' THEN 'TABLE' + WHEN 'v' THEN 'VIEW' + WHEN 'S' THEN 'SEQUENCE' + ELSE NULL + END as typ + FROM pg_class + WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + RETURN; +END; +$$ LANGUAGE PLPGSQL; +SELECT pgstac_admin_owns(); + +CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; + +GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; + +ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; + +GRANT pgstac_read TO pgstac_ingest; +GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; + +SET ROLE pgstac_admin; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; +RESET ROLE; + +SET ROLE pgstac_ingest; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; +RESET ROLE; + +SET SEARCH_PATH TO pgstac, public; + +-- PgSTAC references PostGIS without schema qualification and runs with +-- search_path pinned to "pgstac, public", so PostGIS has to be reachable from +-- there. Fail here rather than part way through creating the schema. +DO $$ + BEGIN + IF to_regtype('geometry') IS NULL THEN + RAISE EXCEPTION 'PostGIS is not reachable from search_path "pgstac, public"' + USING HINT = 'PgSTAC requires the postgis extension in the public schema.'; + END IF; + END +$$; + +SET ROLE pgstac_admin; + +DO $$ + BEGIN + DROP FUNCTION IF EXISTS analyze_items; + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; +DO $$ + BEGIN + DROP FUNCTION IF EXISTS validate_constraints; + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; +-- Drain the queue here, where the functions its entries name still have the +-- signatures they were queued against and pgstac_admin owns the objects any +-- queued DDL touches. Statistics updates are discarded instead: +-- 998_idempotent_post recalculates every partition regardless. +DO $$ + BEGIN + DELETE FROM query_queue WHERE query LIKE 'SELECT update_partition_stats(%'; + PERFORM run_queued_queries_intransaction(); + EXCEPTION WHEN undefined_table OR undefined_function THEN + RAISE NOTICE 'No query queue to drain.'; + END +$$; + +-- Return type or argument list differs from an earlier release, which +-- CREATE OR REPLACE cannot change. +DROP FUNCTION IF EXISTS run_or_queue(text); +DROP FUNCTION IF EXISTS update_partition_stats_q(text, boolean); +DROP FUNCTION IF EXISTS update_partition_stats(text, boolean); +DROP FUNCTION IF EXISTS maintain_index(text, text, boolean, boolean, boolean); +DROP FUNCTION IF EXISTS queryable_indexes(text, boolean); + +-- Install these idempotently as migrations do not put them before trying to modify the collections table + + +CREATE OR REPLACE FUNCTION collection_geom(content jsonb) +RETURNS geometry AS $$ + WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) + SELECT + st_makeenvelope( + (box->>0)::float, + (box->>1)::float, + (box->>2)::float, + (box->>3)::float, + 4326 + ) + FROM box; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) +RETURNS timestamptz AS $$ + SELECT + CASE + WHEN + (content->'extent'->'temporal'->'interval'->0->>0) IS NULL + THEN '-infinity'::timestamptz + ELSE + (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz + END + ; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) +RETURNS timestamptz AS $$ + SELECT + CASE + WHEN + (content->'extent'->'temporal'->'interval'->0->>1) IS NULL + THEN 'infinity'::timestamptz + ELSE + (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz + END + ; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE TABLE IF NOT EXISTS migrations ( + version text PRIMARY KEY, + datetime timestamptz DEFAULT clock_timestamp() NOT NULL +); + +CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ + SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ + INSERT INTO pgstac.migrations (version) VALUES ($1) + ON CONFLICT DO NOTHING + RETURNING version; +$$ LANGUAGE SQL; + + +CREATE TABLE IF NOT EXISTS pgstac_settings ( + name text PRIMARY KEY, + value text NOT NULL +); + +CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ +DECLARE + retval boolean; +BEGIN + EXECUTE format($q$ + SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) + $q$, + $1 + ) INTO retval; + RETURN retval; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ +SELECT COALESCE( + nullif(conf->>_setting, ''), + nullif(current_setting(concat('pgstac.',_setting), TRUE),''), + nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') +); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ +SELECT COALESCE( + nullif(conf->>_setting, ''), + nullif(current_setting(concat('pgstac.',_setting), TRUE),''), + nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), + 'FALSE' +)::boolean; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ + SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ + SELECT pgstac.get_setting_bool('additional_properties'); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ + SELECT pgstac.get_setting_bool('readonly', conf); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ + SELECT pgstac.get_setting('context', conf); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ + SELECT pgstac.get_setting('context_estimated_count', conf)::int; +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS context_estimated_cost(); +CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ + SELECT pgstac.get_setting('context_estimated_cost', conf)::float; +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS context_stats_ttl(); +CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ + SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ + SELECT extract(epoch FROM $1::interval)::text || ' s'; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; + +CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ + SELECT abs(extract(epoch from age(a,b)) * 1000); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ + SELECT t2s(coalesce( + get_setting('queue_timeout'), + '1h' + ))::interval; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ +DECLARE +debug boolean := current_setting('pgstac.debug', true); +BEGIN + IF debug THEN + RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); + RETURN TRUE; + END IF; + RETURN FALSE; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ +SELECT CASE + WHEN $1 IS NULL THEN TRUE + WHEN cardinality($1)<1 THEN TRUE +ELSE FALSE +END; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ + SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) + RETURNS text[] AS $$ + SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) + RETURNS text[] AS $$ + SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ +SELECT ARRAY( + SELECT $1[i] + FROM generate_subscripts($1,1) AS s(i) + ORDER BY i DESC +); +$$ LANGUAGE SQL STRICT IMMUTABLE; + +DROP TABLE IF EXISTS query_queue; +CREATE TABLE query_queue ( + query text PRIMARY KEY, + added timestamptz DEFAULT now() +); + +DROP TABLE IF EXISTS query_queue_history; +CREATE TABLE query_queue_history( + query text, + added timestamptz NOT NULL, + finished timestamptz NOT NULL DEFAULT now(), + error text +); + +CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ +DECLARE + qitem query_queue%ROWTYPE; + timeout_ts timestamptz; + error text; + cnt int := 0; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; + IF NOT FOUND THEN + EXIT; + END IF; + cnt := cnt + 1; + BEGIN + RAISE NOTICE 'RUNNING QUERY: %', qitem.query; + EXECUTE qitem.query; + EXCEPTION WHEN others THEN + error := format('%s | %s', SQLERRM, SQLSTATE); + END; + INSERT INTO query_queue_history (query, added, finished, error) + VALUES (qitem.query, qitem.added, clock_timestamp(), error); + COMMIT; + END LOOP; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ +DECLARE + qitem query_queue%ROWTYPE; + timeout_ts timestamptz; + error text; + cnt int := 0; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; + IF NOT FOUND THEN + RETURN cnt; + END IF; + cnt := cnt + 1; + BEGIN + qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); + RAISE NOTICE 'RUNNING QUERY: %', qitem.query; + + EXECUTE qitem.query; + EXCEPTION WHEN others THEN + error := format('%s | %s', SQLERRM, SQLSTATE); + RAISE WARNING '%', error; + END; + INSERT INTO query_queue_history (query, added, finished, error) + VALUES (qitem.query, qitem.added, clock_timestamp(), error); + END LOOP; + RETURN cnt; +END; +$$ LANGUAGE PLPGSQL; + + +-- TRUE if the query ran, FALSE if it was queued, so callers can tell whether +-- its effects are visible in this transaction. +CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS boolean AS $$ +DECLARE + use_queue boolean := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; +BEGIN + IF get_setting_bool('debug') THEN + RAISE NOTICE '%', query; + END IF; + IF use_queue THEN + INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; + RETURN FALSE; + END IF; + EXECUTE query; + RETURN TRUE; +END; +$$ LANGUAGE PLPGSQL; + + + +DROP FUNCTION IF EXISTS check_pgstac_settings; +CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ +DECLARE + settingval text; + sysmem bigint := pg_size_bytes(_sysmem); + effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); + shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); + work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); + max_connections int := current_setting('max_connections', TRUE); + maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); + seq_page_cost float := current_setting('seq_page_cost', TRUE); + random_page_cost float := current_setting('random_page_cost', TRUE); + temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); + r record; +BEGIN + IF _sysmem IS NULL THEN + RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; + ELSE + IF effective_cache_size < (sysmem * 0.5) THEN + RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); + ELSIF effective_cache_size > (sysmem * 0.75) THEN + RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); + ELSE + RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); + END IF; + + IF shared_buffers < (sysmem * 0.2) THEN + RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); + ELSIF shared_buffers > (sysmem * 0.3) THEN + RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); + ELSE + RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); + END IF; + shared_buffers = sysmem * 0.3; + IF maintenance_work_mem < (sysmem * 0.2) THEN + RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); + ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN + RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); + ELSE + RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); + END IF; + + IF work_mem * max_connections > shared_buffers THEN + RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); + ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN + RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); + ELSE + RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); + END IF; + + IF random_page_cost / seq_page_cost != 1.1 THEN + RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; + ELSE + RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; + END IF; + + IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN + RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); + END IF; + END IF; + + RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; + RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; + + FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP + RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; + END LOOP; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; + ELSE + RAISE NOTICE 'pg_cron % is installed', settingval; + END IF; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; + ELSE + RAISE NOTICE 'pgstattuple % is installed', settingval; + END IF; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; + ELSE + RAISE NOTICE 'pg_stat_statements % is installed', settingval; + IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN + RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; + END IF; + END IF; + + -- Undrained queue means stale statistics and constraints, silently. + SELECT count(*) INTO settingval FROM query_queue; + IF settingval::bigint > 0 THEN + RAISE WARNING '% queries are waiting in query_queue. Run "CALL run_queued_queries();" (pypgstac runqueue) to drain it -- until then partition statistics and constraints are stale.', settingval; + END IF; + +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; +CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ + SELECT floor(($1->>0)::float)::int; +$$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ + SELECT ($1->>0)::float; +$$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ + SELECT ($1->>0)::timestamptz; +$$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ + SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; +$$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ + SELECT + CASE jsonb_typeof($1) + WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) + ELSE ARRAY[$1->>0] + END + ; +$$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ +SELECT CASE jsonb_array_length(_bbox) + WHEN 4 THEN + ST_SetSRID(ST_MakeEnvelope( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float, + (_bbox->>3)::float + ),4326) + WHEN 6 THEN + ST_SetSRID(ST_3DMakeBox( + ST_MakePoint( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float + ), + ST_MakePoint( + (_bbox->>3)::float, + (_bbox->>4)::float, + (_bbox->>5)::float + ) + ),4326) + ELSE null END; +; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ + SELECT jsonb_build_array( + st_xmin(_geom), + st_ymin(_geom), + st_xmax(_geom), + st_ymax(_geom) + ); +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ + SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ + SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ + WITH RECURSIVE t AS ( + SELECT e FROM explode_dotpaths(j) e + UNION ALL + SELECT e[1:cardinality(e)-1] + FROM t + WHERE cardinality(e)>1 + ) SELECT e FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ +DECLARE +BEGIN + IF cardinality(path) > 1 THEN + FOR i IN 1..(cardinality(path)-1) LOOP + IF j #> path[:i] IS NULL THEN + j := jsonb_set_lax(j, path[:i], '{}', TRUE); + END IF; + END LOOP; + END IF; + RETURN jsonb_set_lax(j, path, val, true); + +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + + + +CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ +DECLARE + includes jsonb := f-> 'include'; + outj jsonb := '{}'::jsonb; + path text[]; +BEGIN + IF + includes IS NULL + OR jsonb_array_length(includes) = 0 + THEN + RETURN j; + ELSE + includes := includes || ( + CASE WHEN j ? 'collection' THEN + '["id","collection"]' + ELSE + '["id"]' + END)::jsonb; + FOR path IN SELECT explode_dotpaths(includes) LOOP + outj := jsonb_set_nested(outj, path, j #> path); + END LOOP; + END IF; + RETURN outj; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ +DECLARE + excludes jsonb := f-> 'exclude'; + outj jsonb := j; + path text[]; +BEGIN + IF + excludes IS NULL + OR jsonb_array_length(excludes) = 0 + THEN + RETURN j; + ELSE + FOR path IN SELECT explode_dotpaths(excludes) LOOP + outj := outj #- path; + END LOOP; + END IF; + RETURN outj; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ + SELECT jsonb_exclude(jsonb_include(j, f), f); +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ + SELECT + CASE + WHEN _a = '"𒍟※"'::jsonb THEN NULL + WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b + WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN + ( + SELECT + jsonb_strip_nulls( + jsonb_object_agg( + key, + merge_jsonb(a.value, b.value) + ) + ) + FROM + jsonb_each(coalesce(_a,'{}'::jsonb)) as a + FULL JOIN + jsonb_each(coalesce(_b,'{}'::jsonb)) as b + USING (key) + ) + WHEN + jsonb_typeof(_a) = 'array' + AND jsonb_typeof(_b) = 'array' + AND jsonb_array_length(_a) = jsonb_array_length(_b) + THEN + ( + SELECT jsonb_agg(m) FROM + ( SELECT + merge_jsonb( + jsonb_array_elements(_a), + jsonb_array_elements(_b) + ) as m + ) as l + ) + ELSE _a + END + ; +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ + SELECT + CASE + + WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb + WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a + WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb + WHEN _a = _b THEN NULL + WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN + ( + SELECT + jsonb_strip_nulls( + jsonb_object_agg( + key, + strip_jsonb(a.value, b.value) + ) + ) + FROM + jsonb_each(_a) as a + FULL JOIN + jsonb_each(_b) as b + USING (key) + ) + WHEN + jsonb_typeof(_a) = 'array' + AND jsonb_typeof(_b) = 'array' + AND jsonb_array_length(_a) = jsonb_array_length(_b) + THEN + ( + SELECT jsonb_agg(m) FROM + ( SELECT + strip_jsonb( + jsonb_array_elements(_a), + jsonb_array_elements(_b) + ) as m + ) as l + ) + ELSE _a + END + ; +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ + SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ + SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ + SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ + SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ + SELECT nullif_jsonbnullempty(greatest(a, b)); +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ + SELECT COALESCE($1,$2); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( + SFUNC = first_notnull_sfunc, + STYPE = anyelement +); + +CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( + STYPE = jsonb, + SFUNC = jsonb_concat_ignorenull, + FINALFUNC = jsonb_array_unique +); + +CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( + STYPE = jsonb, + SFUNC = jsonb_least +); + +CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( + STYPE = jsonb, + SFUNC = jsonb_greatest +); +/* looks for a geometry in a stac item first from geometry and falling back to bbox */ +CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ +SELECT + CASE + WHEN value ? 'intersects' THEN + ST_GeomFromGeoJSON(value->>'intersects') + WHEN value ? 'geometry' THEN + ST_GeomFromGeoJSON(value->>'geometry') + WHEN value ? 'bbox' THEN + pgstac.bbox_geom(value->'bbox') + ELSE NULL + END as geometry +; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION stac_daterange( + value jsonb +) RETURNS tstzrange AS $$ +DECLARE + props jsonb := value; + dt timestamptz; + edt timestamptz; +BEGIN + IF props ? 'properties' THEN + props := props->'properties'; + END IF; + IF + props ? 'start_datetime' + AND props->>'start_datetime' IS NOT NULL + AND props ? 'end_datetime' + AND props->>'end_datetime' IS NOT NULL + THEN + dt := props->>'start_datetime'; + edt := props->>'end_datetime'; + IF dt > edt THEN + RAISE EXCEPTION 'start_datetime must be < end_datetime'; + END IF; + ELSE + dt := props->>'datetime'; + edt := props->>'datetime'; + END IF; + IF dt is NULL OR edt IS NULL THEN + RAISE NOTICE 'DT: %, EDT: %', dt, edt; + RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; + END IF; + RETURN tstzrange(dt, edt, '[]'); +END; +$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ + SELECT lower(stac_daterange(value)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ + SELECT upper(stac_daterange(value)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE TABLE IF NOT EXISTS stac_extensions( + url text PRIMARY KEY, + content jsonb +); +CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ + SELECT jsonb_build_object( + 'type', 'Feature', + 'stac_version', content->'stac_version', + 'assets', content->'item_assets', + 'collection', content->'id' + ); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE TABLE IF NOT EXISTS collections ( + key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, + content JSONB NOT NULL, + base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, + geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, + datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, + end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, + private jsonb, + partition_trunc text CHECK (partition_trunc IN ('year', 'month')) +); + +CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ + SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ +DECLARE + out collections%ROWTYPE; +BEGIN + UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ON CONFLICT (id) DO + UPDATE + SET content=EXCLUDED.content + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +-- SECURITY DEFINER: the delete trigger drops partition tables, which are owned +-- by pgstac_admin. +CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ +BEGIN + DELETE FROM collections WHERE id = _id; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _id USING ERRCODE = 'no_data_found'; + END IF; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ + SELECT content FROM collections + WHERE id=$1 + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ + SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + collection_base_partition text := concat('_items_', OLD.key); +BEGIN + -- Tables before rows: check_partition takes these locks in the same order, + -- and the reverse deadlocks against a concurrent partition create. + EXECUTE format($q$ + DROP TABLE IF EXISTS %I CASCADE; + DELETE FROM partition_stats WHERE collection=%L; + $q$, + collection_base_partition, + OLD.id + ); + RETURN OLD; +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections +FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); +CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ + SELECT concat(n, c); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE TABLE queryables ( + id bigint GENERATED ALWAYS AS identity PRIMARY KEY, + name text NOT NULL, + collection_ids text[], -- used to determine what partitions to create indexes on + definition jsonb, + property_path text, + property_wrapper text, + property_index_type text +); +CREATE INDEX queryables_name_idx ON queryables (name); +CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); +CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); + +CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ +DECLARE + allcollections text[]; +BEGIN + RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; + IF NEW.collection_ids IS NOT NULL THEN + IF EXISTS ( + SELECT 1 + FROM unnest(NEW.collection_ids) c + LEFT JOIN + collections + ON (collections.id = c) + WHERE collections.id IS NULL + ) THEN + RAISE foreign_key_violation USING MESSAGE = format( + 'One or more collections in %s do not exist.', NEW.collection_ids + ); + RETURN NULL; + END IF; + END IF; + IF TG_OP = 'INSERT' THEN + IF EXISTS ( + SELECT 1 FROM queryables q + WHERE + q.name = NEW.name + AND ( + q.collection_ids && NEW.collection_ids + OR + q.collection_ids IS NULL + OR + NEW.collection_ids IS NULL + ) + ) THEN + RAISE unique_violation USING MESSAGE = format( + 'There is already a queryable for %s for a collection in %s: %s', + NEW.name, + NEW.collection_ids, + (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE + q.name = NEW.name + AND ( + q.collection_ids && NEW.collection_ids + OR + q.collection_ids IS NULL + OR + NEW.collection_ids IS NULL + )) + ); + RETURN NULL; + END IF; + END IF; + IF TG_OP = 'UPDATE' THEN + IF EXISTS ( + SELECT 1 FROM queryables q + WHERE + q.id != NEW.id + AND + q.name = NEW.name + AND ( + q.collection_ids && NEW.collection_ids + OR + q.collection_ids IS NULL + OR + NEW.collection_ids IS NULL + ) + ) THEN + RAISE unique_violation + USING MESSAGE = format( + 'There is already a queryable for %s for a collection in %s', + NEW.name, + NEW.collection_ids + ); + RETURN NULL; + END IF; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER queryables_constraint_insert_trigger +BEFORE INSERT ON queryables +FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); + +CREATE TRIGGER queryables_constraint_update_trigger +BEFORE UPDATE ON queryables +FOR EACH ROW +WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) +EXECUTE PROCEDURE queryables_constraint_triggerfunc(); + + +CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ + SELECT string_agg( + quote_literal(v), + '->' + ) FROM unnest(arr) v; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + + + +CREATE OR REPLACE FUNCTION queryable( + IN dotpath text, + OUT path text, + OUT expression text, + OUT wrapper text, + OUT nulled_wrapper text +) AS $$ +DECLARE + q RECORD; + path_elements text[]; +BEGIN + dotpath := replace(dotpath, 'properties.', ''); + IF dotpath = 'start_datetime' THEN + dotpath := 'datetime'; + END IF; + IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN + path := dotpath; + expression := dotpath; + wrapper := NULL; + RETURN; + END IF; + + SELECT * INTO q FROM queryables + WHERE + name=dotpath + OR name = 'properties.' || dotpath + OR name = replace(dotpath, 'properties.', '') + ; + IF q.property_wrapper IS NULL THEN + IF q.definition->>'type' = 'number' THEN + wrapper := 'to_float'; + nulled_wrapper := wrapper; + ELSIF q.definition->>'format' = 'date-time' THEN + wrapper := 'to_tstz'; + nulled_wrapper := wrapper; + ELSE + nulled_wrapper := NULL; + wrapper := 'to_text'; + END IF; + ELSE + wrapper := q.property_wrapper; + nulled_wrapper := wrapper; + END IF; + IF q.property_path IS NOT NULL THEN + path := q.property_path; + ELSE + path_elements := string_to_array(dotpath, '.'); + IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN + path := format('content->%s', array_to_path(path_elements)); + ELSIF path_elements[1] = 'properties' THEN + path := format('content->%s', array_to_path(path_elements)); + ELSE + path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); + END IF; + END IF; + expression := format('%I(%s)', wrapper, path); + RETURN; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + +CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ + DECLARE + BEGIN + IF collection_ids IS NULL THEN + RETURN QUERY SELECT id FROM collections; + END IF; + RETURN QUERY SELECT unnest(collection_ids); + END; +$$ LANGUAGE PLPGSQL STABLE; + +CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ +DECLARE +BEGIN + def := btrim(def, ' \n\t'); + def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); + RETURN def; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ + DECLARE + out text; + BEGIN + IF q.name = 'id' THEN + out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; + ELSIF q.name = 'datetime' THEN + out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; + ELSIF q.name = 'geometry' THEN + out := 'CREATE INDEX ON %I USING gist (geometry)'; + ELSE + out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, + lower(COALESCE(q.property_index_type, 'BTREE')), + lower(COALESCE(q.property_wrapper, 'to_text')), + q.name + ); + END IF; + RETURN btrim(out, ' \n\t'); + END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +DROP VIEW IF EXISTS pgstac_indexes; +CREATE VIEW pgstac_indexes AS +SELECT + i.schemaname, + i.tablename, + i.indexname, + regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, + COALESCE( + (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], + (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], + CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END + ) AS field, + pg_table_size(i.indexname::text) as index_size, + pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty +FROM + pg_indexes i +WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; + +DROP VIEW IF EXISTS pgstac_index_stats; +CREATE VIEW pgstac_indexes_stats AS +SELECT + i.schemaname, + i.tablename, + i.indexname, + indexdef, + COALESCE( + (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], + (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], + CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END + ) AS field, + pg_table_size(i.indexname::text) as index_size, + pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, + n_distinct, + most_common_vals::text::text[], + most_common_freqs::text::text[], + histogram_bounds::text::text[], + correlation +FROM + pg_indexes i + LEFT JOIN pg_stats s ON (s.tablename = i.indexname) +WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; + +CREATE OR REPLACE FUNCTION queryable_indexes( + IN treeroot text DEFAULT 'items', + IN changes boolean DEFAULT FALSE, + OUT collection text, + OUT partition text, + OUT field text, + OUT indexname text, + OUT existing_idx text, + OUT queryable_idx text, + OUT queryable_id bigint +) RETURNS SETOF RECORD AS $$ +WITH p AS ( + SELECT + relid::text as partition, + replace(replace( + CASE + WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) + ELSE pg_get_expr(parent.relpartbound, parent.oid) + END, + 'FOR VALUES IN (''',''), ''')', + '' + ) AS collection + FROM pg_partition_tree(treeroot) + JOIN pg_class c ON (relid::regclass = c.oid) + JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) + ), i AS ( + SELECT + partition, + indexname, + regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, + COALESCE( + (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], + (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], + CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END + ) AS field + FROM + pg_indexes + JOIN p ON (tablename=partition) + ), q AS ( + SELECT + name AS field, + collection, + partition, + format(indexdef(queryables), partition) as qidx, + queryables.id as qid + FROM queryables, unnest_collection(queryables.collection_ids) collection + JOIN p USING (collection) + WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') + ) + SELECT + collection, + partition, + field, + indexname, + iidx as existing_idx, + qidx as queryable_idx, + qid as queryable_id + FROM i FULL JOIN q USING (field, partition) + WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; +; +$$ LANGUAGE SQL; + +-- SECURITY DEFINER, so the index statement is built here from the queryables +-- row; a caller supplied one would run with the privileges of the schema owner. +CREATE OR REPLACE FUNCTION maintain_index( + _partition text, + _indexname text, + _queryable_id bigint, + dropindexes boolean DEFAULT FALSE, + rebuildindexes boolean DEFAULT FALSE, + idxconcurrently boolean DEFAULT FALSE +) RETURNS VOID AS $$ +DECLARE + _queryable_idx text; +BEGIN + -- Runs elevated, so it may only touch partitions of items. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(_partition)) THEN + RETURN; + END IF; + IF _queryable_id IS NOT NULL AND _partition IS NOT NULL THEN + SELECT format(indexdef(q), _partition) INTO _queryable_idx + FROM queryables q WHERE q.id = _queryable_id; + END IF; + IF _indexname IS NOT NULL THEN + IF dropindexes OR _queryable_idx IS NOT NULL THEN + EXECUTE format('DROP INDEX IF EXISTS %I;', _indexname); + ELSIF rebuildindexes THEN + IF idxconcurrently THEN + EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', _indexname); + ELSE + EXECUTE format('REINDEX INDEX %I;', _indexname); + END IF; + END IF; + END IF; + IF _queryable_idx IS NOT NULL THEN + IF idxconcurrently THEN + EXECUTE replace(_queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); + ELSE EXECUTE _queryable_idx; + END IF; + END IF; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + + + +set check_function_bodies to off; +CREATE OR REPLACE FUNCTION maintain_partition_queries( + part text DEFAULT 'items', + dropindexes boolean DEFAULT FALSE, + rebuildindexes boolean DEFAULT FALSE, + idxconcurrently boolean DEFAULT FALSE +) RETURNS SETOF text AS $$ +DECLARE + rec record; + q text; +BEGIN + FOR rec IN ( + SELECT * FROM queryable_indexes(part,true) + ) LOOP + q := format( + 'SELECT maintain_index( + %L,%L,%L,%L,%L,%L + );', + rec.partition, + rec.indexname, + rec.queryable_id, + dropindexes, + rebuildindexes, + idxconcurrently + ); + RAISE NOTICE 'Q: %', q; + RETURN NEXT q; + END LOOP; + RETURN; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION maintain_partitions( + part text DEFAULT 'items', + dropindexes boolean DEFAULT FALSE, + rebuildindexes boolean DEFAULT FALSE +) RETURNS VOID AS $$ + WITH t AS ( + SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q + ) SELECT count(*) FROM t; +$$ LANGUAGE SQL; + + +CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ +DECLARE +BEGIN + PERFORM maintain_partitions(); + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables +FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); + + +CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ +DECLARE +BEGIN + -- Build up queryables if the input contains valid collection ids or is empty + IF EXISTS ( + SELECT 1 FROM collections + WHERE + _collection_ids IS NULL + OR cardinality(_collection_ids) = 0 + OR id = ANY(_collection_ids) + ) + THEN + RETURN ( + WITH base AS ( + SELECT + unnest(collection_ids) as collection_id, + name, + coalesce(definition, '{"type":"string"}'::jsonb) as definition + FROM queryables + WHERE + _collection_ids IS NULL OR + _collection_ids = '{}'::text[] OR + _collection_ids && collection_ids + UNION ALL + SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition + FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] + ), g AS ( + SELECT + name, + first_notnull(definition) as definition, + jsonb_array_unique_merge(definition->'enum') as enum, + jsonb_min(definition->'minimum') as minimum, + jsonb_min(definition->'maxiumn') as maximum + FROM base + GROUP BY 1 + ) + SELECT + jsonb_build_object( + '$schema', 'http://json-schema.org/draft-07/schema#', + '$id', '', + 'type', 'object', + 'title', 'STAC Queryables.', + 'properties', jsonb_object_agg( + name, + definition + || + jsonb_strip_nulls(jsonb_build_object( + 'enum', enum, + 'minimum', minimum, + 'maximum', maximum + )) + ), + 'additionalProperties', pgstac.additional_properties() + ) + FROM g + ); + ELSE + RETURN NULL; + END IF; +END; +$$ LANGUAGE PLPGSQL STABLE; + +CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ + SELECT + CASE + WHEN _collection IS NULL THEN get_queryables(NULL::text[]) + ELSE get_queryables(ARRAY[_collection]) + END + ; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ + SELECT get_queryables(NULL::text[]); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ + SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; + + +CREATE OR REPLACE VIEW stac_extension_queryables AS +SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; + + +CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ +DECLARE + q text; + _partition text; + explain_json json; + psize float; + estrows float; +BEGIN + SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; + + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) + INTO explain_json; + psize := explain_json->0->'Plan'->'Plan Rows'; + estrows := _tablesample * .01 * psize; + IF estrows < minrows THEN + _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); + RAISE NOTICE '%', (psize / estrows) / 100; + END IF; + RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; + + q := format( + $q$ + WITH q AS ( + SELECT * FROM queryables + WHERE + collection_ids IS NULL + OR %L = ANY(collection_ids) + ), t AS ( + SELECT + content->'properties' AS properties + FROM + %I + TABLESAMPLE SYSTEM(%L) + ), p AS ( + SELECT DISTINCT ON (key) + key, + value, + s.definition + FROM t + JOIN LATERAL jsonb_each(properties) ON TRUE + LEFT JOIN q ON (q.name=key) + LEFT JOIN stac_extension_queryables s ON (s.name=key) + WHERE q.definition IS NULL + ) + SELECT + %L, + key, + COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, + CASE + WHEN definition->>'type' = 'integer' THEN 'to_int' + WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' + WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' + ELSE 'to_text' + END + FROM p; + $q$, + _collection, + _partition, + _tablesample, + _collection + ); + RETURN QUERY EXECUTE q; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ + SELECT + array_agg(collection), + name, + definition, + property_wrapper + FROM + collections + JOIN LATERAL + missing_queryables(id, _tablesample) c + ON TRUE + GROUP BY + 2,3,4 + ORDER BY 2,1 + ; +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate jsonb, + relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) +) RETURNS tstzrange AS $$ +DECLARE + timestrs text[]; + s timestamptz; + e timestamptz; +BEGIN + timestrs := + CASE + WHEN _indate ? 'timestamp' THEN + ARRAY[_indate->>'timestamp'] + WHEN _indate ? 'interval' THEN + to_text_array(_indate->'interval') + WHEN jsonb_typeof(_indate) = 'array' THEN + to_text_array(_indate) + ELSE + regexp_split_to_array( + _indate->>0, + '/' + ) + END; + RAISE NOTICE 'TIMESTRS %', timestrs; + IF cardinality(timestrs) = 1 THEN + IF timestrs[1] ILIKE 'P%' THEN + RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); + END IF; + s := timestrs[1]::timestamptz; + RETURN tstzrange(s, s, '[]'); + END IF; + + IF cardinality(timestrs) != 2 THEN + RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; + END IF; + + IF timestrs[1] = '..' OR timestrs[1] = '' THEN + s := '-infinity'::timestamptz; + e := timestrs[2]::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] = '..' OR timestrs[2] = '' THEN + s := timestrs[1]::timestamptz; + e := 'infinity'::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN + e := timestrs[2]::timestamptz; + s := e - upper(timestrs[1])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN + s := timestrs[1]::timestamptz; + e := s + upper(timestrs[2])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + s := timestrs[1]::timestamptz; + e := timestrs[2]::timestamptz; + + RETURN tstzrange(s,e,'[)'); + + RETURN NULL; + +END; +$$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate text, + relative_base timestamptz DEFAULT CURRENT_TIMESTAMP +) RETURNS tstzrange AS $$ + SELECT parse_dtrange(to_jsonb(_indate), relative_base); +$$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + ll text := 'datetime'; + lh text := 'end_datetime'; + rrange tstzrange; + rl text; + rh text; + outq text; +BEGIN + rrange := parse_dtrange(args->1); + RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; + op := lower(op); + rl := format('%L::timestamptz', lower(rrange)); + rh := format('%L::timestamptz', upper(rrange)); + outq := CASE op + WHEN 't_before' THEN 'lh < rl' + WHEN 't_after' THEN 'll > rh' + WHEN 't_meets' THEN 'lh = rl' + WHEN 't_metby' THEN 'll = rh' + WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' + WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' + WHEN 't_starts' THEN 'll = rl AND lh < rh' + WHEN 't_startedby' THEN 'll = rl AND lh > rh' + WHEN 't_during' THEN 'll > rl AND lh < rh' + WHEN 't_contains' THEN 'll < rl AND lh > rh' + WHEN 't_finishes' THEN 'll > rl AND lh = rh' + WHEN 't_finishedby' THEN 'll < rl AND lh = rh' + WHEN 't_equals' THEN 'll = rl AND lh = rh' + WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' + WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' + WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' + END; + outq := regexp_replace(outq, '\mll\M', ll); + outq := regexp_replace(outq, '\mlh\M', lh); + outq := regexp_replace(outq, '\mrl\M', rl); + outq := regexp_replace(outq, '\mrh\M', rh); + outq := format('(%s)', outq); + RETURN outq; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + + + +CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + geom text; + j jsonb := args->1; +BEGIN + op := lower(op); + RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; + IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN + RAISE EXCEPTION 'Spatial Operator % Not Supported', op; + END IF; + op := regexp_replace(op, '^s_', 'st_'); + IF op = 'intersects' THEN + op := 'st_intersects'; + END IF; + -- Convert geometry to WKB string + IF j ? 'type' AND j ? 'coordinates' THEN + geom := st_geomfromgeojson(j)::text; + ELSIF jsonb_typeof(j) = 'array' THEN + geom := bbox_geom(j)::text; + END IF; + + RETURN format('%s(geometry, %L::geometry)', op, geom); +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ +-- Translates anything passed in through the deprecated "query" into equivalent CQL2 +WITH t AS ( + SELECT key as property, value as ops + FROM jsonb_each(q) +), t2 AS ( + SELECT property, (jsonb_each(ops)).* + FROM t WHERE jsonb_typeof(ops) = 'object' + UNION ALL + SELECT property, 'eq', ops + FROM t WHERE jsonb_typeof(ops) != 'object' +) +SELECT + jsonb_strip_nulls(jsonb_build_object( + 'op', 'and', + 'args', jsonb_agg( + jsonb_build_object( + 'op', key, + 'args', jsonb_build_array( + jsonb_build_object('property',property), + value + ) + ) + ) + ) +) as qcql FROM t2 +; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + +CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ +DECLARE + args jsonb; + ret jsonb; +BEGIN + RAISE NOTICE 'CQL1_TO_CQL2: %', j; + IF j ? 'filter' THEN + RETURN cql1_to_cql2(j->'filter'); + END IF; + IF j ? 'property' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'array' THEN + SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; + RETURN args; + END IF; + IF jsonb_typeof(j) = 'number' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'string' THEN + RETURN j; + END IF; + + IF jsonb_typeof(j) = 'object' THEN + SELECT jsonb_build_object( + 'op', key, + 'args', cql1_to_cql2(value) + ) INTO ret + FROM jsonb_each(j) + WHERE j IS NOT NULL; + RETURN ret; + END IF; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE STRICT; + +CREATE TABLE cql2_ops ( + op text PRIMARY KEY, + template text, + types text[] +); +INSERT INTO cql2_ops (op, template, types) VALUES + ('eq', '%s = %s', NULL), + ('neq', '%s != %s', NULL), + ('ne', '%s != %s', NULL), + ('!=', '%s != %s', NULL), + ('<>', '%s != %s', NULL), + ('lt', '%s < %s', NULL), + ('lte', '%s <= %s', NULL), + ('gt', '%s > %s', NULL), + ('gte', '%s >= %s', NULL), + ('le', '%s <= %s', NULL), + ('ge', '%s >= %s', NULL), + ('=', '%s = %s', NULL), + ('<', '%s < %s', NULL), + ('<=', '%s <= %s', NULL), + ('>', '%s > %s', NULL), + ('>=', '%s >= %s', NULL), + ('like', '%s LIKE %s', NULL), + ('ilike', '%s ILIKE %s', NULL), + ('+', '%s + %s', NULL), + ('-', '%s - %s', NULL), + ('*', '%s * %s', NULL), + ('/', '%s / %s', NULL), + ('not', 'NOT (%s)', NULL), + ('between', '%s BETWEEN %s AND %s', NULL), + ('isnull', '%s IS NULL', NULL), + ('upper', 'upper(%s)', NULL), + ('lower', 'lower(%s)', NULL), + ('casei', 'upper(%s)', NULL), + ('accenti', 'unaccent(%s)', NULL) +ON CONFLICT (op) DO UPDATE + SET + template = EXCLUDED.template +; + + +CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ +#variable_conflict use_variable +DECLARE + args jsonb := j->'args'; + arg jsonb; + op text := lower(j->>'op'); + cql2op RECORD; + literal text; + _wrapper text; + leftarg text; + rightarg text; + prop text; + extra_props bool := pgstac.additional_properties(); +BEGIN + IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN + RETURN NULL; + END IF; + RAISE NOTICE 'CQL2_QUERY: %', j; + + -- check if all properties are represented in the queryables + IF NOT extra_props THEN + FOR prop IN + SELECT DISTINCT p->>0 + FROM jsonb_path_query(j, 'strict $.**.property') p + WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') + LOOP + IF (queryable(prop)).nulled_wrapper IS NULL THEN + RAISE EXCEPTION 'Term % is not found in queryables.', prop; + END IF; + END LOOP; + END IF; + + IF j ? 'filter' THEN + RETURN cql2_query(j->'filter'); + END IF; + + IF j ? 'upper' THEN + RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); + END IF; + + IF j ? 'lower' THEN + RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); + END IF; + + -- Temporal Query + IF op ilike 't_%' or op = 'anyinteracts' THEN + RETURN temporal_op_query(op, args); + END IF; + + -- If property is a timestamp convert it to text to use with + -- general operators + IF j ? 'timestamp' THEN + RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); + END IF; + IF j ? 'interval' THEN + RAISE EXCEPTION 'Please use temporal operators when using intervals.'; + RETURN NONE; + END IF; + + -- Spatial Query + IF op ilike 's_%' or op = 'intersects' THEN + RETURN spatial_op_query(op, args); + END IF; + + IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN + IF args->0 ? 'property' THEN + leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); + END IF; + IF args->1 ? 'property' THEN + rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); + END IF; + RETURN FORMAT( + '%s %s %s', + COALESCE(leftarg, quote_literal(to_text_array(args->0))), + CASE op + WHEN 'a_equals' THEN '=' + WHEN 'a_contains' THEN '@>' + WHEN 'a_contained_by' THEN '<@' + WHEN 'a_overlaps' THEN '&&' + END, + COALESCE(rightarg, quote_literal(to_text_array(args->1))) + ); + END IF; + + IF op = 'in' THEN + RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; + args := jsonb_build_array(args->0) || (args->1); + RAISE NOTICE 'IN2 : %', args; + END IF; + + + + IF op = 'between' THEN + args = jsonb_build_array( + args->0, + args->1, + args->2 + ); + END IF; + + -- Make sure that args is an array and run cql2_query on + -- each element of the array + RAISE NOTICE 'ARGS PRE: %', args; + IF j ? 'args' THEN + IF jsonb_typeof(args) != 'array' THEN + args := jsonb_build_array(args); + END IF; + + IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN + wrapper := NULL; + ELSE + -- if any of the arguments are a property, try to get the property_wrapper + FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP + RAISE NOTICE 'Arg: %', arg; + wrapper := (queryable(arg->>'property')).nulled_wrapper; + RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; + IF wrapper IS NOT NULL THEN + EXIT; + END IF; + END LOOP; + + -- if the property was not in queryables, see if any args were numbers + IF + wrapper IS NULL + AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') + THEN + wrapper := 'to_float'; + END IF; + wrapper := coalesce(wrapper, 'to_text'); + END IF; + + SELECT jsonb_agg(cql2_query(a, wrapper)) + INTO args + FROM jsonb_array_elements(args) a; + END IF; + RAISE NOTICE 'ARGS: %', args; + + IF op IN ('and', 'or') THEN + RETURN + format( + '(%s)', + array_to_string(to_text_array(args), format(' %s ', upper(op))) + ); + END IF; + + IF op = 'in' THEN + RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); + RETURN format( + '%s IN (%s)', + to_text(args->0), + array_to_string((to_text_array(args))[2:], ',') + ); + END IF; + + -- Look up template from cql2_ops + IF j ? 'op' THEN + SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; + IF FOUND THEN + -- If specific index set in queryables for a property cast other arguments to that type + + RETURN format( + cql2op.template, + VARIADIC (to_text_array(args)) + ); + ELSE + RAISE EXCEPTION 'Operator % Not Supported.', op; + END IF; + END IF; + + + IF wrapper IS NOT NULL THEN + RAISE NOTICE 'Wrapping % with %', j, wrapper; + IF j ? 'property' THEN + RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); + ELSE + RETURN format('%I(%L)', wrapper, j); + END IF; + ELSIF j ? 'property' THEN + RETURN quote_ident(j->>'property'); + END IF; + + RETURN quote_literal(to_text(j)); +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION paging_dtrange( + j jsonb +) RETURNS tstzrange AS $$ +DECLARE + op text; + filter jsonb := j->'filter'; + dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); + sdate timestamptz := '-infinity'::timestamptz; + edate timestamptz := 'infinity'::timestamptz; + jpitem jsonb; +BEGIN + + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP + op := lower(jpitem->>'op'); + dtrange := parse_dtrange(jpitem->'args'->1); + IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN + sdate := greatest(sdate,'-infinity'); + edate := least(edate, upper(dtrange)); + ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN + edate := least(edate, 'infinity'); + sdate := greatest(sdate, lower(dtrange)); + ELSIF op IN ('=', 'eq') THEN + edate := least(edate, upper(dtrange)); + sdate := greatest(sdate, lower(dtrange)); + END IF; + RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; + END LOOP; + END IF; + IF sdate > edate THEN + RETURN 'empty'::tstzrange; + END IF; + RETURN tstzrange(sdate,edate, '[]'); +END; +$$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION paging_collections( + IN j jsonb +) RETURNS text[] AS $$ +DECLARE + filter jsonb := j->'filter'; + jpitem jsonb; + op text; + args jsonb; + arg jsonb; + collections text[]; +BEGIN + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP + RAISE NOTICE 'JPITEM: %', jpitem; + op := jpitem->>'op'; + args := jpitem->'args'; + IF op IN ('=', 'eq', 'in') THEN + FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP + IF jsonb_typeof(arg) IN ('string', 'array') THEN + RAISE NOTICE 'arg: %, collections: %', arg, collections; + IF collections IS NULL OR collections = '{}'::text[] THEN + collections := to_text_array(arg); + ELSE + collections := array_intersection(collections, to_text_array(arg)); + END IF; + END IF; + END LOOP; + END IF; + END LOOP; + END IF; + IF collections = '{}'::text[] THEN + RETURN NULL; + END IF; + RETURN collections; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; +CREATE TABLE items ( + id text NOT NULL, + geometry geometry NOT NULL, + collection text NOT NULL, + datetime timestamptz NOT NULL, + end_datetime timestamptz NOT NULL, + content JSONB NOT NULL, + private jsonb +) +PARTITION BY LIST (collection) +; + +CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); +CREATE INDEX "geometry_idx" ON items USING GIST (geometry); + +CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; + +ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; + +CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ +DECLARE + p text; + t timestamptz := clock_timestamp(); +BEGIN + RAISE NOTICE 'Updating partition stats %', t; + -- Ordered: each iteration holds a partition_stats row lock until commit. + FOR p IN SELECT DISTINCT partition + FROM newdata n JOIN partition_stats p + ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) + ORDER BY 1 + LOOP + PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); + END LOOP; + IF TG_OP IN ('DELETE','UPDATE') THEN + DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; + END IF; + RAISE NOTICE 't: % %', t, clock_timestamp() - t; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; + +CREATE TRIGGER items_after_insert_trigger +AFTER INSERT ON items +REFERENCING NEW TABLE AS newdata +FOR EACH STATEMENT +EXECUTE FUNCTION partition_after_triggerfunc(); + +CREATE TRIGGER items_after_update_trigger +AFTER DELETE ON items +REFERENCING OLD TABLE AS newdata +FOR EACH STATEMENT +EXECUTE FUNCTION partition_after_triggerfunc(); + +CREATE TRIGGER items_after_delete_trigger +AFTER UPDATE ON items +REFERENCING NEW TABLE AS newdata +FOR EACH STATEMENT +EXECUTE FUNCTION partition_after_triggerfunc(); + + +CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ + SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ + SELECT + content->>'id' as id, + stac_geom(content) as geometry, + content->>'collection' as collection, + stac_datetime(content) as datetime, + stac_end_datetime(content) as end_datetime, + content_slim(content) as content, + null::jsonb as private + ; +$$ LANGUAGE SQL STABLE; + +CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ +DECLARE + includes jsonb := fields->'include'; + excludes jsonb := fields->'exclude'; +BEGIN + IF f IS NULL THEN + RETURN NULL; + END IF; + + + IF + jsonb_typeof(excludes) = 'array' + AND jsonb_array_length(excludes)>0 + AND excludes ? f + THEN + RETURN FALSE; + END IF; + + IF + ( + jsonb_typeof(includes) = 'array' + AND jsonb_array_length(includes) > 0 + AND includes ? f + ) OR + ( + includes IS NULL + OR jsonb_typeof(includes) = 'null' + OR jsonb_array_length(includes) = 0 + ) + THEN + RETURN TRUE; + END IF; + + RETURN FALSE; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); +CREATE OR REPLACE FUNCTION content_hydrate( + _item jsonb, + _base_item jsonb, + fields jsonb DEFAULT '{}'::jsonb +) RETURNS jsonb AS $$ + SELECT merge_jsonb( + jsonb_fields(_item, fields), + jsonb_fields(_base_item, fields) + ); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ +DECLARE + geom jsonb; + bbox jsonb; + output jsonb; + content jsonb; + base_item jsonb := _collection.base_item; +BEGIN + IF include_field('geometry', fields) THEN + geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; + END IF; + output := content_hydrate( + jsonb_build_object( + 'id', _item.id, + 'geometry', geom, + 'collection', _item.collection, + 'type', 'Feature' + ) || _item.content, + _collection.base_item, + fields + ); + + RETURN output; +END; +$$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_nonhydrated( + _item items, + fields jsonb DEFAULT '{}'::jsonb +) RETURNS jsonb AS $$ +DECLARE + geom jsonb; + bbox jsonb; + output jsonb; +BEGIN + IF include_field('geometry', fields) THEN + geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; + END IF; + output := jsonb_build_object( + 'id', _item.id, + 'geometry', geom, + 'collection', _item.collection, + 'type', 'Feature' + ) || _item.content; + RETURN output; +END; +$$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ + SELECT content_hydrate( + _item, + (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), + fields + ); +$$ LANGUAGE SQL STABLE; + + +CREATE UNLOGGED TABLE items_staging ( + content JSONB NOT NULL +); +CREATE UNLOGGED TABLE items_staging_ignore ( + content JSONB NOT NULL +); +CREATE UNLOGGED TABLE items_staging_upsert ( + content JSONB NOT NULL +); + +CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ +DECLARE + part text; + ts timestamptz := clock_timestamp(); + nrows int; +BEGIN + RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; + + FOR part IN WITH t AS ( + SELECT + n.content->>'collection' as collection, + stac_daterange(n.content->'properties') as dtr, + partition_trunc + FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) + ), p AS ( + SELECT + collection, + COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, + tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, + tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange + FROM t + GROUP BY 1,2 + -- Ordered: check_partition holds DDL and row locks until commit. + ) SELECT check_partition(collection, dtrange, edtrange) FROM ( + SELECT * FROM p ORDER BY collection, d + ) ordered LOOP + RAISE NOTICE 'Partition %', part; + END LOOP; + + RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; + DROP TABLE IF EXISTS tmpdata; + CREATE TEMP TABLE tmpdata ON COMMIT DROP AS + SELECT + (content_dehydrate(content)).* + FROM newdata; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; + + RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; + IF TG_TABLE_NAME = 'items_staging' THEN + INSERT INTO items + SELECT * FROM tmpdata; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; + ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN + INSERT INTO items + SELECT * FROM tmpdata + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; + ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN + -- Locked in a fixed order first, so concurrent upserts over an + -- overlapping id set cannot deadlock. A bare DELETE gives no ordering; + -- ORDER BY ... FOR UPDATE does, because LockRows sits above the sort. + WITH locked AS ( + SELECT o.collection, o.id + FROM tmpdata s + JOIN items o ON (o.id = s.id AND o.collection = s.collection) + WHERE o IS DISTINCT FROM s + ORDER BY o.collection, o.id + FOR UPDATE OF o + ) + DELETE FROM items i + USING locked l + WHERE i.collection = l.collection AND i.id = l.id + ; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; + INSERT INTO items AS t + SELECT * FROM tmpdata + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS nrows = ROW_COUNT; + RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; + END IF; + + RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; + EXECUTE format('DELETE FROM %I', TG_TABLE_NAME); + RAISE NOTICE 'Done. %', clock_timestamp() - ts; + + RETURN NULL; + +END; +$$ LANGUAGE PLPGSQL; + + +CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + +CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + +CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + + +CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS +$$ +DECLARE + i items%ROWTYPE; +BEGIN + SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; + RETURN i; +END; +$$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ + SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); +$$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ +DECLARE +out items%ROWTYPE; +BEGIN + DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL; + +--/* +CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging (content) VALUES (data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ +DECLARE + old items %ROWTYPE; + out items%ROWTYPE; +BEGIN + PERFORM delete_item(content->>'id', content->>'collection'); + PERFORM create_item(content); +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging_upsert (content) VALUES (data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging (content) + SELECT * FROM jsonb_array_elements(data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging_upsert (content) + SELECT * FROM jsonb_array_elements(data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ + SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb + FROM items WHERE collection=$1; + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ + SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) + FROM items WHERE collection=$1; +; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; + +-- Recalculates partition statistics before aggregating them; the observed +-- ranges are only maintained automatically when update_collection_extent is on. +-- return_target, not use_json_null: collection_extent returns NULL when it +-- cannot compute a full extent, and JSON null is not a valid STAC extent. +CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ +UPDATE collections + SET content = jsonb_set_lax( + content, + '{extent}'::text[], + collection_extent(id, TRUE), + true, + 'return_target' + ) +; +$$ LANGUAGE SQL; +-- collection and partition_dtrange describe the partition itself and are +-- written synchronously by check_partition. dtrange, edtrange and spatial +-- describe the data in the partition and may be updated asynchronously +-- through the query queue. +CREATE TABLE partition_stats ( + partition text PRIMARY KEY, + collection text, + partition_dtrange tstzrange, + dtrange tstzrange, + edtrange tstzrange, + spatial geometry, + last_updated timestamptz, + keys text[] +) WITH (FILLFACTOR=90); + +CREATE INDEX partition_stats_collection_idx ON partition_stats (collection); + + +CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ + WITH t AS ( + SELECT regexp_matches( + expr, + E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' + ) AS m + ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; + +CREATE OR REPLACE FUNCTION get_tstz_constraint(reloid oid, colname text) RETURNS tstzrange AS $$ +DECLARE + expr text := NULL; + m text[]; + ts_lower timestamptz := NULL; + ts_upper timestamptz := NULL; + lower_inclusive text := '['; + upper_inclusive text := ']'; + ts timestamptz; +BEGIN + SELECT INTO expr + string_agg(def, ' AND ') + FROM pg_constraint JOIN LATERAL pg_get_constraintdef(oid) AS def ON TRUE + WHERE + conrelid = reloid + AND contype = 'c' + AND def LIKE '%' || colname || '%' + ; + + IF expr IS NULL THEN + RETURN NULL; + END IF; + + RAISE DEBUG 'Constraint expression for % on %: %', colname, reloid::regclass, expr; + -- collect all constraints for the specified column + FOR m IN SELECT regexp_matches(expr, '[ (]' || colname || $expr$\s*([<>=]{1,2})\s*'([0-9 :.+\-]+)'$expr$, 'g') LOOP + ts := m[2]::timestamptz; + IF m[1] IN ('>', '>=') + THEN + IF ts_lower IS NULL OR ts > ts_lower OR (ts = ts_lower AND m[1] = '>') THEN + ts_lower := ts; + lower_inclusive := CASE WHEN m[1] = '>' THEN '(' ELSE '[' END; + END IF; + ELSIF m[1] IN ('<', '<=') + THEN + IF ts_upper IS NULL OR ts < ts_upper OR (ts = ts_upper AND m[1] = '<') THEN + ts_upper := ts; + upper_inclusive := CASE WHEN m[1] = '<' THEN ')' ELSE ']' END; + END IF; + END IF; + END LOOP; + RAISE DEBUG 'Constraint % for %: % %', colname, reloid::regclass, ts_lower, ts_upper; + RETURN tstzrange(ts_lower, ts_upper, lower_inclusive || upper_inclusive); +END; +$$ LANGUAGE plpgsql STRICT STABLE; + +CREATE OR REPLACE FUNCTION get_partition_name(relid regclass) RETURNS text AS $$ + SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; +$$ LANGUAGE SQL STABLE STRICT; + +-- Resolve a partition name, bare or pgstac qualified, against pgstac rather +-- than the search_path. NULL if there is no such table, or if the name is +-- qualified with a different schema. +CREATE OR REPLACE FUNCTION partition_oid(_partition text) RETURNS oid AS $$ + SELECT CASE + WHEN cardinality(parts) > 1 AND parts[cardinality(parts) - 1] <> 'pgstac' THEN NULL + ELSE to_regclass(format('pgstac.%I', parts[cardinality(parts)])) + END + FROM parse_ident(_partition) AS parts; +$$ LANGUAGE SQL STABLE STRICT; + +-- Catalog metadata for one partition, touching one relation. pg_partition_tree +-- would lock every partition in the tree. No rows unless _partition is a leaf +-- partition of items. +CREATE OR REPLACE FUNCTION partition_catalog_meta(_partition text) +RETURNS TABLE ( + collection text, + partition_dtrange tstzrange, + constraint_dtrange tstzrange, + constraint_edtrange tstzrange +) AS $$ +DECLARE + _oid oid; + _parent oid; + _expr text; + _inf tstzrange := tstzrange('-infinity', 'infinity', '[]'); + _dtrange tstzrange; +BEGIN + _oid := partition_oid(_partition); + -- Leaves only, matching partitions_view: an intermediate partition has no + -- meaningful range of its own. + IF _oid IS NULL OR EXISTS (SELECT 1 FROM pg_inherits WHERE inhparent = _oid) THEN + RETURN; + END IF; + + SELECT inhparent INTO _parent FROM pg_inherits WHERE inhrelid = _oid; + + -- Partitions of items only. The SECURITY DEFINER functions below use this + -- to decide what they may alter, so any other relation must return nothing. + IF _parent IS NULL + OR ( + _parent <> 'pgstac.items'::regclass + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits + WHERE inhrelid = _parent AND inhparent = 'pgstac.items'::regclass + ) + ) + THEN + RETURN; + END IF; + + -- A partition of a sub-partitioned collection carries a datetime range + -- bound; its collection is on the parent. A direct partition of items + -- carries the collection itself. + IF _parent = 'pgstac.items'::regclass THEN + SELECT pg_get_expr(relpartbound, oid) INTO _expr FROM pg_class WHERE oid = _oid; + ELSE + SELECT pg_get_expr(relpartbound, oid) INTO _expr FROM pg_class WHERE oid = _parent; + END IF; + + SELECT COALESCE( + constraint_tstzrange(pg_get_expr(relpartbound, oid)), + _inf + ) INTO _dtrange FROM pg_class WHERE oid = _oid; + + RETURN QUERY SELECT + replace(replace(_expr, 'FOR VALUES IN (''', ''), ''')', ''), + _dtrange, + COALESCE(get_tstz_constraint(_oid, 'datetime'), _dtrange, _inf), + COALESCE(get_tstz_constraint(_oid, 'end_datetime'), _inf); +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE VIEW partition_sys_meta AS +SELECT + partition, + replace( + replace( + CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, + 'FOR VALUES IN (''', + '' + ), + ''')', + '' + ) AS collection, + level, + c.reltuples, + c.relhastriggers, + partition_dtrange, + COALESCE( + get_tstz_constraint(c.oid, 'datetime'), + partition_dtrange, + inf_range + ) as constraint_dtrange, + COALESCE( + get_tstz_constraint(c.oid, 'end_datetime'), + inf_range + ) as constraint_edtrange +FROM + pg_partition_tree('items') + JOIN pg_class c ON (relid::regclass = c.oid) + JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) + JOIN LATERAL get_partition_name(relid) AS partition ON TRUE + JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE + JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE + JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE + JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE +WHERE isleaf +; + +-- partition_sys_meta plus the statistics tracked alongside each partition. +CREATE OR REPLACE VIEW partitions_view AS +SELECT + sm.*, + ps.dtrange, + ps.edtrange, + ps.spatial, + ps.last_updated +FROM partition_sys_meta sm + LEFT JOIN pgstac.partition_stats ps USING (partition) +; + +CREATE VIEW partitions AS +SELECT * FROM partitions_view; + + +-- Returns TRUE if the stats update ran, FALSE if it was queued. +CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS boolean AS $$ +DECLARE +BEGIN + RETURN run_or_queue( + format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) + ); +END; +$$ LANGUAGE PLPGSQL; + +-- _extent NULL follows pgstac.update_collection_extent; TRUE computes the +-- spatial extent regardless, for callers that are about to read it. +CREATE OR REPLACE FUNCTION update_partition_stats( + _partition text, + istrigger boolean default false, + _extent boolean default NULL +) RETURNS VOID AS $$ +DECLARE + dtrange tstzrange; + edtrange tstzrange; + cdtrange tstzrange; + cedtrange tstzrange; + extent geometry; + collection text; + pdtrange tstzrange; + auto_extent boolean := get_setting_bool('update_collection_extent'); + do_extent boolean := COALESCE(_extent, auto_extent); +BEGIN + -- Cannot be STRICT: _extent is three valued, and STRICT would skip the + -- body whenever it is NULL. + IF _partition IS NULL OR istrigger IS NULL THEN + RETURN; + END IF; + RAISE NOTICE 'Updating stats for %.', _partition; + + SELECT m.collection, m.partition_dtrange, m.constraint_dtrange, m.constraint_edtrange + INTO collection, pdtrange, cdtrange, cedtrange + FROM partition_catalog_meta(_partition) m; + + -- A queued update can outlive the partition it names. + IF NOT FOUND THEN + RAISE NOTICE 'Partition % no longer exists, skipping stats update.', _partition; + RETURN; + END IF; + + -- Taken before the partition is read. The constraint rebuild below needs + -- ACCESS EXCLUSIVE, and escalating to that mid-transaction deadlocks + -- against another session doing the same. SHARE UPDATE EXCLUSIVE conflicts + -- with itself but not with readers. + IF NOT istrigger THEN + EXECUTE format('LOCK TABLE %I IN SHARE UPDATE EXCLUSIVE MODE', _partition); + END IF; + + -- The observed ranges feed the constraint tightening below and collection + -- extents. When neither will read them, only the partition's identity is + -- written, which is what keeps it visible to search. + IF NOT istrigger OR do_extent THEN + -- st_extent visits every geometry, so it is only run when wanted. + IF do_extent THEN + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]'), + st_extent(geometry)::geometry + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange, extent; + RAISE DEBUG 'Extent: %', extent; + ELSE + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]') + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange; + END IF; + + INSERT INTO partition_stats + (partition, collection, partition_dtrange, dtrange, edtrange, spatial, last_updated) + VALUES (_partition, collection, pdtrange, dtrange, edtrange, extent, now()) + ON CONFLICT (partition) DO + UPDATE SET + collection=EXCLUDED.collection, + partition_dtrange=EXCLUDED.partition_dtrange, + dtrange=EXCLUDED.dtrange, + edtrange=EXCLUDED.edtrange, + spatial=COALESCE(EXCLUDED.spatial, partition_stats.spatial), + last_updated=EXCLUDED.last_updated + ; + ELSE + INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES (_partition, collection, pdtrange) + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + END IF; + + RAISE NOTICE 'Checking if we need to modify constraints...'; + RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; + IF + (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) + AND NOT istrigger + THEN + RAISE NOTICE 'Modifying Constraints'; + RAISE NOTICE 'Existing % %', cdtrange, cedtrange; + RAISE NOTICE 'New % %', dtrange, edtrange; + PERFORM drop_table_constraints(_partition); + PERFORM create_table_constraints(_partition, dtrange, edtrange); + END IF; + -- auto_extent, not do_extent: a caller that passed _extent aggregates the + -- extent itself, and update_collection_extents would then be updating + -- collections from inside its own UPDATE of collections. + RAISE NOTICE 'Checking if we need to update collection extents.'; + IF auto_extent THEN + RAISE NOTICE 'updating collection extent for %', collection; + PERFORM run_or_queue(format($q$ + UPDATE collections + SET content = jsonb_set_lax( + content, + '{extent}'::text[], + collection_extent(%L, FALSE), + true, + 'use_json_null' + ) WHERE id=%L + ; + $q$, collection, collection)); + ELSE + RAISE NOTICE 'Not updating collection extent for %', collection; + END IF; + +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; + + + +CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ +DECLARE + c RECORD; + parent_name text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + parent_name := format('_items_%s', c.key); + + + IF c.partition_trunc = 'year' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); + ELSE + partition_name := parent_name; + partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); + END IF; + IF partition_range IS NULL THEN + partition_range := tstzrange( + date_trunc(c.partition_trunc::text, dt), + date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval + ); + END IF; + RETURN; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ +DECLARE + q text; + _oid oid := partition_oid(t); +BEGIN + IF _oid IS NULL THEN + RETURN NULL; + END IF; + -- Only partitions of items. This runs elevated, so without the check it + -- would alter any table pgstac_admin owns that the caller names. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(t)) THEN + RETURN NULL; + END IF; + -- Reduce to the bare name so the ALTER statements below quote it correctly + -- even when the caller passed a schema qualified name. + t := get_partition_name(_oid); + FOR q IN SELECT FORMAT( + $q$ + ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; + $q$, + t, + conname + ) FROM pg_constraint + WHERE conrelid=_oid AND contype='c' + LOOP + EXECUTE q; + END LOOP; + RETURN t; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ +DECLARE + q text; + _oid oid := partition_oid(t); +BEGIN + IF _oid IS NULL THEN + RETURN NULL; + END IF; + -- Only partitions of items. This runs elevated, so without the check it + -- would alter any table pgstac_admin owns that the caller names. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(t)) THEN + RETURN NULL; + END IF; + -- Reduce to the bare name so the ALTER statements below quote it correctly + -- even when the caller passed a schema qualified name. + t := get_partition_name(_oid); + RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; + IF _dtrange = 'empty' AND _edtrange = 'empty' THEN + q :=format( + $q$ + DO $block$ + BEGIN + ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + + + + EXCEPTION WHEN others THEN + RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; + END; + $block$; + $q$, + t, + format('%s_dt', t), + t, + format('%s_dt', t), + t, + format('%s_dt', t), + t + ); + ELSE + q :=format( + $q$ + DO $block$ + BEGIN + + ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK ( + (datetime >= %L) + AND (datetime <= %L) + AND (end_datetime >= %L) + AND (end_datetime <= %L) + ) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + + + + EXCEPTION WHEN others THEN + RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; + END; + $block$; + $q$, + t, + format('%s_dt', t), + t, + format('%s_dt', t), + lower(_dtrange), + upper(_dtrange), + lower(_edtrange), + upper(_edtrange), + t, + format('%s_dt', t), + t + ); + END IF; + -- Run, not queued: the queue runner is not a definer, so queued DDL + -- executes as whoever drains it. Defer by queueing a call to this function. + EXECUTE q; + RETURN t; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + + +CREATE OR REPLACE FUNCTION check_partition( + _collection text, + _dtrange tstzrange, + _edtrange tstzrange +) RETURNS text AS $$ +DECLARE + c RECORD; + pm RECORD; + _partition_name text; + _partition_dtrange tstzrange; + _constraint_dtrange tstzrange; + _constraint_edtrange tstzrange; + q text; + err_context text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + + IF c.partition_trunc IS NOT NULL THEN + _partition_dtrange := tstzrange( + date_trunc(c.partition_trunc, lower(_dtrange)), + date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, + '[)' + ); + ELSE + _partition_dtrange := '[-infinity, infinity]'::tstzrange; + END IF; + + IF NOT _partition_dtrange @> _dtrange THEN + RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; + END IF; + + + IF c.partition_trunc = 'year' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); + ELSE + _partition_name := format('_items_%s', c.key); + END IF; + + -- Constraint ranges are maintained asynchronously, so they come from the + -- catalog rather than partition_stats. + SELECT ps.partition, m.constraint_dtrange, m.constraint_edtrange + INTO pm + FROM partition_stats ps + JOIN LATERAL partition_catalog_meta(ps.partition) m ON TRUE + WHERE ps.collection = _collection AND ps.partition_dtrange @> _dtrange + LIMIT 1; + IF FOUND THEN + RAISE NOTICE '% % %', _edtrange, _dtrange, pm; + _constraint_edtrange := + tstzrange( + least( + lower(_edtrange), + nullif(lower(pm.constraint_edtrange), '-infinity') + ), + greatest( + upper(_edtrange), + nullif(upper(pm.constraint_edtrange), 'infinity') + ), + '[]' + ); + _constraint_dtrange := + tstzrange( + least( + lower(_dtrange), + nullif(lower(pm.constraint_dtrange), '-infinity') + ), + greatest( + upper(_dtrange), + nullif(upper(pm.constraint_dtrange), 'infinity') + ), + '[]' + ); + + IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN + RETURN pm.partition; + ELSE + PERFORM drop_table_constraints(_partition_name); + END IF; + ELSE + _constraint_edtrange := _edtrange; + _constraint_dtrange := _dtrange; + END IF; + RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; + RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; + IF c.partition_trunc IS NULL THEN + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + GRANT ALL ON %I to pgstac_ingest; + $q$, + _partition_name, + _collection, + concat(_partition_name,'_pk'), + _partition_name, + _partition_name + ); + ELSE + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); + CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + GRANT ALL ON %I TO pgstac_ingest; + $q$, + format('_items_%s', c.key), + _collection, + _partition_name, + format('_items_%s', c.key), + lower(_partition_dtrange), + upper(_partition_dtrange), + format('%s_pk', _partition_name), + _partition_name, + _partition_name + ); + END IF; + + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', _partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + -- The _constraint_ ranges are the union of the existing constraint and the + -- incoming batch, so they hold for rows already present as well as the ones + -- about to be added. Queueable: rebuilding validates the partition under an + -- ACCESS EXCLUSIVE lock. + PERFORM run_or_queue(format( + 'SELECT create_table_constraints(%L, %L, %L);', + _partition_name, + _constraint_dtrange, + _constraint_edtrange + )); + PERFORM maintain_partitions(_partition_name); + -- Search finds partitions through partition_stats, so the row has to exist + -- before this transaction commits. A queued stats update has not written it. + IF NOT update_partition_stats_q(_partition_name, true) THEN + INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES (_partition_name, _collection, _partition_dtrange) + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + END IF; + RETURN _partition_name; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + + +CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ +DECLARE + c RECORD; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + IF triggered THEN + RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; + ELSE + RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; + IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN + RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; + RETURN _collection; + END IF; + END IF; + + IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN + EXECUTE format( + $q$ + CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; + DROP TABLE IF EXISTS %I CASCADE; + DELETE FROM partition_stats WHERE collection = %L; + WITH p AS ( + SELECT + collection, + CASE + WHEN %L IS NULL THEN '-infinity'::timestamptz + ELSE date_trunc(%L, datetime) + END as d, + tstzrange(min(datetime),max(datetime),'[]') as dtrange, + tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange + FROM changepartitionstaging + GROUP BY 1,2 + ) SELECT check_partition(collection, dtrange, edtrange) FROM p; + INSERT INTO items SELECT * FROM changepartitionstaging; + DROP TABLE changepartitionstaging; + $q$, + concat('_items_', c.key), + concat('_items_', c.key), + _collection, + c.partition_trunc, + c.partition_trunc + ); + END IF; + RETURN _collection; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + q text; + partition_name text := format('_items_%s', NEW.key); + partition_exists boolean := false; + partition_empty boolean := true; + err_context text; + loadtemp boolean := FALSE; +BEGIN + RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN + PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); + END IF; + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE TRIGGER collections_trigger AFTER +INSERT +OR +UPDATE ON collections +FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); + +CREATE OR REPLACE FUNCTION chunker( + IN _where text, + OUT s timestamptz, + OUT e timestamptz +) RETURNS SETOF RECORD AS $$ +DECLARE + explain jsonb; +BEGIN + IF _where IS NULL THEN + _where := ' TRUE '; + END IF; + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) + INTO explain; + RAISE DEBUG 'EXPLAIN: %', explain; + + RETURN QUERY + WITH t AS ( + SELECT j->>0 as p FROM + jsonb_path_query( + explain, + 'strict $.**."Relation Name" ? (@ != null)' + ) j + ), + parts AS ( + -- = ANY(array) uses the partition_stats primary key; the planner + -- cannot estimate jsonb_path_query, so a join plans as a full scan. + SELECT + date_trunc('month', lower(partition_dtrange)) as sdate, + date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate + FROM partition_stats + WHERE + partition = ANY (ARRAY(SELECT p FROM t)) + AND partition_dtrange IS NOT NULL + AND partition_dtrange != 'empty'::tstzrange + ), + times AS ( + SELECT sdate FROM parts + UNION + SELECT edate FROM parts + ), + uniq AS ( + SELECT DISTINCT sdate FROM times ORDER BY sdate + ), + last AS ( + SELECT sdate, lead(sdate, 1) over () as edate FROM uniq + ) + SELECT sdate, edate FROM last WHERE edate IS NOT NULL; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION partition_queries( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN partitions text[] DEFAULT NULL +) RETURNS SETOF text AS $$ +DECLARE + query text; + sdate timestamptz; + edate timestamptz; +BEGIN +IF _where IS NULL OR trim(_where) = '' THEN + _where = ' TRUE '; +END IF; +RAISE NOTICE 'Getting chunks for % %', _where, _orderby; +IF _orderby ILIKE 'datetime d%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, + _where, + _orderby + ); + END LOOP; +ELSIF _orderby ILIKE 'datetime a%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, + _where, + _orderby + ); + END LOOP; +ELSE + query := format($q$ + SELECT * FROM items + WHERE %s + ORDER BY %s + $q$, _where, _orderby + ); + + RETURN NEXT query; + RETURN; +END IF; + +RETURN; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION partition_query_view( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN _limit int DEFAULT 10 +) RETURNS text AS $$ + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total LIMIT %s + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ), + _limit + )) + ELSE NULL + END FROM p; +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION q_to_tsquery (jinput jsonb) + RETURNS tsquery + AS $$ +DECLARE + input text; + processed_text text; + temp_text text; + quote_array text[]; + placeholder text := '@QUOTE@'; +BEGIN + IF jsonb_typeof(jinput) = 'string' THEN + input := jinput->>0; + ELSIF jsonb_typeof(jinput) = 'array' THEN + input := array_to_string( + array(select jsonb_array_elements_text(jinput)), + ' OR ' + ); + ELSE + RAISE EXCEPTION 'Input must be a string or an array of strings.'; + END IF; + -- Extract all quoted phrases and store in array + quote_array := regexp_matches(input, '"[^"]*"', 'g'); + + -- Replace each quoted part with a unique placeholder if there are any quoted phrases + IF array_length(quote_array, 1) IS NOT NULL THEN + processed_text := input; + FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP + processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); + END LOOP; + ELSE + processed_text := input; + END IF; + + -- Replace non-quoted text using regular expressions + + -- , -> | + processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); + + -- and -> & + processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); + + -- or -> | + processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); + + -- + -> + processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start + processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere + + -- - -> ! + processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start + processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere + + -- terms separated with spaces are assumed to represent adjacent terms. loop through these + -- occurrences and replace them with the adjacency operator (<->) + LOOP + temp_text := regexp_replace(processed_text, '([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_]+)(?!\s*[&|<>])', '\1 <-> \2', 'g'); + IF temp_text = processed_text THEN + EXIT; -- No more replacements were made + END IF; + processed_text := temp_text; + END LOOP; + + + -- Replace placeholders back with quoted phrases if there were any + IF array_length(quote_array, 1) IS NOT NULL THEN + FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP + processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); + END LOOP; + END IF; + + -- Print processed_text to the console for debugging purposes + RAISE NOTICE 'processed_text: %', processed_text; + + RETURN to_tsquery('english', processed_text); +END; +$$ +LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ +DECLARE + where_segments text[]; + _where text; + dtrange tstzrange; + collections text[]; + geom geometry; + sdate timestamptz; + edate timestamptz; + filterlang text; + filter jsonb := j->'filter'; + ft_query tsquery; +BEGIN + IF j ? 'ids' THEN + where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); + END IF; + + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + where_segments := where_segments || format('collection = ANY (%L) ', collections); + END IF; + + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); + + where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', + edate, + sdate + ); + END IF; + + IF j ? 'q' THEN + ft_query := q_to_tsquery(j->'q'); + where_segments := where_segments || format( + $quote$ + ( + to_tsvector('english', content->'properties'->>'description') || + to_tsvector('english', coalesce(content->'properties'->>'title', '')) || + to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) + ) @@ %L + $quote$, + ft_query + ); + END IF; + + geom := stac_geom(j); + IF geom IS NOT NULL THEN + where_segments := where_segments || format('st_intersects(geometry, %L)',geom); + END IF; + + filterlang := COALESCE( + j->>'filter-lang', + get_setting('default_filter_lang', j->'conf') + ); + IF NOT filter @? '$.**.op' THEN + filterlang := 'cql-json'; + END IF; + + IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN + RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; + END IF; + + IF j ? 'query' AND j ? 'filter' THEN + RAISE EXCEPTION 'Can only use either query or filter at one time.'; + END IF; + + IF j ? 'query' THEN + filter := query_to_cql2(j->'query'); + ELSIF filterlang = 'cql-json' THEN + filter := cql1_to_cql2(filter); + END IF; + RAISE NOTICE 'FILTER: %', filter; + where_segments := where_segments || cql2_query(filter); + IF cardinality(where_segments) < 1 THEN + RETURN ' TRUE '; + END IF; + + _where := array_to_string(array_remove(where_segments, NULL), ' AND '); + + IF _where IS NULL OR BTRIM(_where) = '' THEN + RETURN ' TRUE '; + END IF; + RETURN _where; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN NOT reverse THEN d + WHEN d = 'ASC' THEN 'DESC' + WHEN d = 'DESC' THEN 'ASC' + END + FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN d = 'ASC' AND prev THEN '<=' + WHEN d = 'DESC' AND prev THEN '>=' + WHEN d = 'ASC' THEN '>=' + WHEN d = 'DESC' THEN '<=' + END + FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION sort_sqlorderby( + _search jsonb DEFAULT NULL, + reverse boolean DEFAULT FALSE +) RETURNS text AS $$ + WITH sortby AS ( + SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort + ), withid AS ( + SELECT CASE + WHEN sort @? '$[*] ? (@.field == "id")' THEN sort + ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb + END as sort + FROM sortby + ), withid_rows AS ( + SELECT jsonb_array_elements(sort) as value FROM withid + ),sorts AS ( + SELECT + coalesce( + (queryable(value->>'field')).expression + ) as key, + parse_sort_dir(value->>'direction', reverse) as dir + FROM withid_rows + ) + SELECT array_to_string( + array_agg(concat(key, ' ', dir)), + ', ' + ) FROM sorts; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ + SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION get_token_val_str( + _field text, + _item items +) RETURNS text AS $$ +DECLARE + q text; + literal text; +BEGIN + q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); + EXECUTE q INTO literal USING _item; + RETURN literal; +END; +$$ LANGUAGE PLPGSQL; + + + +CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ +DECLARE + _itemid text := _token; + _collectionid text; +BEGIN + IF _token IS NULL THEN + RETURN; + END IF; + RAISE NOTICE 'Looking for token: %', _token; + prev := FALSE; + IF _token ILIKE 'prev:%' THEN + _itemid := replace(_token, 'prev:',''); + prev := TRUE; + ELSIF _token ILIKE 'next:%' THEN + _itemid := replace(_token, 'next:', ''); + END IF; + SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); + IF FOUND THEN + _itemid := replace(_itemid, concat(_collectionid,':'), ''); + SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; + ELSE + SELECT * INTO item FROM items WHERE id=_itemid; + END IF; + IF item IS NULL THEN + RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; + END IF; + RETURN; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + + +CREATE OR REPLACE FUNCTION get_token_filter( + _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, + token_item items DEFAULT NULL, + prev boolean DEFAULT FALSE, + inclusive boolean DEFAULT FALSE +) RETURNS text AS $$ +DECLARE + ltop text := '<'; + gtop text := '>'; + dir text; + sort record; + orfilter text := ''; + orfilters text[] := '{}'::text[]; + andfilters text[] := '{}'::text[]; + output text; + token_where text; +BEGIN + IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN + _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; + END IF; + _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); + RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; + IF inclusive THEN + orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); + END IF; + + FOR sort IN + WITH s1 AS ( + SELECT + _row, + (queryable(value->>'field')).expression as _field, + (value->>'field' = 'id') as _isid, + get_sort_dir(value) as _dir + FROM jsonb_array_elements(_sortby) + WITH ORDINALITY AS t(value, _row) + ) + SELECT + _row, + _field, + _dir, + get_token_val_str(_field, token_item) as _val + FROM s1 + WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) + LOOP + orfilter := NULL; + RAISE NOTICE 'SORT: %', sort; + IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN + orfilter := format($f$( + (%s %s %s) OR (%s IS NULL) + )$f$, + sort._field, + ltop, + sort._val, + sort._val + ); + ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN + RAISE NOTICE '< but null'; + orfilter := format('%s IS NOT NULL', sort._field); + ELSIF sort._val IS NULL THEN + RAISE NOTICE '> but null'; + ELSE + orfilter := format($f$( + (%s %s %s) OR (%s IS NULL) + )$f$, + sort._field, + gtop, + sort._val, + sort._field + ); + END IF; + RAISE NOTICE 'ORFILTER: %', orfilter; + + IF orfilter IS NOT NULL THEN + IF sort._row = 1 THEN + orfilters := orfilters || orfilter; + ELSE + orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); + END IF; + END IF; + IF sort._val IS NOT NULL THEN + andfilters := andfilters || format('%s = %s', sort._field, sort._val); + ELSE + andfilters := andfilters || format('%s IS NULL', sort._field); + END IF; + END LOOP; + + output := array_to_string(orfilters, ' OR '); + + token_where := concat('(',coalesce(output,'true'),')'); + IF trim(token_where) = '' THEN + token_where := NULL; + END IF; + RAISE NOTICE 'TOKEN_WHERE: %',token_where; + RETURN token_where; + END; +$$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE +; + +CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ + SELECT md5(concat(($1 - '{token,limit,context,includes,excludes}'::text[])::text,$2::text)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; +DROP FUNCTION IF EXISTS search_tohash(jsonb); + +CREATE TABLE IF NOT EXISTS searches( + hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, + search jsonb NOT NULL, + _where text, + orderby text, + lastused timestamptz DEFAULT now(), + usecount bigint DEFAULT 0, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL +); + +CREATE TABLE IF NOT EXISTS search_wheres( + id bigint generated always as identity primary key, + _where text NOT NULL, + lastused timestamptz DEFAULT now(), + usecount bigint DEFAULT 0, + statslastupdated timestamptz, + estimated_count bigint, + estimated_cost float, + time_to_estimate float, + total_count bigint, + time_to_count float, + partitions text[] +); + +CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); +CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); + +CREATE OR REPLACE FUNCTION where_stats( + inwhere text, + updatestats boolean default false, + conf jsonb default null +) RETURNS search_wheres AS $$ +DECLARE + t timestamptz; + i interval; + explain_json jsonb; + sw search_wheres%ROWTYPE; + inwhere_hash text := md5(inwhere); + _context text := lower(context(conf)); + _stats_ttl interval := context_stats_ttl(conf); + _estimated_cost_threshold float := context_estimated_cost(conf); + _estimated_count_threshold int := context_estimated_count(conf); + ro bool := pgstac.readonly(conf); +BEGIN + -- If updatestats is true then set ttl to 0 + IF updatestats THEN + RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; + _stats_ttl := '0'::interval; + END IF; + + -- If we don't need to calculate context, just return + IF _context = 'off' THEN + sw._where = inwhere; + RETURN sw; + END IF; + + -- Unlocked read. A fresh hit only bumps bookkeeping counters, and that is + -- the common case when identical searches run concurrently. + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; + + -- Within ttl: bump usage counters and return. The bump skips locked rows so + -- identical searches do not serialize; a missed increment is harmless. + -- sw.id, not "sw IS NOT NULL": a composite is only IS NOT NULL when every + -- field is, and search_wheres.partitions is never populated. + IF + sw.id IS NOT NULL + AND sw.statslastupdated IS NOT NULL + AND sw.total_count IS NOT NULL + AND now() - sw.statslastupdated <= _stats_ttl + THEN + RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; + IF NOT ro THEN + UPDATE search_wheres SET + lastused = now(), + usecount = search_wheres.usecount + 1 + WHERE id = ( + SELECT id FROM search_wheres + WHERE md5(_where) = inwhere_hash + FOR UPDATE SKIP LOCKED + ); + END IF; + RAISE DEBUG 'Returning cached counts. %', sw; + RETURN sw; + END IF; + + -- Missing or stale, so lock the row to compute once, then re-check + -- freshness in case another session finished while we waited. + IF NOT ro THEN + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; + IF + sw.statslastupdated IS NOT NULL + AND sw.total_count IS NOT NULL + AND now() - sw.statslastupdated <= _stats_ttl + THEN + RAISE DEBUG 'Another process refreshed stats while we waited: %', sw; + UPDATE search_wheres SET + lastused = now(), + usecount = search_wheres.usecount + 1 + WHERE md5(_where) = inwhere_hash + RETURNING * INTO sw; + RETURN sw; + END IF; + END IF; + + -- Calculate estimated cost and rows + -- Use explain to get estimated count/cost + IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN + RAISE DEBUG 'Calculating estimated stats'; + t := clock_timestamp(); + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) + INTO explain_json; + RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; + i := clock_timestamp() - t; + + sw.estimated_count := (explain_json->0->'Plan'->>'Plan Rows')::bigint; + sw.estimated_cost := (explain_json->0->'Plan'->>'Total Cost')::float; + sw.time_to_estimate := extract(epoch from i); + END IF; + + RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; + RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; + + -- If context is set to auto and the costs are within the threshold return the estimated costs + IF + _context = 'auto' + AND sw.estimated_count >= _estimated_count_threshold + AND sw.estimated_cost >= _estimated_cost_threshold + THEN + IF NOT ro THEN + INSERT INTO search_wheres ( + _where, + lastused, + usecount, + statslastupdated, + estimated_count, + estimated_cost, + time_to_estimate, + total_count, + time_to_count + ) VALUES ( + inwhere, + now(), + 1, + now(), + sw.estimated_count, + sw.estimated_cost, + sw.time_to_estimate, + null, + null + ) ON CONFLICT ((md5(_where))) + DO UPDATE SET + lastused = EXCLUDED.lastused, + usecount = search_wheres.usecount + 1, + statslastupdated = EXCLUDED.statslastupdated, + estimated_count = EXCLUDED.estimated_count, + estimated_cost = EXCLUDED.estimated_cost, + time_to_estimate = EXCLUDED.time_to_estimate, + total_count = EXCLUDED.total_count, + time_to_count = EXCLUDED.time_to_count + RETURNING * INTO sw; + END IF; + RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; + RETURN sw; + END IF; + + -- Calculate Actual Count + t := clock_timestamp(); + RAISE NOTICE 'Calculating actual count...'; + EXECUTE format( + 'SELECT count(*) FROM items WHERE %s', + inwhere + ) INTO sw.total_count; + i := clock_timestamp() - t; + RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; + sw.time_to_count := extract(epoch FROM i); + + IF NOT ro THEN + INSERT INTO search_wheres ( + _where, + lastused, + usecount, + statslastupdated, + estimated_count, + estimated_cost, + time_to_estimate, + total_count, + time_to_count + ) VALUES ( + inwhere, + now(), + 1, + now(), + sw.estimated_count, + sw.estimated_cost, + sw.time_to_estimate, + sw.total_count, + sw.time_to_count + ) ON CONFLICT ((md5(_where))) + DO UPDATE SET + lastused = EXCLUDED.lastused, + usecount = search_wheres.usecount + 1, + statslastupdated = EXCLUDED.statslastupdated, + estimated_count = EXCLUDED.estimated_count, + estimated_cost = EXCLUDED.estimated_cost, + time_to_estimate = EXCLUDED.time_to_estimate, + total_count = EXCLUDED.total_count, + time_to_count = EXCLUDED.time_to_count + RETURNING * INTO sw; + END IF; + RAISE DEBUG 'Returning with actual count. %', sw; + RETURN sw; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION search_query( + _search jsonb = '{}'::jsonb, + updatestats boolean = false, + _metadata jsonb = '{}'::jsonb +) RETURNS searches AS $$ +DECLARE + search searches%ROWTYPE; + cached_search searches%ROWTYPE; + pexplain jsonb; + t timestamptz; + i interval; + doupdate boolean := FALSE; + insertfound boolean := FALSE; + ro boolean := pgstac.readonly(); + found_search text; +BEGIN + RAISE NOTICE 'SEARCH: %', _search; + -- Calculate hash, where clause, and order by statement + search.search := _search; + search.metadata := _metadata; + search.hash := search_hash(_search, _metadata); + search._where := stac_search_to_where(_search); + search.orderby := sort_sqlorderby(_search); + search.lastused := now(); + search.usecount := 1; + + -- If we are in read only mode, directly return search + IF ro THEN + RETURN search; + END IF; + + RAISE NOTICE 'Updating Statistics for search: %s', search; + -- Update statistics for times used and and when last used + -- If the entry is locked, rather than waiting, skip updating the stats + INSERT INTO searches (search, lastused, usecount, metadata) + VALUES (search.search, now(), 1, search.metadata) + ON CONFLICT DO NOTHING + RETURNING * INTO cached_search + ; + + IF NOT FOUND OR cached_search IS NULL THEN + UPDATE searches SET + lastused = now(), + usecount = searches.usecount + 1 + WHERE hash = ( + SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED + ) + RETURNING * INTO cached_search + ; + END IF; + + IF cached_search IS NOT NULL THEN + cached_search._where = search._where; + cached_search.orderby = search.orderby; + RETURN cached_search; + END IF; + RETURN search; + +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION search_fromhash( + _hash text +) RETURNS searches AS $$ + SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); +$$ LANGUAGE SQL STRICT; + +CREATE OR REPLACE FUNCTION search_rows( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN partitions text[] DEFAULT NULL, + IN _limit int DEFAULT 10 +) RETURNS SETOF items AS $$ +DECLARE + base_query text; + query text; + sdate timestamptz; + edate timestamptz; + n int; + records_left int := _limit; + timer timestamptz := clock_timestamp(); + full_timer timestamptz := clock_timestamp(); +BEGIN +IF _where IS NULL OR trim(_where) = '' THEN + _where = ' TRUE '; +END IF; +RAISE NOTICE 'Getting chunks for % %', _where, _orderby; + +base_query := $q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + LIMIT %L +$q$; + +IF _orderby ILIKE 'datetime d%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP + RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); + query := format( + base_query, + sdate, + edate, + _where, + _orderby, + records_left + ); + RAISE DEBUG 'QUERY: %', query; + timer := clock_timestamp(); + RETURN QUERY EXECUTE query; + + GET DIAGNOSTICS n = ROW_COUNT; + records_left := records_left - n; + RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); + timer := clock_timestamp(); + IF records_left <= 0 THEN + RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); + RETURN; + END IF; + END LOOP; +ELSIF _orderby ILIKE 'datetime a%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP + RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); + query := format( + base_query, + sdate, + edate, + _where, + _orderby, + records_left + ); + RAISE DEBUG 'QUERY: %', query; + timer := clock_timestamp(); + RETURN QUERY EXECUTE query; + + GET DIAGNOSTICS n = ROW_COUNT; + records_left := records_left - n; + RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); + timer := clock_timestamp(); + IF records_left <= 0 THEN + RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); + RETURN; + END IF; + END LOOP; +ELSE + query := format($q$ + SELECT * FROM items + WHERE %s + ORDER BY %s + LIMIT %L + $q$, _where, _orderby, _limit + ); + RAISE DEBUG 'QUERY: %', query; + timer := clock_timestamp(); + RETURN QUERY EXECUTE query; + RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); +END IF; +RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); +RETURN; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + + +CREATE UNLOGGED TABLE format_item_cache( + id text, + collection text, + fields text, + hydrated bool, + output jsonb, + lastused timestamptz DEFAULT now(), + usecount int DEFAULT 1, + timetoformat float, + PRIMARY KEY (collection, id, fields, hydrated) +); +CREATE INDEX ON format_item_cache (lastused); + +CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ +DECLARE + cache bool := get_setting_bool('format_cache'); + _output jsonb := null; + t timestamptz := clock_timestamp(); +BEGIN + IF cache THEN + SELECT output INTO _output FROM format_item_cache + WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; + END IF; + IF _output IS NULL THEN + IF _hydrated THEN + _output := content_hydrate(_item, _fields); + ELSE + _output := content_nonhydrated(_item, _fields); + END IF; + END IF; + IF cache THEN + INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) + VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) + ON CONFLICT(collection, id, fields, hydrated) DO + UPDATE + SET lastused=now(), usecount = format_item_cache.usecount + 1 + ; + END IF; + RETURN _output; + +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ +DECLARE + searches searches%ROWTYPE; + _where text; + orderby text; + search_where search_wheres%ROWTYPE; + total_count bigint; + token record; + token_prev boolean; + token_item items%ROWTYPE; + token_where text; + full_where text; + init_ts timestamptz := clock_timestamp(); + timer timestamptz := clock_timestamp(); + hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); + prev text; + next text; + context jsonb; + collection jsonb; + out_records jsonb; + out_len int; + _limit int := coalesce((_search->>'limit')::int, 10); + _querylimit int; + _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); + has_prev boolean := FALSE; + has_next boolean := FALSE; + links jsonb := '[]'::jsonb; + base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); +BEGIN + searches := search_query(_search); + _where := searches._where; + orderby := searches.orderby; + search_where := where_stats(_where); + total_count := coalesce(search_where.total_count, search_where.estimated_count); + RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; + token := get_token_record(_search->>'token'); + RAISE NOTICE '***TOKEN: %', token; + _querylimit := _limit + 1; + IF token IS NOT NULL THEN + token_prev := token.prev; + token_item := token.item; + token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); + RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); + IF token_prev THEN -- if we are using a prev token, we know has_next is true + RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; + has_next := TRUE; + orderby := sort_sqlorderby(_search, TRUE); + ELSE + RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; + has_prev := TRUE; + + END IF; + ELSE -- if there was no token, we know there is no prev + RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; + has_prev := FALSE; + END IF; + + full_where := concat_ws(' AND ', _where, token_where); + RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; + RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); + timer := clock_timestamp(); + + IF hydrate THEN + RAISE NOTICE 'Getting hydrated data.'; + ELSE + RAISE NOTICE 'Getting non-hydrated data.'; + END IF; + RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); + RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); + timer := clock_timestamp(); + SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records + FROM search_rows( + full_where, + orderby, + search_where.partitions, + _querylimit + ) as i; + + RAISE NOTICE 'Time to fetch rows %', age_ms(timer); + timer := clock_timestamp(); + + + IF token_prev THEN + out_records := flip_jsonb_array(out_records); + END IF; + + RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); + RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; + RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; + RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; + + -- REMOVE records that were from our token + IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN + out_records := out_records - 0; + ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN + out_records := out_records - -1; + END IF; + + out_len := jsonb_array_length(out_records); + + IF out_len = _limit + 1 THEN + IF token_prev THEN + has_prev := TRUE; + out_records := out_records - 0; + ELSE + has_next := TRUE; + out_records := out_records - -1; + END IF; + END IF; + + + links := links || jsonb_build_object( + 'rel', 'root', + 'type', 'application/json', + 'href', base_url + ) || jsonb_build_object( + 'rel', 'self', + 'type', 'application/json', + 'href', concat(base_url, '/search') + ); + + IF has_next THEN + next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); + RAISE NOTICE 'HAS NEXT | %', next; + links := links || jsonb_build_object( + 'rel', 'next', + 'type', 'application/geo+json', + 'method', 'GET', + 'href', concat(base_url, '/search?token=next:', next) + ); + END IF; + + IF has_prev THEN + prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); + RAISE NOTICE 'HAS PREV | %', prev; + links := links || jsonb_build_object( + 'rel', 'prev', + 'type', 'application/geo+json', + 'method', 'GET', + 'href', concat(base_url, '/search?token=prev:', prev) + ); + END IF; + + RAISE NOTICE 'Time to get prev/next %', age_ms(timer); + timer := clock_timestamp(); + + + collection := jsonb_build_object( + 'type', 'FeatureCollection', + 'features', coalesce(out_records, '[]'::jsonb), + 'links', links + ); + + + + IF context(_search->'conf') != 'off' THEN + collection := collection || jsonb_strip_nulls(jsonb_build_object( + 'numberMatched', total_count, + 'numberReturned', coalesce(jsonb_array_length(out_records), 0) + )); + ELSE + collection := collection || jsonb_strip_nulls(jsonb_build_object( + 'numberReturned', coalesce(jsonb_array_length(out_records), 0) + )); + END IF; + + IF get_setting_bool('timing', _search->'conf') THEN + collection = collection || jsonb_build_object('timing', age_ms(init_ts)); + END IF; + + RAISE NOTICE 'Time to build final json %', age_ms(timer); + timer := clock_timestamp(); + + RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); + RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; + RETURN collection; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ +DECLARE + curs refcursor; + searches searches%ROWTYPE; + _where text; + _orderby text; + q text; + +BEGIN + searches := search_query(_search); + _where := searches._where; + _orderby := searches.orderby; + + OPEN curs FOR + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ) + )) + ELSE NULL + END FROM p; + RETURN curs; +END; +$$ LANGUAGE PLPGSQL; +CREATE OR REPLACE VIEW collections_asitems AS +SELECT + id, + geometry, + 'collections' AS collection, + datetime, + end_datetime, + jsonb_build_object( + 'properties', content - '{links,assets,stac_version,stac_extensions}', + 'links', content->'links', + 'assets', content->'assets', + 'stac_version', content->'stac_version', + 'stac_extensions', content->'stac_extensions' + ) AS content, + content as collectionjson +FROM collections; + + +CREATE OR REPLACE FUNCTION collection_search_matched( + IN _search jsonb DEFAULT '{}'::jsonb, + OUT matched bigint +) RETURNS bigint AS $$ +DECLARE + _where text := stac_search_to_where(_search); +BEGIN + EXECUTE format( + $query$ + SELECT + count(*) + FROM + collections_asitems + WHERE %s + ; + $query$, + _where + ) INTO matched; + RETURN; +END; +$$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION collection_search_rows( + _search jsonb DEFAULT '{}'::jsonb +) RETURNS SETOF jsonb AS $$ +DECLARE + _where text := stac_search_to_where(_search); + _limit int := coalesce((_search->>'limit')::int, 10); + _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); + _orderby text; + _offset int := COALESCE((_search->>'offset')::int, 0); +BEGIN + _orderby := sort_sqlorderby( + jsonb_build_object( + 'sortby', + coalesce( + _search->'sortby', + '[{"field": "id", "direction": "asc"}]'::jsonb + ) + ) + ); + RETURN QUERY EXECUTE format( + $query$ + SELECT + jsonb_fields(collectionjson, %L) as c + FROM + collections_asitems + WHERE %s + ORDER BY %s + LIMIT %L + OFFSET %L + ; + $query$, + _fields, + _where, + _orderby, + _limit, + _offset + ); +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION collection_search( + _search jsonb DEFAULT '{}'::jsonb +) RETURNS jsonb AS $$ +DECLARE + out_records jsonb; + number_matched bigint := collection_search_matched(_search); + number_returned bigint; + _limit int := coalesce((_search->>'limit')::float::int, 10); + _offset int := coalesce((_search->>'offset')::float::int, 0); + links jsonb := '[]'; + ret jsonb; + base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); + prevoffset int; + nextoffset int; +BEGIN + SELECT + coalesce(jsonb_agg(c), '[]') + INTO out_records + FROM collection_search_rows(_search) c; + + number_returned := jsonb_array_length(out_records); + RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; + + + + IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links + nextoffset := least(_offset + _limit, number_matched - 1); + prevoffset := greatest(_offset - _limit, 0); + + IF _offset > 0 THEN + links := links || jsonb_build_object( + 'rel', 'prev', + 'type', 'application/json', + 'method', 'GET' , + 'href', base_url, + 'body', jsonb_build_object('offset', prevoffset), + 'merge', TRUE + ); + END IF; + + IF (_offset + _limit < number_matched) THEN + links := links || jsonb_build_object( + 'rel', 'next', + 'type', 'application/json', + 'method', 'GET' , + 'href', base_url, + 'body', jsonb_build_object('offset', nextoffset), + 'merge', TRUE + ); + END IF; + + END IF; + + ret := jsonb_build_object( + 'collections', out_records, + 'numberMatched', number_matched, + 'numberReturned', number_returned, + 'links', links + ); + RETURN ret; + +END; +$$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; +SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ +WITH t AS ( + SELECT + 20037508.3427892 as merc_max, + -20037508.3427892 as merc_min, + (2 * 20037508.3427892) / (2 ^ zoom) as tile_size +) +SELECT st_makeenvelope( + merc_min + (tile_size * x), + merc_max - (tile_size * (y + 1)), + merc_min + (tile_size * (x + 1)), + merc_max - (tile_size * y), + 3857 +) FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; + + +CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ +SELECT age(clock_timestamp(), transaction_timestamp()); +$$ LANGUAGE SQL; +SET SEARCH_PATH to pgstac, public; + +DROP FUNCTION IF EXISTS geometrysearch; +CREATE OR REPLACE FUNCTION geometrysearch( + IN geom geometry, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered + IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items +) RETURNS jsonb AS $$ +DECLARE + search searches%ROWTYPE; + curs refcursor; + _where text; + query text; + iter_record items%ROWTYPE; + out_records jsonb := '{}'::jsonb[]; + exit_flag boolean := FALSE; + counter int := 1; + scancounter int := 1; + remaining_limit int := _scanlimit; + tilearea float; + unionedgeom geometry; + clippedgeom geometry; + unionedgeom_area float := 0; + prev_area float := 0; + excludes text[]; + includes text[]; + +BEGIN + DROP TABLE IF EXISTS pgstac_results; + CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; + + -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false + IF ST_GeometryType(geom) !~* 'polygon' THEN + RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; + skipcovered = FALSE; + exitwhenfull = FALSE; + END IF; + + -- If skipcovered is true then you will always want to exit when the passed in geometry is full + IF skipcovered THEN + exitwhenfull := TRUE; + END IF; + + search := search_fromhash(queryhash); + + IF search IS NULL THEN + RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; + END IF; + + tilearea := st_area(geom); + _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); + + + FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP + query := format('%s LIMIT %L', query, remaining_limit); + RAISE NOTICE '%', query; + OPEN curs FOR EXECUTE query; + LOOP + FETCH curs INTO iter_record; + EXIT WHEN NOT FOUND; + IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations + clippedgeom := st_intersection(geom, iter_record.geometry); + + IF unionedgeom IS NULL THEN + unionedgeom := clippedgeom; + ELSE + unionedgeom := st_union(unionedgeom, clippedgeom); + END IF; + + unionedgeom_area := st_area(unionedgeom); + + IF skipcovered AND prev_area = unionedgeom_area THEN + scancounter := scancounter + 1; + CONTINUE; + END IF; + + prev_area := unionedgeom_area; + + RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); + END IF; + RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); + INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); + + IF counter >= _limit + OR scancounter > _scanlimit + OR ftime() > _timelimit + OR (exitwhenfull AND unionedgeom_area >= tilearea) + THEN + exit_flag := TRUE; + EXIT; + END IF; + counter := counter + 1; + scancounter := scancounter + 1; + + END LOOP; + CLOSE curs; + EXIT WHEN exit_flag; + remaining_limit := _scanlimit - scancounter; + END LOOP; + + SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; + + RETURN jsonb_build_object( + 'type', 'FeatureCollection', + 'features', coalesce(out_records, '[]'::jsonb) + ); +END; +$$ LANGUAGE PLPGSQL; + +DROP FUNCTION IF EXISTS geojsonsearch; +CREATE OR REPLACE FUNCTION geojsonsearch( + IN geojson jsonb, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, + IN skipcovered boolean DEFAULT TRUE +) RETURNS jsonb AS $$ + SELECT * FROM geometrysearch( + st_geomfromgeojson(geojson), + queryhash, + fields, + _scanlimit, + _limit, + _timelimit, + exitwhenfull, + skipcovered + ); +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS xyzsearch; +CREATE OR REPLACE FUNCTION xyzsearch( + IN _x int, + IN _y int, + IN _z int, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, + IN skipcovered boolean DEFAULT TRUE +) RETURNS jsonb AS $$ + SELECT * FROM geometrysearch( + st_transform(tileenvelope(_z, _x, _y), 4326), + queryhash, + fields, + _scanlimit, + _limit, + _timelimit, + exitwhenfull, + skipcovered + ); +$$ LANGUAGE SQL; + +CREATE OR REPLACE PROCEDURE analyze_items() AS $$ +DECLARE + q text; + timeout_ts timestamptz; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q + FROM pg_stat_user_tables + WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; + IF NOT FOUND THEN + EXIT; + END IF; + RAISE NOTICE '%', q; + EXECUTE q; + COMMIT; + END LOOP; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ +DECLARE + q text; +BEGIN + FOR q IN + SELECT + FORMAT( + 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', + nsp.nspname, + cls.relname, + con.conname + ) + + FROM pg_constraint AS con + JOIN pg_class AS cls + ON con.conrelid = cls.oid + JOIN pg_namespace AS nsp + ON cls.relnamespace = nsp.oid + WHERE convalidated = FALSE AND contype in ('c','f') + AND nsp.nspname = 'pgstac' + LOOP + RAISE NOTICE '%', q; + PERFORM run_or_queue(q); + COMMIT; + END LOOP; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ +DECLARE + geom_extent geometry; + mind timestamptz; + maxd timestamptz; + extent jsonb; + _partition text; +BEGIN + IF runupdate THEN + -- Not queued: the aggregate below reads what this writes. Ordered by + -- partition, as every other multi-partition writer is. + FOR _partition IN + SELECT partition FROM partition_stats + WHERE collection=_collection + ORDER BY partition + LOOP + PERFORM update_partition_stats(_partition, false, true); + END LOOP; + END IF; + SELECT + min(lower(dtrange)), + max(upper(edtrange)), + st_extent(spatial) + INTO + mind, + maxd, + geom_extent + FROM partition_stats + WHERE collection=_collection; + + IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN + extent := jsonb_build_object( + 'spatial', jsonb_build_object( + 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) + ), + 'temporal', jsonb_build_object( + 'interval', to_jsonb(array[array[mind, maxd]]) + ) + ); + RETURN extent; + END IF; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL; + + +-- Reconcile partition_stats against the partition tree: an identity row for +-- every partition, and no rows for partitions that are gone. dtrange, edtrange +-- and spatial are left alone; update_partition_stats maintains those. Used by +-- the idempotent install and as a repair path. +CREATE OR REPLACE FUNCTION sync_partition_stats() RETURNS VOID AS $$ +BEGIN + -- Ordered by partition, as every other writer of these rows is. + INSERT INTO partition_stats (partition, collection, partition_dtrange) + SELECT partition, collection, partition_dtrange FROM partitions_view + ORDER BY partition + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + + DELETE FROM partition_stats ps + WHERE ps.partition IN ( + SELECT partition FROM partition_stats stale + WHERE NOT EXISTS ( + SELECT 1 FROM partitions_view pv WHERE pv.partition = stale.partition + ) + ORDER BY partition + FOR UPDATE + ); +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; +DO $$ + BEGIN + INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES + ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); + EXCEPTION WHEN unique_violation THEN + RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES + ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); + EXCEPTION WHEN unique_violation THEN + RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES + ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); + EXCEPTION WHEN unique_violation THEN + RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DELETE FROM queryables a USING queryables b + WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; + + +INSERT INTO pgstac_settings (name, value) VALUES + ('context', 'off'), + ('context_estimated_count', '100000'), + ('context_estimated_cost', '100000'), + ('context_stats_ttl', '1 day'), + ('default_filter_lang', 'cql2-json'), + ('additional_properties', 'true'), + ('use_queue', 'false'), + ('queue_timeout', '10 minutes'), + ('update_collection_extent', 'false'), + ('format_cache', 'false'), + ('readonly', 'false') +ON CONFLICT DO NOTHING +; + + +INSERT INTO cql2_ops (op, template, types) VALUES + ('eq', '%s = %s', NULL), + ('neq', '%s != %s', NULL), + ('ne', '%s != %s', NULL), + ('!=', '%s != %s', NULL), + ('<>', '%s != %s', NULL), + ('lt', '%s < %s', NULL), + ('lte', '%s <= %s', NULL), + ('gt', '%s > %s', NULL), + ('gte', '%s >= %s', NULL), + ('le', '%s <= %s', NULL), + ('ge', '%s >= %s', NULL), + ('=', '%s = %s', NULL), + ('<', '%s < %s', NULL), + ('<=', '%s <= %s', NULL), + ('>', '%s > %s', NULL), + ('>=', '%s >= %s', NULL), + ('like', '%s LIKE %s', NULL), + ('ilike', '%s ILIKE %s', NULL), + ('+', '%s + %s', NULL), + ('-', '%s - %s', NULL), + ('*', '%s * %s', NULL), + ('/', '%s / %s', NULL), + ('not', 'NOT (%s)', NULL), + ('between', '%s BETWEEN %s AND %s', NULL), + ('isnull', '%s IS NULL', NULL), + ('upper', 'upper(%s)', NULL), + ('lower', 'lower(%s)', NULL), + ('casei', 'upper(%s)', NULL), + ('accenti', 'unaccent(%s)', NULL) +ON CONFLICT (op) DO UPDATE + SET + template = EXCLUDED.template +; + + +ALTER FUNCTION to_text COST 5000; +ALTER FUNCTION to_float COST 5000; +ALTER FUNCTION to_int COST 5000; +ALTER FUNCTION to_tstz COST 5000; +ALTER FUNCTION to_text_array COST 5000; + +ALTER FUNCTION drop_table_constraints SECURITY DEFINER; +ALTER FUNCTION create_table_constraints SECURITY DEFINER; +ALTER FUNCTION check_partition SECURITY DEFINER; +ALTER FUNCTION repartition SECURITY DEFINER; +ALTER FUNCTION maintain_index SECURITY DEFINER; + +-- Elevated only where ownership of pgstac_admin's objects is required. +-- Everything else relies on table privileges, which any role inheriting +-- pgstac_ingest or pgstac_read already has. +ALTER FUNCTION where_stats SECURITY INVOKER; +ALTER FUNCTION search_query SECURITY INVOKER; +ALTER FUNCTION format_item SECURITY INVOKER; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +GRANT ALL ON SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON SCHEMA pgstac to pgstac_admin; + +-- pgstac_read role limited to using function apis +GRANT EXECUTE ON FUNCTION search TO pgstac_read; +GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; +GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; +GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; +GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; + +-- The caches maintained by where_stats, search_query and format_item. +GRANT SELECT, INSERT, UPDATE ON search_wheres TO pgstac_read; +GRANT SELECT, INSERT, UPDATE ON searches TO pgstac_read; +GRANT SELECT, INSERT, UPDATE ON format_item_cache TO pgstac_read; + + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; + +REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; +GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; + +REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; +GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; + +-- PostgreSQL grants EXECUTE to PUBLIC on every new function, so each definer +-- is revoked and granted back to the roles that need it. Keep in step with the +-- ALTER FUNCTION ... SECURITY DEFINER statements above; the pgtap suite fails +-- if a definer is left PUBLIC executable. +REVOKE ALL PRIVILEGES ON FUNCTION + drop_table_constraints, + create_table_constraints, + check_partition, + repartition, + maintain_index, + delete_collection +FROM public; + +GRANT EXECUTE ON FUNCTION + drop_table_constraints, + create_table_constraints, + check_partition, + repartition, + maintain_index, + delete_collection +TO pgstac_ingest; + +RESET ROLE; + +SET ROLE pgstac_ingest; + +-- Search finds partitions through partition_stats, so this must be synchronous +-- rather than queued. +SELECT sync_partition_stats(); + +-- Repairs observed ranges and CHECK constraints for every partition, ordered +-- by partition as every other writer of these rows is. The most expensive part +-- of an install on a large catalog: run with pgstac.use_queue on (pypgstac +-- --usequeue) and drain with pypgstac runqueue to keep it off the migration. +SELECT update_partition_stats_q(partition) FROM partitions_view ORDER BY partition; +SELECT set_version('0.9.12'); diff --git a/src/pgstac/pgstac.sql b/src/pgstac/pgstac.sql index af4ad13a..6d71b2b4 100644 --- a/src/pgstac/pgstac.sql +++ b/src/pgstac/pgstac.sql @@ -132,6 +132,19 @@ ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FU RESET ROLE; SET SEARCH_PATH TO pgstac, public; + +-- PgSTAC references PostGIS without schema qualification and runs with +-- search_path pinned to "pgstac, public", so PostGIS has to be reachable from +-- there. Fail here rather than part way through creating the schema. +DO $$ + BEGIN + IF to_regtype('geometry') IS NULL THEN + RAISE EXCEPTION 'PostGIS is not reachable from search_path "pgstac, public"' + USING HINT = 'PgSTAC requires the postgis extension in the public schema.'; + END IF; + END +$$; + SET ROLE pgstac_admin; DO $$ @@ -148,6 +161,26 @@ DO $$ RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; +-- Drain the queue here, where the functions its entries name still have the +-- signatures they were queued against and pgstac_admin owns the objects any +-- queued DDL touches. Statistics updates are discarded instead: +-- 998_idempotent_post recalculates every partition regardless. +DO $$ + BEGIN + DELETE FROM query_queue WHERE query LIKE 'SELECT update_partition_stats(%'; + PERFORM run_queued_queries_intransaction(); + EXCEPTION WHEN undefined_table OR undefined_function THEN + RAISE NOTICE 'No query queue to drain.'; + END +$$; + +-- Return type or argument list differs from an earlier release, which +-- CREATE OR REPLACE cannot change. +DROP FUNCTION IF EXISTS run_or_queue(text); +DROP FUNCTION IF EXISTS update_partition_stats_q(text, boolean); +DROP FUNCTION IF EXISTS update_partition_stats(text, boolean); +DROP FUNCTION IF EXISTS maintain_index(text, text, boolean, boolean, boolean); +DROP FUNCTION IF EXISTS queryable_indexes(text, boolean); -- Install these idempotently as migrations do not put them before trying to modify the collections table @@ -406,19 +439,21 @@ END; $$ LANGUAGE PLPGSQL; -CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ +-- TRUE if the query ran, FALSE if it was queued, so callers can tell whether +-- its effects are visible in this transaction. +CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS boolean AS $$ DECLARE - use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; + use_queue boolean := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; - ELSE - EXECUTE query; + RETURN FALSE; END IF; - RETURN; + EXECUTE query; + RETURN TRUE; END; $$ LANGUAGE PLPGSQL; @@ -516,6 +551,12 @@ BEGIN END IF; END IF; + -- Undrained queue means stale statistics and constraints, silently. + SELECT count(*) INTO settingval FROM query_queue; + IF settingval::bigint > 0 THEN + RAISE WARNING '% queries are waiting in query_queue. Run "CALL run_queued_queries();" (pypgstac runqueue) to drain it -- until then partition statistics and constraints are stale.', settingval; + END IF; + END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ @@ -907,13 +948,16 @@ CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; +-- SECURITY DEFINER: the delete trigger drops partition tables, which are owned +-- by pgstac_admin. CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ -DECLARE - out collections%ROWTYPE; BEGIN - DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; + DELETE FROM collections WHERE id = _id; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _id USING ERRCODE = 'no_data_found'; + END IF; END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ @@ -931,15 +975,14 @@ CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN + -- Tables before rows: check_partition takes these locks in the same order, + -- and the reverse deadlocks against a concurrent partition create. EXECUTE format($q$ - DELETE FROM partition_stats WHERE partition IN ( - SELECT partition FROM partition_sys_meta - WHERE collection=%L - ); DROP TABLE IF EXISTS %I CASCADE; + DELETE FROM partition_stats WHERE collection=%L; $q$, - OLD.id, - collection_base_partition + collection_base_partition, + OLD.id ); RETURN OLD; END; @@ -1215,7 +1258,8 @@ CREATE OR REPLACE FUNCTION queryable_indexes( OUT field text, OUT indexname text, OUT existing_idx text, - OUT queryable_idx text + OUT queryable_idx text, + OUT queryable_id bigint ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT @@ -1249,7 +1293,8 @@ WITH p AS ( name AS field, collection, partition, - format(indexdef(queryables), partition) as qidx + format(indexdef(queryables), partition) as qidx, + queryables.id as qid FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') @@ -1260,40 +1305,53 @@ WITH p AS ( field, indexname, iidx as existing_idx, - qidx as queryable_idx + qidx as queryable_idx, + qid as queryable_id FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; +-- SECURITY DEFINER, so the index statement is built here from the queryables +-- row; a caller supplied one would run with the privileges of the schema owner. CREATE OR REPLACE FUNCTION maintain_index( - indexname text, - queryable_idx text, + _partition text, + _indexname text, + _queryable_id bigint, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE + _queryable_idx text; BEGIN - IF indexname IS NOT NULL THEN - IF dropindexes OR queryable_idx IS NOT NULL THEN - EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); + -- Runs elevated, so it may only touch partitions of items. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(_partition)) THEN + RETURN; + END IF; + IF _queryable_id IS NOT NULL AND _partition IS NOT NULL THEN + SELECT format(indexdef(q), _partition) INTO _queryable_idx + FROM queryables q WHERE q.id = _queryable_id; + END IF; + IF _indexname IS NOT NULL THEN + IF dropindexes OR _queryable_idx IS NOT NULL THEN + EXECUTE format('DROP INDEX IF EXISTS %I;', _indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN - EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); + EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', _indexname); ELSE - EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); + EXECUTE format('REINDEX INDEX %I;', _indexname); END IF; END IF; END IF; - IF queryable_idx IS NOT NULL THEN + IF _queryable_idx IS NOT NULL THEN IF idxconcurrently THEN - EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); - ELSE EXECUTE queryable_idx; + EXECUTE replace(_queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); + ELSE EXECUTE _queryable_idx; END IF; END IF; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; @@ -1313,10 +1371,11 @@ BEGIN ) LOOP q := format( 'SELECT maintain_index( - %L,%L,%L,%L,%L + %L,%L,%L,%L,%L,%L );', + rec.partition, rec.indexname, - rec.queryable_idx, + rec.queryable_id, dropindexes, rebuildindexes, idxconcurrently @@ -2060,9 +2119,11 @@ DECLARE t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; + -- Ordered: each iteration holds a partition_stats row lock until commit. FOR p IN SELECT DISTINCT partition - FROM newdata n JOIN partition_sys_meta p + FROM newdata n JOIN partition_stats p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) + ORDER BY 1 LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; @@ -2072,7 +2133,7 @@ BEGIN RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items @@ -2229,8 +2290,6 @@ CREATE UNLOGGED TABLE items_staging_upsert ( CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE - p record; - _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; @@ -2251,7 +2310,10 @@ BEGIN tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 - ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP + -- Ordered: check_partition holds DDL and row locks until commit. + ) SELECT check_partition(collection, dtrange, edtrange) FROM ( + SELECT * FROM p ORDER BY collection, d + ) ordered LOOP RAISE NOTICE 'Partition %', part; END LOOP; @@ -2277,11 +2339,20 @@ BEGIN GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN - DELETE FROM items i USING tmpdata s - WHERE - i.id = s.id - AND i.collection = s.collection - AND i IS DISTINCT FROM s + -- Locked in a fixed order first, so concurrent upserts over an + -- overlapping id set cannot deadlock. A bare DELETE gives no ordering; + -- ORDER BY ... FOR UPDATE does, because LockRows sits above the sort. + WITH locked AS ( + SELECT o.collection, o.id + FROM tmpdata s + JOIN items o ON (o.id = s.id AND o.collection = s.collection) + WHERE o IS DISTINCT FROM s + ORDER BY o.collection, o.id + FOR UPDATE OF o + ) + DELETE FROM items i + USING locked l + WHERE i.collection = l.collection AND i.id = l.id ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; @@ -2293,7 +2364,7 @@ BEGIN END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; - DELETE FROM items_staging; + EXECUTE format('DELETE FROM %I', TG_TABLE_NAME); RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; @@ -2377,19 +2448,29 @@ CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; +-- Recalculates partition statistics before aggregating them; the observed +-- ranges are only maintained automatically when update_collection_extent is on. +-- return_target, not use_json_null: collection_extent returns NULL when it +-- cannot compute a full extent, and JSON null is not a valid STAC extent. CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], - collection_extent(id, FALSE), + collection_extent(id, TRUE), true, - 'use_json_null' + 'return_target' ) ; $$ LANGUAGE SQL; +-- collection and partition_dtrange describe the partition itself and are +-- written synchronously by check_partition. dtrange, edtrange and spatial +-- describe the data in the partition and may be updated asynchronously +-- through the query queue. CREATE TABLE partition_stats ( partition text PRIMARY KEY, + collection text, + partition_dtrange tstzrange, dtrange tstzrange, edtrange tstzrange, spatial geometry, @@ -2397,7 +2478,7 @@ CREATE TABLE partition_stats ( keys text[] ) WITH (FILLFACTOR=90); -CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); +CREATE INDEX partition_stats_collection_idx ON partition_stats (collection); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ @@ -2460,6 +2541,80 @@ CREATE OR REPLACE FUNCTION get_partition_name(relid regclass) RETURNS text AS $$ SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; $$ LANGUAGE SQL STABLE STRICT; +-- Resolve a partition name, bare or pgstac qualified, against pgstac rather +-- than the search_path. NULL if there is no such table, or if the name is +-- qualified with a different schema. +CREATE OR REPLACE FUNCTION partition_oid(_partition text) RETURNS oid AS $$ + SELECT CASE + WHEN cardinality(parts) > 1 AND parts[cardinality(parts) - 1] <> 'pgstac' THEN NULL + ELSE to_regclass(format('pgstac.%I', parts[cardinality(parts)])) + END + FROM parse_ident(_partition) AS parts; +$$ LANGUAGE SQL STABLE STRICT; + +-- Catalog metadata for one partition, touching one relation. pg_partition_tree +-- would lock every partition in the tree. No rows unless _partition is a leaf +-- partition of items. +CREATE OR REPLACE FUNCTION partition_catalog_meta(_partition text) +RETURNS TABLE ( + collection text, + partition_dtrange tstzrange, + constraint_dtrange tstzrange, + constraint_edtrange tstzrange +) AS $$ +DECLARE + _oid oid; + _parent oid; + _expr text; + _inf tstzrange := tstzrange('-infinity', 'infinity', '[]'); + _dtrange tstzrange; +BEGIN + _oid := partition_oid(_partition); + -- Leaves only, matching partitions_view: an intermediate partition has no + -- meaningful range of its own. + IF _oid IS NULL OR EXISTS (SELECT 1 FROM pg_inherits WHERE inhparent = _oid) THEN + RETURN; + END IF; + + SELECT inhparent INTO _parent FROM pg_inherits WHERE inhrelid = _oid; + + -- Partitions of items only. The SECURITY DEFINER functions below use this + -- to decide what they may alter, so any other relation must return nothing. + IF _parent IS NULL + OR ( + _parent <> 'pgstac.items'::regclass + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits + WHERE inhrelid = _parent AND inhparent = 'pgstac.items'::regclass + ) + ) + THEN + RETURN; + END IF; + + -- A partition of a sub-partitioned collection carries a datetime range + -- bound; its collection is on the parent. A direct partition of items + -- carries the collection itself. + IF _parent = 'pgstac.items'::regclass THEN + SELECT pg_get_expr(relpartbound, oid) INTO _expr FROM pg_class WHERE oid = _oid; + ELSE + SELECT pg_get_expr(relpartbound, oid) INTO _expr FROM pg_class WHERE oid = _parent; + END IF; + + SELECT COALESCE( + constraint_tstzrange(pg_get_expr(relpartbound, oid)), + _inf + ) INTO _dtrange FROM pg_class WHERE oid = _oid; + + RETURN QUERY SELECT + replace(replace(_expr, 'FOR VALUES IN (''', ''), ''')', ''), + _dtrange, + COALESCE(get_tstz_constraint(_oid, 'datetime'), _dtrange, _inf), + COALESCE(get_tstz_constraint(_oid, 'end_datetime'), _inf); +END; +$$ LANGUAGE PLPGSQL STABLE; + + CREATE OR REPLACE VIEW partition_sys_meta AS SELECT partition, @@ -2489,86 +2644,47 @@ FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) - LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE - JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE - JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE WHERE isleaf ; +-- partition_sys_meta plus the statistics tracked alongside each partition. CREATE OR REPLACE VIEW partitions_view AS SELECT - (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, - replace( - replace( - CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, - 'FOR VALUES IN (''', - '' - ), - ''')', - '' - ) AS collection, - level, - c.reltuples, - c.relhastriggers, - partition_dtrange, - COALESCE( - get_tstz_constraint(c.oid, 'datetime'), - partition_dtrange, - inf_range - ) as constraint_dtrange, - COALESCE( - get_tstz_constraint(c.oid, 'end_datetime'), - inf_range - ) as constraint_edtrange, - dtrange, - edtrange, - spatial, - last_updated -FROM - pg_partition_tree('items') - JOIN pg_class c ON (relid::regclass = c.oid) - JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) - LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') - JOIN LATERAL get_partition_name(relid) AS partition ON TRUE - JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE - JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE - JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE - JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE - JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE - JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE - LEFT JOIN pgstac.partition_stats USING (partition) -WHERE isleaf + sm.*, + ps.dtrange, + ps.edtrange, + ps.spatial, + ps.last_updated +FROM partition_sys_meta sm + LEFT JOIN pgstac.partition_stats ps USING (partition) ; -CREATE MATERIALIZED VIEW partitions AS +CREATE VIEW partitions AS SELECT * FROM partitions_view; -CREATE UNIQUE INDEX ON partitions (partition); - -CREATE MATERIALIZED VIEW partition_steps AS -SELECT - partition as name, - date_trunc('month',lower(partition_dtrange)) as sdate, - date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate - FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange - ORDER BY dtrange ASC -; -CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ +-- Returns TRUE if the stats update ran, FALSE if it was queued. +CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS boolean AS $$ DECLARE BEGIN - PERFORM run_or_queue( + RETURN run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; -CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ +-- _extent NULL follows pgstac.update_collection_extent; TRUE computes the +-- spatial extent regardless, for callers that are about to read it. +CREATE OR REPLACE FUNCTION update_partition_stats( + _partition text, + istrigger boolean default false, + _extent boolean default NULL +) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; @@ -2576,34 +2692,87 @@ DECLARE cedtrange tstzrange; extent geometry; collection text; + pdtrange tstzrange; + auto_extent boolean := get_setting_bool('update_collection_extent'); + do_extent boolean := COALESCE(_extent, auto_extent); BEGIN + -- Cannot be STRICT: _extent is three valued, and STRICT would skip the + -- body whenever it is NULL. + IF _partition IS NULL OR istrigger IS NULL THEN + RETURN; + END IF; RAISE NOTICE 'Updating stats for %.', _partition; - EXECUTE format( - $q$ - SELECT - tstzrange(min(datetime), max(datetime),'[]'), - tstzrange(min(end_datetime), max(end_datetime), '[]') - FROM %I - $q$, - _partition - ) INTO dtrange, edtrange; - EXECUTE format('ANALYZE %I;', _partition); - extent := st_estimatedextent('pgstac', _partition, 'geometry'); - RAISE DEBUG 'Estimated Extent: %', extent; - INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) - SELECT _partition, dtrange, edtrange, extent, now() - ON CONFLICT (partition) DO - UPDATE SET - dtrange=EXCLUDED.dtrange, - edtrange=EXCLUDED.edtrange, - spatial=EXCLUDED.spatial, - last_updated=EXCLUDED.last_updated - ; - SELECT - constraint_dtrange, constraint_edtrange, pv.collection - INTO cdtrange, cedtrange, collection - FROM partitions_view pv WHERE partition = _partition; + SELECT m.collection, m.partition_dtrange, m.constraint_dtrange, m.constraint_edtrange + INTO collection, pdtrange, cdtrange, cedtrange + FROM partition_catalog_meta(_partition) m; + + -- A queued update can outlive the partition it names. + IF NOT FOUND THEN + RAISE NOTICE 'Partition % no longer exists, skipping stats update.', _partition; + RETURN; + END IF; + + -- Taken before the partition is read. The constraint rebuild below needs + -- ACCESS EXCLUSIVE, and escalating to that mid-transaction deadlocks + -- against another session doing the same. SHARE UPDATE EXCLUSIVE conflicts + -- with itself but not with readers. + IF NOT istrigger THEN + EXECUTE format('LOCK TABLE %I IN SHARE UPDATE EXCLUSIVE MODE', _partition); + END IF; + + -- The observed ranges feed the constraint tightening below and collection + -- extents. When neither will read them, only the partition's identity is + -- written, which is what keeps it visible to search. + IF NOT istrigger OR do_extent THEN + -- st_extent visits every geometry, so it is only run when wanted. + IF do_extent THEN + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]'), + st_extent(geometry)::geometry + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange, extent; + RAISE DEBUG 'Extent: %', extent; + ELSE + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]') + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange; + END IF; + + INSERT INTO partition_stats + (partition, collection, partition_dtrange, dtrange, edtrange, spatial, last_updated) + VALUES (_partition, collection, pdtrange, dtrange, edtrange, extent, now()) + ON CONFLICT (partition) DO + UPDATE SET + collection=EXCLUDED.collection, + partition_dtrange=EXCLUDED.partition_dtrange, + dtrange=EXCLUDED.dtrange, + edtrange=EXCLUDED.edtrange, + spatial=COALESCE(EXCLUDED.spatial, partition_stats.spatial), + last_updated=EXCLUDED.last_updated + ; + ELSE + INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES (_partition, collection, pdtrange) + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + END IF; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; @@ -2617,10 +2786,11 @@ BEGIN PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; - REFRESH MATERIALIZED VIEW partitions; - REFRESH MATERIALIZED VIEW partition_steps; + -- auto_extent, not do_extent: a caller that passed _extent aggregates the + -- extent itself, and update_collection_extents would then be updating + -- collections from inside its own UPDATE of collections. RAISE NOTICE 'Checking if we need to update collection extents.'; - IF get_setting_bool('update_collection_extent') THEN + IF auto_extent THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections @@ -2638,7 +2808,8 @@ BEGIN END IF; END; -$$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; + CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ @@ -2676,10 +2847,19 @@ $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; + _oid oid := partition_oid(t); BEGIN - IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN + IF _oid IS NULL THEN RETURN NULL; END IF; + -- Only partitions of items. This runs elevated, so without the check it + -- would alter any table pgstac_admin owns that the caller names. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(t)) THEN + RETURN NULL; + END IF; + -- Reduce to the bare name so the ALTER statements below quote it correctly + -- even when the caller passed a schema qualified name. + t := get_partition_name(_oid); FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; @@ -2687,21 +2867,30 @@ BEGIN t, conname ) FROM pg_constraint - WHERE conrelid=t::regclass::oid AND contype='c' + WHERE conrelid=_oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; + _oid oid := partition_oid(t); BEGIN - IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN + IF _oid IS NULL THEN RETURN NULL; END IF; + -- Only partitions of items. This runs elevated, so without the check it + -- would alter any table pgstac_admin owns that the caller names. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(t)) THEN + RETURN NULL; + END IF; + -- Reduce to the bare name so the ALTER statements below quote it correctly + -- even when the caller passed a schema qualified name. + t := get_partition_name(_oid); RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( @@ -2772,10 +2961,12 @@ BEGIN t ); END IF; - PERFORM run_or_queue(q); + -- Run, not queued: the queue runner is not a definer, so queued DDL + -- executes as whoever drains it. Defer by queueing a call to this function. + EXECUTE q; RETURN t; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION check_partition( @@ -2791,7 +2982,6 @@ DECLARE _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; - deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; @@ -2822,7 +3012,14 @@ BEGIN _partition_name := format('_items_%s', c.key); END IF; - SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; + -- Constraint ranges are maintained asynchronously, so they come from the + -- catalog rather than partition_stats. + SELECT ps.partition, m.constraint_dtrange, m.constraint_edtrange + INTO pm + FROM partition_stats ps + JOIN LATERAL partition_catalog_meta(ps.partition) m ON TRUE + WHERE ps.collection = _collection AND ps.partition_dtrange @> _dtrange + LIMIT 1; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := @@ -2905,20 +3102,38 @@ BEGIN RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; + -- The _constraint_ ranges are the union of the existing constraint and the + -- incoming batch, so they hold for rows already present as well as the ones + -- about to be added. Queueable: rebuilding validates the partition under an + -- ACCESS EXCLUSIVE lock. + PERFORM run_or_queue(format( + 'SELECT create_table_constraints(%L, %L, %L);', + _partition_name, + _constraint_dtrange, + _constraint_edtrange + )); PERFORM maintain_partitions(_partition_name); - PERFORM update_partition_stats_q(_partition_name, true); - REFRESH MATERIALIZED VIEW partitions; - REFRESH MATERIALIZED VIEW partition_steps; + -- Search finds partitions through partition_stats, so the row has to exist + -- before this transaction commits. A queued stats update has not written it. + IF NOT update_partition_stats_q(_partition_name, true) THEN + INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES (_partition_name, _collection, _partition_dtrange) + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + END IF; RETURN _partition_name; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; - q text; - from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN @@ -2939,6 +3154,7 @@ BEGIN $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; + DELETE FROM partition_stats WHERE collection = %L; WITH p AS ( SELECT collection, @@ -2956,13 +3172,14 @@ BEGIN $q$, concat('_items_', c.key), concat('_items_', c.key), + _collection, c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE @@ -3012,7 +3229,16 @@ BEGIN ) j ), parts AS ( - SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) + -- = ANY(array) uses the partition_stats primary key; the planner + -- cannot estimate jsonb_path_query, so a join plans as a full scan. + SELECT + date_trunc('month', lower(partition_dtrange)) as sdate, + date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate + FROM partition_stats + WHERE + partition = ANY (ARRAY(SELECT p FROM t)) + AND partition_dtrange IS NOT NULL + AND partition_dtrange != 'empty'::tstzrange ), times AS ( SELECT sdate FROM parts @@ -3531,7 +3757,6 @@ DECLARE t timestamptz; i interval; explain_json jsonb; - partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); @@ -3552,36 +3777,54 @@ BEGIN RETURN sw; END IF; - -- Get any stats that we have. - IF NOT ro THEN - -- If there is a lock where another process is - -- updating the stats, wait so that we don't end up calculating a bunch of times. - SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; - ELSE - SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; - END IF; + -- Unlocked read. A fresh hit only bumps bookkeeping counters, and that is + -- the common case when identical searches run concurrently. + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; - -- If there is a cached row, figure out if we need to update + -- Within ttl: bump usage counters and return. The bump skips locked rows so + -- identical searches do not serialize; a missed increment is harmless. + -- sw.id, not "sw IS NOT NULL": a composite is only IS NOT NULL when every + -- field is, and search_wheres.partitions is never populated. IF - sw IS NOT NULL + sw.id IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN - -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN - RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 - WHERE md5(_where) = inwhere_hash - RETURNING * INTO sw; + WHERE id = ( + SELECT id FROM search_wheres + WHERE md5(_where) = inwhere_hash + FOR UPDATE SKIP LOCKED + ); END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; + -- Missing or stale, so lock the row to compute once, then re-check + -- freshness in case another session finished while we waited. + IF NOT ro THEN + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; + IF + sw.statslastupdated IS NOT NULL + AND sw.total_count IS NOT NULL + AND now() - sw.statslastupdated <= _stats_ttl + THEN + RAISE DEBUG 'Another process refreshed stats while we waited: %', sw; + UPDATE search_wheres SET + lastused = now(), + usecount = search_wheres.usecount + 1 + WHERE md5(_where) = inwhere_hash + RETURNING * INTO sw; + RETURN sw; + END IF; + END IF; + -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN @@ -3592,8 +3835,8 @@ BEGIN RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; - sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; - sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; + sw.estimated_count := (explain_json->0->'Plan'->>'Plan Rows')::bigint; + sw.estimated_cost := (explain_json->0->'Plan'->>'Total Cost')::float; sw.time_to_estimate := extract(epoch from i); END IF; @@ -4500,10 +4743,18 @@ DECLARE mind timestamptz; maxd timestamptz; extent jsonb; + _partition text; BEGIN IF runupdate THEN - PERFORM update_partition_stats_q(partition) - FROM partitions_view WHERE collection=_collection; + -- Not queued: the aggregate below reads what this writes. Ordered by + -- partition, as every other multi-partition writer is. + FOR _partition IN + SELECT partition FROM partition_stats + WHERE collection=_collection + ORDER BY partition + LOOP + PERFORM update_partition_stats(_partition, false, true); + END LOOP; END IF; SELECT min(lower(dtrange)), @@ -4513,7 +4764,7 @@ BEGIN mind, maxd, geom_extent - FROM partitions_view + FROM partition_stats WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN @@ -4530,6 +4781,37 @@ BEGIN RETURN NULL; END; $$ LANGUAGE PLPGSQL; + + +-- Reconcile partition_stats against the partition tree: an identity row for +-- every partition, and no rows for partitions that are gone. dtrange, edtrange +-- and spatial are left alone; update_partition_stats maintains those. Used by +-- the idempotent install and as a repair path. +CREATE OR REPLACE FUNCTION sync_partition_stats() RETURNS VOID AS $$ +BEGIN + -- Ordered by partition, as every other writer of these rows is. + INSERT INTO partition_stats (partition, collection, partition_dtrange) + SELECT partition, collection, partition_dtrange FROM partitions_view + ORDER BY partition + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + + DELETE FROM partition_stats ps + WHERE ps.partition IN ( + SELECT partition FROM partition_stats stale + WHERE NOT EXISTS ( + SELECT 1 FROM partitions_view pv WHERE pv.partition = stale.partition + ) + ORDER BY partition + FOR UPDATE + ); +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES @@ -4619,17 +4901,19 @@ ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; -ALTER FUNCTION update_partition_stats SECURITY DEFINER; -ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; -ALTER FUNCTION where_stats SECURITY DEFINER; -ALTER FUNCTION search_query SECURITY DEFINER; -ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; +-- Elevated only where ownership of pgstac_admin's objects is required. +-- Everything else relies on table privileges, which any role inheriting +-- pgstac_ingest or pgstac_read already has. +ALTER FUNCTION where_stats SECURITY INVOKER; +ALTER FUNCTION search_query SECURITY INVOKER; +ALTER FUNCTION format_item SECURITY INVOKER; + GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; @@ -4641,6 +4925,11 @@ GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; +-- The caches maintained by where_stats, search_query and format_item. +GRANT SELECT, INSERT, UPDATE ON search_wheres TO pgstac_read; +GRANT SELECT, INSERT, UPDATE ON searches TO pgstac_read; +GRANT SELECT, INSERT, UPDATE ON format_item_cache TO pgstac_read; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; @@ -4652,8 +4941,39 @@ GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; +-- PostgreSQL grants EXECUTE to PUBLIC on every new function, so each definer +-- is revoked and granted back to the roles that need it. Keep in step with the +-- ALTER FUNCTION ... SECURITY DEFINER statements above; the pgtap suite fails +-- if a definer is left PUBLIC executable. +REVOKE ALL PRIVILEGES ON FUNCTION + drop_table_constraints, + create_table_constraints, + check_partition, + repartition, + maintain_index, + delete_collection +FROM public; + +GRANT EXECUTE ON FUNCTION + drop_table_constraints, + create_table_constraints, + check_partition, + repartition, + maintain_index, + delete_collection +TO pgstac_ingest; + RESET ROLE; SET ROLE pgstac_ingest; -SELECT update_partition_stats_q(partition) FROM partitions_view; -SELECT set_version('0.9.11'); + +-- Search finds partitions through partition_stats, so this must be synchronous +-- rather than queued. +SELECT sync_partition_stats(); + +-- Repairs observed ranges and CHECK constraints for every partition, ordered +-- by partition as every other writer of these rows is. The most expensive part +-- of an install on a large catalog: run with pgstac.use_queue on (pypgstac +-- --usequeue) and drain with pypgstac runqueue to keep it off the migration. +SELECT update_partition_stats_q(partition) FROM partitions_view ORDER BY partition; +SELECT set_version('0.9.12'); diff --git a/src/pgstac/sql/000_idempotent_pre.sql b/src/pgstac/sql/000_idempotent_pre.sql index 9f558e4b..3452edc4 100644 --- a/src/pgstac/sql/000_idempotent_pre.sql +++ b/src/pgstac/sql/000_idempotent_pre.sql @@ -132,6 +132,19 @@ ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FU RESET ROLE; SET SEARCH_PATH TO pgstac, public; + +-- PgSTAC references PostGIS without schema qualification and runs with +-- search_path pinned to "pgstac, public", so PostGIS has to be reachable from +-- there. Fail here rather than part way through creating the schema. +DO $$ + BEGIN + IF to_regtype('geometry') IS NULL THEN + RAISE EXCEPTION 'PostGIS is not reachable from search_path "pgstac, public"' + USING HINT = 'PgSTAC requires the postgis extension in the public schema.'; + END IF; + END +$$; + SET ROLE pgstac_admin; DO $$ @@ -148,6 +161,26 @@ DO $$ RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; +-- Drain the queue here, where the functions its entries name still have the +-- signatures they were queued against and pgstac_admin owns the objects any +-- queued DDL touches. Statistics updates are discarded instead: +-- 998_idempotent_post recalculates every partition regardless. +DO $$ + BEGIN + DELETE FROM query_queue WHERE query LIKE 'SELECT update_partition_stats(%'; + PERFORM run_queued_queries_intransaction(); + EXCEPTION WHEN undefined_table OR undefined_function THEN + RAISE NOTICE 'No query queue to drain.'; + END +$$; + +-- Return type or argument list differs from an earlier release, which +-- CREATE OR REPLACE cannot change. +DROP FUNCTION IF EXISTS run_or_queue(text); +DROP FUNCTION IF EXISTS update_partition_stats_q(text, boolean); +DROP FUNCTION IF EXISTS update_partition_stats(text, boolean); +DROP FUNCTION IF EXISTS maintain_index(text, text, boolean, boolean, boolean); +DROP FUNCTION IF EXISTS queryable_indexes(text, boolean); -- Install these idempotently as migrations do not put them before trying to modify the collections table diff --git a/src/pgstac/sql/001_core.sql b/src/pgstac/sql/001_core.sql index 46d71dbd..c78bb5b5 100644 --- a/src/pgstac/sql/001_core.sql +++ b/src/pgstac/sql/001_core.sql @@ -213,19 +213,21 @@ END; $$ LANGUAGE PLPGSQL; -CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ +-- TRUE if the query ran, FALSE if it was queued, so callers can tell whether +-- its effects are visible in this transaction. +CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS boolean AS $$ DECLARE - use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; + use_queue boolean := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; - ELSE - EXECUTE query; + RETURN FALSE; END IF; - RETURN; + EXECUTE query; + RETURN TRUE; END; $$ LANGUAGE PLPGSQL; @@ -323,5 +325,11 @@ BEGIN END IF; END IF; + -- Undrained queue means stale statistics and constraints, silently. + SELECT count(*) INTO settingval FROM query_queue; + IF settingval::bigint > 0 THEN + RAISE WARNING '% queries are waiting in query_queue. Run "CALL run_queued_queries();" (pypgstac runqueue) to drain it -- until then partition statistics and constraints are stale.', settingval; + END IF; + END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; diff --git a/src/pgstac/sql/002_collections.sql b/src/pgstac/sql/002_collections.sql index 8447d27e..dcbac969 100644 --- a/src/pgstac/sql/002_collections.sql +++ b/src/pgstac/sql/002_collections.sql @@ -49,13 +49,16 @@ CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; +-- SECURITY DEFINER: the delete trigger drops partition tables, which are owned +-- by pgstac_admin. CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ -DECLARE - out collections%ROWTYPE; BEGIN - DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; + DELETE FROM collections WHERE id = _id; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _id USING ERRCODE = 'no_data_found'; + END IF; END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ @@ -73,15 +76,14 @@ CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN + -- Tables before rows: check_partition takes these locks in the same order, + -- and the reverse deadlocks against a concurrent partition create. EXECUTE format($q$ - DELETE FROM partition_stats WHERE partition IN ( - SELECT partition FROM partition_sys_meta - WHERE collection=%L - ); DROP TABLE IF EXISTS %I CASCADE; + DELETE FROM partition_stats WHERE collection=%L; $q$, - OLD.id, - collection_base_partition + collection_base_partition, + OLD.id ); RETURN OLD; END; diff --git a/src/pgstac/sql/002a_queryables.sql b/src/pgstac/sql/002a_queryables.sql index bbc92142..20564a84 100644 --- a/src/pgstac/sql/002a_queryables.sql +++ b/src/pgstac/sql/002a_queryables.sql @@ -266,7 +266,8 @@ CREATE OR REPLACE FUNCTION queryable_indexes( OUT field text, OUT indexname text, OUT existing_idx text, - OUT queryable_idx text + OUT queryable_idx text, + OUT queryable_id bigint ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT @@ -300,7 +301,8 @@ WITH p AS ( name AS field, collection, partition, - format(indexdef(queryables), partition) as qidx + format(indexdef(queryables), partition) as qidx, + queryables.id as qid FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') @@ -311,40 +313,53 @@ WITH p AS ( field, indexname, iidx as existing_idx, - qidx as queryable_idx + qidx as queryable_idx, + qid as queryable_id FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; +-- SECURITY DEFINER, so the index statement is built here from the queryables +-- row; a caller supplied one would run with the privileges of the schema owner. CREATE OR REPLACE FUNCTION maintain_index( - indexname text, - queryable_idx text, + _partition text, + _indexname text, + _queryable_id bigint, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE + _queryable_idx text; BEGIN - IF indexname IS NOT NULL THEN - IF dropindexes OR queryable_idx IS NOT NULL THEN - EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); + -- Runs elevated, so it may only touch partitions of items. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(_partition)) THEN + RETURN; + END IF; + IF _queryable_id IS NOT NULL AND _partition IS NOT NULL THEN + SELECT format(indexdef(q), _partition) INTO _queryable_idx + FROM queryables q WHERE q.id = _queryable_id; + END IF; + IF _indexname IS NOT NULL THEN + IF dropindexes OR _queryable_idx IS NOT NULL THEN + EXECUTE format('DROP INDEX IF EXISTS %I;', _indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN - EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); + EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', _indexname); ELSE - EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); + EXECUTE format('REINDEX INDEX %I;', _indexname); END IF; END IF; END IF; - IF queryable_idx IS NOT NULL THEN + IF _queryable_idx IS NOT NULL THEN IF idxconcurrently THEN - EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); - ELSE EXECUTE queryable_idx; + EXECUTE replace(_queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); + ELSE EXECUTE _queryable_idx; END IF; END IF; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; @@ -364,10 +379,11 @@ BEGIN ) LOOP q := format( 'SELECT maintain_index( - %L,%L,%L,%L,%L + %L,%L,%L,%L,%L,%L );', + rec.partition, rec.indexname, - rec.queryable_idx, + rec.queryable_id, dropindexes, rebuildindexes, idxconcurrently diff --git a/src/pgstac/sql/003a_items.sql b/src/pgstac/sql/003a_items.sql index 88924d17..420a44fc 100644 --- a/src/pgstac/sql/003a_items.sql +++ b/src/pgstac/sql/003a_items.sql @@ -23,9 +23,11 @@ DECLARE t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; + -- Ordered: each iteration holds a partition_stats row lock until commit. FOR p IN SELECT DISTINCT partition - FROM newdata n JOIN partition_sys_meta p + FROM newdata n JOIN partition_stats p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) + ORDER BY 1 LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; @@ -35,7 +37,7 @@ BEGIN RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items @@ -192,8 +194,6 @@ CREATE UNLOGGED TABLE items_staging_upsert ( CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE - p record; - _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; @@ -214,7 +214,10 @@ BEGIN tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 - ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP + -- Ordered: check_partition holds DDL and row locks until commit. + ) SELECT check_partition(collection, dtrange, edtrange) FROM ( + SELECT * FROM p ORDER BY collection, d + ) ordered LOOP RAISE NOTICE 'Partition %', part; END LOOP; @@ -240,11 +243,20 @@ BEGIN GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN - DELETE FROM items i USING tmpdata s - WHERE - i.id = s.id - AND i.collection = s.collection - AND i IS DISTINCT FROM s + -- Locked in a fixed order first, so concurrent upserts over an + -- overlapping id set cannot deadlock. A bare DELETE gives no ordering; + -- ORDER BY ... FOR UPDATE does, because LockRows sits above the sort. + WITH locked AS ( + SELECT o.collection, o.id + FROM tmpdata s + JOIN items o ON (o.id = s.id AND o.collection = s.collection) + WHERE o IS DISTINCT FROM s + ORDER BY o.collection, o.id + FOR UPDATE OF o + ) + DELETE FROM items i + USING locked l + WHERE i.collection = l.collection AND i.id = l.id ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; @@ -256,7 +268,7 @@ BEGIN END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; - DELETE FROM items_staging; + EXECUTE format('DELETE FROM %I', TG_TABLE_NAME); RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; @@ -340,14 +352,18 @@ CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; +-- Recalculates partition statistics before aggregating them; the observed +-- ranges are only maintained automatically when update_collection_extent is on. +-- return_target, not use_json_null: collection_extent returns NULL when it +-- cannot compute a full extent, and JSON null is not a valid STAC extent. CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], - collection_extent(id, FALSE), + collection_extent(id, TRUE), true, - 'use_json_null' + 'return_target' ) ; $$ LANGUAGE SQL; diff --git a/src/pgstac/sql/003b_partitions.sql b/src/pgstac/sql/003b_partitions.sql index 9b726a3a..e4c72e6f 100644 --- a/src/pgstac/sql/003b_partitions.sql +++ b/src/pgstac/sql/003b_partitions.sql @@ -1,5 +1,11 @@ +-- collection and partition_dtrange describe the partition itself and are +-- written synchronously by check_partition. dtrange, edtrange and spatial +-- describe the data in the partition and may be updated asynchronously +-- through the query queue. CREATE TABLE partition_stats ( partition text PRIMARY KEY, + collection text, + partition_dtrange tstzrange, dtrange tstzrange, edtrange tstzrange, spatial geometry, @@ -7,7 +13,7 @@ CREATE TABLE partition_stats ( keys text[] ) WITH (FILLFACTOR=90); -CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); +CREATE INDEX partition_stats_collection_idx ON partition_stats (collection); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ @@ -70,6 +76,80 @@ CREATE OR REPLACE FUNCTION get_partition_name(relid regclass) RETURNS text AS $$ SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; $$ LANGUAGE SQL STABLE STRICT; +-- Resolve a partition name, bare or pgstac qualified, against pgstac rather +-- than the search_path. NULL if there is no such table, or if the name is +-- qualified with a different schema. +CREATE OR REPLACE FUNCTION partition_oid(_partition text) RETURNS oid AS $$ + SELECT CASE + WHEN cardinality(parts) > 1 AND parts[cardinality(parts) - 1] <> 'pgstac' THEN NULL + ELSE to_regclass(format('pgstac.%I', parts[cardinality(parts)])) + END + FROM parse_ident(_partition) AS parts; +$$ LANGUAGE SQL STABLE STRICT; + +-- Catalog metadata for one partition, touching one relation. pg_partition_tree +-- would lock every partition in the tree. No rows unless _partition is a leaf +-- partition of items. +CREATE OR REPLACE FUNCTION partition_catalog_meta(_partition text) +RETURNS TABLE ( + collection text, + partition_dtrange tstzrange, + constraint_dtrange tstzrange, + constraint_edtrange tstzrange +) AS $$ +DECLARE + _oid oid; + _parent oid; + _expr text; + _inf tstzrange := tstzrange('-infinity', 'infinity', '[]'); + _dtrange tstzrange; +BEGIN + _oid := partition_oid(_partition); + -- Leaves only, matching partitions_view: an intermediate partition has no + -- meaningful range of its own. + IF _oid IS NULL OR EXISTS (SELECT 1 FROM pg_inherits WHERE inhparent = _oid) THEN + RETURN; + END IF; + + SELECT inhparent INTO _parent FROM pg_inherits WHERE inhrelid = _oid; + + -- Partitions of items only. The SECURITY DEFINER functions below use this + -- to decide what they may alter, so any other relation must return nothing. + IF _parent IS NULL + OR ( + _parent <> 'pgstac.items'::regclass + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits + WHERE inhrelid = _parent AND inhparent = 'pgstac.items'::regclass + ) + ) + THEN + RETURN; + END IF; + + -- A partition of a sub-partitioned collection carries a datetime range + -- bound; its collection is on the parent. A direct partition of items + -- carries the collection itself. + IF _parent = 'pgstac.items'::regclass THEN + SELECT pg_get_expr(relpartbound, oid) INTO _expr FROM pg_class WHERE oid = _oid; + ELSE + SELECT pg_get_expr(relpartbound, oid) INTO _expr FROM pg_class WHERE oid = _parent; + END IF; + + SELECT COALESCE( + constraint_tstzrange(pg_get_expr(relpartbound, oid)), + _inf + ) INTO _dtrange FROM pg_class WHERE oid = _oid; + + RETURN QUERY SELECT + replace(replace(_expr, 'FOR VALUES IN (''', ''), ''')', ''), + _dtrange, + COALESCE(get_tstz_constraint(_oid, 'datetime'), _dtrange, _inf), + COALESCE(get_tstz_constraint(_oid, 'end_datetime'), _inf); +END; +$$ LANGUAGE PLPGSQL STABLE; + + CREATE OR REPLACE VIEW partition_sys_meta AS SELECT partition, @@ -99,86 +179,47 @@ FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) - LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE - JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE - JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE WHERE isleaf ; +-- partition_sys_meta plus the statistics tracked alongside each partition. CREATE OR REPLACE VIEW partitions_view AS SELECT - (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, - replace( - replace( - CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, - 'FOR VALUES IN (''', - '' - ), - ''')', - '' - ) AS collection, - level, - c.reltuples, - c.relhastriggers, - partition_dtrange, - COALESCE( - get_tstz_constraint(c.oid, 'datetime'), - partition_dtrange, - inf_range - ) as constraint_dtrange, - COALESCE( - get_tstz_constraint(c.oid, 'end_datetime'), - inf_range - ) as constraint_edtrange, - dtrange, - edtrange, - spatial, - last_updated -FROM - pg_partition_tree('items') - JOIN pg_class c ON (relid::regclass = c.oid) - JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) - LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') - JOIN LATERAL get_partition_name(relid) AS partition ON TRUE - JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE - JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE - JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE - JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE - JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE - JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE - LEFT JOIN pgstac.partition_stats USING (partition) -WHERE isleaf + sm.*, + ps.dtrange, + ps.edtrange, + ps.spatial, + ps.last_updated +FROM partition_sys_meta sm + LEFT JOIN pgstac.partition_stats ps USING (partition) ; -CREATE MATERIALIZED VIEW partitions AS +CREATE VIEW partitions AS SELECT * FROM partitions_view; -CREATE UNIQUE INDEX ON partitions (partition); - -CREATE MATERIALIZED VIEW partition_steps AS -SELECT - partition as name, - date_trunc('month',lower(partition_dtrange)) as sdate, - date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate - FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange - ORDER BY dtrange ASC -; -CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ +-- Returns TRUE if the stats update ran, FALSE if it was queued. +CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS boolean AS $$ DECLARE BEGIN - PERFORM run_or_queue( + RETURN run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; -CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ +-- _extent NULL follows pgstac.update_collection_extent; TRUE computes the +-- spatial extent regardless, for callers that are about to read it. +CREATE OR REPLACE FUNCTION update_partition_stats( + _partition text, + istrigger boolean default false, + _extent boolean default NULL +) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; @@ -186,34 +227,87 @@ DECLARE cedtrange tstzrange; extent geometry; collection text; + pdtrange tstzrange; + auto_extent boolean := get_setting_bool('update_collection_extent'); + do_extent boolean := COALESCE(_extent, auto_extent); BEGIN + -- Cannot be STRICT: _extent is three valued, and STRICT would skip the + -- body whenever it is NULL. + IF _partition IS NULL OR istrigger IS NULL THEN + RETURN; + END IF; RAISE NOTICE 'Updating stats for %.', _partition; - EXECUTE format( - $q$ - SELECT - tstzrange(min(datetime), max(datetime),'[]'), - tstzrange(min(end_datetime), max(end_datetime), '[]') - FROM %I - $q$, - _partition - ) INTO dtrange, edtrange; - EXECUTE format('ANALYZE %I;', _partition); - extent := st_estimatedextent('pgstac', _partition, 'geometry'); - RAISE DEBUG 'Estimated Extent: %', extent; - INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) - SELECT _partition, dtrange, edtrange, extent, now() - ON CONFLICT (partition) DO - UPDATE SET - dtrange=EXCLUDED.dtrange, - edtrange=EXCLUDED.edtrange, - spatial=EXCLUDED.spatial, - last_updated=EXCLUDED.last_updated - ; - SELECT - constraint_dtrange, constraint_edtrange, pv.collection - INTO cdtrange, cedtrange, collection - FROM partitions_view pv WHERE partition = _partition; + SELECT m.collection, m.partition_dtrange, m.constraint_dtrange, m.constraint_edtrange + INTO collection, pdtrange, cdtrange, cedtrange + FROM partition_catalog_meta(_partition) m; + + -- A queued update can outlive the partition it names. + IF NOT FOUND THEN + RAISE NOTICE 'Partition % no longer exists, skipping stats update.', _partition; + RETURN; + END IF; + + -- Taken before the partition is read. The constraint rebuild below needs + -- ACCESS EXCLUSIVE, and escalating to that mid-transaction deadlocks + -- against another session doing the same. SHARE UPDATE EXCLUSIVE conflicts + -- with itself but not with readers. + IF NOT istrigger THEN + EXECUTE format('LOCK TABLE %I IN SHARE UPDATE EXCLUSIVE MODE', _partition); + END IF; + + -- The observed ranges feed the constraint tightening below and collection + -- extents. When neither will read them, only the partition's identity is + -- written, which is what keeps it visible to search. + IF NOT istrigger OR do_extent THEN + -- st_extent visits every geometry, so it is only run when wanted. + IF do_extent THEN + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]'), + st_extent(geometry)::geometry + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange, extent; + RAISE DEBUG 'Extent: %', extent; + ELSE + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]') + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange; + END IF; + + INSERT INTO partition_stats + (partition, collection, partition_dtrange, dtrange, edtrange, spatial, last_updated) + VALUES (_partition, collection, pdtrange, dtrange, edtrange, extent, now()) + ON CONFLICT (partition) DO + UPDATE SET + collection=EXCLUDED.collection, + partition_dtrange=EXCLUDED.partition_dtrange, + dtrange=EXCLUDED.dtrange, + edtrange=EXCLUDED.edtrange, + spatial=COALESCE(EXCLUDED.spatial, partition_stats.spatial), + last_updated=EXCLUDED.last_updated + ; + ELSE + INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES (_partition, collection, pdtrange) + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + END IF; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; @@ -227,10 +321,11 @@ BEGIN PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; - REFRESH MATERIALIZED VIEW partitions; - REFRESH MATERIALIZED VIEW partition_steps; + -- auto_extent, not do_extent: a caller that passed _extent aggregates the + -- extent itself, and update_collection_extents would then be updating + -- collections from inside its own UPDATE of collections. RAISE NOTICE 'Checking if we need to update collection extents.'; - IF get_setting_bool('update_collection_extent') THEN + IF auto_extent THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections @@ -248,7 +343,8 @@ BEGIN END IF; END; -$$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; + CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ @@ -286,10 +382,19 @@ $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; + _oid oid := partition_oid(t); BEGIN - IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN + IF _oid IS NULL THEN RETURN NULL; END IF; + -- Only partitions of items. This runs elevated, so without the check it + -- would alter any table pgstac_admin owns that the caller names. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(t)) THEN + RETURN NULL; + END IF; + -- Reduce to the bare name so the ALTER statements below quote it correctly + -- even when the caller passed a schema qualified name. + t := get_partition_name(_oid); FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; @@ -297,21 +402,30 @@ BEGIN t, conname ) FROM pg_constraint - WHERE conrelid=t::regclass::oid AND contype='c' + WHERE conrelid=_oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; + _oid oid := partition_oid(t); BEGIN - IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN + IF _oid IS NULL THEN RETURN NULL; END IF; + -- Only partitions of items. This runs elevated, so without the check it + -- would alter any table pgstac_admin owns that the caller names. + IF NOT EXISTS (SELECT 1 FROM partition_catalog_meta(t)) THEN + RETURN NULL; + END IF; + -- Reduce to the bare name so the ALTER statements below quote it correctly + -- even when the caller passed a schema qualified name. + t := get_partition_name(_oid); RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( @@ -382,10 +496,12 @@ BEGIN t ); END IF; - PERFORM run_or_queue(q); + -- Run, not queued: the queue runner is not a definer, so queued DDL + -- executes as whoever drains it. Defer by queueing a call to this function. + EXECUTE q; RETURN t; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION check_partition( @@ -401,7 +517,6 @@ DECLARE _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; - deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; @@ -432,7 +547,14 @@ BEGIN _partition_name := format('_items_%s', c.key); END IF; - SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; + -- Constraint ranges are maintained asynchronously, so they come from the + -- catalog rather than partition_stats. + SELECT ps.partition, m.constraint_dtrange, m.constraint_edtrange + INTO pm + FROM partition_stats ps + JOIN LATERAL partition_catalog_meta(ps.partition) m ON TRUE + WHERE ps.collection = _collection AND ps.partition_dtrange @> _dtrange + LIMIT 1; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := @@ -515,20 +637,38 @@ BEGIN RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; + -- The _constraint_ ranges are the union of the existing constraint and the + -- incoming batch, so they hold for rows already present as well as the ones + -- about to be added. Queueable: rebuilding validates the partition under an + -- ACCESS EXCLUSIVE lock. + PERFORM run_or_queue(format( + 'SELECT create_table_constraints(%L, %L, %L);', + _partition_name, + _constraint_dtrange, + _constraint_edtrange + )); PERFORM maintain_partitions(_partition_name); - PERFORM update_partition_stats_q(_partition_name, true); - REFRESH MATERIALIZED VIEW partitions; - REFRESH MATERIALIZED VIEW partition_steps; + -- Search finds partitions through partition_stats, so the row has to exist + -- before this transaction commits. A queued stats update has not written it. + IF NOT update_partition_stats_q(_partition_name, true) THEN + INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES (_partition_name, _collection, _partition_dtrange) + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + END IF; RETURN _partition_name; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; - q text; - from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN @@ -549,6 +689,7 @@ BEGIN $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; + DELETE FROM partition_stats WHERE collection = %L; WITH p AS ( SELECT collection, @@ -566,13 +707,14 @@ BEGIN $q$, concat('_items_', c.key), concat('_items_', c.key), + _collection, c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER; +$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE diff --git a/src/pgstac/sql/004_search.sql b/src/pgstac/sql/004_search.sql index a00f569e..16ec1f25 100644 --- a/src/pgstac/sql/004_search.sql +++ b/src/pgstac/sql/004_search.sql @@ -23,7 +23,16 @@ BEGIN ) j ), parts AS ( - SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) + -- = ANY(array) uses the partition_stats primary key; the planner + -- cannot estimate jsonb_path_query, so a join plans as a full scan. + SELECT + date_trunc('month', lower(partition_dtrange)) as sdate, + date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate + FROM partition_stats + WHERE + partition = ANY (ARRAY(SELECT p FROM t)) + AND partition_dtrange IS NOT NULL + AND partition_dtrange != 'empty'::tstzrange ), times AS ( SELECT sdate FROM parts @@ -542,7 +551,6 @@ DECLARE t timestamptz; i interval; explain_json jsonb; - partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); @@ -563,36 +571,54 @@ BEGIN RETURN sw; END IF; - -- Get any stats that we have. - IF NOT ro THEN - -- If there is a lock where another process is - -- updating the stats, wait so that we don't end up calculating a bunch of times. - SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; - ELSE - SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; - END IF; + -- Unlocked read. A fresh hit only bumps bookkeeping counters, and that is + -- the common case when identical searches run concurrently. + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; - -- If there is a cached row, figure out if we need to update + -- Within ttl: bump usage counters and return. The bump skips locked rows so + -- identical searches do not serialize; a missed increment is harmless. + -- sw.id, not "sw IS NOT NULL": a composite is only IS NOT NULL when every + -- field is, and search_wheres.partitions is never populated. IF - sw IS NOT NULL + sw.id IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN - -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN - RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 - WHERE md5(_where) = inwhere_hash - RETURNING * INTO sw; + WHERE id = ( + SELECT id FROM search_wheres + WHERE md5(_where) = inwhere_hash + FOR UPDATE SKIP LOCKED + ); END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; + -- Missing or stale, so lock the row to compute once, then re-check + -- freshness in case another session finished while we waited. + IF NOT ro THEN + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; + IF + sw.statslastupdated IS NOT NULL + AND sw.total_count IS NOT NULL + AND now() - sw.statslastupdated <= _stats_ttl + THEN + RAISE DEBUG 'Another process refreshed stats while we waited: %', sw; + UPDATE search_wheres SET + lastused = now(), + usecount = search_wheres.usecount + 1 + WHERE md5(_where) = inwhere_hash + RETURNING * INTO sw; + RETURN sw; + END IF; + END IF; + -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN @@ -603,8 +629,8 @@ BEGIN RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; - sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; - sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; + sw.estimated_count := (explain_json->0->'Plan'->>'Plan Rows')::bigint; + sw.estimated_cost := (explain_json->0->'Plan'->>'Total Cost')::float; sw.time_to_estimate := extract(epoch from i); END IF; diff --git a/src/pgstac/sql/997_maintenance.sql b/src/pgstac/sql/997_maintenance.sql index df1175d2..8a02c257 100644 --- a/src/pgstac/sql/997_maintenance.sql +++ b/src/pgstac/sql/997_maintenance.sql @@ -55,10 +55,18 @@ DECLARE mind timestamptz; maxd timestamptz; extent jsonb; + _partition text; BEGIN IF runupdate THEN - PERFORM update_partition_stats_q(partition) - FROM partitions_view WHERE collection=_collection; + -- Not queued: the aggregate below reads what this writes. Ordered by + -- partition, as every other multi-partition writer is. + FOR _partition IN + SELECT partition FROM partition_stats + WHERE collection=_collection + ORDER BY partition + LOOP + PERFORM update_partition_stats(_partition, false, true); + END LOOP; END IF; SELECT min(lower(dtrange)), @@ -68,7 +76,7 @@ BEGIN mind, maxd, geom_extent - FROM partitions_view + FROM partition_stats WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN @@ -85,3 +93,34 @@ BEGIN RETURN NULL; END; $$ LANGUAGE PLPGSQL; + + +-- Reconcile partition_stats against the partition tree: an identity row for +-- every partition, and no rows for partitions that are gone. dtrange, edtrange +-- and spatial are left alone; update_partition_stats maintains those. Used by +-- the idempotent install and as a repair path. +CREATE OR REPLACE FUNCTION sync_partition_stats() RETURNS VOID AS $$ +BEGIN + -- Ordered by partition, as every other writer of these rows is. + INSERT INTO partition_stats (partition, collection, partition_dtrange) + SELECT partition, collection, partition_dtrange FROM partitions_view + ORDER BY partition + ON CONFLICT (partition) DO UPDATE + SET collection = EXCLUDED.collection, + partition_dtrange = EXCLUDED.partition_dtrange + WHERE + partition_stats.collection IS DISTINCT FROM EXCLUDED.collection + OR partition_stats.partition_dtrange IS DISTINCT FROM EXCLUDED.partition_dtrange + ; + + DELETE FROM partition_stats ps + WHERE ps.partition IN ( + SELECT partition FROM partition_stats stale + WHERE NOT EXISTS ( + SELECT 1 FROM partitions_view pv WHERE pv.partition = stale.partition + ) + ORDER BY partition + FOR UPDATE + ); +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; diff --git a/src/pgstac/sql/998_idempotent_post.sql b/src/pgstac/sql/998_idempotent_post.sql index 2a6cad7c..6f49ea9b 100644 --- a/src/pgstac/sql/998_idempotent_post.sql +++ b/src/pgstac/sql/998_idempotent_post.sql @@ -87,17 +87,19 @@ ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; -ALTER FUNCTION update_partition_stats SECURITY DEFINER; -ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; -ALTER FUNCTION where_stats SECURITY DEFINER; -ALTER FUNCTION search_query SECURITY DEFINER; -ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; +-- Elevated only where ownership of pgstac_admin's objects is required. +-- Everything else relies on table privileges, which any role inheriting +-- pgstac_ingest or pgstac_read already has. +ALTER FUNCTION where_stats SECURITY INVOKER; +ALTER FUNCTION search_query SECURITY INVOKER; +ALTER FUNCTION format_item SECURITY INVOKER; + GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; @@ -109,6 +111,11 @@ GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; +-- The caches maintained by where_stats, search_query and format_item. +GRANT SELECT, INSERT, UPDATE ON search_wheres TO pgstac_read; +GRANT SELECT, INSERT, UPDATE ON searches TO pgstac_read; +GRANT SELECT, INSERT, UPDATE ON format_item_cache TO pgstac_read; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; @@ -120,7 +127,38 @@ GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; +-- PostgreSQL grants EXECUTE to PUBLIC on every new function, so each definer +-- is revoked and granted back to the roles that need it. Keep in step with the +-- ALTER FUNCTION ... SECURITY DEFINER statements above; the pgtap suite fails +-- if a definer is left PUBLIC executable. +REVOKE ALL PRIVILEGES ON FUNCTION + drop_table_constraints, + create_table_constraints, + check_partition, + repartition, + maintain_index, + delete_collection +FROM public; + +GRANT EXECUTE ON FUNCTION + drop_table_constraints, + create_table_constraints, + check_partition, + repartition, + maintain_index, + delete_collection +TO pgstac_ingest; + RESET ROLE; SET ROLE pgstac_ingest; -SELECT update_partition_stats_q(partition) FROM partitions_view; + +-- Search finds partitions through partition_stats, so this must be synchronous +-- rather than queued. +SELECT sync_partition_stats(); + +-- Repairs observed ranges and CHECK constraints for every partition, ordered +-- by partition as every other writer of these rows is. The most expensive part +-- of an install on a large catalog: run with pgstac.use_queue on (pypgstac +-- --usequeue) and drain with pypgstac runqueue to keep it off the migration. +SELECT update_partition_stats_q(partition) FROM partitions_view ORDER BY partition; diff --git a/src/pgstac/sql/999_version.sql b/src/pgstac/sql/999_version.sql index 24240500..9bbbda62 100644 --- a/src/pgstac/sql/999_version.sql +++ b/src/pgstac/sql/999_version.sql @@ -1 +1 @@ -SELECT set_version('0.9.11'); +SELECT set_version('0.9.12'); diff --git a/src/pgstac/tests/basic/crud_functions.sql.out b/src/pgstac/tests/basic/crud_functions.sql.out index 44ec404c..2a8edd9c 100644 --- a/src/pgstac/tests/basic/crud_functions.sql.out +++ b/src/pgstac/tests/basic/crud_functions.sql.out @@ -32,18 +32,18 @@ SELECT * FROM items WHERE collection='pgstactest-crudtest'; -- Check to see if extent got updated SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; -{"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2020-01-01T00:00:00+00:00"]]}} - + {"spatial": {"bbox": [[-85.379245, 30.933949, -85.308201, 31.003555]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2020-01-01T00:00:00+00:00"]]}} -- Update item with new datetime that is in a different partition SELECT update_item((SELECT content || '{"properties":{"datetime":"2023-01-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); + SELECT * FROM items WHERE collection='pgstactest-crudtest'; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2023-01-01 00:00:00+00 | 2023-01-01 00:00:00+00 | {"properties": {"datetime": "2023-01-01 00:00:00Z"}} | -- Check to see if extent got updated SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; -{"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2023-01-01T00:00:00+00:00", "2023-01-01T00:00:00+00:00"]]}} + {"spatial": {"bbox": [[-85.379245, 30.933949, -85.308201, 31.003555]]}, "temporal": {"interval": [["2023-01-01T00:00:00+00:00", "2023-01-01T00:00:00+00:00"]]}} -- Update item with new datetime that is in a different partition SELECT upsert_item((SELECT content || '{"properties":{"datetime":"2023-02-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); @@ -97,8 +97,9 @@ SELECT get_setting_bool('update_collection_extent'); SELECT update_item((SELECT content || '{"properties":{"datetime":"2024-01-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); + SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; - {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2023-02-01T00:00:00+00:00", "2023-02-01T00:00:00+00:00"]]}} + {"spatial": {"bbox": [[-85.379245, 30.933949, -85.308201, 31.003555]]}, "temporal": {"interval": [["2023-02-01T00:00:00+00:00", "2023-02-01T00:00:00+00:00"]]}} -- check formatting of temporal extent SELECT collection_temporal_extent('pgstactest-crudtest'); diff --git a/src/pgstac/tests/basic/partition_stats.sql b/src/pgstac/tests/basic/partition_stats.sql new file mode 100644 index 00000000..c50063ec --- /dev/null +++ b/src/pgstac/tests/basic/partition_stats.sql @@ -0,0 +1,123 @@ +-- Tests for partition metadata tracked on partition_stats, which is what +-- search uses to find partitions. +SET ROLE pgstac_ingest; +SET pgstac.use_queue=FALSE; +SET pgstac.update_collection_extent=FALSE; + +CREATE TEMP TABLE ps_items AS +SELECT jsonb_build_object( + 'id', concat('pgstactest-pstats-', (row_number() over ())::text), + 'collection', 'pgstactest-pstats', + 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, + 'properties', jsonb_build_object('datetime', g::text) +) as content FROM generate_series('2020-01-01'::timestamptz, '2020-04-01'::timestamptz, '1 week'::interval) g; + +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-pstats"}', 'month'); +INSERT INTO items_staging (content) SELECT content FROM ps_items; + +-- every leaf partition has an identity row +SELECT count(*) = 0 AS every_partition_has_identity FROM partitions_view pv +WHERE NOT EXISTS ( + SELECT 1 FROM partition_stats ps + WHERE ps.partition = pv.partition + AND ps.collection IS NOT NULL + AND ps.partition_dtrange IS NOT NULL +); + +-- and no rows for partitions that do not exist +SELECT count(*) = 0 AS no_orphan_rows FROM partition_stats ps +WHERE NOT EXISTS (SELECT 1 FROM partitions_view pv WHERE pv.partition = ps.partition); + +-- identity columns agree with the catalog +SELECT count(*) = 0 AS identity_matches_catalog +FROM partitions_view pv JOIN partition_stats ps USING (partition) +WHERE ps.collection IS DISTINCT FROM pv.collection + OR ps.partition_dtrange IS DISTINCT FROM pv.partition_dtrange; + +-- partition_oid name handling +SELECT + partition_oid(partition) IS NOT NULL AS bare_name_resolves, + partition_oid('pgstac.' || partition) IS NOT NULL AS qualified_resolves, + partition_oid('pgstac."' || partition || '"') IS NOT NULL AS quoted_resolves, + partition_oid('public.' || partition) IS NULL AS other_schema_rejected +FROM partition_stats WHERE collection = 'pgstactest-pstats' ORDER BY partition LIMIT 1; + +SELECT partition_oid('_items_no_such_partition') IS NULL AS missing_is_null; + +-- partition_catalog_meta returns nothing for a partition that does not exist, +-- and nothing for an intermediate (non-leaf) partition +SELECT count(*) = 0 AS meta_missing_is_empty FROM partition_catalog_meta('_items_no_such_partition'); + +SELECT count(*) = 0 AS meta_skips_intermediate +FROM collections c, LATERAL partition_catalog_meta(format('_items_%s', c.key)) +WHERE c.id = 'pgstactest-pstats'; + +-- an update for a partition that no longer exists must not create a row +SELECT update_partition_stats('_items_no_such_partition'); +SELECT count(*) = 0 AS no_row_for_missing_partition +FROM partition_stats WHERE partition = '_items_no_such_partition'; + +-- run_or_queue / update_partition_stats_q report whether they ran +SET pgstac.use_queue=FALSE; +SELECT run_or_queue('SELECT 1') AS runs_inline_when_queue_off; +SET pgstac.use_queue=TRUE; +SELECT run_or_queue('SELECT 2') AS queues_when_queue_on; +DELETE FROM query_queue; +SET pgstac.use_queue=FALSE; + +-- chunker bands cover the full data range of the collection +SELECT + min(s) = date_trunc('month', min_dt) AS bands_start_at_first_month, + max(e) > max_dt AS bands_cover_last_item +FROM chunker($q$ collection = 'pgstactest-pstats' $q$), +LATERAL (SELECT min(datetime) AS min_dt, max(datetime) AS max_dt FROM items WHERE collection='pgstactest-pstats') d +GROUP BY min_dt, max_dt; + +-- searching the collection finds every item +SELECT jsonb_array_length(search('{"collections":["pgstactest-pstats"],"limit":100}')->'features') + = (SELECT count(*) FROM items WHERE collection='pgstactest-pstats') AS search_returns_all; + +-- sync_partition_stats removes orphans and restores missing identity rows +INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES ('_items_orphan_row', 'pgstactest-pstats', '[-infinity,infinity]'::tstzrange); +DELETE FROM partition_stats WHERE partition = ( + SELECT partition FROM partition_stats WHERE collection='pgstactest-pstats' ORDER BY partition LIMIT 1 +); +SELECT sync_partition_stats(); +SELECT count(*) = 0 AS sync_pruned_orphan FROM partition_stats WHERE partition = '_items_orphan_row'; +SELECT count(*) = 0 AS sync_restored_identity FROM partitions_view pv +WHERE NOT EXISTS ( + SELECT 1 FROM partition_stats ps + WHERE ps.partition = pv.partition + AND ps.collection IS NOT NULL + AND ps.partition_dtrange IS NOT NULL +); + +-- the staging tables each clear themselves +INSERT INTO items_staging_upsert (content) SELECT content FROM ps_items LIMIT 1; +SELECT count(*) AS items_staging_upsert_cleared FROM items_staging_upsert; +INSERT INTO items_staging_ignore (content) SELECT content FROM ps_items LIMIT 1; +SELECT count(*) AS items_staging_ignore_cleared FROM items_staging_ignore; + +-- deleting the collection removes its partition_stats rows. delete_collection +-- is SECURITY DEFINER, so pgstac_ingest can drop the partition tables it owns. +SELECT delete_collection('pgstactest-pstats'); +SELECT count(*) AS stats_rows_after_collection_delete +FROM partition_stats WHERE collection = 'pgstactest-pstats'; + +-- queued mode: the identity row still exists before the queue is drained +SET pgstac.use_queue=TRUE; +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-pstats-q"}', 'month'); +INSERT INTO items_staging (content) +SELECT content || '{"collection":"pgstactest-pstats-q"}'::jsonb FROM ps_items; + +SELECT count(*) > 0 AS queued_partitions_visible +FROM partition_stats WHERE collection = 'pgstactest-pstats-q' AND partition_dtrange IS NOT NULL; + +SELECT count(*) = 0 AS queued_observed_ranges_deferred +FROM partition_stats WHERE collection = 'pgstactest-pstats-q' AND dtrange IS NOT NULL; + +SELECT jsonb_array_length(search('{"collections":["pgstactest-pstats-q"],"limit":100}')->'features') + = (SELECT count(*) FROM items WHERE collection='pgstactest-pstats-q') AS queued_search_returns_all; + +SET pgstac.use_queue=FALSE; diff --git a/src/pgstac/tests/basic/partition_stats.sql.out b/src/pgstac/tests/basic/partition_stats.sql.out new file mode 100644 index 00000000..4814dbe3 --- /dev/null +++ b/src/pgstac/tests/basic/partition_stats.sql.out @@ -0,0 +1,166 @@ +-- Tests for partition metadata tracked on partition_stats, which is what +-- search uses to find partitions. +SET ROLE pgstac_ingest; +SET +SET pgstac.use_queue=FALSE; +SET +SET pgstac.update_collection_extent=FALSE; +SET +CREATE TEMP TABLE ps_items AS +SELECT jsonb_build_object( + 'id', concat('pgstactest-pstats-', (row_number() over ())::text), + 'collection', 'pgstactest-pstats', + 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, + 'properties', jsonb_build_object('datetime', g::text) +) as content FROM generate_series('2020-01-01'::timestamptz, '2020-04-01'::timestamptz, '1 week'::interval) g; +SELECT 14 +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-pstats"}', 'month'); +INSERT 0 1 +INSERT INTO items_staging (content) SELECT content FROM ps_items; +INSERT 0 14 +-- every leaf partition has an identity row +SELECT count(*) = 0 AS every_partition_has_identity FROM partitions_view pv +WHERE NOT EXISTS ( + SELECT 1 FROM partition_stats ps + WHERE ps.partition = pv.partition + AND ps.collection IS NOT NULL + AND ps.partition_dtrange IS NOT NULL +); + t + +-- and no rows for partitions that do not exist +SELECT count(*) = 0 AS no_orphan_rows FROM partition_stats ps +WHERE NOT EXISTS (SELECT 1 FROM partitions_view pv WHERE pv.partition = ps.partition); + t + +-- identity columns agree with the catalog +SELECT count(*) = 0 AS identity_matches_catalog +FROM partitions_view pv JOIN partition_stats ps USING (partition) +WHERE ps.collection IS DISTINCT FROM pv.collection + OR ps.partition_dtrange IS DISTINCT FROM pv.partition_dtrange; + t + +-- partition_oid name handling +SELECT + partition_oid(partition) IS NOT NULL AS bare_name_resolves, + partition_oid('pgstac.' || partition) IS NOT NULL AS qualified_resolves, + partition_oid('pgstac."' || partition || '"') IS NOT NULL AS quoted_resolves, + partition_oid('public.' || partition) IS NULL AS other_schema_rejected +FROM partition_stats WHERE collection = 'pgstactest-pstats' ORDER BY partition LIMIT 1; + t | t | t | t + +SELECT partition_oid('_items_no_such_partition') IS NULL AS missing_is_null; + t + +-- partition_catalog_meta returns nothing for a partition that does not exist, +-- and nothing for an intermediate (non-leaf) partition +SELECT count(*) = 0 AS meta_missing_is_empty FROM partition_catalog_meta('_items_no_such_partition'); + t + +SELECT count(*) = 0 AS meta_skips_intermediate +FROM collections c, LATERAL partition_catalog_meta(format('_items_%s', c.key)) +WHERE c.id = 'pgstactest-pstats'; + t + +-- an update for a partition that no longer exists must not create a row +SELECT update_partition_stats('_items_no_such_partition'); + + +SELECT count(*) = 0 AS no_row_for_missing_partition +FROM partition_stats WHERE partition = '_items_no_such_partition'; + t + +-- run_or_queue / update_partition_stats_q report whether they ran +SET pgstac.use_queue=FALSE; +SET +SELECT run_or_queue('SELECT 1') AS runs_inline_when_queue_off; + t + +SET pgstac.use_queue=TRUE; +SET +SELECT run_or_queue('SELECT 2') AS queues_when_queue_on; + f + +DELETE FROM query_queue; +DELETE 1 +SET pgstac.use_queue=FALSE; +SET +-- chunker bands cover the full data range of the collection +SELECT + min(s) = date_trunc('month', min_dt) AS bands_start_at_first_month, + max(e) > max_dt AS bands_cover_last_item +FROM chunker($q$ collection = 'pgstactest-pstats' $q$), +LATERAL (SELECT min(datetime) AS min_dt, max(datetime) AS max_dt FROM items WHERE collection='pgstactest-pstats') d +GROUP BY min_dt, max_dt; + t | t + +-- searching the collection finds every item +SELECT jsonb_array_length(search('{"collections":["pgstactest-pstats"],"limit":100}')->'features') + = (SELECT count(*) FROM items WHERE collection='pgstactest-pstats') AS search_returns_all; + t + +-- sync_partition_stats removes orphans and restores missing identity rows +INSERT INTO partition_stats (partition, collection, partition_dtrange) + VALUES ('_items_orphan_row', 'pgstactest-pstats', '[-infinity,infinity]'::tstzrange); +INSERT 0 1 +DELETE FROM partition_stats WHERE partition = ( + SELECT partition FROM partition_stats WHERE collection='pgstactest-pstats' ORDER BY partition LIMIT 1 +); +DELETE 1 +SELECT sync_partition_stats(); + + +SELECT count(*) = 0 AS sync_pruned_orphan FROM partition_stats WHERE partition = '_items_orphan_row'; + t + +SELECT count(*) = 0 AS sync_restored_identity FROM partitions_view pv +WHERE NOT EXISTS ( + SELECT 1 FROM partition_stats ps + WHERE ps.partition = pv.partition + AND ps.collection IS NOT NULL + AND ps.partition_dtrange IS NOT NULL +); + t + +-- the staging tables each clear themselves +INSERT INTO items_staging_upsert (content) SELECT content FROM ps_items LIMIT 1; +INSERT 0 1 +SELECT count(*) AS items_staging_upsert_cleared FROM items_staging_upsert; + 0 + +INSERT INTO items_staging_ignore (content) SELECT content FROM ps_items LIMIT 1; +INSERT 0 1 +SELECT count(*) AS items_staging_ignore_cleared FROM items_staging_ignore; + 0 + +-- deleting the collection removes its partition_stats rows. delete_collection +-- is SECURITY DEFINER, so pgstac_ingest can drop the partition tables it owns. +SELECT delete_collection('pgstactest-pstats'); + + +SELECT count(*) AS stats_rows_after_collection_delete +FROM partition_stats WHERE collection = 'pgstactest-pstats'; + 0 + +-- queued mode: the identity row still exists before the queue is drained +SET pgstac.use_queue=TRUE; +SET +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-pstats-q"}', 'month'); +INSERT 0 1 +INSERT INTO items_staging (content) +SELECT content || '{"collection":"pgstactest-pstats-q"}'::jsonb FROM ps_items; +INSERT 0 14 +SELECT count(*) > 0 AS queued_partitions_visible +FROM partition_stats WHERE collection = 'pgstactest-pstats-q' AND partition_dtrange IS NOT NULL; + t + +SELECT count(*) = 0 AS queued_observed_ranges_deferred +FROM partition_stats WHERE collection = 'pgstactest-pstats-q' AND dtrange IS NOT NULL; + t + +SELECT jsonb_array_length(search('{"collections":["pgstactest-pstats-q"],"limit":100}')->'features') + = (SELECT count(*) FROM items WHERE collection='pgstactest-pstats-q') AS queued_search_returns_all; + t + +SET pgstac.use_queue=FALSE; +SET diff --git a/src/pgstac/tests/basic/partitions.sql b/src/pgstac/tests/basic/partitions.sql index 73fdf7f8..c7348c6b 100644 --- a/src/pgstac/tests/basic/partitions.sql +++ b/src/pgstac/tests/basic/partitions.sql @@ -109,4 +109,4 @@ SELECT content || '{"collection":"pgstactest-partitioned-oddballs"}'::jsonb FROM SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-oddballs'; -SELECT collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM partitions WHERE collection='pgstactest-partitioned-oddballs'; +SELECT collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM partitions WHERE collection='pgstactest-partitioned-oddballs' ORDER BY partition_dtrange; diff --git a/src/pgstac/tests/basic/partitions.sql.out b/src/pgstac/tests/basic/partitions.sql.out index 8f30734c..e4571d2a 100644 --- a/src/pgstac/tests/basic/partitions.sql.out +++ b/src/pgstac/tests/basic/partitions.sql.out @@ -117,10 +117,10 @@ SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q' and --check that collection extents have been updated SELECT id, content->'extent' FROM collections WHERE id LIKE 'pgstactest-partitioned%' ORDER BY id; - pgstactest-partitioned | {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} - pgstactest-partitioned-month | {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} - pgstactest-partitioned-q | {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} - pgstactest-partitioned-year | {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} + pgstactest-partitioned | {"spatial": {"bbox": [[-85.379245, 30.933949, -85.308201, 31.003555]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} + pgstactest-partitioned-month | {"spatial": {"bbox": [[-85.379245, 30.933949, -85.308201, 31.003555]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} + pgstactest-partitioned-q | {"spatial": {"bbox": [[-85.379245, 30.933949, -85.308201, 31.003555]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} + pgstactest-partitioned-year | {"spatial": {"bbox": [[-85.379245, 30.933949, -85.308201, 31.003555]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} --check that values for datetimes that are non 4 digit or that have very high precision are ingesting correctly and that partitioning is working for them SET pgstac.use_queue=FALSE; @@ -160,31 +160,31 @@ INSERT 0 108 SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-oddballs'; 26 -SELECT collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM partitions WHERE collection='pgstactest-partitioned-oddballs'; - pgstactest-partitioned-oddballs | ["2020-09-01 00:00:00+00","2020-10-01 00:00:00+00") | ["2020-09-01 00:00:00+00","2020-10-01 00:00:00+00") | [-infinity,infinity] | ["2020-09-02 00:00:00+00","2020-09-30 00:00:00+00"] | ["2020-09-02 00:00:00+00","2020-09-30 00:00:00+00"] - pgstactest-partitioned-oddballs | ["10000-01-01 00:00:00+00","10000-02-01 00:00:00+00") | ["10000-01-01 00:00:00+00","10000-02-01 00:00:00+00") | [-infinity,infinity] | ["10000-01-01 00:00:00+00","10000-01-01 00:00:00+00"] | ["10000-01-01 00:00:00+00","10000-01-01 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-05-01 00:00:00+00","2020-06-01 00:00:00+00") | ["2020-05-01 00:00:00+00","2020-06-01 00:00:00+00") | [-infinity,infinity] | ["2020-05-06 00:00:00+00","2020-05-27 00:00:00+00"] | ["2020-05-06 00:00:00+00","2020-05-27 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-01-01 00:00:00+00","2021-02-01 00:00:00+00") | ["2021-01-01 00:00:00+00","2021-02-01 00:00:00+00") | [-infinity,infinity] | ["2021-01-06 00:00:00+00","2021-01-27 00:00:00+00"] | ["2021-01-06 00:00:00+00","2021-01-27 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-07-01 00:00:00+00","2020-08-01 00:00:00+00") | ["2020-07-01 00:00:00+00","2020-08-01 00:00:00+00") | [-infinity,infinity] | ["2020-07-01 00:00:00+00","2020-07-29 00:00:00+00"] | ["2020-07-01 00:00:00+00","2020-07-29 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-12-01 00:00:00+00","2021-01-01 00:00:00+00") | ["2020-12-01 00:00:00+00","2021-01-01 00:00:00+00") | [-infinity,infinity] | ["2020-12-02 00:00:00+00","2020-12-30 00:00:00+00"] | ["2020-12-02 00:00:00+00","2020-12-30 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-04-01 00:00:00+00","2021-05-01 00:00:00+00") | ["2021-04-01 00:00:00+00","2021-05-01 00:00:00+00") | [-infinity,infinity] | ["2021-04-07 00:00:00+00","2021-04-28 00:00:00+00"] | ["2021-04-07 00:00:00+00","2021-04-28 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-03-01 00:00:00+00","2021-04-01 00:00:00+00") | ["2021-03-01 00:00:00+00","2021-04-01 00:00:00+00") | [-infinity,infinity] | ["2021-03-03 00:00:00+00","2021-03-31 00:00:00+00"] | ["2021-03-03 00:00:00+00","2021-03-31 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-10-01 00:00:00+00","2021-11-01 00:00:00+00") | ["2021-10-01 00:00:00+00","2021-11-01 00:00:00+00") | [-infinity,infinity] | ["2021-10-06 00:00:00+00","2021-10-27 00:00:00+00"] | ["2021-10-06 00:00:00+00","2021-10-27 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-03-01 00:00:00+00","2020-04-01 00:00:00+00") | ["2020-03-01 00:00:00+00","2020-04-01 00:00:00+00") | [-infinity,infinity] | ["2020-03-04 00:00:00+00","2020-03-25 00:00:00+00"] | ["2020-03-04 00:00:00+00","2020-03-25 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2000-01-01 00:00:00+00","2000-02-01 00:00:00+00") | ["2000-01-01 00:00:00+00","2000-02-01 00:00:00+00") | [-infinity,infinity] | ["2000-01-01 00:00:00.123899+00","2000-01-01 00:00:00.123899+00"] | ["2000-01-01 00:00:00.123899+00","99999-01-01 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-10-01 00:00:00+00","2020-11-01 00:00:00+00") | ["2020-10-01 00:00:00+00","2020-11-01 00:00:00+00") | [-infinity,infinity] | ["2020-10-07 00:00:00+00","2020-10-28 00:00:00+00"] | ["2020-10-07 00:00:00+00","2020-10-28 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-09-01 00:00:00+00","2021-10-01 00:00:00+00") | ["2021-09-01 00:00:00+00","2021-10-01 00:00:00+00") | [-infinity,infinity] | ["2021-09-01 00:00:00+00","2021-09-29 00:00:00+00"] | ["2021-09-01 00:00:00+00","2021-09-29 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-01-01 00:00:00+00","2020-02-01 00:00:00+00") | ["2020-01-01 00:00:00+00","2020-02-01 00:00:00+00") | [-infinity,infinity] | ["2020-01-01 00:00:00+00","2020-01-29 00:00:00+00"] | ["2020-01-01 00:00:00+00","2020-01-29 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-07-01 00:00:00+00","2021-08-01 00:00:00+00") | ["2021-07-01 00:00:00+00","2021-08-01 00:00:00+00") | [-infinity,infinity] | ["2021-07-07 00:00:00+00","2021-07-28 00:00:00+00"] | ["2021-07-07 00:00:00+00","2021-07-28 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-08-01 00:00:00+00","2021-09-01 00:00:00+00") | ["2021-08-01 00:00:00+00","2021-09-01 00:00:00+00") | [-infinity,infinity] | ["2021-08-04 00:00:00+00","2021-08-25 00:00:00+00"] | ["2021-08-04 00:00:00+00","2021-08-25 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-11-01 00:00:00+00","2020-12-01 00:00:00+00") | ["2020-11-01 00:00:00+00","2020-12-01 00:00:00+00") | [-infinity,infinity] | ["2020-11-04 00:00:00+00","2020-11-25 00:00:00+00"] | ["2020-11-04 00:00:00+00","2020-11-25 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-11-01 00:00:00+00","2021-12-01 00:00:00+00") | ["2021-11-01 00:00:00+00","2021-12-01 00:00:00+00") | [-infinity,infinity] | ["2021-11-03 00:00:00+00","2021-11-24 00:00:00+00"] | ["2021-11-03 00:00:00+00","2021-11-24 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-02-01 00:00:00+00","2020-03-01 00:00:00+00") | ["2020-02-01 00:00:00+00","2020-03-01 00:00:00+00") | [-infinity,infinity] | ["2020-02-05 00:00:00+00","2020-02-26 00:00:00+00"] | ["2020-02-05 00:00:00+00","2020-02-26 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-04-01 00:00:00+00","2020-05-01 00:00:00+00") | ["2020-04-01 00:00:00+00","2020-05-01 00:00:00+00") | [-infinity,infinity] | ["2020-04-01 00:00:00+00","2020-04-29 00:00:00+00"] | ["2020-04-01 00:00:00+00","2020-04-29 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-12-01 00:00:00+00","2022-01-01 00:00:00+00") | ["2021-12-01 00:00:00+00","2022-01-01 00:00:00+00") | [-infinity,infinity] | ["2021-12-01 00:00:00+00","2021-12-29 00:00:00+00"] | ["2021-12-01 00:00:00+00","2021-12-29 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-02-01 00:00:00+00","2021-03-01 00:00:00+00") | ["2021-02-01 00:00:00+00","2021-03-01 00:00:00+00") | [-infinity,infinity] | ["2021-02-03 00:00:00+00","2021-02-24 00:00:00+00"] | ["2021-02-03 00:00:00+00","2021-02-24 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-06-01 00:00:00+00","2020-07-01 00:00:00+00") | ["2020-06-01 00:00:00+00","2020-07-01 00:00:00+00") | [-infinity,infinity] | ["2020-06-03 00:00:00+00","2020-06-24 00:00:00+00"] | ["2020-06-03 00:00:00+00","2020-06-24 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2020-08-01 00:00:00+00","2020-09-01 00:00:00+00") | ["2020-08-01 00:00:00+00","2020-09-01 00:00:00+00") | [-infinity,infinity] | ["2020-08-05 00:00:00+00","2020-08-26 00:00:00+00"] | ["2020-08-05 00:00:00+00","2020-08-26 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-06-01 00:00:00+00","2021-07-01 00:00:00+00") | ["2021-06-01 00:00:00+00","2021-07-01 00:00:00+00") | [-infinity,infinity] | ["2021-06-02 00:00:00+00","2021-06-30 00:00:00+00"] | ["2021-06-02 00:00:00+00","2021-06-30 00:00:00+00"] - pgstactest-partitioned-oddballs | ["2021-05-01 00:00:00+00","2021-06-01 00:00:00+00") | ["2021-05-01 00:00:00+00","2021-06-01 00:00:00+00") | [-infinity,infinity] | ["2021-05-05 00:00:00+00","2021-05-26 00:00:00+00"] | ["2021-05-05 00:00:00+00","2021-05-26 00:00:00+00"] +SELECT collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM partitions WHERE collection='pgstactest-partitioned-oddballs' ORDER BY partition_dtrange; + pgstactest-partitioned-oddballs | ["2000-01-01 00:00:00+00","2000-02-01 00:00:00+00") | ["2000-01-01 00:00:00.123899+00","2000-01-01 00:00:00.123899+00"] | ["2000-01-01 00:00:00.123899+00","99999-01-01 00:00:00+00"] | ["2000-01-01 00:00:00.123899+00","2000-01-01 00:00:00.123899+00"] | ["2000-01-01 00:00:00.123899+00","99999-01-01 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-01-01 00:00:00+00","2020-02-01 00:00:00+00") | ["2020-01-01 00:00:00+00","2020-01-29 00:00:00+00"] | ["2020-01-01 00:00:00+00","2020-01-29 00:00:00+00"] | ["2020-01-01 00:00:00+00","2020-01-29 00:00:00+00"] | ["2020-01-01 00:00:00+00","2020-01-29 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-02-01 00:00:00+00","2020-03-01 00:00:00+00") | ["2020-02-05 00:00:00+00","2020-02-26 00:00:00+00"] | ["2020-02-05 00:00:00+00","2020-02-26 00:00:00+00"] | ["2020-02-05 00:00:00+00","2020-02-26 00:00:00+00"] | ["2020-02-05 00:00:00+00","2020-02-26 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-03-01 00:00:00+00","2020-04-01 00:00:00+00") | ["2020-03-04 00:00:00+00","2020-03-25 00:00:00+00"] | ["2020-03-04 00:00:00+00","2020-03-25 00:00:00+00"] | ["2020-03-04 00:00:00+00","2020-03-25 00:00:00+00"] | ["2020-03-04 00:00:00+00","2020-03-25 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-04-01 00:00:00+00","2020-05-01 00:00:00+00") | ["2020-04-01 00:00:00+00","2020-04-29 00:00:00+00"] | ["2020-04-01 00:00:00+00","2020-04-29 00:00:00+00"] | ["2020-04-01 00:00:00+00","2020-04-29 00:00:00+00"] | ["2020-04-01 00:00:00+00","2020-04-29 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-05-01 00:00:00+00","2020-06-01 00:00:00+00") | ["2020-05-06 00:00:00+00","2020-05-27 00:00:00+00"] | ["2020-05-06 00:00:00+00","2020-05-27 00:00:00+00"] | ["2020-05-06 00:00:00+00","2020-05-27 00:00:00+00"] | ["2020-05-06 00:00:00+00","2020-05-27 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-06-01 00:00:00+00","2020-07-01 00:00:00+00") | ["2020-06-03 00:00:00+00","2020-06-24 00:00:00+00"] | ["2020-06-03 00:00:00+00","2020-06-24 00:00:00+00"] | ["2020-06-03 00:00:00+00","2020-06-24 00:00:00+00"] | ["2020-06-03 00:00:00+00","2020-06-24 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-07-01 00:00:00+00","2020-08-01 00:00:00+00") | ["2020-07-01 00:00:00+00","2020-07-29 00:00:00+00"] | ["2020-07-01 00:00:00+00","2020-07-29 00:00:00+00"] | ["2020-07-01 00:00:00+00","2020-07-29 00:00:00+00"] | ["2020-07-01 00:00:00+00","2020-07-29 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-08-01 00:00:00+00","2020-09-01 00:00:00+00") | ["2020-08-05 00:00:00+00","2020-08-26 00:00:00+00"] | ["2020-08-05 00:00:00+00","2020-08-26 00:00:00+00"] | ["2020-08-05 00:00:00+00","2020-08-26 00:00:00+00"] | ["2020-08-05 00:00:00+00","2020-08-26 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-09-01 00:00:00+00","2020-10-01 00:00:00+00") | ["2020-09-02 00:00:00+00","2020-09-30 00:00:00+00"] | ["2020-09-02 00:00:00+00","2020-09-30 00:00:00+00"] | ["2020-09-02 00:00:00+00","2020-09-30 00:00:00+00"] | ["2020-09-02 00:00:00+00","2020-09-30 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-10-01 00:00:00+00","2020-11-01 00:00:00+00") | ["2020-10-07 00:00:00+00","2020-10-28 00:00:00+00"] | ["2020-10-07 00:00:00+00","2020-10-28 00:00:00+00"] | ["2020-10-07 00:00:00+00","2020-10-28 00:00:00+00"] | ["2020-10-07 00:00:00+00","2020-10-28 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-11-01 00:00:00+00","2020-12-01 00:00:00+00") | ["2020-11-04 00:00:00+00","2020-11-25 00:00:00+00"] | ["2020-11-04 00:00:00+00","2020-11-25 00:00:00+00"] | ["2020-11-04 00:00:00+00","2020-11-25 00:00:00+00"] | ["2020-11-04 00:00:00+00","2020-11-25 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2020-12-01 00:00:00+00","2021-01-01 00:00:00+00") | ["2020-12-02 00:00:00+00","2020-12-30 00:00:00+00"] | ["2020-12-02 00:00:00+00","2020-12-30 00:00:00+00"] | ["2020-12-02 00:00:00+00","2020-12-30 00:00:00+00"] | ["2020-12-02 00:00:00+00","2020-12-30 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-01-01 00:00:00+00","2021-02-01 00:00:00+00") | ["2021-01-06 00:00:00+00","2021-01-27 00:00:00+00"] | ["2021-01-06 00:00:00+00","2021-01-27 00:00:00+00"] | ["2021-01-06 00:00:00+00","2021-01-27 00:00:00+00"] | ["2021-01-06 00:00:00+00","2021-01-27 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-02-01 00:00:00+00","2021-03-01 00:00:00+00") | ["2021-02-03 00:00:00+00","2021-02-24 00:00:00+00"] | ["2021-02-03 00:00:00+00","2021-02-24 00:00:00+00"] | ["2021-02-03 00:00:00+00","2021-02-24 00:00:00+00"] | ["2021-02-03 00:00:00+00","2021-02-24 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-03-01 00:00:00+00","2021-04-01 00:00:00+00") | ["2021-03-03 00:00:00+00","2021-03-31 00:00:00+00"] | ["2021-03-03 00:00:00+00","2021-03-31 00:00:00+00"] | ["2021-03-03 00:00:00+00","2021-03-31 00:00:00+00"] | ["2021-03-03 00:00:00+00","2021-03-31 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-04-01 00:00:00+00","2021-05-01 00:00:00+00") | ["2021-04-07 00:00:00+00","2021-04-28 00:00:00+00"] | ["2021-04-07 00:00:00+00","2021-04-28 00:00:00+00"] | ["2021-04-07 00:00:00+00","2021-04-28 00:00:00+00"] | ["2021-04-07 00:00:00+00","2021-04-28 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-05-01 00:00:00+00","2021-06-01 00:00:00+00") | ["2021-05-05 00:00:00+00","2021-05-26 00:00:00+00"] | ["2021-05-05 00:00:00+00","2021-05-26 00:00:00+00"] | ["2021-05-05 00:00:00+00","2021-05-26 00:00:00+00"] | ["2021-05-05 00:00:00+00","2021-05-26 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-06-01 00:00:00+00","2021-07-01 00:00:00+00") | ["2021-06-02 00:00:00+00","2021-06-30 00:00:00+00"] | ["2021-06-02 00:00:00+00","2021-06-30 00:00:00+00"] | ["2021-06-02 00:00:00+00","2021-06-30 00:00:00+00"] | ["2021-06-02 00:00:00+00","2021-06-30 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-07-01 00:00:00+00","2021-08-01 00:00:00+00") | ["2021-07-07 00:00:00+00","2021-07-28 00:00:00+00"] | ["2021-07-07 00:00:00+00","2021-07-28 00:00:00+00"] | ["2021-07-07 00:00:00+00","2021-07-28 00:00:00+00"] | ["2021-07-07 00:00:00+00","2021-07-28 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-08-01 00:00:00+00","2021-09-01 00:00:00+00") | ["2021-08-04 00:00:00+00","2021-08-25 00:00:00+00"] | ["2021-08-04 00:00:00+00","2021-08-25 00:00:00+00"] | ["2021-08-04 00:00:00+00","2021-08-25 00:00:00+00"] | ["2021-08-04 00:00:00+00","2021-08-25 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-09-01 00:00:00+00","2021-10-01 00:00:00+00") | ["2021-09-01 00:00:00+00","2021-09-29 00:00:00+00"] | ["2021-09-01 00:00:00+00","2021-09-29 00:00:00+00"] | ["2021-09-01 00:00:00+00","2021-09-29 00:00:00+00"] | ["2021-09-01 00:00:00+00","2021-09-29 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-10-01 00:00:00+00","2021-11-01 00:00:00+00") | ["2021-10-06 00:00:00+00","2021-10-27 00:00:00+00"] | ["2021-10-06 00:00:00+00","2021-10-27 00:00:00+00"] | ["2021-10-06 00:00:00+00","2021-10-27 00:00:00+00"] | ["2021-10-06 00:00:00+00","2021-10-27 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-11-01 00:00:00+00","2021-12-01 00:00:00+00") | ["2021-11-03 00:00:00+00","2021-11-24 00:00:00+00"] | ["2021-11-03 00:00:00+00","2021-11-24 00:00:00+00"] | ["2021-11-03 00:00:00+00","2021-11-24 00:00:00+00"] | ["2021-11-03 00:00:00+00","2021-11-24 00:00:00+00"] + pgstactest-partitioned-oddballs | ["2021-12-01 00:00:00+00","2022-01-01 00:00:00+00") | ["2021-12-01 00:00:00+00","2021-12-29 00:00:00+00"] | ["2021-12-01 00:00:00+00","2021-12-29 00:00:00+00"] | ["2021-12-01 00:00:00+00","2021-12-29 00:00:00+00"] | ["2021-12-01 00:00:00+00","2021-12-29 00:00:00+00"] + pgstactest-partitioned-oddballs | ["10000-01-01 00:00:00+00","10000-02-01 00:00:00+00") | ["10000-01-01 00:00:00+00","10000-01-01 00:00:00+00"] | ["10000-01-01 00:00:00+00","10000-01-01 00:00:00+00"] | ["10000-01-01 00:00:00+00","10000-01-01 00:00:00+00"] | ["10000-01-01 00:00:00+00","10000-01-01 00:00:00+00"] diff --git a/src/pgstac/tests/basic/search_path.sql b/src/pgstac/tests/basic/search_path.sql index 74259a06..263c9242 100644 --- a/src/pgstac/tests/basic/search_path.sql +++ b/src/pgstac/tests/basic/search_path.sql @@ -26,15 +26,11 @@ SELECT * FROM partitions WHERE collection='pgstactest-searchpath' ORDER BY parti CREATE TEMP TABLE sp_partitions_view AS SELECT * FROM partitions_view WHERE collection='pgstactest-searchpath' ORDER BY partition; -CREATE TEMP TABLE sp_partition_steps AS -SELECT * FROM partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) ORDER BY name; - -- Verify we have data SELECT count(*) > 0 AS has_partition_sys_meta FROM sp_partition_sys_meta; SELECT count(*) > 0 AS has_partition_stats FROM sp_partition_stats; SELECT count(*) > 0 AS has_partitions FROM sp_partitions; SELECT count(*) > 0 AS has_partitions_view FROM sp_partitions_view; -SELECT count(*) > 0 AS has_partition_steps FROM sp_partition_steps; -- Now remove pgstac from search_path SET search_path TO public; @@ -107,22 +103,5 @@ SELECT count(*) = 0 AS partitions_view_data_match FROM ( SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM pgstac.partitions_view WHERE collection='pgstactest-searchpath') ) diff; --- partition_steps: compare counts and key columns -SELECT ( - SELECT count(*) FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) -) = ( - SELECT count(*) FROM sp_partition_steps -) AS partition_steps_count_match; - -SELECT count(*) = 0 AS partition_steps_data_match FROM ( - (SELECT name, sdate, edate FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) - EXCEPT - SELECT name, sdate, edate FROM sp_partition_steps) - UNION ALL - (SELECT name, sdate, edate FROM sp_partition_steps - EXCEPT - SELECT name, sdate, edate FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta)) -) diff; - -- Restore search_path SET search_path TO pgstac, public; diff --git a/src/pgstac/tests/basic/search_path.sql.out b/src/pgstac/tests/basic/search_path.sql.out index 30718c0d..db5117bf 100644 --- a/src/pgstac/tests/basic/search_path.sql.out +++ b/src/pgstac/tests/basic/search_path.sql.out @@ -29,9 +29,6 @@ SELECT 5 CREATE TEMP TABLE sp_partitions_view AS SELECT * FROM partitions_view WHERE collection='pgstactest-searchpath' ORDER BY partition; SELECT 5 -CREATE TEMP TABLE sp_partition_steps AS -SELECT * FROM partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) ORDER BY name; -SELECT 5 -- Verify we have data SELECT count(*) > 0 AS has_partition_sys_meta FROM sp_partition_sys_meta; t @@ -45,9 +42,6 @@ SELECT count(*) > 0 AS has_partitions FROM sp_partitions; SELECT count(*) > 0 AS has_partitions_view FROM sp_partitions_view; t -SELECT count(*) > 0 AS has_partition_steps FROM sp_partition_steps; - t - -- Now remove pgstac from search_path SET search_path TO public; SET @@ -127,25 +121,6 @@ SELECT count(*) = 0 AS partitions_view_data_match FROM ( ) diff; t --- partition_steps: compare counts and key columns -SELECT ( - SELECT count(*) FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) -) = ( - SELECT count(*) FROM sp_partition_steps -) AS partition_steps_count_match; - t - -SELECT count(*) = 0 AS partition_steps_data_match FROM ( - (SELECT name, sdate, edate FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) - EXCEPT - SELECT name, sdate, edate FROM sp_partition_steps) - UNION ALL - (SELECT name, sdate, edate FROM sp_partition_steps - EXCEPT - SELECT name, sdate, edate FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta)) -) diff; - t - -- Restore search_path SET search_path TO pgstac, public; SET diff --git a/src/pgstac/tests/migration/snapshot.sql b/src/pgstac/tests/migration/snapshot.sql new file mode 100644 index 00000000..5ab64735 --- /dev/null +++ b/src/pgstac/tests/migration/snapshot.sql @@ -0,0 +1,75 @@ +-- Build a populated database on the version under test and record what a +-- migration has to preserve. Run against the OLD version, before migrating. +SET SEARCH_PATH TO pgstac, public; + +INSERT INTO collections (content) VALUES ( + '{"id":"mig-flat","type":"Collection", + "extent":{"spatial":{"bbox":[[-180,-90,180,90]]}, + "temporal":{"interval":[["2020-01-01T00:00:00Z",null]]}}}' +); +INSERT INTO collections (content, partition_trunc) VALUES ( + '{"id":"mig-month","type":"Collection", + "extent":{"spatial":{"bbox":[[-180,-90,180,90]]}, + "temporal":{"interval":[["2020-01-01T00:00:00Z",null]]}}}', + 'month' +); +INSERT INTO collections (content, partition_trunc) VALUES ( + '{"id":"mig-year","type":"Collection", + "extent":{"spatial":{"bbox":[[-180,-90,180,90]]}, + "temporal":{"interval":[["2020-01-01T00:00:00Z",null]]}}}', + 'year' +); + +-- Spread over two years so the partitioned collections get several partitions, +-- and give a tenth of the items a long duration so end_datetime is not simply +-- equal to datetime. +INSERT INTO items_staging (content) +SELECT jsonb_build_object( + 'id', c || '-' || g, + 'collection', c, + 'type', 'Feature', + 'geometry', jsonb_build_object( + 'type','Point', + 'coordinates', jsonb_build_array((g % 60) - 30, (g % 40) - 20) + ), + 'properties', jsonb_build_object( + 'datetime', dt::text, + 'end_datetime', ( + dt + CASE WHEN g % 10 = 0 THEN '20 days'::interval ELSE '1 hour'::interval END + )::text + ) +) +FROM generate_series(1, 150) g, + unnest(ARRAY['mig-flat','mig-month','mig-year']) c, + LATERAL (SELECT '2020-01-01'::timestamptz + ((g * 5) || ' days')::interval AS dt) d; + +-- Kept outside the pgstac schema so the migration cannot touch it. +DROP TABLE IF EXISTS public.migration_snapshot; +CREATE TABLE public.migration_snapshot AS +SELECT + (SELECT count(*) FROM items) AS items, + (SELECT count(*) FROM collections) AS collections, + (SELECT count(*) FROM partitions_view) AS partitions, + (SELECT jsonb_object_agg(id, content->'extent') FROM collections) AS extents, + (SELECT jsonb_object_agg(c, n) FROM ( + SELECT collection AS c, count(*) AS n FROM items GROUP BY collection + ) x) AS items_per_collection, + (SELECT jsonb_object_agg(c, n) FROM ( + SELECT c, jsonb_array_length( + search(jsonb_build_object('collections', jsonb_build_array(c), 'limit', 500))->'features' + ) AS n + FROM unnest(ARRAY['mig-flat','mig-month','mig-year']) c + ) y) AS search_per_collection, + (SELECT jsonb_object_agg(c, n) FROM ( + SELECT c, jsonb_array_length( + search(jsonb_build_object( + 'collections', jsonb_build_array(c), + 'datetime', '2020-06-01T00:00:00Z/2020-09-01T00:00:00Z', + 'limit', 500 + ))->'features' + ) AS n + FROM unnest(ARRAY['mig-flat','mig-month','mig-year']) c + ) z) AS temporal_search_per_collection +; + +SELECT * FROM public.migration_snapshot; diff --git a/src/pgstac/tests/migration/verify.sql b/src/pgstac/tests/migration/verify.sql new file mode 100644 index 00000000..a42b7e3e --- /dev/null +++ b/src/pgstac/tests/migration/verify.sql @@ -0,0 +1,131 @@ +-- Compare the migrated database against the snapshot taken on the old version. +-- Every check raises, so ON_ERROR_STOP makes the suite fail on any regression. +SET SEARCH_PATH TO pgstac, public; + +DO $$ +DECLARE + s public.migration_snapshot%ROWTYPE; + now_items bigint; + now_collections bigint; + now_partitions bigint; + now_extents jsonb; + now_items_per jsonb; + now_search_per jsonb; + now_temporal_per jsonb; + missing bigint; + orphans bigint; + unconstrained bigint; +BEGIN + SELECT * INTO s FROM public.migration_snapshot; + + SELECT count(*) INTO now_items FROM items; + SELECT count(*) INTO now_collections FROM collections; + SELECT count(*) INTO now_partitions FROM partitions_view; + SELECT jsonb_object_agg(id, content->'extent') INTO now_extents FROM collections; + + SELECT jsonb_object_agg(c, n) INTO now_items_per FROM ( + SELECT collection AS c, count(*) AS n FROM items GROUP BY collection + ) x; + SELECT jsonb_object_agg(c, n) INTO now_search_per FROM ( + SELECT c, jsonb_array_length( + search(jsonb_build_object('collections', jsonb_build_array(c), 'limit', 500))->'features' + ) AS n + FROM unnest(ARRAY['mig-flat','mig-month','mig-year']) c + ) y; + SELECT jsonb_object_agg(c, n) INTO now_temporal_per FROM ( + SELECT c, jsonb_array_length( + search(jsonb_build_object( + 'collections', jsonb_build_array(c), + 'datetime', '2020-06-01T00:00:00Z/2020-09-01T00:00:00Z', + 'limit', 500 + ))->'features' + ) AS n + FROM unnest(ARRAY['mig-flat','mig-month','mig-year']) c + ) z; + + -- Nothing may be lost. + IF now_items <> s.items THEN + RAISE EXCEPTION 'items changed across migration: % -> %', s.items, now_items; + END IF; + IF now_collections <> s.collections THEN + RAISE EXCEPTION 'collections changed across migration: % -> %', s.collections, now_collections; + END IF; + IF now_partitions <> s.partitions THEN + RAISE EXCEPTION 'partitions changed across migration: % -> %', s.partitions, now_partitions; + END IF; + IF now_items_per IS DISTINCT FROM s.items_per_collection THEN + RAISE EXCEPTION 'items per collection changed: % -> %', s.items_per_collection, now_items_per; + END IF; + + -- Search must return exactly what it did before, including the temporal + -- overlap query, which depends on partition pruning and the CHECK + -- constraints the migration rebuilds. + IF now_search_per IS DISTINCT FROM s.search_per_collection THEN + RAISE EXCEPTION 'search results changed: % -> %', s.search_per_collection, now_search_per; + END IF; + IF now_temporal_per IS DISTINCT FROM s.temporal_search_per_collection THEN + RAISE EXCEPTION 'temporal search results changed: % -> %', + s.temporal_search_per_collection, now_temporal_per; + END IF; + + -- A valid extent must never be replaced, in particular not with JSON null. + IF now_extents IS DISTINCT FROM s.extents THEN + RAISE EXCEPTION 'collection extents changed: % -> %', s.extents, now_extents; + END IF; + + -- Search finds partitions through partition_stats, so every partition needs + -- an identity row and there must be no rows for partitions that are gone. + SELECT count(*) INTO missing FROM partitions_view pv + WHERE NOT EXISTS ( + SELECT 1 FROM partition_stats ps + WHERE ps.partition = pv.partition + AND ps.collection IS NOT NULL + AND ps.partition_dtrange IS NOT NULL + ); + IF missing > 0 THEN + RAISE EXCEPTION '% partitions have no identity row in partition_stats', missing; + END IF; + + SELECT count(*) INTO orphans FROM partition_stats ps + WHERE NOT EXISTS (SELECT 1 FROM partitions_view pv WHERE pv.partition = ps.partition); + IF orphans > 0 THEN + RAISE EXCEPTION '% partition_stats rows reference partitions that do not exist', orphans; + END IF; + + -- The migration is what repairs the CHECK constraints that 0.9.11 left off + -- the staging path. Without a validated end_datetime constraint a partition + -- cannot be pruned by an end_datetime predicate. + SELECT count(*) INTO unconstrained FROM partitions_view + WHERE constraint_edtrange = '[-infinity,infinity]'::tstzrange; + IF unconstrained > 0 THEN + RAISE EXCEPTION '% partitions still have no end_datetime constraint after migration', unconstrained; + END IF; + + RAISE NOTICE 'migration verified: % items, % partitions, extents and search results unchanged', + now_items, now_partitions; +END; +$$; + +-- Ingest has to keep working afterwards, including a batch outside the +-- constraints the migration just built, which check_partition must widen. +INSERT INTO items_staging (content) VALUES ( + '{"id":"post-migrate","collection":"mig-month","type":"Feature", + "geometry":{"type":"Point","coordinates":[5,5]}, + "properties":{"datetime":"2031-07-01T00:00:00Z"}}' +); + +DO $$ +DECLARE + found int; +BEGIN + SELECT jsonb_array_length( + search('{"collections":["mig-month"],"ids":["post-migrate"]}')->'features' + ) INTO found; + IF found <> 1 THEN + RAISE EXCEPTION 'item ingested after migration is not searchable (found %)', found; + END IF; + RAISE NOTICE 'post-migration ingest verified'; +END; +$$; + +DROP TABLE public.migration_snapshot; diff --git a/src/pgstac/tests/pgtap.sql b/src/pgstac/tests/pgtap.sql index a2819eaa..ab7fe3f7 100644 --- a/src/pgstac/tests/pgtap.sql +++ b/src/pgstac/tests/pgtap.sql @@ -17,12 +17,13 @@ CREATE EXTENSION IF NOT EXISTS pgtap; SET SEARCH_PATH TO pgstac, pgtap, public; -- Plan the tests. -SELECT plan(229); +SELECT plan(241); --SELECT * FROM no_plan(); -- Run the tests. -- Core +\i tests/pgtap/000_security.sql \i tests/pgtap/001_core.sql \i tests/pgtap/001a_jsonutils.sql \i tests/pgtap/001b_cursorutils.sql diff --git a/src/pgstac/tests/pgtap/000_security.sql b/src/pgstac/tests/pgtap/000_security.sql new file mode 100644 index 00000000..4f380947 --- /dev/null +++ b/src/pgstac/tests/pgtap/000_security.sql @@ -0,0 +1,86 @@ +-- A SECURITY DEFINER function runs with the privileges of its owner, and +-- PostgreSQL grants EXECUTE to PUBLIC on every new function. Without an +-- explicit REVOKE, any role in the database can use one to act as the schema +-- owner. + +SELECT is_empty( + $$ + SELECT proname FROM pg_proc + WHERE pronamespace='pgstac'::regnamespace + AND prosecdef + AND has_function_privilege('public', oid, 'EXECUTE') + $$, + 'no SECURITY DEFINER function in pgstac is executable by PUBLIC' +); + +-- Without a pinned search_path a definer function resolves unqualified names +-- against whatever the caller set. +SELECT is_empty( + $$ + SELECT proname FROM pg_proc + WHERE pronamespace='pgstac'::regnamespace + AND prosecdef + AND NOT COALESCE(proconfig::text ~ 'search_path', FALSE) + $$, + 'every SECURITY DEFINER function in pgstac pins its search_path' +); + +-- maintain_index runs elevated, so it takes a queryables row and builds the +-- index statement itself rather than accepting one. +SELECT has_function( + 'pgstac'::name, + 'maintain_index'::name, + ARRAY['text','text','bigint','boolean','boolean','boolean'] +); +SELECT hasnt_function( + 'pgstac'::name, + 'maintain_index'::name, + ARRAY['text','text','boolean','boolean','boolean'] +); + +-- What a read only role may do. The first is the premise of the two that +-- follow: pgstac_read has no write privileges of its own, so anything it +-- manages to change it changed through a definer function. +SET ROLE pgstac_read; + +SELECT throws_ok( + 'DELETE FROM pgstac.collections', + '42501', + NULL, + 'pgstac_read cannot write collections directly' +); + +SELECT throws_ok( + $$ SELECT pgstac.delete_collection('anything') $$, + '42501', + NULL, + 'pgstac_read cannot drop a collection through delete_collection' +); + +SELECT throws_ok( + $$ SELECT pgstac.maintain_index('_items_1', NULL, NULL) $$, + '42501', + NULL, + 'pgstac_read cannot run DDL through maintain_index' +); + +-- where_stats, search_query and format_item run as invokers, so pgstac_read +-- needs direct grants on the caches they maintain. +SELECT lives_ok( + $$ SELECT pgstac.search('{}') $$, + 'pgstac_read can still search' +); + +RESET ROLE; + +-- The elevated DDL functions decide what they may alter by asking +-- partition_catalog_meta, so it must describe partitions of items and nothing +-- else. +SELECT is_empty( + $$ SELECT * FROM pgstac.partition_catalog_meta('collections') $$, + 'partition_catalog_meta rejects a table that is not a partition of items' +); +SELECT is_empty( + $$ SELECT * FROM pgstac.partition_catalog_meta('queryables') $$, + 'partition_catalog_meta rejects queryables' +); diff --git a/src/pgstac/tests/pgtap/001_core.sql b/src/pgstac/tests/pgtap/001_core.sql index ee8ecbbf..a1b150de 100644 --- a/src/pgstac/tests/pgtap/001_core.sql +++ b/src/pgstac/tests/pgtap/001_core.sql @@ -28,13 +28,18 @@ SELECT lives_ok( ); RESET pgstac.context; -SELECT is_definer('update_partition_stats'); -SELECT is_definer('partition_after_triggerfunc'); SELECT is_definer('drop_table_constraints'); SELECT is_definer('create_table_constraints'); SELECT is_definer('check_partition'); SELECT is_definer('repartition'); -SELECT is_definer('where_stats'); -SELECT is_definer('search_query'); -SELECT is_definer('format_item'); SELECT is_definer('maintain_index'); +SELECT is_definer('delete_collection'); + +-- Everything that does not need ownership of pgstac_admin's objects runs as +-- the invoker and relies on table privileges instead. +SELECT isnt_definer('update_partition_stats'); +SELECT isnt_definer('partition_after_triggerfunc'); +SELECT isnt_definer('sync_partition_stats'); +SELECT isnt_definer('where_stats'); +SELECT isnt_definer('search_query'); +SELECT isnt_definer('format_item'); diff --git a/src/pypgstac/pyproject.toml b/src/pypgstac/pyproject.toml index e958554e..28b58846 100644 --- a/src/pypgstac/pyproject.toml +++ b/src/pypgstac/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pypgstac" -version = "0.9.11" +version = "0.9.12" description = "Schema, functions and a python library for storing and accessing STAC collections and items in PostgreSQL" readme = "README.md" requires-python = ">=3.8" diff --git a/src/pypgstac/src/pypgstac/load.py b/src/pypgstac/src/pypgstac/load.py index 467676a0..cb1668e9 100644 --- a/src/pypgstac/src/pypgstac/load.py +++ b/src/pypgstac/src/pypgstac/load.py @@ -476,10 +476,21 @@ def load_partition( "Available modes are insert, ignore, upsert, and delsert." f"You entered {insert_mode}.", ) - logger.debug("Updating Partition Stats") + + # Outside the transaction above: this takes ACCESS EXCLUSIVE, and + # holding that alongside the load's EXCLUSIVE lock blocks readers + # for the combined window. Logged, not raised: the rows are + # committed and the retry wrapping this would re-COPY them. Only + # the observed ranges lag, not the identity columns search uses. + logger.debug("Updating Partition Stats") + try: cur.execute("SELECT update_partition_stats_q(%s);", (partition.name,)) logger.debug(cur.statusmessage) - logger.debug(f"Rows affected: {cur.rowcount}") + except psycopg.Error as e: + logger.warning( + f"Could not update partition stats for {partition.name}: {e}. " + "Data was loaded; run update_partition_stats to refresh.", + ) logger.debug( f"Copying data for {partition} took {time.perf_counter() - t} seconds", ) @@ -521,7 +532,7 @@ def _partition_update(self, item: Dict[str, Any]) -> str: as end_datetime_range_min, nullif(upper(constraint_edtrange),'infinity') as end_datetime_range_max - FROM partition_sys_meta WHERE partition=%s; + FROM partition_catalog_meta(%s); """, [partition_name], ), diff --git a/src/pypgstac/src/pypgstac/migrate.py b/src/pypgstac/src/pypgstac/migrate.py index ea4c1244..e0995e3c 100644 --- a/src/pypgstac/src/pypgstac/migrate.py +++ b/src/pypgstac/src/pypgstac/migrate.py @@ -1,4 +1,5 @@ """Utilities to help migrate pgstac schema.""" + import glob import logging import os @@ -36,7 +37,8 @@ def __init__(self, path: str, f: str, t: str) -> None: def parse_filename(self, filename: str) -> List[str]: """Get version numbers from filename.""" filename = os.path.splitext(os.path.basename(filename))[0].replace( - "pgstac.", "", + "pgstac.", + "", ) return filename.split("-") @@ -96,7 +98,7 @@ def migrations(self) -> List[str]: def get_sql(file: str) -> str: """Get sql from a file as a string.""" sqlstrs = [] - file = re.sub("[0-9]+[.][0-9]+[.][0-9]+-dev","unreleased",file) + file = re.sub("[0-9]+[.][0-9]+[.][0-9]+-dev", "unreleased", file) fp = os.path.join(migrations_dir, file) file_handle: Any = open(fp) @@ -118,7 +120,7 @@ def run_migration(self, toversion: Optional[str] = None) -> str: if toversion is None: toversion = __version__ files = [] - if re.search(r"-dev$",toversion): + if re.search(r"-dev$", toversion): logger.info("using unreleased version") toversion = "unreleased" @@ -126,7 +128,7 @@ def run_migration(self, toversion: Optional[str] = None) -> str: map( int, [ - self.db.pg_version[i:i + 2] + self.db.pg_version[i : i + 2] for i in range(0, len(self.db.pg_version), 2) ], ), @@ -149,6 +151,7 @@ def run_migration(self, toversion: Optional[str] = None) -> str: conn = self.db.connect() + queued = 0 with conn.cursor() as cur: conn.autocommit = False for file in files: @@ -160,6 +163,19 @@ def run_migration(self, toversion: Optional[str] = None) -> str: logger.debug(f"Database migrated to {toversion}") + # Inside the migration's transaction: querying after the commit + # below leaves this connection idle in transaction, holding locks. + # Two statements, not one guarded by CASE: query_queue does not + # exist in the versions a chain starts from, and names resolve at + # parse time, so even an unreachable reference aborts the + # migration. + cur.execute("SELECT to_regclass('pgstac.query_queue') IS NOT NULL;") + exists = cur.fetchone() + if exists and exists[0]: + cur.execute("SELECT count(*) FROM pgstac.query_queue;") + row = cur.fetchone() + queued = row[0] if row else 0 + newversion = self.db.version if conn is not None: if newversion == toversion: @@ -172,4 +188,13 @@ def run_migration(self, toversion: Optional[str] = None) -> str: logger.debug(f"New Version: {newversion}") + # With pgstac.use_queue on, statistics and constraint maintenance are + # deferred, and nothing else reports that they are still pending. + if queued > 0: + logger.warning( + f"{queued} queries are queued after migration. " + "Run 'pypgstac runqueue' until it drains -- until then " + "partition statistics and constraints are stale.", + ) + return newversion diff --git a/src/pypgstac/src/pypgstac/version.py b/src/pypgstac/src/pypgstac/version.py index 9d70cab6..6d4ee16e 100644 --- a/src/pypgstac/src/pypgstac/version.py +++ b/src/pypgstac/src/pypgstac/version.py @@ -1,3 +1,3 @@ """Version.""" -__version__ = "0.9.11" +__version__ = "0.9.12" diff --git a/src/pypgstac/tests/test_concurrency.py b/src/pypgstac/tests/test_concurrency.py new file mode 100644 index 00000000..da518313 --- /dev/null +++ b/src/pypgstac/tests/test_concurrency.py @@ -0,0 +1,701 @@ +"""Concurrent read/write behaviour. + +Covers both ingest paths: the SQL staging tables, and pypgstac's Loader, which +is the primary way data is loaded and takes a different route entirely -- +check_partition, then a COPY (straight into the partition for insert, into a +temp table under LOCK TABLE ... EXCLUSIVE for the other modes), then a stats +update. + +Deadlocks are asserted on the delta of pg_stat_database.deadlocks rather than on +a raised exception: Loader.load_partition retries DeadlockDetected up to ten +times, so an exception based assertion passes even when the server is breaking +deadlocks on every attempt. +""" + +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Optional, Tuple + +import psycopg +from psycopg import sql + +from pypgstac.db import PgstacDB +from pypgstac.load import Loader, Methods + +MONTHS = ["2020-01", "2020-02", "2020-03", "2020-04", "2020-05", "2020-06"] + + +def deadlocks(db: PgstacDB) -> int: + """Deadlocks the server has broken on this database so far.""" + row = db.query_one( + "SELECT deadlocks FROM pg_stat_database WHERE datname = current_database();", + ) + assert isinstance(row, int) + return row + + +def item_content( + collection: str, + ident: str, + month: str, + revision: int = 0, +) -> str: + """A minimal STAC item as a jsonb literal. + + revision changes the stored content, which the upsert path needs in order + to consider a row worth replacing. + """ + return ( + '{"id": "%s", "collection": "%s", "type": "Feature",' + ' "geometry": {"type": "Point", "coordinates": [0, 0]},' + ' "properties": {"datetime": "%s-05T00:00:00Z", "gsd": %d}}' + % (ident, collection, month, revision) + ) + + +def worker(fn: Callable[[psycopg.Connection], Any]) -> Any: + """Run fn on its own connection. psycopg connections are not thread safe.""" + with psycopg.connect(autocommit=True) as conn: + conn.execute("SET search_path TO pgstac, public;") + conn.execute("SET ROLE pgstac_ingest;") + conn.execute("SET pgstac.use_queue TO FALSE;") + return fn(conn) + + +def run_concurrently(fns: List[Callable[[psycopg.Connection], Any]]) -> List[Any]: + """Run each callable on its own connection, all at once.""" + with ThreadPoolExecutor(max_workers=len(fns)) as pool: + futures = [pool.submit(worker, fn) for fn in fns] + return [f.result() for f in futures] + + +def make_collection(db: PgstacDB, collection: str, trunc: Optional[str]) -> None: + """Create a collection, optionally partitioned by datetime.""" + db.query_one( + """ + INSERT INTO collections (content, partition_trunc) + VALUES (jsonb_build_object('id', %s::text), %s::text) + RETURNING id; + """, + [collection, trunc], + ) + + +def stage(conn: psycopg.Connection, table: str, contents: List[str]) -> None: + """Insert a batch into a staging table in a single statement.""" + conn.execute( + sql.SQL("INSERT INTO {} (content) VALUES {}").format( + sql.Identifier(table), + sql.SQL(",").join([sql.SQL("(%s::jsonb)")] * len(contents)), + ), + contents, + ) + + +def test_concurrent_multi_partition_ingest_no_deadlock(db: PgstacDB) -> None: + """Statements spanning the same partitions in different orders. + + Each statement's after-trigger takes a partition_stats row lock per + partition touched and holds it until commit, so every writer has to acquire + them in the same order. + """ + collection = "conc-multi" + make_collection(db, collection, "month") + before = deadlocks(db) + + orders = [MONTHS, list(reversed(MONTHS)), MONTHS[3:] + MONTHS[:3]] + fns: List[Callable[[psycopg.Connection], Any]] = [ + lambda conn, order=order, i=i: stage( + conn, + "items_staging", + [item_content(collection, f"w{i}-{m}", m) for m in order], + ) + for i, order in enumerate(orders) + ] + run_concurrently(fns) + + assert deadlocks(db) == before + count = db.query_one( + "SELECT count(*) FROM items WHERE collection = %s;", + [collection], + ) + assert count == len(orders) * len(MONTHS) + + +def test_concurrent_same_id_upsert_no_deadlock(db: PgstacDB) -> None: + """Two upserts over the same ids in opposite order. + + The upsert path deletes the rows it is about to replace. Without a fixed + lock order the two statements take the same item row locks in opposite + order and deadlock. The revisions differ from what is stored, otherwise the + IS DISTINCT FROM filter would skip every row and lock nothing. + """ + collection = "conc-upsert" + make_collection(db, collection, None) + ids = [f"i{n:03d}" for n in range(40)] + + run_concurrently( + [ + lambda conn: stage( + conn, + "items_staging", + [item_content(collection, i, "2020-01") for i in ids], + ), + ], + ) + before = deadlocks(db) + + forward = [item_content(collection, i, "2020-01", 1) for i in ids] + backward = list(reversed([item_content(collection, i, "2020-01", 2) for i in ids])) + run_concurrently( + [ + lambda conn: stage(conn, "items_staging_upsert", forward), + lambda conn: stage(conn, "items_staging_upsert", backward), + ], + ) + + assert deadlocks(db) == before + count = db.query_one( + "SELECT count(*) FROM items WHERE collection = %s;", + [collection], + ) + assert count == len(ids) + + +def test_collection_delete_vs_concurrent_partition_create(db: PgstacDB) -> None: + """Dropping a collection while another session ingests into it. + + Unlike the other tests here this one does not assert zero deadlocks. The + two operations genuinely conflict: delete_collection takes the collections + row and then the partition tables, while ingest takes the partition tables + and then needs KEY SHARE on that same collections row through + items_collections_fk. Postgres breaking that cycle by aborting one side is + a correct outcome, and it is observed intermittently. + + What must hold is that it resolves safely: both sessions finish, and the + database is left consistent either way. + """ + collection = "conc-delete" + make_collection(db, collection, "month") + run_concurrently( + [ + lambda conn: stage( + conn, + "items_staging", + [item_content(collection, f"seed-{m}", m) for m in MONTHS[:2]], + ), + ], + ) + + def ingest(conn: psycopg.Connection) -> Optional[str]: + try: + stage( + conn, + "items_staging", + [item_content(collection, f"late-{m}", m) for m in MONTHS[2:]], + ) + except psycopg.Error as e: + return type(e).__name__ + return None + + def drop(conn: psycopg.Connection) -> Optional[str]: + try: + conn.execute("SELECT delete_collection(%s);", [collection]) + except psycopg.Error as e: + return type(e).__name__ + return None + + # Neither side may hang; run_concurrently only returns once both have. + results = run_concurrently([ingest, drop]) + assert len(results) == 2 + + survived = db.query_one( + "SELECT count(*) FROM collections WHERE id = %s;", + [collection], + ) + orphans = db.query_one( + """ + SELECT count(*) FROM partition_stats ps + WHERE NOT EXISTS ( + SELECT 1 FROM partitions_view pv WHERE pv.partition = ps.partition + ); + """, + ) + assert orphans == 0, "partition_stats rows left behind for dropped partitions" + + if survived == 0: + # Delete won: nothing of the collection may remain. + assert ( + db.query_one( + "SELECT count(*) FROM partition_stats WHERE collection = %s;", + [collection], + ) + == 0 + ) + assert ( + db.query_one( + "SELECT count(*) FROM items WHERE collection = %s;", + [collection], + ) + == 0 + ) + else: + # Ingest won: every surviving partition is still visible to search. + assert ( + db.query_one( + """ + SELECT count(*) FROM partitions_view pv + WHERE pv.collection = %s AND NOT EXISTS ( + SELECT 1 FROM partition_stats ps + WHERE ps.partition = pv.partition AND ps.collection IS NOT NULL + ); + """, + [collection], + ) + == 0 + ) + + +def test_search_sees_all_committed_items_during_partition_creation( + db: PgstacDB, +) -> None: + """Search finds partitions through partition_stats. + + A partition whose stats row is missing drops out of the datetime bands and + its items disappear from results without an error, so a reader must never + see fewer items than were committed before it started. + """ + collection = "conc-visibility" + make_collection(db, collection, "month") + + for committed, month in enumerate(MONTHS, start=1): + run_concurrently( + [ + lambda conn, month=month: stage( + conn, + "items_staging", + [item_content(collection, f"v-{month}", month)], + ), + ], + ) + found = db.query_one( + """ + SELECT jsonb_array_length( + search(jsonb_build_object( + 'collections', jsonb_build_array(%s::text), 'limit', 100 + )) -> 'features' + ); + """, + [collection], + ) + assert found == committed, ( + f"search returned {found} of {committed} committed items after " + f"creating the {month} partition" + ) + + +def test_fresh_cache_hit_does_not_block_on_locked_stats_row(db: PgstacDB) -> None: + """A search whose cached counts are current must not wait on that row. + + A fresh hit only bumps usage counters; taking FOR UPDATE for that would + serialize identical searches. Holding the row locked in another transaction + makes the outcome deterministic: the reader either skips the lock or waits + until statement_timeout fires. + """ + collection = "conc-search" + make_collection(db, collection, "month") + run_concurrently( + [ + lambda conn: stage( + conn, + "items_staging", + [item_content(collection, f"s-{m}", m) for m in MONTHS], + ), + ], + ) + + search = sql.SQL( + "SELECT search(jsonb_build_object(" + "'collections', jsonb_build_array({}), 'limit', 5));", + ).format(sql.Literal(collection)) + + # Populate search_wheres so there is a fresh row to hit. + run_concurrently( + [ + lambda conn: ( + conn.execute("SET pgstac.context TO 'on';"), + conn.execute(search), + None, + )[-1], + ], + ) + cached = db.query_one( + "SELECT _where FROM search_wheres WHERE total_count IS NOT NULL LIMIT 1;", + ) + assert isinstance(cached, str), "where_stats cached no counts to hit" + + holder = psycopg.connect(autocommit=False) + try: + holder.execute("SET search_path TO pgstac, public;") + holder.execute("SELECT * FROM search_wheres FOR UPDATE;") + + def read(conn: psycopg.Connection) -> None: + conn.execute("SET pgstac.context TO 'on';") + conn.execute("SET statement_timeout TO '10s';") + conn.execute("SELECT where_stats(%s);", [cached]) + + try: + run_concurrently([read]) + except psycopg.errors.QueryCanceled: + raise AssertionError( + "where_stats blocked on a locked search_wheres row; a fresh " + "cache hit must not take a row lock", + ) from None + finally: + holder.rollback() + holder.close() + + +def constraint_ranges(db: PgstacDB, partition: str) -> Tuple[str, str]: + """The CHECK constraint ranges the catalog reports for a partition.""" + row = db.query_one( + """ + SELECT constraint_dtrange::text, constraint_edtrange::text + FROM partitions_view WHERE partition = %s; + """, + [partition], + ) + assert isinstance(row, tuple) + return row[0], row[1] + + +def test_check_constraints_survive_widening(db: PgstacDB) -> None: + """Widening a partition must not leave it without CHECK constraints. + + check_partition drops them when the incoming batch falls outside the + current bounds and queues the rebuild. The staging path has nothing else + that would put them back. + """ + collection = "conc-widen" + make_collection(db, collection, None) + + run_concurrently( + [ + lambda conn: stage( + conn, + "items_staging", + [item_content(collection, "narrow", "2020-06")], + ), + ], + ) + partition = db.query_one( + "SELECT partition FROM partition_stats WHERE collection = %s;", + [collection], + ) + assert isinstance(partition, str) + + # Outside the current bounds in both directions, forcing a widen. + run_concurrently( + [ + lambda conn: stage( + conn, + "items_staging", + [ + item_content(collection, "early", "2019-01"), + item_content(collection, "late", "2021-12"), + ], + ), + ], + ) + + validated = db.query_one( + """ + SELECT count(*) FROM pg_constraint + WHERE conrelid = format('pgstac.%%I', %s::text)::regclass + AND contype = 'c' AND convalidated; + """, + [partition], + ) + assert validated == 1, ( + f"{partition} has {validated} validated CHECK constraints after " + "widening; NOT VALID constraints do not prune" + ) + + dtrange, edtrange = constraint_ranges(db, partition) + assert "2019-01" in dtrange and "2021-12" in dtrange, dtrange + assert "infinity" not in edtrange, ( + f"end_datetime constraint is unbounded ({edtrange}); " + "end_datetime predicates cannot prune this partition" + ) + + +# -------------------------------------------------------------------------- +# Loader path: check_partition, a COPY whose shape depends on the insert mode, +# then update_partition_stats_q. No staging tables involved. +# -------------------------------------------------------------------------- + + +def loader_item(collection: str, ident: str, dt: datetime) -> Dict[str, Any]: + """A minimal STAC item for the Loader.""" + return { + "id": ident, + "type": "Feature", + "collection": collection, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-85.31, 30.93], + [-85.31, 31.00], + [-85.38, 31.00], + [-85.38, 30.93], + [-85.31, 30.93], + ], + ], + }, + "bbox": [-85.38, 30.93, -85.31, 31.00], + "links": [], + "assets": {}, + "properties": {"datetime": dt.isoformat()}, + "stac_version": "1.0.0", + "stac_extensions": [], + } + + +def load( + items: List[Dict[str, Any]], + insert_mode: Methods = Methods.insert, +) -> Optional[str]: + """Load with a Loader on its own connection, as a separate process would. + + The connection is closed on the way out; leaking it leaves the backend idle + in transaction, which blocks the fixture's DROP DATABASE at teardown. + """ + try: + with PgstacDB() as pgdb: + Loader(pgdb).load_items(iter(items), insert_mode=insert_mode) + except Exception as e: # noqa: BLE001 - the exception type is the assertion + return type(e).__name__ + return None + + +def run_loaders(batches: List[Callable[[], Any]]) -> List[Any]: + """Run each loader callable concurrently, one connection each.""" + with ThreadPoolExecutor(max_workers=len(batches)) as pool: + return [f.result() for f in [pool.submit(b) for b in batches]] + + +def month_dt(month: str, day: int = 5) -> datetime: + """A datetime inside the given YYYY-MM.""" + year, mon = month.split("-") + return datetime(int(year), int(mon), day, tzinfo=timezone.utc) + + +def test_loader_concurrent_multi_partition_no_deadlock(db: PgstacDB) -> None: + """Concurrent Loaders converging on the same partitions. + + load_items sorts each chunk by partition, so the loaders walk partitions in + the same order and the contention is per partition rather than a cross + partition ordering problem: for each one they run update_partition_stats + concurrently, which reads the partition and then rebuilds its constraints. + The input orders are varied only so the batches differ. + """ + collection = "loader-multi" + make_collection(db, collection, "month") + before = deadlocks(db) + + orders = [MONTHS, list(reversed(MONTHS)), MONTHS[2:] + MONTHS[:2]] + batches = [ + ( + lambda order=order, i=i: load( + [loader_item(collection, f"L{i}-{m}", month_dt(m)) for m in order], + ) + ) + for i, order in enumerate(orders) + ] + errors = [e for e in run_loaders(batches) if e] + + assert errors == [], f"concurrent loads raised: {errors}" + assert deadlocks(db) == before + count = db.query_one( + "SELECT count(*) FROM items WHERE collection = %s;", + [collection], + ) + assert count == len(orders) * len(MONTHS) + + +def test_loader_concurrent_upsert_same_ids_no_deadlock(db: PgstacDB) -> None: + """Concurrent Loaders upserting the same ids from opposite ends. + + The loader's upsert takes LOCK TABLE ... EXCLUSIVE on the partition before + writing, so the two serialize rather than interleave; this asserts that + holds and that no rows are lost or duplicated. + """ + collection = "loader-upsert" + make_collection(db, collection, None) + dt = month_dt("2020-03") + ids = [f"u{n:03d}" for n in range(50)] + + assert load([loader_item(collection, i, dt) for i in ids]) is None + before = deadlocks(db) + + forward = [loader_item(collection, i, dt) for i in ids] + backward = list(reversed([loader_item(collection, i, dt) for i in ids])) + errors = [ + e + for e in run_loaders( + [ + lambda: load(forward, Methods.upsert), + lambda: load(backward, Methods.upsert), + ], + ) + if e + ] + + assert errors == [], f"concurrent upserts raised: {errors}" + assert deadlocks(db) == before + count = db.query_one( + "SELECT count(*) FROM items WHERE collection = %s;", + [collection], + ) + assert count == len(ids) + + +def test_loader_concurrent_delsert_same_ids_no_deadlock(db: PgstacDB) -> None: + """delsert deletes the rows it is about to rewrite, then inserts. + + That DELETE is unordered, so it is the loader's equivalent of the staging + upsert path and worth asserting on directly. + """ + collection = "loader-delsert" + make_collection(db, collection, None) + dt = month_dt("2020-04") + ids = [f"d{n:03d}" for n in range(50)] + + assert load([loader_item(collection, i, dt) for i in ids]) is None + before = deadlocks(db) + + forward = [loader_item(collection, i, dt) for i in ids] + backward = list(reversed([loader_item(collection, i, dt) for i in ids])) + errors = [ + e + for e in run_loaders( + [ + lambda: load(forward, Methods.delsert), + lambda: load(backward, Methods.delsert), + ], + ) + if e + ] + + assert errors == [], f"concurrent delserts raised: {errors}" + assert deadlocks(db) == before + count = db.query_one( + "SELECT count(*) FROM items WHERE collection = %s;", + [collection], + ) + assert count == len(ids) + + +def test_loader_widening_partition_keeps_check_constraints(db: PgstacDB) -> None: + """The loader path must also leave validated constraints behind. + + check_partition drops them when a batch falls outside the current bounds. + NOT VALID constraints do not prune, so what matters is that a validated one + exists afterwards and covers the data. + """ + collection = "loader-widen" + make_collection(db, collection, None) + + assert load([loader_item(collection, "mid", month_dt("2020-06"))]) is None + partition = db.query_one( + "SELECT partition FROM partition_stats WHERE collection = %s;", + [collection], + ) + assert isinstance(partition, str) + + assert ( + load( + [ + loader_item(collection, "early", month_dt("2019-01")), + loader_item(collection, "late", month_dt("2021-12")), + ], + ) + is None + ) + + validated = db.query_one( + """ + SELECT count(*) FROM pg_constraint + WHERE conrelid = format('pgstac.%%I', %s::text)::regclass + AND contype = 'c' AND convalidated; + """, + [partition], + ) + assert validated == 1, ( + f"{partition} has {validated} validated CHECK constraints after a " + "widening load" + ) + + dtrange, edtrange = constraint_ranges(db, partition) + assert "2019-01" in dtrange and "2021-12" in dtrange, dtrange + assert "infinity" not in edtrange, edtrange + + +def test_loader_does_not_starve_readers(db: PgstacDB) -> None: + """Searches must keep returning while a loader writes. + + Tightening the CHECK constraints takes ACCESS EXCLUSIVE on the partition, + and the loader runs it outside the transaction holding the load's EXCLUSIVE + lock so the two windows do not combine. + """ + collection = "loader-readers" + make_collection(db, collection, "month") + assert ( + load([loader_item(collection, f"seed-{m}", month_dt(m)) for m in MONTHS]) + is None + ) + + search = sql.SQL( + "SELECT search(jsonb_build_object(" + "'collections', jsonb_build_array({}), 'limit', 5));", + ).format(sql.Literal(collection)) + stop = False + waits: List[float] = [] + + def reader(conn: psycopg.Connection) -> None: + conn.execute("SET statement_timeout TO '30s';") + while not stop: + started = time.perf_counter() + conn.execute(search) + waits.append(time.perf_counter() - started) + time.sleep(0.01) + + with ThreadPoolExecutor(max_workers=2) as pool: + reading = pool.submit(worker, reader) + try: + for round_n in range(6): + assert ( + load( + [ + loader_item( + collection, + f"r{round_n}-{m}", + month_dt(m, day=6 + round_n), + ) + for m in MONTHS + ], + Methods.upsert, + ) + is None + ) + finally: + stop = True + reading.result() + + assert waits, "reader never completed a search" + assert max(waits) < 10, ( + f"a search waited {max(waits):.1f}s while the loader ran; readers are " + "being blocked by the loader's locks" + )