From 0dfe82fb7b9bf205f403a13391332d56ccbfe7e5 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Tue, 25 Aug 2026 14:22:36 +0530 Subject: [PATCH 01/20] chore: add state version info and compatibility history in json files --- .github/CODEOWNERS | 10 ++++ .github/rulesets/state-version-approval.json | 49 ++++++++++++++++++ constants/state-versions.json | 53 ++++++++++++++++++++ constants/state_version.go | 35 ++++++++++--- 4 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/rulesets/state-version-approval.json create mode 100644 constants/state-versions.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..88c19d811 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,10 @@ +# CODEOWNERS + +## Ownership and review routing only -- the 2-approval requirement for these paths is a repository ruleset (required reviewer rule on the same teams), since CODEOWNERS with branch protection is satisfied by any one owner. A team needs write access to the repo for CODEOWNERS to resolve it + +## The state configuration and the compat rules pin backward-compatibility semantics forever, so changes need 2 sign-offs from the people who own those semantics. CODEOWNERS takes one pattern per line, so the team repeats; the ruleset groups both paths under a single rule + +/.github/CODEOWNERS @datazip-inc/olake-admins + +/constants/state-versions.json @datazip-inc/state-version-owners +/tests/testutils/compatibility_rules.json @datazip-inc/state-version-owners diff --git a/.github/rulesets/state-version-approval.json b/.github/rulesets/state-version-approval.json new file mode 100644 index 000000000..e67eff3c3 --- /dev/null +++ b/.github/rulesets/state-version-approval.json @@ -0,0 +1,49 @@ +{ + "name": "State version approval", + "target": "branch", + "enforcement": "active", + "conditions": { + "ref_name": { + "include": [ + "refs/heads/master", + "refs/heads/staging" + ], + "exclude": [] + } + }, + "rules": [ + { + "type": "pull_request", + "parameters": { + "required_approving_review_count": 1, + "dismiss_stale_reviews_on_push": true, + "require_code_owner_review": true, + "require_last_push_approval": true, + "required_review_thread_resolution": true, + "required_reviewers": [ + { + "file_patterns": [ + "constants/state-versions.json", + "tests/testutils/compatibility_rules.json" + ], + "minimum_approvals": 2, + "reviewer": { + "id": 0, + "type": "Team" + } + }, + { + "file_patterns": [ + ".github/CODEOWNERS" + ], + "minimum_approvals": 2, + "reviewer": { + "id": 0, + "type": "Team" + } + } + ] + } + } + ] +} diff --git a/constants/state-versions.json b/constants/state-versions.json new file mode 100644 index 000000000..76cca8ba1 --- /dev/null +++ b/constants/state-versions.json @@ -0,0 +1,53 @@ +{ + "latest_state_version": 7, + "baselines": [ + { + "state_version": 0, + "release_tag": "v0.3.11", + "drivers": "*", + "note": "newest pre-versioning release; an absent `version` field reads as 0. Legacy semantics: a string that cannot be parsed as a timestamp collapses to epoch (1970-01-01)" + }, + { + "state_version": 1, + "release_tag": "v0.3.15", + "drivers": "*", + "note": "stricter timestamp parsing: an unparseable string stays a string instead of collapsing to epoch, failing fast rather than corrupting data" + }, + { + "state_version": 2, + "release_tag": "v0.3.16", + "drivers": "mysql", + "note": "consistent MySQL timezone handling: binlog CDC uses TimestampStringLocation to match the connection's timezone, so CDC timestamps agree with Full Refresh" + }, + { + "state_version": 3, + "release_tag": "v0.4.0", + "drivers": "mysql", + "note": "MySQL offset-format timezones (session or global) parse correctly and set the connection timezone instead of falling back to UTC" + }, + { + "state_version": 4, + "release_tag": "v0.6.1", + "drivers": "mysql", + "note": "MySQL unsigned int/integer/bigint map to Int64; earlier they mapped to Int32 and overflowed" + }, + { + "state_version": 5, + "release_tag": "v0.6.5", + "drivers": "mongodb", + "note": "MongoDB BSON DateTime at any depth decodes to UTC time.Time via a custom client registry, preventing json.Marshal crashes for out-of-range years ([0,9999]); top-level DateTimes that previously formatted with the local machine timezone (e.g. \"+05:30\") now always output UTC (\"Z\")" + }, + { + "state_version": 6, + "release_tag": "v0.9.0", + "drivers": "*", + "note": "ReformatInt64 accepts []uint8: numeric values some SQL drivers return as byte slices parse to int64 instead of erroring" + }, + { + "state_version": 7, + "release_tag": "v0.9.2", + "drivers": "s3", + "note": "parquet INT96 maps to Timestamp (the raw 96-bit value was emitted as a string, which disagreed with the inferred Timestamp schema and collapsed the column to String) and unsigned 32-bit widens to Int64, matching pg/mysql (earlier read as signed int32 and mapped to Int32, so values above 2^31-1 wrapped negative); older state keeps both previous behaviors so existing destination columns do not change type on upgrade" + } + ] +} diff --git a/constants/state_version.go b/constants/state_version.go index 373eb36f6..f3008144c 100644 --- a/constants/state_version.go +++ b/constants/state_version.go @@ -1,12 +1,14 @@ package constants +import ( + _ "embed" + "encoding/json" +) + // State version constants for backward compatibility // State files can have different versions to support migration and backward compatibility // when the state file format or behavior changes. -// LatestStateVersion is the current version of the state file format. -// This version is used when creating new state files. -// // Version History: // - Version 0: Legacy format (backward compatibility) // * More lenient date/timestamp parsing behavior @@ -42,9 +44,28 @@ package constants // * Unsigned 32-bit: earlier read as a signed int32 and mapped to Int32, so values above 2^31-1 wrapped negative. Now widened to Int64, matching pg/mysql. // * Older state keeps both previous behaviors so existing destination columns do not change type on upgrade. -// tests/testutils/constants keeps a temporary copy of this value; update it there as well when bumping the version. -// TODO: remove this file after state version is moved to secrets -const LatestStateVersion = 7 +// LatestStateVersion is the current version of the state file format. +// This version is used when creating new state files. +var LatestStateVersion int // Used as the current version of the state when the program is running -var LoadedStateVersion = LatestStateVersion +var LoadedStateVersion int + +//go:embed state-versions.json +var rawStateVersions []byte + +// init initializes static information only: the version this build writes. The version a running +// sync is pinned at comes from its state file, via SetLoadedStateVersion. +func init() { + var doc struct { + LatestStateVersion int `json:"latest_state_version"` + } + if err := json.Unmarshal(rawStateVersions, &doc); err != nil { + panic("constants/state-versions.json is not valid JSON: " + err.Error()) + } + if doc.LatestStateVersion <= 0 { + panic("constants/state-versions.json must set latest_state_version to a positive integer") + } + LatestStateVersion = doc.LatestStateVersion + LoadedStateVersion = LatestStateVersion +} From a3151f4f2f75345f982ea42e675f4638671fbd82 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Fri, 21 Aug 2026 11:52:19 +0530 Subject: [PATCH 02/20] chore: minor changes for smooth tests --- drivers/db2/driver.mk | 2 +- drivers/kafka/docker-compose.yml | 2 + .../docker-entrypoint-initdb.d/01-init.sql | 39 ------------------- drivers/s3/docker-compose.yml | 2 +- 4 files changed, 4 insertions(+), 41 deletions(-) diff --git a/drivers/db2/driver.mk b/drivers/db2/driver.mk index a8a6986ca..243e9117f 100644 --- a/drivers/db2/driver.mk +++ b/drivers/db2/driver.mk @@ -3,7 +3,7 @@ # The db2 container initializes a full instance on first boot; probe slowly. WAIT_RETRIES.db2 := 30 WAIT_SLEEP.db2 := 25 -PROBE.db2 = docker exec db2-test bash -c "su - db2inst1 -c 'db2 connect to TESTDB'" +PROBE.db2 = docker logs db2-test 2>&1 | grep -q "Setup has completed" && docker exec db2-test bash -c "su - db2inst1 -c 'db2 connect to TESTDB'" # Compiling drivers/db2 needs cgo against IBM's clidriver. prepare.db2 finds # an existing install or downloads the build for this OS/arch (mirrors diff --git a/drivers/kafka/docker-compose.yml b/drivers/kafka/docker-compose.yml index 20a7e2f99..bc2b4a95d 100644 --- a/drivers/kafka/docker-compose.yml +++ b/drivers/kafka/docker-compose.yml @@ -24,6 +24,7 @@ services: KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafkaJson:9092,EXTERNAL_HOST://127.0.0.1:29092,EXTERNAL_CONT://host.docker.internal:39092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL_HOST:PLAINTEXT,EXTERNAL_CONT:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 @@ -74,6 +75,7 @@ services: KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafkaAvro:9092,EXTERNAL_HOST://127.0.0.1:29192,EXTERNAL_CONT://host.docker.internal:39192 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL_HOST:PLAINTEXT,EXTERNAL_CONT:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 diff --git a/drivers/mssql/docker-entrypoint-initdb.d/01-init.sql b/drivers/mssql/docker-entrypoint-initdb.d/01-init.sql index c2cda9935..c56f872a0 100644 --- a/drivers/mssql/docker-entrypoint-initdb.d/01-init.sql +++ b/drivers/mssql/docker-entrypoint-initdb.d/01-init.sql @@ -61,42 +61,3 @@ BEGIN @supports_net_changes = 0; END; GO - -------------------------------------------------------------------------------- --- Integration test -------------------------------------------------------------------------------- -IF DB_ID('olake_mssql_test') IS NULL -BEGIN - CREATE DATABASE olake_mssql_test; -END; -GO - -USE olake_mssql_test; -GO - --- Enable CDC at database level -IF EXISTS (SELECT 1 FROM sys.databases WHERE name = 'olake_mssql_test' AND is_cdc_enabled = 0) -BEGIN - EXEC sys.sp_cdc_enable_db; -END; -GO - -------------------------------------------------------------------------------- --- 2PC suite. Its own database, not just its own table: table separation alone --- races -- DROP/CREATE TABLE modify database-scoped shared metadata (system --- catalog, cdc schema) even for separate tables, and the loser transaction --- fails as the deadlock victim (error 1205). -------------------------------------------------------------------------------- -IF DB_ID('olake_mssql_test_2pc') IS NULL -BEGIN - CREATE DATABASE olake_mssql_test_2pc; -END; -GO - -USE olake_mssql_test_2pc; -GO - -IF EXISTS (SELECT 1 FROM sys.databases WHERE name = 'olake_mssql_test_2pc' AND is_cdc_enabled = 0) -BEGIN - EXEC sys.sp_cdc_enable_db; -END; diff --git a/drivers/s3/docker-compose.yml b/drivers/s3/docker-compose.yml index f24dfad7e..dcc1926da 100644 --- a/drivers/s3/docker-compose.yml +++ b/drivers/s3/docker-compose.yml @@ -16,7 +16,7 @@ services: - 9001:9001 - 9000:9000 volumes: - - ../../destination/iceberg/local-test/data/minio-data:/data + - minio-data:/data command: [ "server", "/data", "--console-address", ":9001" ] mc: From db2237a2e498f3d9d43dc1004c2254d978793b43 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Tue, 25 Aug 2026 14:23:55 +0530 Subject: [PATCH 03/20] chore: refactor integration test framework --- .github/workflows/integration-tests.yml | 40 +- tests/db2/db2_test.go | 45 +- tests/db2/db2_util_test.go | 13 +- .../{source.json => source.template.json} | 0 tests/db2/testdata/streams.template.json | 1 + tests/db2/testdata/test_streams.json | 1 - tests/kafka/kafka_test.go | 102 +- tests/kafka/kafka_util_test.go | 39 +- tests/kafka/rebalance_test.go | 125 + .../{source.json => source.template.json} | 2 +- .../kafka/testdata/avro/streams.template.json | 1 + tests/kafka/testdata/avro/test_streams.json | 1 - .../{source.json => source.template.json} | 2 +- .../kafka/testdata/json/streams.template.json | 1 + tests/kafka/testdata/json/test_streams.json | 1 - tests/mongodb/mongodb_test.go | 72 +- tests/mongodb/mongodb_util_test.go | 122 +- .../{source.json => source.template.json} | 4 +- tests/mongodb/testdata/streams.template.json | 1 + tests/mongodb/testdata/test_streams.json | 1 - tests/mssql/mssql_test.go | 44 +- tests/mssql/mssql_util_test.go | 138 +- .../{source.json => source.template.json} | 2 +- tests/mssql/testdata/streams.template.json | 1 + tests/mssql/testdata/test_streams.json | 1 - tests/mysql/mysql_test.go | 84 +- tests/mysql/mysql_util_test.go | 158 +- .../{source.json => source.template.json} | 0 tests/mysql/testdata/streams.template.json | 1 + tests/mysql/testdata/test_streams.json | 1 - tests/oracle/oracle_test.go | 44 +- tests/oracle/oracle_util_test.go | 13 +- .../{source.json => source.template.json} | 0 tests/oracle/testdata/streams.template.json | 1 + tests/oracle/testdata/test_streams.json | 1 - tests/postgres/postgres_test.go | 61 +- tests/postgres/postgres_util_test.go | 72 +- .../{source.json => source.template.json} | 2 +- ...est_streams.json => streams.template.json} | 2 +- tests/s3/s3_test.go | 80 +- tests/s3/s3_util_test.go | 223 +- .../csv/{source.json => source.template.json} | 2 +- tests/s3/testdata/csv/streams.template.json | 1 + tests/s3/testdata/csv/test_streams.json | 1 - .../{source.json => source.template.json} | 2 +- tests/s3/testdata/json/streams.template.json | 1 + tests/s3/testdata/json/test_streams.json | 1 - .../{source.json => source.template.json} | 2 +- .../s3/testdata/parquet/streams.template.json | 1 + tests/s3/testdata/parquet/test_streams.json | 1 - .../xml/{source.json => source.template.json} | 6 +- ...est_streams.json => streams.template.json} | 6 +- tests/testutils/constants/constants.go | 6 - tests/testutils/docker.go | 171 +- tests/testutils/integration/2pc.go | 272 ++ tests/testutils/integration/discover.go | 122 + tests/testutils/integration/iceberg.go | 80 + tests/testutils/integration/integration.go | 179 ++ tests/testutils/integration/parquet.go | 108 + .../{ => integration}/parquet_rolling.go | 23 +- tests/testutils/integration/sync.go | 507 ++++ tests/testutils/integration/verify.go | 468 +++ tests/testutils/performance/benchmarks.go | 113 + tests/testutils/performance/performance.go | 221 ++ tests/testutils/require/require.go | 144 + tests/testutils/source_config.go | 47 +- tests/testutils/state_version.go | 70 + tests/testutils/test_utils.go | 2594 +++-------------- tests/testutils/timing.go | 18 +- tests/testutils/utils.go | 90 + 70 files changed, 3899 insertions(+), 2861 deletions(-) rename tests/db2/testdata/{source.json => source.template.json} (100%) create mode 100644 tests/db2/testdata/streams.template.json delete mode 100644 tests/db2/testdata/test_streams.json create mode 100644 tests/kafka/rebalance_test.go rename tests/kafka/testdata/avro/{source.json => source.template.json} (81%) create mode 100644 tests/kafka/testdata/avro/streams.template.json delete mode 100644 tests/kafka/testdata/avro/test_streams.json rename tests/kafka/testdata/json/{source.json => source.template.json} (75%) create mode 100644 tests/kafka/testdata/json/streams.template.json delete mode 100644 tests/kafka/testdata/json/test_streams.json rename tests/mongodb/testdata/{source.json => source.template.json} (80%) create mode 100644 tests/mongodb/testdata/streams.template.json delete mode 100644 tests/mongodb/testdata/test_streams.json rename tests/mssql/testdata/{source.json => source.template.json} (81%) create mode 100644 tests/mssql/testdata/streams.template.json delete mode 100644 tests/mssql/testdata/test_streams.json rename tests/mysql/testdata/{source.json => source.template.json} (100%) create mode 100644 tests/mysql/testdata/streams.template.json delete mode 100644 tests/mysql/testdata/test_streams.json rename tests/oracle/testdata/{source.json => source.template.json} (100%) create mode 100644 tests/oracle/testdata/streams.template.json delete mode 100644 tests/oracle/testdata/test_streams.json rename tests/postgres/testdata/{source.json => source.template.json} (90%) rename tests/postgres/testdata/{test_streams.json => streams.template.json} (95%) rename tests/s3/testdata/csv/{source.json => source.template.json} (88%) create mode 100644 tests/s3/testdata/csv/streams.template.json delete mode 100644 tests/s3/testdata/csv/test_streams.json rename tests/s3/testdata/json/{source.json => source.template.json} (88%) create mode 100644 tests/s3/testdata/json/streams.template.json delete mode 100644 tests/s3/testdata/json/test_streams.json rename tests/s3/testdata/parquet/{source.json => source.template.json} (90%) create mode 100644 tests/s3/testdata/parquet/streams.template.json delete mode 100644 tests/s3/testdata/parquet/test_streams.json rename tests/s3/testdata/xml/{source.json => source.template.json} (74%) rename tests/s3/testdata/xml/{test_streams.json => streams.template.json} (97%) create mode 100644 tests/testutils/integration/2pc.go create mode 100644 tests/testutils/integration/discover.go create mode 100644 tests/testutils/integration/iceberg.go create mode 100644 tests/testutils/integration/integration.go create mode 100644 tests/testutils/integration/parquet.go rename tests/testutils/{ => integration}/parquet_rolling.go (83%) create mode 100644 tests/testutils/integration/sync.go create mode 100644 tests/testutils/integration/verify.go create mode 100644 tests/testutils/performance/benchmarks.go create mode 100644 tests/testutils/performance/performance.go create mode 100644 tests/testutils/require/require.go create mode 100644 tests/testutils/state_version.go diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8dd4c2a3d..28ab4d4ef 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,8 +1,7 @@ name: Integration Tests on: - # Pushes run the cache jobs only -- the matrix below is gated to pull requests. A branch ref is - # the one scope every PR can restore from, and only a push writes one. + # A push runs the suite post-merge and publishes the caches every pull request restores from. push: branches: - "master" @@ -10,7 +9,7 @@ on: pull_request: branches: - "*" - paths: + paths: &paths - '**/*.go' - '**/*.java' - '**/go.mod' @@ -19,16 +18,15 @@ on: - 'Dockerfile' - '.dockerignore' - 'Makefile' - - '.golangci.yml' - - 'drivers/**.conf' - 'drivers/*/driver.mk' - 'drivers/*/docker-compose.yml' + - 'drivers/**.conf' - 'destination/iceberg/local-test/**' - 'tests/**' + - '.golangci.yml' + - '.github/actions/**' + - '.github/scripts/**' - '.github/workflows/integration-tests.yml' - - '.github/actions/detect-drivers/action.yml' - - '.github/actions/go-caches/action.yml' - - '.github/scripts/changed-in-paths.sh' jobs: # Ungated, like the three cache jobs below it: the environment prompts once per wave of jobs that @@ -96,6 +94,20 @@ jobs: key: ${{ runner.os }}-aptwarm-${{ hashFiles('Dockerfile') }} lookup-only: true + # The run's single approval, last before the matrix so a pre-job failure fails or skips it too -- + # then any re-run re-executes it, and every attempt that reaches the drivers prompts exactly once. + approve: + name: Approve integration tests + needs: [preflight, build-jar, apt-warm, go-cache] + if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.build-jar.result != 'failure' && needs.apt-warm.result != 'failure' && needs.go-cache.result != 'failure' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + environment: ${{ github.event_name == 'pull_request' && 'integration_tests' || '' }} + outputs: + attempt: ${{ github.run_attempt }} + steps: + - run: echo "Approved -- running the driver matrix." + # Only when the jar is missing: preflight already looked up the cache, so an unchanged writer # skips Maven and this whole job. build-jar: @@ -210,16 +222,10 @@ jobs: # plus destination stack, builds its own image and runs every suite for that driver. integration-tests: name: Test ${{ matrix.driver }} - needs: [preflight, build-jar, apt-warm, go-cache] - # Every need above this can legitimately be skipped, and a skipped need would otherwise skip - # this too -- so gate on "not failed" rather than on success. go-cache is exempt entirely: its - # cache warm and db2 clidriver download must not block seven drivers, and its lint and gosec - # failures already surface as their own check. - if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.preflight.result == 'success' && needs.preflight.outputs.drivers != '[]' && needs.build-jar.result != 'failure' && needs.apt-warm.result != 'failure' }} + needs: [preflight, approve] + if: ${{ !cancelled() && needs.approve.result == 'success' }} runs-on: 16gb-runner - # The whole workflow's approval, deliberately on the only job that spends a 16gb runner: one - # prompt releases every driver, and "Re-run failed jobs" re-enters it so a replay costs another. - environment: integration_tests + environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} timeout-minutes: 45 strategy: fail-fast: false diff --git a/tests/db2/db2_test.go b/tests/db2/db2_test.go index 05fcf8265..4199c1b61 100644 --- a/tests/db2/db2_test.go +++ b/tests/db2/db2_test.go @@ -5,21 +5,19 @@ import ( "github.com/datazip-inc/olake/tests/testutils" "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/require" ) // db2BaseConfig returns an IntegrationTest pre-populated with all fields shared -func db2BaseConfig(t *testing.T) *testutils.IntegrationTest { - return &testutils.IntegrationTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.DB2)).WithImagePlatform("linux/amd64"), - Namespace: "DB2INST1", - ExpectedData: ExpectedDB2Data, - DestinationDataTypeSchema: DB2ToDestinationSchema, - ExecuteQuery: ExecuteQuery, - DestinationDB: "db2_testdb_db2inst1", - CursorField: "COL_CURSOR:COL_TIMESTAMP", - PartitionRegex: "/{id, identity}", - ColumnToExclude: "EXCLUDEDCOLUMN", - FilterConfig: `{ +func db2BaseConfig(t *testing.T) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.DB2, "DB2INST1", "db2_testdb_db2inst1", ExecuteQuery, + testutils.WithImagePlatform("linux/amd64")) + require.NoError(t, err, "failed to build the test config") + cfg.CursorField = "COL_CURSOR:COL_TIMESTAMP" + cfg.PartitionRegex = "/{id, identity}" + cfg.ColumnToExclude = "EXCLUDEDCOLUMN" + cfg.FilterConfig = `{ "logical_operator": "And", "conditions": [ { @@ -33,7 +31,12 @@ func db2BaseConfig(t *testing.T) *testutils.IntegrationTest { "value": "2022-07-01T15:30:00.000+00:00" } ] - }`, + }` + + return &integration.Test{ + TestConfig: cfg, + ExpectedData: ExpectedDB2Data, + DestinationDataTypeSchema: DB2ToDestinationSchema, } } @@ -53,3 +56,19 @@ func TestDB22PC(t *testing.T) { t.Parallel() db2BaseConfig(t).Test2PCIntegration(t) } + +// TestDB2Compatibility pins the backward-compatibility contract: the same scenarios run on a released +// baseline image and on this build after the initial load, and the destinations must match. +// See tests/testutils/compatibility.go. +// func TestDB2Compatibility(t *testing.T) { +// t.Parallel() +// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { +// base := db2BaseConfig(t) +// base.ExpectedUpdatedData = ExpectedUpdatedDB2Data +// base.UpdatedDestinationDataTypeSchema = UpdatedDB2ToDestinationSchema +// cfg := &compatibility.Test{IntegrationTest: base} +// // Type tags for compatibility_rules.json's db2 rules; floor and descriptions live there too. +// cfg.ColumnTypes = map[string][]string{"col_decfloat": {"decfloat"}} +// return cfg +// }) +// } diff --git a/tests/db2/db2_util_test.go b/tests/db2/db2_util_test.go index 09a146ad3..2fe7e0aa5 100644 --- a/tests/db2/db2_util_test.go +++ b/tests/db2/db2_util_test.go @@ -10,9 +10,9 @@ import ( "github.com/apache/arrow-go/v18/arrow" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/require" _ "github.com/ibmdb/go_ibm_db" "github.com/jmoiron/sqlx" - "github.com/stretchr/testify/require" ) var ( @@ -25,7 +25,7 @@ var ( func buildDSN(config testutils.SourceConfig) string { dsn := fmt.Sprintf( "HOSTNAME=%s;PORT=%d;DATABASE=%s;UID=%s;PWD=%s;", - config.String("host"), + config.Host("host"), config.Int("port"), config.String("database"), config.String("username"), @@ -80,17 +80,12 @@ func exec(ctx context.Context, db *sqlx.DB, query string) error { func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() - var dsn string - if conf.SourceBaseConfig != nil { - dsn = buildDSN(conf.SourceBaseConfig) - } else { - dsn = "HOSTNAME=localhost;PORT=50000;DATABASE=testdb;UID=db2inst1;PWD=secret1234;" - } + dsn := buildDSN(conf.SourceBaseConfig) db := getDB(ctx, t, dsn) var err error - integrationTestTable := testutils.TestTableName(conf) + integrationTestTable := conf.GetTableName() var query string switch operation { diff --git a/tests/db2/testdata/source.json b/tests/db2/testdata/source.template.json similarity index 100% rename from tests/db2/testdata/source.json rename to tests/db2/testdata/source.template.json diff --git a/tests/db2/testdata/streams.template.json b/tests/db2/testdata/streams.template.json new file mode 100644 index 000000000..42e4d6776 --- /dev/null +++ b/tests/db2/testdata/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"DB2INST1":[{"partition_regex":"","stream_name":"TEST_TABLE_OLAKE_${SUITE}","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["COL_DATE","_olake_timestamp","COL_INT","COL_DOUBLE","COL_BOOL","COL_SMALLINT","_olake_id","COL_BIGINT","COL_CHARACTER","COL_VARGRAPHIC","COL_BLOB","_op_type","COL_CURSOR","COL_VARCHAR","COL_TIME","COL_TIMESTAMP","COL_CHAR","COL_REAL","COL_CLOB","COL_DECIMAL","COL_DECFLOAT","EXCLUDEDCOLUMN","ID","COL_GRAPHIC"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"TEST_TABLE_OLAKE_${SUITE}","namespace":"DB2INST1","type_schema":{"properties":{"COL_BIGINT":{"type":["null","integer"],"destination_column_name":"col_bigint"},"COL_BLOB":{"type":["string","null"],"destination_column_name":"col_blob"},"COL_BOOL":{"type":["boolean","null"],"destination_column_name":"col_bool"},"COL_CHAR":{"type":["string","null"],"destination_column_name":"col_char"},"COL_CHARACTER":{"type":["string","null"],"destination_column_name":"col_character"},"COL_CLOB":{"type":["null","string"],"destination_column_name":"col_clob"},"COL_CURSOR":{"type":["integer","null"],"destination_column_name":"col_cursor"},"COL_DATE":{"type":["timestamp","null"],"destination_column_name":"col_date"},"COL_DECIMAL":{"type":["number","null"],"destination_column_name":"col_decimal"},"COL_DECFLOAT":{"type":["string","null"],"destination_column_name":"col_decfloat"},"COL_DOUBLE":{"type":["number","null"],"destination_column_name":"col_double"},"COL_GRAPHIC":{"type":["string","null"],"destination_column_name":"col_graphic"},"COL_INT":{"type":["integer_small","null"],"destination_column_name":"col_int"},"COL_REAL":{"type":["number_small","null"],"destination_column_name":"col_real"},"COL_SMALLINT":{"type":["null","integer_small"],"destination_column_name":"col_smallint"},"COL_TIME":{"type":["string","null"],"destination_column_name":"col_time"},"COL_TIMESTAMP":{"type":["timestamp","null"],"destination_column_name":"col_timestamp"},"COL_VARCHAR":{"type":["string","null"],"destination_column_name":"col_varchar"},"COL_VARGRAPHIC":{"type":["string","null"],"destination_column_name":"col_vargraphic"},"EXCLUDEDCOLUMN":{"type":["null","integer_small"],"destination_column_name":"excludedcolumn"},"ID":{"type":["integer"],"destination_column_name":"id"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":["ID"],"available_cursor_fields":["ID","COL_CHAR","COL_DECIMAL","COL_DECFLOAT","COL_BOOL","COL_TIMESTAMP","COL_CHARACTER","COL_VARCHAR","COL_DOUBLE","COL_BLOB","COL_TIME","COL_DATE","COL_INT","COL_CLOB","COL_VARGRAPHIC","EXCLUDEDCOLUMN","COL_CURSOR","COL_BIGINT","COL_REAL","COL_SMALLINT","COL_GRAPHIC"],"cursor_field":"COL_BIGINT","sync_mode":"incremental","destination_database":"db2_testdb:db2inst1","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/db2/testdata/test_streams.json b/tests/db2/testdata/test_streams.json deleted file mode 100644 index fe19cd7e8..000000000 --- a/tests/db2/testdata/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"DB2INST1":[{"partition_regex":"","stream_name":"DB2_TEST_TABLE_OLAKE","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["COL_DATE","_olake_timestamp","COL_INT","COL_DOUBLE","COL_BOOL","COL_SMALLINT","_olake_id","COL_BIGINT","COL_CHARACTER","COL_VARGRAPHIC","COL_BLOB","_op_type","COL_CURSOR","COL_VARCHAR","COL_TIME","COL_TIMESTAMP","COL_CHAR","COL_REAL","COL_CLOB","COL_DECIMAL","COL_DECFLOAT","EXCLUDEDCOLUMN","ID","COL_GRAPHIC"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"DB2_TEST_TABLE_OLAKE","namespace":"DB2INST1","type_schema":{"properties":{"COL_BIGINT":{"type":["null","integer"],"destination_column_name":"col_bigint"},"COL_BLOB":{"type":["string","null"],"destination_column_name":"col_blob"},"COL_BOOL":{"type":["boolean","null"],"destination_column_name":"col_bool"},"COL_CHAR":{"type":["string","null"],"destination_column_name":"col_char"},"COL_CHARACTER":{"type":["string","null"],"destination_column_name":"col_character"},"COL_CLOB":{"type":["null","string"],"destination_column_name":"col_clob"},"COL_CURSOR":{"type":["integer","null"],"destination_column_name":"col_cursor"},"COL_DATE":{"type":["timestamp","null"],"destination_column_name":"col_date"},"COL_DECIMAL":{"type":["number","null"],"destination_column_name":"col_decimal"},"COL_DECFLOAT":{"type":["string","null"],"destination_column_name":"col_decfloat"},"COL_DOUBLE":{"type":["number","null"],"destination_column_name":"col_double"},"COL_GRAPHIC":{"type":["string","null"],"destination_column_name":"col_graphic"},"COL_INT":{"type":["integer_small","null"],"destination_column_name":"col_int"},"COL_REAL":{"type":["number_small","null"],"destination_column_name":"col_real"},"COL_SMALLINT":{"type":["null","integer_small"],"destination_column_name":"col_smallint"},"COL_TIME":{"type":["string","null"],"destination_column_name":"col_time"},"COL_TIMESTAMP":{"type":["timestamp","null"],"destination_column_name":"col_timestamp"},"COL_VARCHAR":{"type":["string","null"],"destination_column_name":"col_varchar"},"COL_VARGRAPHIC":{"type":["string","null"],"destination_column_name":"col_vargraphic"},"EXCLUDEDCOLUMN":{"type":["null","integer_small"],"destination_column_name":"excludedcolumn"},"ID":{"type":["integer"],"destination_column_name":"id"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":["ID"],"available_cursor_fields":["ID","COL_CHAR","COL_DECIMAL","COL_DECFLOAT","COL_BOOL","COL_TIMESTAMP","COL_CHARACTER","COL_VARCHAR","COL_DOUBLE","COL_BLOB","COL_TIME","COL_DATE","COL_INT","COL_CLOB","COL_VARGRAPHIC","EXCLUDEDCOLUMN","COL_CURSOR","COL_BIGINT","COL_REAL","COL_SMALLINT","COL_GRAPHIC"],"cursor_field":"COL_BIGINT","sync_mode":"incremental","destination_database":"db2_testdb:db2inst1","destination_table":"db2_test_table_olake","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/kafka/kafka_test.go b/tests/kafka/kafka_test.go index 5443fb0fd..6d343ad9c 100644 --- a/tests/kafka/kafka_test.go +++ b/tests/kafka/kafka_test.go @@ -5,34 +5,29 @@ import ( "github.com/datazip-inc/olake/tests/testutils" "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/require" ) type kafkaFormat struct { name string - cfg *testutils.IntegrationTest + // build runs inside the subtest, not beside it: every name a suite owns is derived from + // t.Name(), so both formats built against the parent would answer to the same one. + build func(t *testing.T) *integration.Test } -func kafkaFormats(t *testing.T) []kafkaFormat { - return []kafkaFormat{ - {name: "JSON-Format", cfg: kafkaJSONBaseConfig(t)}, - {name: "AVRO-Format", cfg: kafkaAvroBaseConfig(t)}, - } +var kafkaFormats = []kafkaFormat{ + {name: "JSON-Format", build: kafkaJSONBaseConfig}, + {name: "AVRO-Format", build: kafkaAvroBaseConfig}, } -func kafkaJSONBaseConfig(t *testing.T) *testutils.IntegrationTest { - return &testutils.IntegrationTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.Kafka), "json"), - Namespace: "topics", - ExpectedData: ExpectedKafkaJSONData, - ExpectedUpdatedData: ExpectedKafkaUpdatedJSONData, - DestinationDataTypeSchema: KafkaToDestinationJSONSchema, - UpdatedDestinationDataTypeSchema: UpdatedKafkaToDestinationJSONSchema, - DefaultCDCColumnsSchema: ExpectedKafkaDefaultCDCColumnsSchema, - ExecuteQuery: ExecuteQueryJSON, - DestinationDB: "kafka_topics", - PartitionRegex: "/{int_value,identity}", - ColumnToExclude: "col_excluded", - FilterConfig: `{ +func kafkaJSONBaseConfig(t *testing.T) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.Kafka, "topics", "kafka_topics", ExecuteQueryJSON, + testutils.WithDataFormat("json")) + require.NoError(t, err, "failed to build the test config") + cfg.PartitionRegex = "/{int_value,identity}" + cfg.ColumnToExclude = "col_excluded" + cfg.FilterConfig = `{ "logical_operator": "And", "conditions": [ { @@ -46,24 +41,25 @@ func kafkaJSONBaseConfig(t *testing.T) *testutils.IntegrationTest { "value": 100.00 } ] - }`, + }` + + return &integration.Test{ + TestConfig: cfg, + ExpectedData: ExpectedKafkaJSONData, + ExpectedUpdatedData: ExpectedKafkaUpdatedJSONData, + DestinationDataTypeSchema: KafkaToDestinationJSONSchema, + UpdatedDestinationDataTypeSchema: UpdatedKafkaToDestinationJSONSchema, + DefaultCDCColumnsSchema: ExpectedKafkaDefaultCDCColumnsSchema, } } -func kafkaAvroBaseConfig(t *testing.T) *testutils.IntegrationTest { - return &testutils.IntegrationTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.Kafka), "avro"), - Namespace: "topics", - ExpectedData: ExpectedKafkaAvroData, - ExpectedUpdatedData: ExpectedKafkaUpdatedAvroData, - DestinationDataTypeSchema: KafkaToDestinationAvroSchema, - UpdatedDestinationDataTypeSchema: UpdatedKafkaToDestinationAvroSchema, - DefaultCDCColumnsSchema: ExpectedKafkaDefaultCDCColumnsSchema, - ExecuteQuery: ExecuteQueryAvro, - DestinationDB: "kafka_topics", - PartitionRegex: "/{int64_value,identity}", - ColumnToExclude: "col_excluded", - FilterConfig: `{ +func kafkaAvroBaseConfig(t *testing.T) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.Kafka, "topics", "kafka_topics", ExecuteQueryAvro, + testutils.WithDataFormat("avro")) + require.NoError(t, err, "failed to build the test config") + cfg.PartitionRegex = "/{int64_value,identity}" + cfg.ColumnToExclude = "col_excluded" + cfg.FilterConfig = `{ "logical_operator": "And", "conditions": [ { @@ -77,24 +73,32 @@ func kafkaAvroBaseConfig(t *testing.T) *testutils.IntegrationTest { "value": 100.00 } ] - }`, + }` + + return &integration.Test{ + TestConfig: cfg, + ExpectedData: ExpectedKafkaAvroData, + ExpectedUpdatedData: ExpectedKafkaUpdatedAvroData, + DestinationDataTypeSchema: KafkaToDestinationAvroSchema, + UpdatedDestinationDataTypeSchema: UpdatedKafkaToDestinationAvroSchema, + DefaultCDCColumnsSchema: ExpectedKafkaDefaultCDCColumnsSchema, } } func TestKafkaDiscover(t *testing.T) { - for _, format := range kafkaFormats(t) { + for _, format := range kafkaFormats { t.Run(format.name, func(t *testing.T) { - format.cfg.TestDiscover(t) + format.build(t).TestDiscover(t) }) } } func TestKafkaSync(t *testing.T) { t.Parallel() - for _, format := range kafkaFormats(t) { + for _, format := range kafkaFormats { t.Run(format.name, func(t *testing.T) { t.Parallel() - format.cfg.TestSync(t) + format.build(t).TestSync(t) }) } } @@ -106,5 +110,21 @@ func TestKafka2PC(t *testing.T) { func TestKafkaRebalance(t *testing.T) { t.Parallel() - kafkaJSONBaseConfig(t).TestRebalance(t) + runRebalanceSuite(t, kafkaJSONBaseConfig(t)) } + +// TestKafkaCompatibility pins the backward-compatibility contract on the JSON format, the same single +// format Test2PCIntegration uses: the suite varies only the binary, and avro would add a +// schema-registry axis to the comparison. See tests/testutils/compatibility.go. +// func TestKafkaCompatibility(t *testing.T) { +// t.Parallel() +// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { +// base := kafkaJSONBaseConfig(t) +// cfg := &compatibility.Test{IntegrationTest: base} +// // The compatibility floor and its story live in compatibility_rules.json's kafka block. +// // Kafka pipelines interfere across groups: discover enumerates the whole broker, so +// // concurrent groups scan (and race the deletion of) each other's topics. +// cfg.SerialGroups = true +// return cfg +// }) +// } diff --git a/tests/kafka/kafka_util_test.go b/tests/kafka/kafka_util_test.go index 2b7d220ee..0393be897 100644 --- a/tests/kafka/kafka_util_test.go +++ b/tests/kafka/kafka_util_test.go @@ -14,8 +14,8 @@ import ( "github.com/apache/arrow-go/v18/arrow" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/require" "github.com/linkedin/goavro/v2" - "github.com/stretchr/testify/require" "github.com/twmb/franz-go/pkg/kadm" "github.com/twmb/franz-go/pkg/kerr" "github.com/twmb/franz-go/pkg/kgo" @@ -23,11 +23,14 @@ import ( ) const ( - partitionCount = 5 - rebalanceBulkMessageCount = 100_000 - rebalanceBulkPartition = int32(0) - rebalanceBulkBatchSize = 500 + partitionCount = 5 + rebalanceBulkMessageCount = 100_000 + rebalanceBulkPartition = int32(0) + rebalanceBulkBatchSize = 500 + // The broker advertises a listener per network: source.json names the one the driver container + // reaches (host.docker.internal:39092), and dialing it from the host fails on the advertised name. kafkaJSONIntegrationBroker = "127.0.0.1:29092" + kafkaAvroIntegrationBroker = "127.0.0.1:29192" avroSchemaRegistryURL = "http://127.0.0.1:8081" schemaRegistryTopic = "_schemas" topicDeletionAttempts = 8 @@ -129,13 +132,8 @@ var ( func ExecuteQueryJSON(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() - topic := testutils.TestTableName(conf) - var kafkaJSONBroker string - if conf.SourceBaseConfig != nil { - kafkaJSONBroker = conf.SourceBaseConfig.String("bootstrap_servers") - } else { - kafkaJSONBroker = kafkaJSONIntegrationBroker - } + topic := conf.GetTableName() + kafkaJSONBroker := kafkaJSONIntegrationBroker // kafka client client, err := kgo.NewClient( @@ -184,7 +182,7 @@ func ExecuteQueryJSON(ctx context.Context, t *testing.T, conf *testutils.TestCon case "insert_rebalance": addRebalanceBulkMessages(ctx, t, client, topic) - startRebalanceTrigger(ctx, t, suiteConsumerGroup(t, conf), topic, conf.HostStatsPath) + startRebalanceTrigger(ctx, t, suiteConsumerGroup(t, conf), topic, conf.GetFilePath("stats.json")) case "stop_rebalance": stopRebalanceTrigger() @@ -220,8 +218,8 @@ func addRebalanceBulkMessages(ctx context.Context, t *testing.T, client *kgo.Cli func suiteConsumerGroup(t *testing.T, conf *testutils.TestConfig) string { t.Helper() - consumerGroupID := testutils.ReadSourceConfig(t, conf.HostSourcePath).String("consumer_group_id") - require.NotEmpty(t, consumerGroupID, "no consumer_group_id in %s", conf.HostSourcePath) + consumerGroupID := conf.SourceBaseConfig.String("consumer_group_id") + require.NotEmpty(t, consumerGroupID, "no consumer_group_id in the source config of suite %q", conf.Suite) return consumerGroupID } @@ -245,7 +243,7 @@ func startRebalanceTrigger(ctx context.Context, t *testing.T, consumerGroupID, t close(done) }() - testutils.WaitForSyncProgress(rebalanceCtx, t, statsPath) + waitForSyncProgress(rebalanceCtx, t, statsPath) if rebalanceCtx.Err() != nil { return } @@ -290,13 +288,8 @@ func stopRebalanceTrigger() { func ExecuteQueryAvro(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() - topic := testutils.TestTableName(conf) - var kafkaAvroBroker string - if conf.SourceBaseConfig != nil { - kafkaAvroBroker = conf.SourceBaseConfig.String("bootstrap_servers") - } else { - kafkaAvroBroker = "127.0.0.1:29192" - } + topic := conf.GetTableName() + kafkaAvroBroker := kafkaAvroIntegrationBroker // kafka client client, err := kgo.NewClient( kgo.SeedBrokers(kafkaAvroBroker), diff --git a/tests/kafka/rebalance_test.go b/tests/kafka/rebalance_test.go new file mode 100644 index 000000000..d5843fa69 --- /dev/null +++ b/tests/kafka/rebalance_test.go @@ -0,0 +1,125 @@ +package kafka + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/require" +) + +// waitForSyncProgress blocks until the running sync has reported its first records in stats.json, +// which is how the rebalance trigger joins the group at a point where the sync is demonstrably +// mid-flight rather than at a guessed offset into a sleep. +func waitForSyncProgress(ctx context.Context, t *testing.T, statsPath string) { + t.Helper() + + require.Eventually(t, func() bool { + if ctx.Err() != nil { + return true + } + + var stats struct { + SyncedRecords int64 `json:"Synced Records"` + } + if err := testutils.UnmarshalFile(statsPath, &stats, false); err != nil { + return false + } + if stats.SyncedRecords > 0 { + t.Logf("sync started: %d records synced", stats.SyncedRecords) + return true + } + return false + }, testutils.SyncTimeout, time.Second) +} + +// runRebalanceSuite drives the consumer-group rebalance recovery test: the bulk topic is synced +// twice while a rival consumer takes partitions away and gives them back, and the destination must +// still hold every message exactly once. +func runRebalanceSuite(t *testing.T, cfg *integration.Test) { + ctx := t.Context() + testTable := cfg.GetTableName() + + t.Run("Sync", func(t *testing.T) { + // 1. Query on test table + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "create") + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "clean") + + // 2. Enable normalization, partition regex, filter and column exclusion in the catalog + if err := testutils.UpdateSelectedStreams(cfg.TestConfig, cfg.Namespace, cfg.PartitionRegex, cfg.FilterConfig, []string{testTable}, cfg.ColumnToExclude); err != nil { + t.Fatalf("failed to enable normalization and partition regex in the catalog: %s", err) + } + t.Logf("Enabled normalization and added partition regex in %s", cfg.GetFilePath("streams.json")) + + // 3. Run the recovery test against the legacy Iceberg writer + recoverFn := func(ctx context.Context, t *testing.T, testTable string) error { + return rebalanceRecovery(ctx, t, cfg, testTable) + } + if err := cfg.IcebergWriter(ctx, t, testTable, false, recoverFn); err != nil { + t.Fatalf("Kafka rebalance test failed: %v", err) + } + + // 4. Clean up + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") + t.Logf("%s rebalance test cleanup", cfg.Driver) + }) +} + +// rebalanceRecovery syncs the bulk topic across a rebalance and asserts the destination holds each +// message once: a consumer that resumes from the wrong offset shows up here as duplicates. +func rebalanceRecovery(ctx context.Context, t *testing.T, cfg *integration.Test, testTable string) error { + t.Log("Starting Kafka rebalance recovery test") + + integration.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + if err := testutils.ResetStateFile(cfg.TestConfig); err != nil { + return fmt.Errorf("failed to reset state file: %s", err) + } + + rebalanceTestCases := []struct { + name string + operation string + }{ + {name: "CDC - first rebalance sync", operation: "insert_rebalance"}, + // Stop the trigger consumer before resuming so it cannot hold partition assignments. + {name: "CDC - second rebalance sync", operation: "stop_rebalance"}, + } + + for _, tc := range rebalanceTestCases { + t.Run(tc.name, func(t *testing.T) { + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, tc.operation) + + if err := runRebalanceSync(ctx, t, cfg); err != nil { + t.Fatalf("%s failed: %v", tc.name, err) + } + }) + } + + integration.VerifyIcebergNoDuplicates(ctx, t, testTable, cfg.TestConfig.DestinationDB, "c", rebalanceBulkMessageCount) + + t.Log("Kafka rebalance recovery test completed successfully") + + integration.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + t.Logf("Dropped Iceberg table: %s", testTable) + + return nil +} + +// runRebalanceSync runs one stateful sync of the bulk topic. +func runRebalanceSync(ctx context.Context, t *testing.T, cfg *integration.Test) error { + t.Helper() + + cmd := testutils.SyncArgs(true, cfg.IcebergDestinationFile(), "--destination-database-prefix", cfg.UniqueID()) + + code, out, err := testutils.RunOlake(ctx, cfg.TestConfig, cmd...) + if err != nil { + return fmt.Errorf("sync exec error: %w\n%s", err, out) + } + if code != 0 { + return testutils.RenderOlakeFailure(code, nil, out) + } + t.Logf("sync completed successfully") + return nil +} diff --git a/tests/kafka/testdata/avro/source.json b/tests/kafka/testdata/avro/source.template.json similarity index 81% rename from tests/kafka/testdata/avro/source.json rename to tests/kafka/testdata/avro/source.template.json index 56f62e650..25b0e7170 100644 --- a/tests/kafka/testdata/avro/source.json +++ b/tests/kafka/testdata/avro/source.template.json @@ -3,7 +3,7 @@ "protocol": { "security_protocol": "PLAINTEXT" }, - "consumer_group_id": "kafka-Avro-integration-test-group", + "consumer_group_id": "kafka-${suite}-test-group", "threads_equal_total_partitions": true, "backoff_retry_count": 3, "schema_registry": { diff --git a/tests/kafka/testdata/avro/streams.template.json b/tests/kafka/testdata/avro/streams.template.json new file mode 100644 index 000000000..74b3a6b9c --- /dev/null +++ b/tests/kafka/testdata/avro/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"topics":[{"partition_regex":"","stream_name":"test_table_olake_${suite}","append_mode":true,"normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["float64_value","_olake_id","boolean","float_value","float32_value","col_excluded","int64_value","_kafka_timestamp","int_value","timestamp_value","_kafka_key","_kafka_offset","int32_value","_op_type","_kafka_partition","string_value","_olake_timestamp"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"test_table_olake_${suite}","namespace":"topics","type_schema":{"properties":{"_kafka_key":{"type":["string"],"destination_column_name":"_kafka_key"},"_kafka_offset":{"type":["integer"],"destination_column_name":"_kafka_offset"},"_kafka_partition":{"type":["integer_small"],"destination_column_name":"_kafka_partition"},"_kafka_timestamp":{"type":["timestamp_milli"],"destination_column_name":"_kafka_timestamp"},"_olake_id":{"type":["null","string"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"boolean":{"type":["boolean"],"destination_column_name":"boolean"},"col_excluded":{"type":["integer_small"],"destination_column_name":"col_excluded"},"float32_value":{"type":["number_small"],"destination_column_name":"float32_value"},"float64_value":{"type":["number"],"destination_column_name":"float64_value"},"float_value":{"type":["number_small"],"destination_column_name":"float_value"},"int32_value":{"type":["integer_small"],"destination_column_name":"int32_value"},"int64_value":{"type":["integer"],"destination_column_name":"int64_value"},"int_value":{"type":["integer_small"],"destination_column_name":"int_value"},"string_value":{"type":["string"],"destination_column_name":"string_value"},"timestamp_value":{"type":["timestamp"],"destination_column_name":"timestamp_value"}}},"supported_sync_modes":["strict_cdc"],"source_defined_primary_key":["_kafka_offset","_kafka_partition"],"available_cursor_fields":[],"sync_mode":"strict_cdc","destination_database":"kafka:topics","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":false,"append_mode":true}}}]} \ No newline at end of file diff --git a/tests/kafka/testdata/avro/test_streams.json b/tests/kafka/testdata/avro/test_streams.json deleted file mode 100644 index 7b20bc992..000000000 --- a/tests/kafka/testdata/avro/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"topics":[{"partition_regex":"","stream_name":"kafka_avro_test_table_olake","append_mode":true,"normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["float64_value","_olake_id","boolean","float_value","float32_value","col_excluded","int64_value","_kafka_timestamp","int_value","timestamp_value","_kafka_key","_kafka_offset","int32_value","_op_type","_kafka_partition","string_value","_olake_timestamp"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"kafka_avro_test_table_olake","namespace":"topics","type_schema":{"properties":{"_kafka_key":{"type":["string"],"destination_column_name":"_kafka_key"},"_kafka_offset":{"type":["integer"],"destination_column_name":"_kafka_offset"},"_kafka_partition":{"type":["integer_small"],"destination_column_name":"_kafka_partition"},"_kafka_timestamp":{"type":["timestamp_milli"],"destination_column_name":"_kafka_timestamp"},"_olake_id":{"type":["null","string"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"boolean":{"type":["boolean"],"destination_column_name":"boolean"},"col_excluded":{"type":["integer_small"],"destination_column_name":"col_excluded"},"float32_value":{"type":["number_small"],"destination_column_name":"float32_value"},"float64_value":{"type":["number"],"destination_column_name":"float64_value"},"float_value":{"type":["number_small"],"destination_column_name":"float_value"},"int32_value":{"type":["integer_small"],"destination_column_name":"int32_value"},"int64_value":{"type":["integer"],"destination_column_name":"int64_value"},"int_value":{"type":["integer_small"],"destination_column_name":"int_value"},"string_value":{"type":["string"],"destination_column_name":"string_value"},"timestamp_value":{"type":["timestamp"],"destination_column_name":"timestamp_value"}}},"supported_sync_modes":["strict_cdc"],"source_defined_primary_key":["_kafka_offset","_kafka_partition"],"available_cursor_fields":[],"sync_mode":"strict_cdc","destination_database":"kafka:topics","destination_table":"kafka_avro_test_table_olake","default_stream_properties":{"normalization":false,"append_mode":true}}}]} \ No newline at end of file diff --git a/tests/kafka/testdata/json/source.json b/tests/kafka/testdata/json/source.template.json similarity index 75% rename from tests/kafka/testdata/json/source.json rename to tests/kafka/testdata/json/source.template.json index 438772329..049e93689 100644 --- a/tests/kafka/testdata/json/source.json +++ b/tests/kafka/testdata/json/source.template.json @@ -3,7 +3,7 @@ "protocol": { "security_protocol": "PLAINTEXT" }, - "consumer_group_id": "kafka-Json-integration-test-group", + "consumer_group_id": "kafka-${suite}-test-group", "threads_equal_total_partitions": true, "backoff_retry_count": 3 } \ No newline at end of file diff --git a/tests/kafka/testdata/json/streams.template.json b/tests/kafka/testdata/json/streams.template.json new file mode 100644 index 000000000..ace208bcd --- /dev/null +++ b/tests/kafka/testdata/json/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"topics":[{"partition_regex":"","stream_name":"test_table_olake_${suite}","append_mode":true,"normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["_olake_id","_kafka_key","_kafka_partition","_kafka_timestamp","int_value","timestamp_value","_kafka_offset","float_value","_olake_timestamp","boolean","string_value","col_excluded","_op_type"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"test_table_olake_${suite}","namespace":"topics","type_schema":{"properties":{"_kafka_key":{"type":["string"],"destination_column_name":"_kafka_key"},"_kafka_offset":{"type":["integer"],"destination_column_name":"_kafka_offset"},"_kafka_partition":{"type":["integer_small"],"destination_column_name":"_kafka_partition"},"_kafka_timestamp":{"type":["timestamp_milli"],"destination_column_name":"_kafka_timestamp"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["null","string"],"destination_column_name":"_op_type","olake_column":true},"boolean":{"type":["boolean"],"destination_column_name":"boolean"},"col_excluded":{"type":["integer"],"destination_column_name":"col_excluded"},"float_value":{"type":["number"],"destination_column_name":"float_value"},"int_value":{"type":["integer"],"destination_column_name":"int_value"},"string_value":{"type":["string"],"destination_column_name":"string_value"},"timestamp_value":{"type":["timestamp"],"destination_column_name":"timestamp_value"}}},"supported_sync_modes":["strict_cdc"],"source_defined_primary_key":["_kafka_offset","_kafka_partition"],"available_cursor_fields":[],"sync_mode":"strict_cdc","destination_database":"kafka:topics","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":false,"append_mode":true}}}]} \ No newline at end of file diff --git a/tests/kafka/testdata/json/test_streams.json b/tests/kafka/testdata/json/test_streams.json deleted file mode 100644 index 626884447..000000000 --- a/tests/kafka/testdata/json/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"topics":[{"partition_regex":"","stream_name":"kafka_json_test_table_olake","append_mode":true,"normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["_olake_id","_kafka_key","_kafka_partition","_kafka_timestamp","int_value","timestamp_value","_kafka_offset","float_value","_olake_timestamp","boolean","string_value","col_excluded","_op_type"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"kafka_json_test_table_olake","namespace":"topics","type_schema":{"properties":{"_kafka_key":{"type":["string"],"destination_column_name":"_kafka_key"},"_kafka_offset":{"type":["integer"],"destination_column_name":"_kafka_offset"},"_kafka_partition":{"type":["integer_small"],"destination_column_name":"_kafka_partition"},"_kafka_timestamp":{"type":["timestamp_milli"],"destination_column_name":"_kafka_timestamp"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["null","string"],"destination_column_name":"_op_type","olake_column":true},"boolean":{"type":["boolean"],"destination_column_name":"boolean"},"col_excluded":{"type":["integer"],"destination_column_name":"col_excluded"},"float_value":{"type":["number"],"destination_column_name":"float_value"},"int_value":{"type":["integer"],"destination_column_name":"int_value"},"string_value":{"type":["string"],"destination_column_name":"string_value"},"timestamp_value":{"type":["timestamp"],"destination_column_name":"timestamp_value"}}},"supported_sync_modes":["strict_cdc"],"source_defined_primary_key":["_kafka_offset","_kafka_partition"],"available_cursor_fields":[],"sync_mode":"strict_cdc","destination_database":"kafka:topics","destination_table":"kafka_json_test_table_olake","default_stream_properties":{"normalization":false,"append_mode":true}}}]} \ No newline at end of file diff --git a/tests/mongodb/mongodb_test.go b/tests/mongodb/mongodb_test.go index 074e45e61..22b853f03 100644 --- a/tests/mongodb/mongodb_test.go +++ b/tests/mongodb/mongodb_test.go @@ -5,22 +5,18 @@ import ( "github.com/datazip-inc/olake/tests/testutils" "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/require" ) // mongodbBaseConfig returns an IntegrationTest pre-populated with all fields shared -func mongodbBaseConfig(t *testing.T) *testutils.IntegrationTest { - return &testutils.IntegrationTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.MongoDB)), - Namespace: "olake_mongodb_test", - ExpectedData: ExpectedMongoData, - DestinationDataTypeSchema: MongoToDestinationSchema, - DefaultCDCColumnsSchema: ExpectedMongoDBDefaultCDCColumnsSchema, - ExecuteQuery: ExecuteQuery, - DestinationDB: "mongodb_olake_mongodb_test", - CursorField: "id_cursor:id_int", - PartitionRegex: "/{_id,identity}", - ColumnToExclude: "excludedColumn", - FilterConfig: `{ +func mongodbBaseConfig(t *testing.T) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.MongoDB, "olake_mongodb_test", "mongodb_olake_mongodb_test", ExecuteQuery) + require.NoError(t, err, "failed to build the test config") + cfg.CursorField = "id_cursor:id_int" + cfg.PartitionRegex = "/{_id,identity}" + cfg.ColumnToExclude = "excludedColumn" + cfg.FilterConfig = `{ "logical_operator": "And", "conditions": [ { @@ -34,7 +30,13 @@ func mongodbBaseConfig(t *testing.T) *testutils.IntegrationTest { "value": "2022-07-01T15:30:00.000+00:00" } ] - }`, + }` + + return &integration.Test{ + TestConfig: cfg, + ExpectedData: ExpectedMongoData, + DestinationDataTypeSchema: MongoToDestinationSchema, + DefaultCDCColumnsSchema: ExpectedMongoDBDefaultCDCColumnsSchema, } } @@ -55,14 +57,36 @@ func TestMongodb2PC(t *testing.T) { mongodbBaseConfig(t).Test2PCIntegration(t) } -func TestMongodbPerformance(t *testing.T) { - config := &testutils.PerformanceTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.MongoDB)), - Namespace: "twitter_data", - BackfillStreams: testutils.GetBackfillStreamsFromCDC(performanceCDCStreams), - CDCStreams: performanceCDCStreams, - ExecuteQuery: ExecuteQuery, - } +// func TestMongodbPerformance(t *testing.T) { +// cfg, err := testutils.NewTestConfig(constants.MongoDB, "twitter_data", "", ExecuteQuery, "") +// require.NoError(t, err, "failed to build the test config") - config.TestPerformance(t) -} +// perf := &performance.Test{ +// TestConfig: cfg, +// BackfillStreams: performance.GetBackfillStreamsFromCDC(performanceCDCStreams), +// CDCStreams: performanceCDCStreams, +// } + +// perf.TestPerformance(t) +// } + +// TestMongodbCompatibility pins the backward-compatibility contract for the driver owning the v5 gate +// (BSON DateTime decoded as UTC time.Time at any depth, constants/state_version.go). v0.6.1 is +// the newest release still on state version 4, so it is the one that exercises it. +// +// _id and _olake_id are volatile here, unlike every other driver: the seed inserts documents +// without an _id, so the server generates a fresh ObjectID per run and _olake_id, which hashes the +// primary key, follows it. Both are still compared by TYPE -- only their values are exempt. +// func TestMongodbCompatibility(t *testing.T) { +// t.Parallel() +// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { +// base := mongodbBaseConfig(t) +// base.ExpectedUpdatedData = ExpectedUpdatedData +// base.UpdatedDestinationDataTypeSchema = UpdatedMongoToDestinationSchema +// cfg := &compatibility.Test{IntegrationTest: base} +// cfg.ExtraVolatileColumns = []string{"_id", "_olake_id"} +// // Type tags for compatibility_rules.json's mongodb rules (G1: id_regex value change at #657). +// cfg.ColumnTypes = map[string][]string{"id_regex": {"regex"}} +// return cfg +// }) +// } diff --git a/tests/mongodb/mongodb_util_test.go b/tests/mongodb/mongodb_util_test.go index 187e4a175..0e24359b3 100644 --- a/tests/mongodb/mongodb_util_test.go +++ b/tests/mongodb/mongodb_util_test.go @@ -9,26 +9,24 @@ import ( "github.com/apache/arrow-go/v18/arrow" "github.com/datazip-inc/olake/tests/testutils" - "github.com/stretchr/testify/require" + "github.com/datazip-inc/olake/tests/testutils/require" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) -// MongoDB connection constants -const ( - MongoDBPort = 27017 - MongoDBDatabase = "olake_mongodb_test" - MongoDBReplicaSet = "rs0" - MongoDBAdminUser = "admin" - MongoDBAdminPass = "password" -) - var ( nestedDoc = bson.M{ "nested_string": "nested_value", "nested_int": 42, + // A BSON DateTime below the top level, which is what the state-version-5 gate governs + // (drivers/mongodb/internal/mon.go: at v>=5 a custom registry decodes it to a UTC + // time.Time, at v<=4 the stock decoder yields a primitive.DateTime). Both marshal to the + // same string for an in-range year -- primitive.DateTime.MarshalJSON already normalizes to + // UTC -- so this pins that the decoder swap did NOT change in-range values. The versions + // only diverge outside [0,9999], where v<=4 fails json.Marshal outright. + "nested_timestamp": time.Date(2023, 1, 1, 12, 0, 0, 0, time.UTC), } ) @@ -39,31 +37,27 @@ var performanceCDCStreams = []string{"tweets_cdc"} func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() - var connStr string + // directConnection because the replica set advertises its member as host.docker.internal, + // which only the driver's container resolves; the harness dials the published port directly. config := conf.SourceBaseConfig - if config != nil { - connStr = fmt.Sprintf( - "mongodb://%s:%s@%s/?authSource=%s&readPreference=%s", - config.String("username"), - config.String("password"), - strings.Join(config.Strings("hosts"), ","), - config.String("authdb"), - config.String("read_preference"), - ) - } else { - connStr = fmt.Sprintf("mongodb://%s:%s@localhost:%d/admin?replicaSet=%s&directConnection=true", - MongoDBAdminUser, MongoDBAdminPass, MongoDBPort, MongoDBReplicaSet) - } + connStr := fmt.Sprintf( + "mongodb://%s:%s@%s/?authSource=%s&readPreference=%s&directConnection=true", + config.String("username"), + config.String("password"), + strings.Join(config.Hosts("hosts"), ","), + config.String("authdb"), + config.String("read_preference"), + ) client, err := mongo.Connect(ctx, options.Client().ApplyURI(connStr)) - require.NoError(t, err, "Failed to connect to MongoDB replica set at localhost:%d", MongoDBPort) + require.NoError(t, err, "failed to connect to mongodb at %s", strings.Join(config.Hosts("hosts"), ",")) defer func() { if err := client.Disconnect(ctx); err != nil { t.Logf("warning: failed to disconnect from MongoDB: %v", err) } }() - integrationTestCollection := testutils.TestTableName(conf) - db := client.Database(MongoDBDatabase) + integrationTestCollection := conf.GetTableName() + db := client.Database(config.String("database")) collection := db.Collection(integrationTestCollection) switch operation { @@ -179,44 +173,44 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, } return - case "bulk_cdc_data_insert": - backfillStreams := testutils.GetBackfillStreamsFromCDC(performanceCDCStreams) - totalRows := 15000000 + // case "bulk_cdc_data_insert": + // backfillStreams := performance.GetBackfillStreamsFromCDC(performanceCDCStreams) + // totalRows := 15000000 - // TODO: insert data in batch - // insert the data into the cdc tables concurrently - err := testutils.Concurrent(ctx, performanceCDCStreams, len(performanceCDCStreams), func(ctx context.Context, cdcStream string, executionNumber int) error { - srcColl := client.Database(config.String("database")).Collection(backfillStreams[executionNumber]) - destColl := client.Database(config.String("database")).Collection(cdcStream) + // // TODO: insert data in batch + // // insert the data into the cdc tables concurrently + // err := testutils.Concurrent(ctx, performanceCDCStreams, len(performanceCDCStreams), func(ctx context.Context, cdcStream string, executionNumber int) error { + // srcColl := client.Database(config.String("database")).Collection(backfillStreams[executionNumber]) + // destColl := client.Database(config.String("database")).Collection(cdcStream) - cursor, err := srcColl.Find(ctx, bson.D{}, options.Find().SetLimit(int64(totalRows))) - if err != nil { - return fmt.Errorf("stream: %s, error: %s", cdcStream, err) - } - defer cursor.Close(ctx) + // cursor, err := srcColl.Find(ctx, bson.D{}, options.Find().SetLimit(int64(totalRows))) + // if err != nil { + // return fmt.Errorf("stream: %s, error: %s", cdcStream, err) + // } + // defer cursor.Close(ctx) - var docs []interface{} - for cursor.Next(ctx) { - var doc bson.M - if err := cursor.Decode(&doc); err != nil { - return err - } - docs = append(docs, doc) - } - if err := cursor.Err(); err != nil { - return err - } - if len(docs) == 0 { - return nil - } - _, err = destColl.InsertMany(ctx, docs) - if err != nil { - return fmt.Errorf("stream: %s, error: %s", cdcStream, err) - } - return nil - }) - require.NoError(t, err, fmt.Sprintf("failed to execute %s operation", operation), err) - return + // var docs []interface{} + // for cursor.Next(ctx) { + // var doc bson.M + // if err := cursor.Decode(&doc); err != nil { + // return err + // } + // docs = append(docs, doc) + // } + // if err := cursor.Err(); err != nil { + // return err + // } + // if len(docs) == 0 { + // return nil + // } + // _, err = destColl.InsertMany(ctx, docs) + // if err != nil { + // return fmt.Errorf("stream: %s, error: %s", cdcStream, err) + // } + // return nil + // }) + // require.NoError(t, err, fmt.Sprintf("failed to execute %s operation", operation), err) + // return } } @@ -266,7 +260,7 @@ var ExpectedMongoData = map[string]interface{}{ "id_bool": true, "created_timestamp": int32(1754905992), "id_regex": `{"Pattern":"test.*","Options":"i"}`, - "id_nested": `{"nested_int":42,"nested_string":"nested_value"}`, + "id_nested": `{"nested_int":42,"nested_string":"nested_value","nested_timestamp":"2023-01-01T12:00:00Z"}`, "id_minkey": `{}`, "id_maxkey": `{}`, "name_varchar": "varchar_val", @@ -280,7 +274,7 @@ var ExpectedUpdatedData = map[string]interface{}{ "id_bool": false, "created_timestamp": int32(1754905699), "id_regex": `{"Pattern":"updated.*","Options":"i"}`, - "id_nested": `{"nested_int":42,"nested_string":"nested_value"}`, + "id_nested": `{"nested_int":42,"nested_string":"nested_value","nested_timestamp":"2023-01-01T12:00:00Z"}`, "id_minkey": `{}`, "id_maxkey": `{}`, "name_varchar": "updated varchar", diff --git a/tests/mongodb/testdata/source.json b/tests/mongodb/testdata/source.template.json similarity index 80% rename from tests/mongodb/testdata/source.json rename to tests/mongodb/testdata/source.template.json index 14660f7a4..708838c40 100644 --- a/tests/mongodb/testdata/source.json +++ b/tests/mongodb/testdata/source.template.json @@ -1,7 +1,7 @@ { "hosts": ["host.docker.internal:27017"], - "username": "mongodb", - "password": "secure_password123", + "username": "admin", + "password": "password", "authdb": "admin", "replica_set": "rs0", "read_preference": "secondaryPreferred", diff --git a/tests/mongodb/testdata/streams.template.json b/tests/mongodb/testdata/streams.template.json new file mode 100644 index 000000000..b726f8493 --- /dev/null +++ b/tests/mongodb/testdata/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"olake_mongodb_test":[{"partition_regex":"","stream_name":"test_table_olake_${suite}","normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["id_regex","name_varchar","_olake_id","_cdc_resume_token","id_maxkey","id_minkey","id_nested","_id","id_bigint","id_double","created_timestamp","id_bool","id_nil","id_int","id_timestamp","id","_cdc_timestamp","_op_type","id_cursor","_olake_timestamp","excludedColumn"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"test_table_olake_${suite}","namespace":"olake_mongodb_test","type_schema":{"properties":{"_cdc_resume_token":{"type":["string","null"],"destination_column_name":"_cdc_resume_token","olake_column":true},"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_id":{"type":["string"],"destination_column_name":"_id"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"created_timestamp":{"type":["integer_small"],"destination_column_name":"created_timestamp"},"excludedColumn":{"type":["integer_small"],"destination_column_name":"excludedcolumn"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_bigint":{"type":["integer"],"destination_column_name":"id_bigint"},"id_bool":{"type":["boolean"],"destination_column_name":"id_bool"},"id_cursor":{"type":["integer_small"],"destination_column_name":"id_cursor"},"id_double":{"type":["number"],"destination_column_name":"id_double"},"id_int":{"type":["integer_small"],"destination_column_name":"id_int"},"id_maxkey":{"type":["unknown"],"destination_column_name":"id_maxkey"},"id_minkey":{"type":["unknown"],"destination_column_name":"id_minkey"},"id_nested":{"type":["object"],"destination_column_name":"id_nested"},"id_nil":{"type":["null"],"destination_column_name":"id_nil"},"id_regex":{"type":["unknown"],"destination_column_name":"id_regex"},"id_timestamp":{"type":["timestamp"],"destination_column_name":"id_timestamp"},"name_varchar":{"type":["string"],"destination_column_name":"name_varchar"}}},"supported_sync_modes":["full_refresh","incremental","cdc","strict_cdc"],"source_defined_primary_key":["_id"],"available_cursor_fields":["name_varchar","id_bigint","id","id_regex","id_nested","_id","id_double","id_nil","excludedColumn","id_maxkey","id_timestamp","id_int","id_minkey","created_timestamp","id_bool","id_cursor"],"sync_mode":"cdc","destination_database":"mongodb:olake_mongodb_test","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":false,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/mongodb/testdata/test_streams.json b/tests/mongodb/testdata/test_streams.json deleted file mode 100644 index 26fa6c938..000000000 --- a/tests/mongodb/testdata/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"olake_mongodb_test":[{"partition_regex":"","stream_name":"mongodb_test_table_olake","normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["id_regex","name_varchar","_olake_id","_cdc_resume_token","id_maxkey","id_minkey","id_nested","_id","id_bigint","id_double","created_timestamp","id_bool","id_nil","id_int","id_timestamp","id","_cdc_timestamp","_op_type","id_cursor","_olake_timestamp","excludedColumn"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"mongodb_test_table_olake","namespace":"olake_mongodb_test","type_schema":{"properties":{"_cdc_resume_token":{"type":["string","null"],"destination_column_name":"_cdc_resume_token","olake_column":true},"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_id":{"type":["string"],"destination_column_name":"_id"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"created_timestamp":{"type":["integer_small"],"destination_column_name":"created_timestamp"},"excludedColumn":{"type":["integer_small"],"destination_column_name":"excludedcolumn"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_bigint":{"type":["integer"],"destination_column_name":"id_bigint"},"id_bool":{"type":["boolean"],"destination_column_name":"id_bool"},"id_cursor":{"type":["integer_small"],"destination_column_name":"id_cursor"},"id_double":{"type":["number"],"destination_column_name":"id_double"},"id_int":{"type":["integer_small"],"destination_column_name":"id_int"},"id_maxkey":{"type":["unknown"],"destination_column_name":"id_maxkey"},"id_minkey":{"type":["unknown"],"destination_column_name":"id_minkey"},"id_nested":{"type":["object"],"destination_column_name":"id_nested"},"id_nil":{"type":["null"],"destination_column_name":"id_nil"},"id_regex":{"type":["unknown"],"destination_column_name":"id_regex"},"id_timestamp":{"type":["timestamp"],"destination_column_name":"id_timestamp"},"name_varchar":{"type":["string"],"destination_column_name":"name_varchar"}}},"supported_sync_modes":["full_refresh","incremental","cdc","strict_cdc"],"source_defined_primary_key":["_id"],"available_cursor_fields":["name_varchar","id_bigint","id","id_regex","id_nested","_id","id_double","id_nil","excludedColumn","id_maxkey","id_timestamp","id_int","id_minkey","created_timestamp","id_bool","id_cursor"],"sync_mode":"cdc","destination_database":"mongodb:olake_mongodb_test","destination_table":"mongodb_test_table_olake","default_stream_properties":{"normalization":false,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/mssql/mssql_test.go b/tests/mssql/mssql_test.go index 61c85b802..a78025935 100644 --- a/tests/mssql/mssql_test.go +++ b/tests/mssql/mssql_test.go @@ -5,23 +5,19 @@ import ( "github.com/datazip-inc/olake/tests/testutils" "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/require" ) // mssqlBaseConfig returns an IntegrationTest pre-populated with all fields shared // by the mssql suites. -func mssqlBaseConfig(t *testing.T) *testutils.IntegrationTest { - return &testutils.IntegrationTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.MSSQL)), - Namespace: "dbo", - ExpectedData: ExpectedMSSQLData, - DestinationDataTypeSchema: MSSQLToDestinationSchema, - DefaultCDCColumnsSchema: ExpectedMSSQLDefaultCDCColumnsSchema, - ExecuteQuery: ExecuteQuery, - ColumnToExclude: "excludedColumn", - DestinationDB: "mssql_olake_mssql_test_dbo", - CursorField: "id_cursor:col_int", - PartitionRegex: "/{id,identity}", - FilterConfig: `{ +func mssqlBaseConfig(t *testing.T) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.MSSQL, "dbo", "mssql_olake_mssql_test_dbo", ExecuteQuery) + require.NoError(t, err, "failed to build the test config") + cfg.ColumnToExclude = "excludedColumn" + cfg.CursorField = "id_cursor:col_int" + cfg.PartitionRegex = "/{id,identity}" + cfg.FilterConfig = `{ "logical_operator": "And", "conditions": [ { @@ -35,7 +31,13 @@ func mssqlBaseConfig(t *testing.T) *testutils.IntegrationTest { "value": "2022-07-01T15:30:00.000+00:00" } ] - }`, + }` + + return &integration.Test{ + TestConfig: cfg, + ExpectedData: ExpectedMSSQLData, + DestinationDataTypeSchema: MSSQLToDestinationSchema, + DefaultCDCColumnsSchema: ExpectedMSSQLDefaultCDCColumnsSchema, } } @@ -55,3 +57,17 @@ func TestMSSQL2PC(t *testing.T) { t.Parallel() mssqlBaseConfig(t).Test2PCIntegration(t) } + +// TestMSSQLCompatibility pins the backward-compatibility contract: the same scenarios run on a released +// baseline image and on this build after the initial load, and the destinations must match. +// See tests/testutils/compatibility.go. +// func TestMSSQLCompatibility(t *testing.T) { +// t.Parallel() +// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { +// base := mssqlBaseConfig(t) +// base.ExpectedUpdatedData = ExpectedUpdatedMSSQLData +// base.UpdatedDestinationDataTypeSchema = MSSQLToDestinationSchema +// cfg := &compatibility.Test{IntegrationTest: base} +// return cfg +// }) +// } diff --git a/tests/mssql/mssql_util_test.go b/tests/mssql/mssql_util_test.go index 019640611..bc483f144 100644 --- a/tests/mssql/mssql_util_test.go +++ b/tests/mssql/mssql_util_test.go @@ -11,9 +11,9 @@ import ( "github.com/apache/arrow-go/v18/arrow" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/require" "github.com/jmoiron/sqlx" _ "github.com/microsoft/go-mssqldb" - "github.com/stretchr/testify/require" ) // cdcMetadataMu serializes CDC enable/disable: both write the server-wide msdb.dbo.cdc_jobs, so two @@ -53,19 +53,16 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, // and fail the loser as the deadlock victim. This has to resolve to the same name // variantSourceOverride writes into the suite's source.json, or olake and these queries end up // in different databases. 01-init.sql provisions each with CDC enabled. - var connStr string - if config := conf.SourceBaseConfig; config != nil { - connStr = fmt.Sprintf("sqlserver://%s:%s@%s:%d?database=%s&encrypt=disable", - config.String("username"), - config.String("password"), - config.String("host"), - config.Int("port"), - testutils.SuiteDatabase(config.String("database"), conf.Suite), - ) - } else { - connStr = fmt.Sprintf("sqlserver://sa:Password!123@localhost:1433?database=%s&encrypt=disable", - testutils.SuiteDatabase("olake_mssql_test", conf.Suite)) - } + config := conf.SourceBaseConfig + dbName := config.String("database") + ensureSuiteDatabase(ctx, t, config, dbName) + connStr := fmt.Sprintf("sqlserver://%s:%s@%s:%d?database=%s&encrypt=disable", + config.String("username"), + config.String("password"), + config.Host("host"), + config.Int("port"), + dbName, + ) db, err := sqlx.ConnectContext(ctx, "sqlserver", connStr) require.NoError(t, err, "failed to connect to mssql") @@ -74,7 +71,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, }() // integration test uses only one stream for testing - integrationTestTable := testutils.TestTableName(conf) + integrationTestTable := conf.GetTableName() // A capture instance is SQL Server’s logical CDC stream for a table. captureInstance := fmt.Sprintf("dbo_%s", integrationTestTable) @@ -162,6 +159,9 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, `, integrationTestTable, captureInstance) require.NoError(t, execCDCMetadata(ctx, t, db, enableTableCDC), "failed to enable CDC on integration test table") + ensureFastCDCPolling(ctx, t, db) + startCDCCapture(ctx, t, db) + // Wait until current_max_lsn >= start_lsn of the capture instance so CDC is ready for sync verifyCDCEnabled(ctx, t, db, captureInstance) @@ -369,6 +369,114 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, } } +// suiteDatabasesEnsured tracks the databases this process has provisioned, so ensureSuiteDatabase +// touches master once per suite, not once per operation. +var ( + suiteDatabasesEnsured = map[string]bool{} + suiteDatabasesEnsuredMu sync.Mutex +) + +// ensureSuiteDatabase creates the suite's CDC-enabled database when the volume lacks it. Lazy and +// harness-owned rather than 01-init.sql: an init script runs only on a fresh volume, and every new +// suite needed a hand-edit there plus a refresh -- this way any volume converges on first touch. +// Runs against master, since the suite connection names a database that cannot exist before it does. +func ensureSuiteDatabase(ctx context.Context, t *testing.T, config testutils.SourceConfig, dbName string) { + t.Helper() + suiteDatabasesEnsuredMu.Lock() + defer suiteDatabasesEnsuredMu.Unlock() + if suiteDatabasesEnsured[dbName] { + return + } + + master, err := sqlx.ConnectContext(ctx, "sqlserver", + fmt.Sprintf("sqlserver://%s:%s@%s:%d?database=master&encrypt=disable", + config.String("username"), config.String("password"), config.Host("host"), config.Int("port"))) + require.NoError(t, err, "failed to connect to master to provision %s", dbName) + defer func() { require.NoError(t, master.Close()) }() + + _, err = master.ExecContext(ctx, fmt.Sprintf(`IF DB_ID(N'%s') IS NULL CREATE DATABASE [%s];`, dbName, dbName)) + require.NoError(t, err, "failed to create database %s", dbName) + // sp_cdc_enable_db acts on the current database; USE binds it within this one batch. Through + // execCDCMetadata for the msdb mutex: enabling CDC writes the same shared job metadata. + require.NoError(t, execCDCMetadata(ctx, t, master, fmt.Sprintf( + `USE [%s]; IF EXISTS (SELECT 1 FROM sys.databases WHERE name = N'%s' AND is_cdc_enabled = 0) EXEC sys.sp_cdc_enable_db;`, + dbName, dbName)), "failed to enable CDC on database %s", dbName) + suiteDatabasesEnsured[dbName] = true +} + +// cdcCaptureJobStatus reports this database's capture job: 1 running, 0 stopped, -1 not registered. +// Pinned to the agent's current session, so rows a previous one left behind do not read as running. +func cdcCaptureJobStatus(ctx context.Context, db *sqlx.DB) (int, error) { + var status int + err := db.QueryRowContext(ctx, ` + DECLARE @job_id UNIQUEIDENTIFIER = ( + SELECT job_id FROM msdb.dbo.sysjobs WHERE name = N'cdc.' + DB_NAME() + N'_capture'); + SELECT CASE + WHEN @job_id IS NULL THEN -1 + WHEN EXISTS ( + SELECT 1 FROM msdb.dbo.sysjobactivity + WHERE job_id = @job_id + AND session_id = (SELECT MAX(session_id) FROM msdb.dbo.syssessions) + AND start_execution_date IS NOT NULL AND stop_execution_date IS NULL) THEN 1 + ELSE 0 END;`).Scan(&status) + return status, err +} + +// startCDCCapture starts the database's capture job and waits for the agent to report it running. +// drop-all's sp_cdc_disable_table stops the job as it removes the last capture instance, and +// re-enabling the table does not start it again -- so without this the LSN verifyCDCEnabled waits +// for never moves. +func startCDCCapture(ctx context.Context, t *testing.T, db *sqlx.DB) { + t.Helper() + const ( + pollInterval = 500 * time.Millisecond + timeout = 30 * time.Second + ) + + deadline := time.Now().Add(timeout) + for { + switch status, err := cdcCaptureJobStatus(ctx, db); { + case err != nil: + t.Logf("startCDCCapture: read capture job status: %s", err) + case status != 0: + return // running, or no job to start + } + // A start the agent is already working on is refused ("already running", "already has a + // pending request"); the agent raises those, so T-SQL cannot trap them and the poll above + // is what settles whether the job came up. + if err := execCDCMetadata(ctx, t, db, ` + DECLARE @job SYSNAME = N'cdc.' + DB_NAME() + N'_capture'; + IF EXISTS (SELECT 1 FROM msdb.dbo.sysjobs WHERE name = @job) + EXEC msdb.dbo.sp_start_job @job_name = @job;`); err != nil { + t.Logf("startCDCCapture: start request refused: %s", err) + } + if time.Now().After(deadline) { + t.Fatalf("CDC capture job did not start within %v", timeout) + } + time.Sleep(pollInterval) + } +} + +// ensureFastCDCPolling drops this database's CDC capture job to the minimum 1s polling interval +// (default 5s) -- the cycle every create / wait-cdc-catchup, and the driver's own catch-up, waits +// out. Only the interval is written: it takes effect on the next start, which is startCDCCapture's +// job, and starting it here too leaves that one racing the agent's own pending request. Best-effort +// -- a job still on 5s is slower, not wrong -- and a no-op once it reports 1s. +func ensureFastCDCPolling(ctx context.Context, t *testing.T, db *sqlx.DB) { + t.Helper() + + var interval int + err := db.QueryRowContext(ctx, + `SELECT pollinginterval FROM msdb.dbo.cdc_jobs WHERE database_id = DB_ID() AND job_type = N'capture'`).Scan(&interval) + if err != nil || interval == 1 { + return // no capture job registered yet, or already fast + } + if err := execCDCMetadata(ctx, t, db, + `EXEC sys.sp_cdc_change_job @job_type = N'capture', @pollinginterval = 1;`); err != nil { + t.Logf("could not lower the CDC capture polling interval (staying on the 5s default): %s", err) + } +} + // verifyCDCEnabled polls until sys.fn_cdc_get_max_lsn() >= start_lsn of the // given capture instance, so the capture instance is ready for CDC sync. func verifyCDCEnabled(ctx context.Context, t *testing.T, db *sqlx.DB, captureInstance string) { diff --git a/tests/mssql/testdata/source.json b/tests/mssql/testdata/source.template.json similarity index 81% rename from tests/mssql/testdata/source.json rename to tests/mssql/testdata/source.template.json index 6ac751331..3598fcdf3 100644 --- a/tests/mssql/testdata/source.json +++ b/tests/mssql/testdata/source.template.json @@ -1,7 +1,7 @@ { "host": "host.docker.internal", "port": 1433, - "database": "olake_mssql_test", + "database": "olake_mssql_test_${suite}", "username": "sa", "password": "Password!123", "ssl": { diff --git a/tests/mssql/testdata/streams.template.json b/tests/mssql/testdata/streams.template.json new file mode 100644 index 000000000..88adcf335 --- /dev/null +++ b/tests/mssql/testdata/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"dbo":[{"partition_regex":"","stream_name":"test_table_olake_${suite}","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["col_sql_variant","id","col_uniqueidentifier","col_datetime","col_nchar","col_char","col_bigint","col_datetime2_nullable","col_bit","col_ntext","created_at","_cdc_timestamp","col_varchar","col_tinyint","excludedColumn","col_smalldatetime","col_int_nullable","id_cursor","col_image","col_hierarchyid","_olake_timestamp","col_datetimeoffset","col_smallint","col_nvarchar","col_int","col_sysname","_olake_id","col_varchar_nullable","col_float","col_date","col_decimal","col_smallmoney","col_text","col_xml","col_real","_cdc_seqval","col_time","_op_type","col_datetime2","col_numeric","_cdc_start_lsn","col_money"],"sync_new_columns":true}},{"partition_regex":"","stream_name":"systranschemas","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["endlsn","_cdc_timestamp","_op_type","_cdc_seqval","typeid","_olake_id","tabid","_cdc_start_lsn","startlsn","_olake_timestamp"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"test_table_olake_${suite}","namespace":"dbo","type_schema":{"properties":{"_cdc_seqval":{"type":["string","null"],"destination_column_name":"_cdc_seqval","olake_column":true},"_cdc_start_lsn":{"type":["string","null"],"destination_column_name":"_cdc_start_lsn","olake_column":true},"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["null","timestamp_micro"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"col_bigint":{"type":["integer"],"destination_column_name":"col_bigint"},"col_bit":{"type":["boolean"],"destination_column_name":"col_bit"},"col_char":{"type":["string"],"destination_column_name":"col_char"},"col_date":{"type":["timestamp"],"destination_column_name":"col_date"},"col_datetime":{"type":["timestamp"],"destination_column_name":"col_datetime"},"col_datetime2":{"type":["timestamp_micro"],"destination_column_name":"col_datetime2"},"col_datetime2_nullable":{"type":["timestamp_micro","null"],"destination_column_name":"col_datetime2_nullable"},"col_datetimeoffset":{"type":["timestamp_micro"],"destination_column_name":"col_datetimeoffset"},"col_decimal":{"type":["number"],"destination_column_name":"col_decimal"},"col_float":{"type":["number"],"destination_column_name":"col_float"},"col_hierarchyid":{"type":["string"],"destination_column_name":"col_hierarchyid"},"col_image":{"type":["string"],"destination_column_name":"col_image"},"col_int":{"type":["integer_small"],"destination_column_name":"col_int"},"col_int_nullable":{"type":["integer_small","null"],"destination_column_name":"col_int_nullable"},"col_money":{"type":["number"],"destination_column_name":"col_money"},"col_nchar":{"type":["string"],"destination_column_name":"col_nchar"},"col_ntext":{"type":["string"],"destination_column_name":"col_ntext"},"col_numeric":{"type":["number"],"destination_column_name":"col_numeric"},"col_nvarchar":{"type":["string"],"destination_column_name":"col_nvarchar"},"col_real":{"type":["number_small"],"destination_column_name":"col_real"},"col_smalldatetime":{"type":["timestamp"],"destination_column_name":"col_smalldatetime"},"col_smallint":{"type":["integer_small"],"destination_column_name":"col_smallint"},"col_smallmoney":{"type":["number"],"destination_column_name":"col_smallmoney"},"col_sql_variant":{"type":["string"],"destination_column_name":"col_sql_variant"},"col_sysname":{"type":["string"],"destination_column_name":"col_sysname"},"col_text":{"type":["string"],"destination_column_name":"col_text"},"col_time":{"type":["string"],"destination_column_name":"col_time"},"col_tinyint":{"type":["integer_small"],"destination_column_name":"col_tinyint"},"col_uniqueidentifier":{"type":["string"],"destination_column_name":"col_uniqueidentifier"},"col_varchar":{"type":["string"],"destination_column_name":"col_varchar"},"col_varchar_nullable":{"type":["string","null"],"destination_column_name":"col_varchar_nullable"},"col_xml":{"type":["string"],"destination_column_name":"col_xml"},"created_at":{"type":["timestamp_micro"],"destination_column_name":"created_at"},"excludedColumn":{"type":["integer_small","null"],"destination_column_name":"excludedcolumn"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_cursor":{"type":["integer_small"],"destination_column_name":"id_cursor"}}},"supported_sync_modes":["incremental","cdc","strict_cdc","full_refresh"],"source_defined_primary_key":["id"],"available_cursor_fields":["col_int_nullable","col_varchar","col_time","col_smalldatetime","col_int","col_smallmoney","col_uniqueidentifier","col_sysname","col_varchar_nullable","id_cursor","col_bigint","col_money","col_char","col_nvarchar","col_datetime","col_datetime2_nullable","col_ntext","col_date","col_datetime2","col_datetimeoffset","excludedColumn","col_numeric","col_image","created_at","id","col_smallint","col_float","col_real","col_nchar","col_hierarchyid","col_sql_variant","col_xml","col_tinyint","col_decimal","col_bit","col_text"],"sync_mode":"cdc","destination_database":"mssql_olake_mssql_test_${suite}:dbo","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":true,"append_mode":false}}},{"stream":{"name":"systranschemas","namespace":"dbo","type_schema":{"properties":{"_cdc_seqval":{"type":["string","null"],"destination_column_name":"_cdc_seqval","olake_column":true},"_cdc_start_lsn":{"type":["string","null"],"destination_column_name":"_cdc_start_lsn","olake_column":true},"_cdc_timestamp":{"type":["null","timestamp_micro"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"endlsn":{"type":["string"],"destination_column_name":"endlsn"},"startlsn":{"type":["string"],"destination_column_name":"startlsn"},"tabid":{"type":["integer_small"],"destination_column_name":"tabid"},"typeid":{"type":["integer_small"],"destination_column_name":"typeid"}}},"supported_sync_modes":["incremental","cdc","strict_cdc","full_refresh"],"source_defined_primary_key":[],"available_cursor_fields":["tabid","startlsn","endlsn","typeid"],"sync_mode":"cdc","destination_database":"mssql_olake_mssql_test_${suite}:dbo","destination_table":"systranschemas","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/mssql/testdata/test_streams.json b/tests/mssql/testdata/test_streams.json deleted file mode 100644 index 90a495e04..000000000 --- a/tests/mssql/testdata/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"dbo":[{"partition_regex":"","stream_name":"mssql_test_table_olake","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["col_sql_variant","id","col_uniqueidentifier","col_datetime","col_nchar","col_char","col_bigint","col_datetime2_nullable","col_bit","col_ntext","created_at","_cdc_timestamp","col_varchar","col_tinyint","excludedColumn","col_smalldatetime","col_int_nullable","id_cursor","col_image","col_hierarchyid","_olake_timestamp","col_datetimeoffset","col_smallint","col_nvarchar","col_int","col_sysname","_olake_id","col_varchar_nullable","col_float","col_date","col_decimal","col_smallmoney","col_text","col_xml","col_real","_cdc_seqval","col_time","_op_type","col_datetime2","col_numeric","_cdc_start_lsn","col_money"],"sync_new_columns":true}},{"partition_regex":"","stream_name":"systranschemas","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["endlsn","_cdc_timestamp","_op_type","_cdc_seqval","typeid","_olake_id","tabid","_cdc_start_lsn","startlsn","_olake_timestamp"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"mssql_test_table_olake","namespace":"dbo","type_schema":{"properties":{"_cdc_seqval":{"type":["string","null"],"destination_column_name":"_cdc_seqval","olake_column":true},"_cdc_start_lsn":{"type":["string","null"],"destination_column_name":"_cdc_start_lsn","olake_column":true},"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["null","timestamp_micro"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"col_bigint":{"type":["integer"],"destination_column_name":"col_bigint"},"col_bit":{"type":["boolean"],"destination_column_name":"col_bit"},"col_char":{"type":["string"],"destination_column_name":"col_char"},"col_date":{"type":["timestamp"],"destination_column_name":"col_date"},"col_datetime":{"type":["timestamp"],"destination_column_name":"col_datetime"},"col_datetime2":{"type":["timestamp_micro"],"destination_column_name":"col_datetime2"},"col_datetime2_nullable":{"type":["timestamp_micro","null"],"destination_column_name":"col_datetime2_nullable"},"col_datetimeoffset":{"type":["timestamp_micro"],"destination_column_name":"col_datetimeoffset"},"col_decimal":{"type":["number"],"destination_column_name":"col_decimal"},"col_float":{"type":["number"],"destination_column_name":"col_float"},"col_hierarchyid":{"type":["string"],"destination_column_name":"col_hierarchyid"},"col_image":{"type":["string"],"destination_column_name":"col_image"},"col_int":{"type":["integer_small"],"destination_column_name":"col_int"},"col_int_nullable":{"type":["integer_small","null"],"destination_column_name":"col_int_nullable"},"col_money":{"type":["number"],"destination_column_name":"col_money"},"col_nchar":{"type":["string"],"destination_column_name":"col_nchar"},"col_ntext":{"type":["string"],"destination_column_name":"col_ntext"},"col_numeric":{"type":["number"],"destination_column_name":"col_numeric"},"col_nvarchar":{"type":["string"],"destination_column_name":"col_nvarchar"},"col_real":{"type":["number_small"],"destination_column_name":"col_real"},"col_smalldatetime":{"type":["timestamp"],"destination_column_name":"col_smalldatetime"},"col_smallint":{"type":["integer_small"],"destination_column_name":"col_smallint"},"col_smallmoney":{"type":["number"],"destination_column_name":"col_smallmoney"},"col_sql_variant":{"type":["string"],"destination_column_name":"col_sql_variant"},"col_sysname":{"type":["string"],"destination_column_name":"col_sysname"},"col_text":{"type":["string"],"destination_column_name":"col_text"},"col_time":{"type":["string"],"destination_column_name":"col_time"},"col_tinyint":{"type":["integer_small"],"destination_column_name":"col_tinyint"},"col_uniqueidentifier":{"type":["string"],"destination_column_name":"col_uniqueidentifier"},"col_varchar":{"type":["string"],"destination_column_name":"col_varchar"},"col_varchar_nullable":{"type":["string","null"],"destination_column_name":"col_varchar_nullable"},"col_xml":{"type":["string"],"destination_column_name":"col_xml"},"created_at":{"type":["timestamp_micro"],"destination_column_name":"created_at"},"excludedColumn":{"type":["integer_small","null"],"destination_column_name":"excludedcolumn"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_cursor":{"type":["integer_small"],"destination_column_name":"id_cursor"}}},"supported_sync_modes":["incremental","cdc","strict_cdc","full_refresh"],"source_defined_primary_key":["id"],"available_cursor_fields":["col_int_nullable","col_varchar","col_time","col_smalldatetime","col_int","col_smallmoney","col_uniqueidentifier","col_sysname","col_varchar_nullable","id_cursor","col_bigint","col_money","col_char","col_nvarchar","col_datetime","col_datetime2_nullable","col_ntext","col_date","col_datetime2","col_datetimeoffset","excludedColumn","col_numeric","col_image","created_at","id","col_smallint","col_float","col_real","col_nchar","col_hierarchyid","col_sql_variant","col_xml","col_tinyint","col_decimal","col_bit","col_text"],"sync_mode":"cdc","destination_database":"mssql_olake_mssql_test:dbo","destination_table":"mssql_test_table_olake","default_stream_properties":{"normalization":true,"append_mode":false}}},{"stream":{"name":"systranschemas","namespace":"dbo","type_schema":{"properties":{"_cdc_seqval":{"type":["string","null"],"destination_column_name":"_cdc_seqval","olake_column":true},"_cdc_start_lsn":{"type":["string","null"],"destination_column_name":"_cdc_start_lsn","olake_column":true},"_cdc_timestamp":{"type":["null","timestamp_micro"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"endlsn":{"type":["string"],"destination_column_name":"endlsn"},"startlsn":{"type":["string"],"destination_column_name":"startlsn"},"tabid":{"type":["integer_small"],"destination_column_name":"tabid"},"typeid":{"type":["integer_small"],"destination_column_name":"typeid"}}},"supported_sync_modes":["incremental","cdc","strict_cdc","full_refresh"],"source_defined_primary_key":[],"available_cursor_fields":["tabid","startlsn","endlsn","typeid"],"sync_mode":"cdc","destination_database":"mssql_olake_mssql_test:dbo","destination_table":"systranschemas","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/mysql/mysql_test.go b/tests/mysql/mysql_test.go index 59d333f18..898ffa7e7 100644 --- a/tests/mysql/mysql_test.go +++ b/tests/mysql/mysql_test.go @@ -5,23 +5,20 @@ import ( "github.com/datazip-inc/olake/tests/testutils" "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/performance" + "github.com/datazip-inc/olake/tests/testutils/require" ) // mysqlBaseConfig returns an IntegrationTest pre-populated with all fields shared // by the mysql suites. -func mysqlBaseConfig(t *testing.T) *testutils.IntegrationTest { - return &testutils.IntegrationTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.MySQL)), - Namespace: "olake_mysql_test", - ExpectedData: ExpectedMySQLData, - DestinationDataTypeSchema: MySQLToDestinationSchema, - DefaultCDCColumnsSchema: ExpectedMySQLDefaultCDCColumnsSchema, - ExecuteQuery: ExecuteQuery, - DestinationDB: "mysql_olake_mysql_test", - CursorField: "id_cursor:id_smallint", - PartitionRegex: "/{id,identity}", - ColumnToExclude: "excludedColumn", - FilterConfig: `{ +func mysqlBaseConfig(t *testing.T) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.MySQL, "olake_mysql_test", "mysql_olake_mysql_test", ExecuteQuery) + require.NoError(t, err, "failed to build the test config") + cfg.CursorField = "id_cursor:id_smallint" + cfg.PartitionRegex = "/{id,identity}" + cfg.ColumnToExclude = "excludedColumn" + cfg.FilterConfig = `{ "logical_operator": "And", "conditions": [ { @@ -35,7 +32,13 @@ func mysqlBaseConfig(t *testing.T) *testutils.IntegrationTest { "value": "2022-07-01T15:30:00.000+00:00" } ] - }`, + }` + + return &integration.Test{ + TestConfig: cfg, + ExpectedData: ExpectedMySQLData, + DestinationDataTypeSchema: MySQLToDestinationSchema, + DefaultCDCColumnsSchema: ExpectedMySQLDefaultCDCColumnsSchema, } } @@ -57,13 +60,54 @@ func TestMySQL2PC(t *testing.T) { } func TestMySQLPerformance(t *testing.T) { - config := &testutils.PerformanceTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.MySQL)), - Namespace: "benchmark", - BackfillStreams: testutils.GetBackfillStreamsFromCDC(performanceCDCStreams), + cfg, err := testutils.NewTestConfig(t, constants.MySQL, "benchmark", "", ExecuteQuery) + require.NoError(t, err, "failed to build the test config") + + perf := &performance.Test{ + TestConfig: cfg, + BackfillStreams: performance.GetBackfillStreamsFromCDC(performanceCDCStreams), CDCStreams: performanceCDCStreams, - ExecuteQuery: ExecuteQuery, } - config.TestPerformance(t) + perf.TestPerformance(t) } + +// TestMySQLCompatibility pins the backward-compatibility contract for the driver that owns three of the +// six version gates -- the binlog timestamp location (v2), the timezone offset (v3) and the +// UNSIGNED widening (v4), see constants/state_version.go. Note that a passing run is the +// contract HOLDING: the candidate reading a state file at version N reproduces version N's types, +// so it agrees with the baseline. A diff here means a gate stopped firing. +// +// Baseline defaults to the newest release; OLAKE_COMPATIBILITY_BASELINE picks another tag, image or +// commit. v0.4.0 is the newest release still on state version 3, so it is the one that exercises +// the UNSIGNED gate. +// func TestMySQLCompatibility(t *testing.T) { +// t.Parallel() +// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { +// base := mysqlBaseConfig(t) +// base.ExpectedUpdatedData = ExpectedUpdatedData +// base.UpdatedDestinationDataTypeSchema = EvolvedMySQLToDestinationSchema +// cfg := &compatibility.Test{IntegrationTest: base} +// // Every known mysql finding, as data (COMPAT_RESULTS_v2.md). The ExcludeBelow columns are +// // the ones #940 ("fix CDC charset corruption for utf16/ucs2/latin1 columns", v0.7.2) added +// // as its own regression test: an older baseline hands their raw bytes to the Iceberg +// // writer as invalid UTF-8, the gRPC marshal fails, and the driver retries on a doubling +// // backoff that looks like a hang -- a hard fail, so they stay out of the seed data +// // entirely. The AssertValueFrom columns synced fine all along but changed value form at +// // the named release, so below it they are compared by type only: SET columns emitted the +// // numeric bitmask on the binlog path before #940 (M1), ENUMs serialized differently before +// // v0.3.9 (M2), and DECIMAL/NUMERIC round-tripped through float32 before v0.3.7 (M3). +// // The closure reads SeedExcludedColumns at call time; RunBackwardCompatibility fills it in after +// // resolving the rules above against the baseline. +// cfg.SupportsSeedExclusion = true +// base.TestConfig.ExecuteQuery = func(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { +// ExecuteQueryExcluding(ctx, t, conf, operation, cfg.SeedExcludedColumns) +// } +// // The filter stays on. It used to be cleared here because v0.4.0 synced the id=999 row +// // that HEAD filtered away -- an 8-vs-7 row count that masked everything behind it. That +// // was the input format, not the binary: filter_config arrived in v0.6.0, so v0.4.0 never +// // saw the key. RunBackwardCompatibility now writes the baseline's own input generation, which +// // hands a pre-v0.6.0 baseline the legacy `filter` string both binaries honor identically. +// return cfg +// }) +// } diff --git a/tests/mysql/mysql_util_test.go b/tests/mysql/mysql_util_test.go index 9a0c8836e..495814b83 100644 --- a/tests/mysql/mysql_util_test.go +++ b/tests/mysql/mysql_util_test.go @@ -4,14 +4,16 @@ import ( "context" "fmt" "math" + "strings" "testing" "time" "github.com/apache/arrow-go/v18/arrow" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/performance" + "github.com/datazip-inc/olake/tests/testutils/require" _ "github.com/go-sql-driver/mysql" "github.com/jmoiron/sqlx" - "github.com/stretchr/testify/require" ) // performanceCDCStreams is the CDC stream set the performance suite drives, shared between the @@ -21,21 +23,65 @@ var performanceCDCStreams = []string{"trips_cdc", "fhv_trips_cdc"} // ExecuteQuery executes MySQL queries for testing based on the operation type func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() + ExecuteQueryExcluding(ctx, t, conf, operation, nil) +} - var connStr, database string - if config := conf.SourceBaseConfig; config != nil { - database = config.String("database") - // the mysql driver spells its single host "hosts" - connStr = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", - config.String("username"), - config.String("password"), - config.String("hosts"), - config.Int("port"), - database) - } else { - database = "olake_mysql_test" - connStr = fmt.Sprintf("mysql:secret1234@tcp(localhost:3306)/%s?parseTime=true", database) +// versionedSeedColumns are the columns TestMySQLCompatibility can leave out of the seed data for old +// baselines (CompatibilityColumnRule.ExcludeBelow); every other suite seeds all of them. +var versionedSeedColumns = []struct { + name, ddl, value, filteredValue, updateExpr string +}{ + {"name_ucs2", "name_ucs2 VARCHAR(100) CHARACTER SET ucs2", "'ucs2_val'", "'filtered ucs2'", "name_ucs2 = 'updated ucs2'"}, + {"name_utf16le", "name_utf16le VARCHAR(100) CHARACTER SET utf16le", "'utf16le_val'", "'filtered utf16le'", "name_utf16le = 'updated utf16le'"}, + {"grade", "grade ENUM('naïve','café','résumé') CHARACTER SET latin1", "'naïve'", "'naïve'", "grade = 'café'"}, +} + +// seedColumnFragments renders the versioned columns NOT being excluded as the fragments each seed +// statement splices in after name_latin1; excluding nothing reproduces the full fixture. +func seedColumnFragments(t *testing.T, excluded []string) (ddl, cols, vals, filteredVals, updates string) { + t.Helper() + supported := make([]string, 0, len(versionedSeedColumns)) + for _, col := range versionedSeedColumns { + supported = append(supported, col.name) + } + drop, err := testutils.SeedColumnsExcluded(excluded, supported) + require.NoError(t, err, "mysql seed exclusion") + + var names, values, filtered, sets []string + for _, col := range versionedSeedColumns { + if drop[col.name] { + continue + } + ddl += "\n\t\t" + col.ddl + "," + names = append(names, col.name) + values = append(values, col.value) + filtered = append(filtered, col.filteredValue) + sets = append(sets, col.updateExpr) } + if len(names) == 0 { + return "", "", "", "", "" + } + join := func(parts []string) string { return " " + strings.Join(parts, ", ") + "," } + return ddl, join(names), join(values), join(filtered), join(sets) +} + +// ExecuteQueryExcluding is ExecuteQuery with columns left out of the seed DDL and DML entirely -- +// the compatibility suite's seed exclusion for columns an old baseline cannot sync at any price. +func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string, excludedColumns []string) { + t.Helper() + + seedDDL, seedCols, seedVals, seedFilteredVals, seedUpdates := seedColumnFragments(t, excludedColumns) + + var connStr, database string + config := conf.SourceBaseConfig + database = config.String("database") + // the mysql driver spells its single host "hosts" + connStr = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + config.String("username"), + config.String("password"), + config.Host("hosts"), + config.Int("port"), + database) db, err := sqlx.ConnectContext(ctx, "mysql", connStr) require.NoError(t, err, "failed to connect to mysql") defer func() { @@ -43,7 +89,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, }() // integration test uses only one stream for testing - integrationTestTable := testutils.TestTableName(conf) + integrationTestTable := conf.GetTableName() var query string switch operation { @@ -91,15 +137,12 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, name_bool TINYINT(1) DEFAULT '1', status ENUM('active','inactive','pending') DEFAULT NULL, priority ENUM('low','medium','high') DEFAULT 'low', - name_latin1 VARCHAR(100) CHARACTER SET latin1, - name_ucs2 VARCHAR(100) CHARACTER SET ucs2, - name_utf16le VARCHAR(100) CHARACTER SET utf16le, - grade ENUM('naïve','café','résumé') CHARACTER SET latin1, + name_latin1 VARCHAR(100) CHARACTER SET latin1,%s tags SET('sports','music','gaming','reading') DEFAULT NULL, permissions SET('read','write','execute') CHARACTER SET latin1 DEFAULT NULL, PRIMARY KEY (id), excludedColumn INT - )`, integrationTestTable) + )`, integrationTestTable, seedDDL) case "drop": query = fmt.Sprintf("DROP TABLE IF EXISTS %s", integrationTestTable) @@ -113,7 +156,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, query = fmt.Sprintf("DELETE FROM %s", integrationTestTable) case "add": - insertTestData(ctx, t, db, integrationTestTable) + insertTestData(ctx, t, db, integrationTestTable, excludedColumns) return // Early return since we handle all inserts in the helper function case "insert": @@ -132,12 +175,12 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1, name_ucs2, name_utf16le, grade, + name_latin1,%s tags, permissions, excludedColumn ) VALUES ( 6, 6, 123456789012345, - 100, 101, 102, 103, + 100, 4294967295, 102, 4294967294, 5001, 5002, 101, 102, 50, 51, 255, 65535, @@ -149,10 +192,10 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, 'mediumtext_val', 'longtext_val', '2023-01-01 12:00:00', '2023-01-01 12:00:00', 1, 'long_varchar_val', 1, 'active', 'high', - 'latin1_val', 'ucs2_val', 'utf16le_val', 'naïve', + 'latin1_val',%s 'sports,reading', 'read,write', 101 - )`, integrationTestTable) + )`, integrationTestTable, seedCols, seedVals) _, err = db.ExecContext(ctx, query) require.NoError(t, err, "Failed to execute %s operation", operation) // insert a filtered doc, it would be filtered out by the filter, won't be synced into the destination @@ -171,7 +214,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1, name_ucs2, name_utf16le, grade, + name_latin1,%s tags, permissions, excludedColumn ) VALUES ( @@ -188,10 +231,10 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, 'filtered medium', 'filtered long', '2022-06-15 10:00:00', '2021-06-15 10:00:00', 0, 'filtered long varchar', 0, 'inactive', 'low', - 'filtered latin1', 'filtered ucs2', 'filtered utf16le', 'naïve', + 'filtered latin1',%s 'music', 'execute', 200 - )`, integrationTestTable) + )`, integrationTestTable, seedCols, seedFilteredVals) _, err = db.ExecContext(ctx, filteredQuery) require.NoError(t, err, "Failed to insert filtered test data row") return @@ -212,11 +255,11 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1, name_ucs2, name_utf16le, grade, + name_latin1,%s tags, permissions ) VALUES ( 7, 7, 123456789012345, - 100, 101, 102, 103, + 100, 4294967295, 102, 4294967294, 5001, 5002, 101, 102, 50, 51, 255, 65535, @@ -228,17 +271,17 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, 'mediumtext_val', 'longtext_val', '2023-01-01 12:00:00', '2023-01-01 12:00:00', 1, 'long_varchar_val', 1, 'active', 'high', - 'latin1_val', 'ucs2_val', 'utf16le_val', 'naïve', + 'latin1_val',%s 'sports,reading', 'read,write' - )`, integrationTestTable) + )`, integrationTestTable, seedCols, seedVals) case "update": query = fmt.Sprintf(` UPDATE %s SET id_cursor = NULL, id_bigint = 987654321098765, - id_int = 200, id_int_unsigned = 201, - id_integer = 202, id_integer_unsigned = 203, + id_int = 200, id_int_unsigned = 4294967293, + id_integer = 202, id_integer_unsigned = 4294967292, id_mediumint = 6001, id_mediumint_unsigned = 6002, id_smallint = 201, id_smallint_unsigned = 202, id_tinyint = 60, id_tinyint_unsigned = 61, @@ -258,18 +301,17 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, created_timestamp = '2024-07-01 15:30:00', is_active = 0, long_varchar = 'updated long...', name_bool = 0, status = 'pending', priority = 'low', - name_latin1 = 'updated latin1', name_ucs2 = 'updated ucs2', - name_utf16le = 'updated utf16le', grade = 'café', + name_latin1 = 'updated latin1',%s tags = 'gaming,reading', permissions = 'read,write,execute', excludedColumn = 102, includedColumn = 202 - WHERE id = 1`, integrationTestTable) + WHERE id = 1`, integrationTestTable, seedUpdates) case "delete": query = fmt.Sprintf("DELETE FROM %s WHERE id = 1", integrationTestTable) case "setup_cdc": - backfillStreams := testutils.GetBackfillStreamsFromCDC(performanceCDCStreams) + backfillStreams := performance.GetBackfillStreamsFromCDC(performanceCDCStreams) // truncate the cdc tables for idx, cdcStream := range performanceCDCStreams { _, err := db.ExecContext(ctx, fmt.Sprintf("TRUNCATE TABLE %s", cdcStream)) @@ -293,14 +335,11 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, return case "bulk_cdc_data_insert": - backfillStreams := testutils.GetBackfillStreamsFromCDC(performanceCDCStreams) + backfillStreams := performance.GetBackfillStreamsFromCDC(performanceCDCStreams) // insert the data into the cdc tables concurrently err := testutils.Concurrent(ctx, performanceCDCStreams, len(performanceCDCStreams), func(ctx context.Context, cdcStream string, executionNumber int) error { - _, err = db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s SELECT * FROM %s LIMIT 15000000", cdcStream, backfillStreams[executionNumber])) - if err != nil { - return err - } - return nil + _, err := db.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s SELECT * FROM %s LIMIT 15000000", cdcStream, backfillStreams[executionNumber])) + return err }) require.NoError(t, err, fmt.Sprintf("failed to execute %s operation", operation), err) return @@ -317,9 +356,10 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, } // insertTestData inserts test data into the specified table -func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName string) { +func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName string, excludedColumns []string) { t.Helper() + _, seedCols, seedVals, seedFilteredVals, _ := seedColumnFragments(t, excludedColumns) for i := 1; i <= 5; i++ { query := fmt.Sprintf(` INSERT INTO %s ( @@ -335,12 +375,12 @@ func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName st name_char, name_varchar, name_text, name_tinytext, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1, name_ucs2, name_utf16le, grade, + name_latin1,%s tags, permissions, excludedColumn ) VALUES ( %d, %d, 123456789012345, - 100, 101, 102, 103, + 100, 4294967295, 102, 4294967294, 5001, 5002, 101, 102, 50, 51, 255, 65535, @@ -351,10 +391,10 @@ func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName st 'c', 'varchar_val', 'text_val', 'tinytext_val', 'mediumtext_val', 'longtext_val', '2023-01-01 12:00:00', '2023-01-01 12:00:00', 1, 'long_varchar_val', 1, 'active', 'high', - 'latin1_val', 'ucs2_val', 'utf16le_val', 'naïve', + 'latin1_val',%s 'sports,reading', 'read,write', 100 - )`, tableName, i, i) + )`, tableName, seedCols, i, i, seedVals) _, err := db.ExecContext(ctx, query) require.NoError(t, err, "Failed to insert test data row %d", i) @@ -374,7 +414,7 @@ func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName st name_char, name_varchar, name_text, name_tinytext, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1, name_ucs2, name_utf16le, grade, + name_latin1,%s tags, permissions, excludedColumn ) VALUES ( @@ -390,22 +430,28 @@ func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName st 'x', 'filtered_val', 'filtered text', 'filtered tiny', 'filtered medium', 'filtered long', '2021-06-15 10:00:00', '2021-06-15 10:00:00', 0, 'filtered long varchar', 0, 'inactive', 'low', - 'filtered latin1', 'filtered ucs2', 'filtered utf16le', 'naïve', + 'filtered latin1',%s 'music', 'execute', 200 - )`, tableName) + )`, tableName, seedCols, seedFilteredVals) _, err := db.ExecContext(ctx, filteredQuery) require.NoError(t, err, "Failed to insert filtered test data row") } +// The id_int_unsigned / id_integer_unsigned values are deliberately ABOVE int32 range (max +// INT UNSIGNED is 4294967295). That is what makes state version 4 observable: at v>=4 the driver +// reinterprets the raw bits as uint32 and the value survives as int64, while at v<=3 it strips the +// "unsigned " prefix, maps to Int32, and the same bits read back as -1 / -2 -- the overflow +// constants/state_version.go's version 4 note describes. Keep them above 2^31-1; small values +// make the gate invisible because they fit in both types. // TODO: olake has no uint64 data type, so the id_bigint_unsigned_* values past MaxInt64 pin what // olake writes today, not what MySQL stored. var ExpectedMySQLData = map[string]interface{}{ "id_bigint": int64(123456789012345), "id_int": int32(100), - "id_int_unsigned": int64(101), + "id_int_unsigned": int64(4294967295), "id_integer": int32(102), - "id_integer_unsigned": int64(103), + "id_integer_unsigned": int64(4294967294), "id_mediumint": int32(5001), "id_mediumint_unsigned": int32(5002), "id_smallint": int32(101), @@ -456,9 +502,9 @@ var ExpectedMySQLData = map[string]interface{}{ var ExpectedUpdatedData = map[string]interface{}{ "id_bigint": int64(987654321098765), "id_int": int64(200), - "id_int_unsigned": int64(201), + "id_int_unsigned": int64(4294967293), "id_integer": int32(202), - "id_integer_unsigned": int64(203), + "id_integer_unsigned": int64(4294967292), "id_mediumint": int32(6001), "id_mediumint_unsigned": int32(6002), "id_smallint": int32(201), diff --git a/tests/mysql/testdata/source.json b/tests/mysql/testdata/source.template.json similarity index 100% rename from tests/mysql/testdata/source.json rename to tests/mysql/testdata/source.template.json diff --git a/tests/mysql/testdata/streams.template.json b/tests/mysql/testdata/streams.template.json new file mode 100644 index 000000000..b1f8fc96b --- /dev/null +++ b/tests/mysql/testdata/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"olake_mysql_test":[{"partition_regex":"","stream_name":"test_table_olake_${suite}","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["name_longtext","id_int_unsigned","name_tinytext","id_smallint","_olake_id","price_decimal","_cdc_binlog_file_name","_olake_timestamp","name_bool","name_latin1","_cdc_timestamp","excludedColumn","id_mediumint_unsigned","id_tinyint_unsigned","status","_op_type","_cdc_binlog_file_pos","name_varchar","id_tinyint","price_double","created_date","id","name_ucs2","id_bigint","created_timestamp","grade","id_integer_unsigned","price_numeric","tags","id_smallint_unsigned","name_text","price_float","priority","name_utf16le","id_integer","long_varchar","price_double_precision","id_mediumint","permissions","id_cursor","price_real","name_char","amount_decimal_9_2","name_mediumtext","id_int","is_active","id_tinyint_unsigned_max","id_smallint_unsigned_max","id_mediumint_unsigned_max","id_mediumint_unsigned_signbit","id_int_unsigned_max","id_bigint_unsigned","id_bigint_unsigned_signbit","id_bigint_unsigned_max"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"test_table_olake_${suite}","namespace":"olake_mysql_test","type_schema":{"properties":{"_cdc_binlog_file_name":{"type":["string","null"],"destination_column_name":"_cdc_binlog_file_name","olake_column":true},"_cdc_binlog_file_pos":{"type":["integer","null"],"destination_column_name":"_cdc_binlog_file_pos","olake_column":true},"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"amount_decimal_9_2":{"type":["number","null"],"destination_column_name":"amount_decimal_9_2"},"created_date":{"type":["timestamp","null"],"destination_column_name":"created_date"},"created_timestamp":{"type":["timestamp","null"],"destination_column_name":"created_timestamp"},"excludedColumn":{"type":["integer_small","null"],"destination_column_name":"excludedcolumn"},"grade":{"type":["string","null"],"destination_column_name":"grade"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_bigint":{"type":["integer","null"],"destination_column_name":"id_bigint"},"id_bigint_unsigned":{"type":["integer","null"],"destination_column_name":"id_bigint_unsigned"},"id_bigint_unsigned_max":{"type":["integer","null"],"destination_column_name":"id_bigint_unsigned_max"},"id_bigint_unsigned_signbit":{"type":["integer","null"],"destination_column_name":"id_bigint_unsigned_signbit"},"id_cursor":{"type":["integer_small","null"],"destination_column_name":"id_cursor"},"id_int":{"type":["integer_small","null"],"destination_column_name":"id_int"},"id_int_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_int_unsigned"},"id_int_unsigned_max":{"type":["integer_small","null"],"destination_column_name":"id_int_unsigned_max"},"id_integer":{"type":["integer_small","null"],"destination_column_name":"id_integer"},"id_integer_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_integer_unsigned"},"id_mediumint":{"type":["integer_small","null"],"destination_column_name":"id_mediumint"},"id_mediumint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_mediumint_unsigned"},"id_mediumint_unsigned_max":{"type":["integer_small","null"],"destination_column_name":"id_mediumint_unsigned_max"},"id_mediumint_unsigned_signbit":{"type":["integer_small","null"],"destination_column_name":"id_mediumint_unsigned_signbit"},"id_smallint":{"type":["integer_small","null"],"destination_column_name":"id_smallint"},"id_smallint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_smallint_unsigned"},"id_smallint_unsigned_max":{"type":["integer_small","null"],"destination_column_name":"id_smallint_unsigned_max"},"id_tinyint":{"type":["integer_small","null"],"destination_column_name":"id_tinyint"},"id_tinyint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_tinyint_unsigned"},"id_tinyint_unsigned_max":{"type":["integer_small","null"],"destination_column_name":"id_tinyint_unsigned_max"},"is_active":{"type":["integer_small","null"],"destination_column_name":"is_active"},"long_varchar":{"type":["string","null"],"destination_column_name":"long_varchar"},"name_bool":{"type":["integer_small","null"],"destination_column_name":"name_bool"},"name_char":{"type":["string","null"],"destination_column_name":"name_char"},"name_latin1":{"type":["string","null"],"destination_column_name":"name_latin1"},"name_longtext":{"type":["null","string"],"destination_column_name":"name_longtext"},"name_mediumtext":{"type":["string","null"],"destination_column_name":"name_mediumtext"},"name_text":{"type":["string","null"],"destination_column_name":"name_text"},"name_tinytext":{"type":["string","null"],"destination_column_name":"name_tinytext"},"name_ucs2":{"type":["string","null"],"destination_column_name":"name_ucs2"},"name_utf16le":{"type":["string","null"],"destination_column_name":"name_utf16le"},"name_varchar":{"type":["string","null"],"destination_column_name":"name_varchar"},"permissions":{"type":["string","null"],"destination_column_name":"permissions"},"price_decimal":{"type":["number","null"],"destination_column_name":"price_decimal"},"price_double":{"type":["number","null"],"destination_column_name":"price_double"},"price_double_precision":{"type":["number","null"],"destination_column_name":"price_double_precision"},"price_float":{"type":["number_small","null"],"destination_column_name":"price_float"},"price_numeric":{"type":["number","null"],"destination_column_name":"price_numeric"},"price_real":{"type":["number","null"],"destination_column_name":"price_real"},"priority":{"type":["null","string"],"destination_column_name":"priority"},"status":{"type":["string","null"],"destination_column_name":"status"},"tags":{"type":["null","string"],"destination_column_name":"tags"}}},"supported_sync_modes":["strict_cdc","full_refresh","incremental","cdc"],"source_defined_primary_key":["id"],"available_cursor_fields":["id","name_varchar","name_tinytext","tags","excludedColumn","id_integer_unsigned","id_mediumint","name_char","priority","permissions","id_smallint_unsigned","name_utf16le","id_smallint","price_double_precision","price_real","name_latin1","id_int","id_cursor","id_mediumint_unsigned","created_date","created_timestamp","id_tinyint","id_tinyint_unsigned","amount_decimal_9_2","long_varchar","grade","id_bigint","id_integer","price_decimal","price_double","name_text","name_mediumtext","name_ucs2","id_int_unsigned","price_float","price_numeric","name_longtext","is_active","name_bool","status","id_tinyint_unsigned_max","id_smallint_unsigned_max","id_mediumint_unsigned_max","id_mediumint_unsigned_signbit","id_int_unsigned_max","id_bigint_unsigned","id_bigint_unsigned_signbit","id_bigint_unsigned_max"],"sync_mode":"cdc","destination_database":"mysql:olake_mysql_test","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/mysql/testdata/test_streams.json b/tests/mysql/testdata/test_streams.json deleted file mode 100644 index 13b5ec96f..000000000 --- a/tests/mysql/testdata/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"olake_mysql_test":[{"partition_regex":"","stream_name":"mysql_test_table_olake","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["name_longtext","id_int_unsigned","name_tinytext","id_smallint","_olake_id","price_decimal","_cdc_binlog_file_name","_olake_timestamp","name_bool","name_latin1","_cdc_timestamp","excludedColumn","id_mediumint_unsigned","id_tinyint_unsigned","status","_op_type","_cdc_binlog_file_pos","name_varchar","id_tinyint","price_double","created_date","id","name_ucs2","id_bigint","created_timestamp","grade","id_integer_unsigned","price_numeric","tags","id_smallint_unsigned","name_text","price_float","priority","name_utf16le","id_integer","long_varchar","price_double_precision","id_mediumint","permissions","id_cursor","price_real","name_char","amount_decimal_9_2","name_mediumtext","id_int","is_active","id_tinyint_unsigned_max","id_smallint_unsigned_max","id_mediumint_unsigned_max","id_mediumint_unsigned_signbit","id_int_unsigned_max","id_bigint_unsigned","id_bigint_unsigned_signbit","id_bigint_unsigned_max"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"mysql_test_table_olake","namespace":"olake_mysql_test","type_schema":{"properties":{"_cdc_binlog_file_name":{"type":["string","null"],"destination_column_name":"_cdc_binlog_file_name","olake_column":true},"_cdc_binlog_file_pos":{"type":["integer","null"],"destination_column_name":"_cdc_binlog_file_pos","olake_column":true},"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"amount_decimal_9_2":{"type":["number","null"],"destination_column_name":"amount_decimal_9_2"},"created_date":{"type":["timestamp","null"],"destination_column_name":"created_date"},"created_timestamp":{"type":["timestamp","null"],"destination_column_name":"created_timestamp"},"excludedColumn":{"type":["integer_small","null"],"destination_column_name":"excludedcolumn"},"grade":{"type":["string","null"],"destination_column_name":"grade"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_bigint":{"type":["integer","null"],"destination_column_name":"id_bigint"},"id_bigint_unsigned":{"type":["integer","null"],"destination_column_name":"id_bigint_unsigned"},"id_bigint_unsigned_max":{"type":["integer","null"],"destination_column_name":"id_bigint_unsigned_max"},"id_bigint_unsigned_signbit":{"type":["integer","null"],"destination_column_name":"id_bigint_unsigned_signbit"},"id_cursor":{"type":["integer_small","null"],"destination_column_name":"id_cursor"},"id_int":{"type":["integer_small","null"],"destination_column_name":"id_int"},"id_int_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_int_unsigned"},"id_int_unsigned_max":{"type":["integer_small","null"],"destination_column_name":"id_int_unsigned_max"},"id_integer":{"type":["integer_small","null"],"destination_column_name":"id_integer"},"id_integer_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_integer_unsigned"},"id_mediumint":{"type":["integer_small","null"],"destination_column_name":"id_mediumint"},"id_mediumint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_mediumint_unsigned"},"id_mediumint_unsigned_max":{"type":["integer_small","null"],"destination_column_name":"id_mediumint_unsigned_max"},"id_mediumint_unsigned_signbit":{"type":["integer_small","null"],"destination_column_name":"id_mediumint_unsigned_signbit"},"id_smallint":{"type":["integer_small","null"],"destination_column_name":"id_smallint"},"id_smallint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_smallint_unsigned"},"id_smallint_unsigned_max":{"type":["integer_small","null"],"destination_column_name":"id_smallint_unsigned_max"},"id_tinyint":{"type":["integer_small","null"],"destination_column_name":"id_tinyint"},"id_tinyint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_tinyint_unsigned"},"id_tinyint_unsigned_max":{"type":["integer_small","null"],"destination_column_name":"id_tinyint_unsigned_max"},"is_active":{"type":["integer_small","null"],"destination_column_name":"is_active"},"long_varchar":{"type":["string","null"],"destination_column_name":"long_varchar"},"name_bool":{"type":["integer_small","null"],"destination_column_name":"name_bool"},"name_char":{"type":["string","null"],"destination_column_name":"name_char"},"name_latin1":{"type":["string","null"],"destination_column_name":"name_latin1"},"name_longtext":{"type":["null","string"],"destination_column_name":"name_longtext"},"name_mediumtext":{"type":["string","null"],"destination_column_name":"name_mediumtext"},"name_text":{"type":["string","null"],"destination_column_name":"name_text"},"name_tinytext":{"type":["string","null"],"destination_column_name":"name_tinytext"},"name_ucs2":{"type":["string","null"],"destination_column_name":"name_ucs2"},"name_utf16le":{"type":["string","null"],"destination_column_name":"name_utf16le"},"name_varchar":{"type":["string","null"],"destination_column_name":"name_varchar"},"permissions":{"type":["string","null"],"destination_column_name":"permissions"},"price_decimal":{"type":["number","null"],"destination_column_name":"price_decimal"},"price_double":{"type":["number","null"],"destination_column_name":"price_double"},"price_double_precision":{"type":["number","null"],"destination_column_name":"price_double_precision"},"price_float":{"type":["number_small","null"],"destination_column_name":"price_float"},"price_numeric":{"type":["number","null"],"destination_column_name":"price_numeric"},"price_real":{"type":["number","null"],"destination_column_name":"price_real"},"priority":{"type":["null","string"],"destination_column_name":"priority"},"status":{"type":["string","null"],"destination_column_name":"status"},"tags":{"type":["null","string"],"destination_column_name":"tags"}}},"supported_sync_modes":["strict_cdc","full_refresh","incremental","cdc"],"source_defined_primary_key":["id"],"available_cursor_fields":["id","name_varchar","name_tinytext","tags","excludedColumn","id_integer_unsigned","id_mediumint","name_char","priority","permissions","id_smallint_unsigned","name_utf16le","id_smallint","price_double_precision","price_real","name_latin1","id_int","id_cursor","id_mediumint_unsigned","created_date","created_timestamp","id_tinyint","id_tinyint_unsigned","amount_decimal_9_2","long_varchar","grade","id_bigint","id_integer","price_decimal","price_double","name_text","name_mediumtext","name_ucs2","id_int_unsigned","price_float","price_numeric","name_longtext","is_active","name_bool","status","id_tinyint_unsigned_max","id_smallint_unsigned_max","id_mediumint_unsigned_max","id_mediumint_unsigned_signbit","id_int_unsigned_max","id_bigint_unsigned","id_bigint_unsigned_signbit","id_bigint_unsigned_max"],"sync_mode":"cdc","destination_database":"mysql:olake_mysql_test","destination_table":"mysql_test_table_olake","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/oracle/oracle_test.go b/tests/oracle/oracle_test.go index 919702b13..dc3259a7c 100644 --- a/tests/oracle/oracle_test.go +++ b/tests/oracle/oracle_test.go @@ -5,22 +5,19 @@ import ( "github.com/datazip-inc/olake/tests/testutils" "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/require" ) // oracleBaseConfig returns an IntegrationTest pre-populated with all fields shared // by the oracle suites. -func oracleBaseConfig(t *testing.T) *testutils.IntegrationTest { - return &testutils.IntegrationTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.Oracle)), - Namespace: "MYUSER", - ExpectedData: ExpectedOracleData, - DestinationDataTypeSchema: OracleToDestinationSchema, - ExecuteQuery: ExecuteQuery, - DestinationDB: "oracle_myuser", - CursorField: "COL_CURSOR:COL_SMALLINT", - PartitionRegex: "/{id, identity}", - ColumnToExclude: "EXCLUDEDCOLUMN", - FilterConfig: `{ +func oracleBaseConfig(t *testing.T) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.Oracle, "MYUSER", "oracle_myuser", ExecuteQuery) + require.NoError(t, err, "failed to build the test config") + cfg.CursorField = "COL_CURSOR:COL_SMALLINT" + cfg.PartitionRegex = "/{id, identity}" + cfg.ColumnToExclude = "EXCLUDEDCOLUMN" + cfg.FilterConfig = `{ "logical_operator": "And", "conditions": [ { @@ -34,7 +31,12 @@ func oracleBaseConfig(t *testing.T) *testutils.IntegrationTest { "value": "2022-07-01T15:30:00.000+00:00" } ] - }`, + }` + + return &integration.Test{ + TestConfig: cfg, + ExpectedData: ExpectedOracleData, + DestinationDataTypeSchema: OracleToDestinationSchema, } } @@ -54,3 +56,19 @@ func TestOracle2PC(t *testing.T) { t.Parallel() oracleBaseConfig(t).Test2PCIntegration(t) } + +// TestOracleCompatibility pins the backward-compatibility contract: the same scenarios run on a released +// baseline image and on this build after the initial load, and the destinations must match. +// See tests/testutils/compatibility.go. +// func TestOracleCompatibility(t *testing.T) { +// t.Parallel() +// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { +// base := oracleBaseConfig(t) +// base.ExpectedUpdatedData = ExpectedUpdatedOracleData +// base.UpdatedDestinationDataTypeSchema = UpdatedOracleToDestinationSchema +// cfg := &compatibility.Test{IntegrationTest: base} +// // No floor declared: oracle images exist for every sweep baseline. If a first sweep finds +// // an unrunnable band, declare it here with its reason. +// return cfg +// }) +// } diff --git a/tests/oracle/oracle_util_test.go b/tests/oracle/oracle_util_test.go index 72e333118..9fe5d0c0b 100644 --- a/tests/oracle/oracle_util_test.go +++ b/tests/oracle/oracle_util_test.go @@ -8,9 +8,9 @@ import ( "github.com/apache/arrow-go/v18/arrow" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/require" "github.com/jmoiron/sqlx" go_ora "github.com/sijms/go-ora/v2" - "github.com/stretchr/testify/require" ) // connectionString builds the go-ora URL from source.json. It mirrors the oracle driver's own @@ -29,7 +29,7 @@ func connectionString(config testutils.SourceConfig) string { } quotedUsername := fmt.Sprintf("%q", config.String("username")) return go_ora.BuildUrl( - config.String("host"), + config.Host("host"), config.Int("port"), config.String("service_name"), quotedUsername, @@ -41,12 +41,7 @@ func connectionString(config testutils.SourceConfig) string { func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() - var connStr string - if conf.SourceBaseConfig != nil { - connStr = connectionString(conf.SourceBaseConfig) - } else { - connStr = "oracle://myuser:secret1234@localhost:1521/orcl" - } + connStr := connectionString(conf.SourceBaseConfig) db, err := sqlx.ConnectContext(ctx, "oracle", connStr) require.NoError(t, err, "failed to connect to oracle") @@ -54,7 +49,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, require.NoError(t, db.Close()) }() - integrationTestTable := testutils.TestTableName(conf) + integrationTestTable := conf.GetTableName() var query string switch operation { diff --git a/tests/oracle/testdata/source.json b/tests/oracle/testdata/source.template.json similarity index 100% rename from tests/oracle/testdata/source.json rename to tests/oracle/testdata/source.template.json diff --git a/tests/oracle/testdata/streams.template.json b/tests/oracle/testdata/streams.template.json new file mode 100644 index 000000000..d6baece3a --- /dev/null +++ b/tests/oracle/testdata/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"MYUSER":[{"partition_regex":"","stream_name":"TEST_TABLE_OLAKE_${SUITE}","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["EXCLUDEDCOLUMN","COL_SMALLINT","COL_VARCHAR2","COL_TIMESTAMPTZ","COL_CHARACTER","COL_DECIMAL","COL_CLOB","COL_FLOAT","COL_TIMESTAMP","COL_DOUBLE_PRECISION","_olake_timestamp","COL_NCLOB","ID","COL_CURSOR","COL_DATE","COL_CHAR","_op_type","COL_BIGINT","COL_INT","COL_BLOB","COL_TIMESTAMPLTZ","_olake_id","COL_INTEGER"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"TEST_TABLE_OLAKE_${SUITE}","namespace":"MYUSER","type_schema":{"properties":{"COL_BIGINT":{"type":["integer","null"],"destination_column_name":"col_bigint"},"COL_BLOB":{"type":["string","null"],"destination_column_name":"col_blob"},"COL_CHAR":{"type":["string","null"],"destination_column_name":"col_char"},"COL_CHARACTER":{"type":["string","null"],"destination_column_name":"col_character"},"COL_CLOB":{"type":["string","null"],"destination_column_name":"col_clob"},"COL_CURSOR":{"type":["integer","null"],"destination_column_name":"col_cursor"},"COL_DATE":{"type":["timestamp_micro","null"],"destination_column_name":"col_date"},"COL_DECIMAL":{"type":["number","null"],"destination_column_name":"col_decimal"},"COL_DOUBLE_PRECISION":{"type":["number","null"],"destination_column_name":"col_double_precision"},"COL_FLOAT":{"type":["number_small","null"],"destination_column_name":"col_float"},"COL_INT":{"type":["integer_small","null"],"destination_column_name":"col_int"},"COL_INTEGER":{"type":["integer","null"],"destination_column_name":"col_integer"},"COL_NCLOB":{"type":["string","null"],"destination_column_name":"col_nclob"},"COL_SMALLINT":{"type":["null","integer_small"],"destination_column_name":"col_smallint"},"COL_TIMESTAMP":{"type":["timestamp_micro","null"],"destination_column_name":"col_timestamp"},"COL_TIMESTAMPLTZ":{"type":["timestamp_micro","null"],"destination_column_name":"col_timestampltz"},"COL_TIMESTAMPTZ":{"type":["timestamp_micro","null"],"destination_column_name":"col_timestamptz"},"COL_VARCHAR2":{"type":["string","null"],"destination_column_name":"col_varchar2"},"EXCLUDEDCOLUMN":{"type":["integer","null"],"destination_column_name":"excludedcolumn"},"ID":{"type":["number"],"destination_column_name":"id"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":["ID"],"available_cursor_fields":["COL_TIMESTAMPTZ","COL_TIMESTAMPLTZ","COL_CURSOR","COL_CHARACTER","EXCLUDEDCOLUMN","COL_INT","ID","COL_DATE","COL_DECIMAL","COL_DOUBLE_PRECISION","COL_SMALLINT","COL_CLOB","COL_NCLOB","COL_TIMESTAMP","COL_INTEGER","COL_BIGINT","COL_CHAR","COL_VARCHAR2","COL_FLOAT","COL_BLOB"],"cursor_field":"COL_BIGINT","sync_mode":"incremental","destination_database":"oracle:myuser","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/oracle/testdata/test_streams.json b/tests/oracle/testdata/test_streams.json deleted file mode 100644 index f025c841b..000000000 --- a/tests/oracle/testdata/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"MYUSER":[{"partition_regex":"","stream_name":"ORACLE_TEST_TABLE_OLAKE","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["EXCLUDEDCOLUMN","COL_SMALLINT","COL_VARCHAR2","COL_TIMESTAMPTZ","COL_CHARACTER","COL_DECIMAL","COL_CLOB","COL_FLOAT","COL_TIMESTAMP","COL_DOUBLE_PRECISION","_olake_timestamp","COL_NCLOB","ID","COL_CURSOR","COL_DATE","COL_CHAR","_op_type","COL_BIGINT","COL_INT","COL_BLOB","COL_TIMESTAMPLTZ","_olake_id","COL_INTEGER"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"ORACLE_TEST_TABLE_OLAKE","namespace":"MYUSER","type_schema":{"properties":{"COL_BIGINT":{"type":["integer","null"],"destination_column_name":"col_bigint"},"COL_BLOB":{"type":["string","null"],"destination_column_name":"col_blob"},"COL_CHAR":{"type":["string","null"],"destination_column_name":"col_char"},"COL_CHARACTER":{"type":["string","null"],"destination_column_name":"col_character"},"COL_CLOB":{"type":["string","null"],"destination_column_name":"col_clob"},"COL_CURSOR":{"type":["integer","null"],"destination_column_name":"col_cursor"},"COL_DATE":{"type":["timestamp_micro","null"],"destination_column_name":"col_date"},"COL_DECIMAL":{"type":["number","null"],"destination_column_name":"col_decimal"},"COL_DOUBLE_PRECISION":{"type":["number","null"],"destination_column_name":"col_double_precision"},"COL_FLOAT":{"type":["number_small","null"],"destination_column_name":"col_float"},"COL_INT":{"type":["integer_small","null"],"destination_column_name":"col_int"},"COL_INTEGER":{"type":["integer","null"],"destination_column_name":"col_integer"},"COL_NCLOB":{"type":["string","null"],"destination_column_name":"col_nclob"},"COL_SMALLINT":{"type":["null","integer_small"],"destination_column_name":"col_smallint"},"COL_TIMESTAMP":{"type":["timestamp_micro","null"],"destination_column_name":"col_timestamp"},"COL_TIMESTAMPLTZ":{"type":["timestamp_micro","null"],"destination_column_name":"col_timestampltz"},"COL_TIMESTAMPTZ":{"type":["timestamp_micro","null"],"destination_column_name":"col_timestamptz"},"COL_VARCHAR2":{"type":["string","null"],"destination_column_name":"col_varchar2"},"EXCLUDEDCOLUMN":{"type":["integer","null"],"destination_column_name":"excludedcolumn"},"ID":{"type":["number"],"destination_column_name":"id"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":["ID"],"available_cursor_fields":["COL_TIMESTAMPTZ","COL_TIMESTAMPLTZ","COL_CURSOR","COL_CHARACTER","EXCLUDEDCOLUMN","COL_INT","ID","COL_DATE","COL_DECIMAL","COL_DOUBLE_PRECISION","COL_SMALLINT","COL_CLOB","COL_NCLOB","COL_TIMESTAMP","COL_INTEGER","COL_BIGINT","COL_CHAR","COL_VARCHAR2","COL_FLOAT","COL_BLOB"],"cursor_field":"COL_BIGINT","sync_mode":"incremental","destination_database":"oracle:myuser","destination_table":"oracle_test_table_olake","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/postgres/postgres_test.go b/tests/postgres/postgres_test.go index feb2d7442..c8994089d 100644 --- a/tests/postgres/postgres_test.go +++ b/tests/postgres/postgres_test.go @@ -5,24 +5,21 @@ import ( "github.com/datazip-inc/olake/tests/testutils" "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/performance" + "github.com/datazip-inc/olake/tests/testutils/require" _ "github.com/lib/pq" ) // postgresBaseConfig returns an IntegrationTest pre-populated with all fields shared // by the postgres suites. -func postgresBaseConfig(t *testing.T) *testutils.IntegrationTest { - return &testutils.IntegrationTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.Postgres)), - Namespace: "public", - ExpectedData: ExpectedPostgresData, - DestinationDataTypeSchema: PostgresToDestinationSchema, - DefaultCDCColumnsSchema: ExpectedPostgresDefaultCDCColumnsSchema, - ExecuteQuery: ExecuteQuery, - DestinationDB: "postgres_postgres_public", - CursorField: "col_cursor:col_int", - PartitionRegex: "/{col_bigserial,identity}", - ColumnToExclude: "excludedcolumn", - FilterConfig: `{ +func postgresBaseConfig(t *testing.T) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.Postgres, "public", "postgres_postgres_public", ExecuteQuery) + require.NoError(t, err, "failed to build the test config") + cfg.CursorField = "col_cursor:col_int" + cfg.PartitionRegex = "/{col_bigserial,identity}" + cfg.ColumnToExclude = "excludedcolumn" + cfg.FilterConfig = `{ "logical_operator": "And", "conditions": [ { @@ -36,7 +33,13 @@ func postgresBaseConfig(t *testing.T) *testutils.IntegrationTest { "value": "2022-07-01T15:30:00.000+00:00" } ] - }`, + }` + + return &integration.Test{ + TestConfig: cfg, + ExpectedData: ExpectedPostgresData, + DestinationDataTypeSchema: PostgresToDestinationSchema, + DefaultCDCColumnsSchema: ExpectedPostgresDefaultCDCColumnsSchema, } } @@ -58,13 +61,31 @@ func TestPostgres2PC(t *testing.T) { } func TestPostgresPerformance(t *testing.T) { - config := &testutils.PerformanceTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.Postgres)), - Namespace: "public", - BackfillStreams: testutils.GetBackfillStreamsFromCDC(performanceCDCStreams), + cfg, err := testutils.NewTestConfig(t, constants.Postgres, "public", "", ExecuteQuery) + require.NoError(t, err, "failed to build the test config") + + perf := &performance.Test{ + TestConfig: cfg, + BackfillStreams: performance.GetBackfillStreamsFromCDC(performanceCDCStreams), CDCStreams: performanceCDCStreams, - ExecuteQuery: ExecuteQuery, } - config.TestPerformance(t) + perf.TestPerformance(t) } + +// TestPostgresCompatibility pins the backward-compatibility contract: the same scenarios run twice in +// parallel -- once entirely on a released baseline image, once handing off to this build after the +// initial load -- and the two destinations must match. The baseline defaults to the newest +// release; OLAKE_COMPATIBILITY_BASELINE picks another tag, image or commit. See tests/testutils/compatibility.go. +// func TestPostgresCompatibility(t *testing.T) { +// t.Parallel() +// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { +// base := postgresBaseConfig(t) +// base.ExpectedUpdatedData = ExpectedUpdatedData +// base.UpdatedDestinationDataTypeSchema = UpdatedPostgresToDestinationSchema +// cfg := &compatibility.Test{IntegrationTest: base} +// // No column rules: postgres compares clean on every reachable baseline (COMPAT_RESULTS_v2.md). +// // The OLAKE_COMPATIBILITY_EXCLUDE_COLUMNS sweep hook lives in RunBackwardCompatibility now. +// return cfg +// }) +// } diff --git a/tests/postgres/postgres_util_test.go b/tests/postgres/postgres_util_test.go index ea12863a4..5f585d47c 100644 --- a/tests/postgres/postgres_util_test.go +++ b/tests/postgres/postgres_util_test.go @@ -3,13 +3,16 @@ package postgres import ( "context" "fmt" + "sync" "testing" "time" "github.com/apache/arrow-go/v18/arrow" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/performance" + "github.com/datazip-inc/olake/tests/testutils/require" "github.com/jmoiron/sqlx" - "github.com/stretchr/testify/require" ) const ( @@ -24,18 +27,21 @@ var performanceCDCStreams = []string{"trips_cdc", "fhv_trips_cdc"} func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() - var connStr string - if config := conf.SourceBaseConfig; config != nil { - connStr = fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=require", - config.String("username"), - config.String("password"), - config.String("host"), - config.Int("port"), - config.String("database"), - ) - } else { - connStr = "postgres://postgres@localhost:5433/postgres?sslmode=disable" + config := conf.SourceBaseConfig + // A config without an ssl block is a remote one (the perf instances); those are reached over + // TLS, which is what this connection assumed before it read the mode from the config at all. + sslMode := config.Sub("ssl").String("mode") + if sslMode == "" { + sslMode = "require" } + connStr := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s", + config.String("username"), + config.String("password"), + config.Host("host"), + config.Int("port"), + config.String("database"), + sslMode, + ) db, ok := sqlx.ConnectContext(ctx, "postgres", connStr) require.NoError(t, ok, "failed to connect to postgres") defer func() { @@ -43,11 +49,14 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, }() // integration test uses only one stream for testing - integrationTestTable := testutils.TestTableName(conf) + integrationTestTable := conf.GetTableName() + replicationSlot := config.Sub("update_method").String("replication_slot") var query string switch operation { case "create": + ensureReplicationSlot(ctx, t, conf, replicationSlot) + query = fmt.Sprintf(` CREATE TABLE IF NOT EXISTS %s ( col_bigint BIGINT, @@ -264,7 +273,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, // insert records in batches batchSize := 300_000 totalRows := 15_000_000 - backfillStreams := testutils.GetBackfillStreamsFromCDC(performanceCDCStreams) + backfillStreams := performance.GetBackfillStreamsFromCDC(performanceCDCStreams) err := testutils.Concurrent(ctx, performanceCDCStreams, len(performanceCDCStreams), func(ctx context.Context, cdcStream string, executionNumber int) error { for offset := 0; offset < totalRows; offset += batchSize { @@ -292,16 +301,16 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, // exceeds the small roll threshold and is split into many files. query = fmt.Sprintf(`INSERT INTO %s (col_text) SELECT md5(random()::text) || md5(random()::text) || md5(random()::text) - FROM generate_series(1, %d)`, integrationTestTable, testutils.RollingSeedRows) + FROM generate_series(1, %d)`, integrationTestTable, integration.RollingSeedRows) case "create-slot": - _, _ = db.ExecContext(ctx, fmt.Sprintf(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = '%s' AND NOT active`, integrationTestTable)) - query = fmt.Sprintf(`SELECT pg_create_logical_replication_slot('%s', 'pgoutput')`, integrationTestTable) + _, _ = db.ExecContext(ctx, fmt.Sprintf(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = '%s' AND NOT active`, replicationSlot)) + query = fmt.Sprintf(`SELECT pg_create_logical_replication_slot('%s', 'pgoutput')`, replicationSlot) case "drop-slot": // Asserted like every other op: the NOT-active guard makes "nothing to drop" a no-op, // so an error here means the cleanup itself is broken. - query = fmt.Sprintf(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = '%s' AND NOT active`, integrationTestTable) + query = fmt.Sprintf(`SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots WHERE slot_name = '%s' AND NOT active`, replicationSlot) default: t.Fatalf("Unsupported operation: %s", operation) @@ -501,3 +510,30 @@ var ExpectedPostgresDefaultCDCColumnsSchema = map[string]string{ "_cdc_timestamp": "timestamp", "_cdc_lsn": "string", } + +// ensureReplicationSlot creates the slot this suite's source config names, once. olake validates +// the CDC config on every command it runs, so the slot has to outlive the whole suite -- hence the +// once: "create" is called again by every subtest that resets the table, and a t.Cleanup registered +// there would drop the slot while the suite is still running. +func ensureReplicationSlot(ctx context.Context, t *testing.T, conf *testutils.TestConfig, slot string) { + t.Helper() + slotsEnsuredMu.Lock() + defer slotsEnsuredMu.Unlock() + if slotsEnsured[slot] { + return + } + slotsEnsured[slot] = true + + ExecuteQuery(ctx, t, conf, "create-slot") + t.Cleanup(func() { + slotsEnsuredMu.Lock() + delete(slotsEnsured, slot) + slotsEnsuredMu.Unlock() + ExecuteQuery(context.WithoutCancel(ctx), t, conf, "drop-slot") + }) +} + +var ( + slotsEnsuredMu sync.Mutex + slotsEnsured = map[string]bool{} +) diff --git a/tests/postgres/testdata/source.json b/tests/postgres/testdata/source.template.json similarity index 90% rename from tests/postgres/testdata/source.json rename to tests/postgres/testdata/source.template.json index 66ba4ae60..c466890ac 100644 --- a/tests/postgres/testdata/source.json +++ b/tests/postgres/testdata/source.template.json @@ -8,7 +8,7 @@ "mode": "disable" }, "update_method": { - "replication_slot": "olake_slot", + "replication_slot": "olake_${suite}", "initial_wait_time": 120, "publication": "olake_publication" }, diff --git a/tests/postgres/testdata/test_streams.json b/tests/postgres/testdata/streams.template.json similarity index 95% rename from tests/postgres/testdata/test_streams.json rename to tests/postgres/testdata/streams.template.json index d5a09a24e..d7547435f 100644 --- a/tests/postgres/testdata/test_streams.json +++ b/tests/postgres/testdata/streams.template.json @@ -1 +1 @@ -{"selected_streams":{"public":[{"partition_regex":"","stream_name":"postgres_test_table_olake","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["col_char","col_varbit","_cdc_lsn","col_bigserial","col_polygon","col_interval","col_timestamp","col_circle","col_numeric","_op_type","col_xml","col_name","col_int2","col_uuid","col_decimal","col_double_precision","excludedcolumn","col_cursor","col_int","col_timestamptz","col_integer","col_character","col_float4","col_json","col_character_varying","_cdc_timestamp","_olake_id","col_bigint","col_point","col_date","col_jsonb","col_bool","col_real","_olake_timestamp","col_text"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"postgres_test_table_olake","namespace":"public","type_schema":{"properties":{"_cdc_lsn":{"type":["string","null"],"destination_column_name":"_cdc_lsn","olake_column":true},"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["null","timestamp_micro"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"col_bigint":{"type":["integer","null"],"destination_column_name":"col_bigint"},"col_bigserial":{"type":["integer"],"destination_column_name":"col_bigserial"},"col_bool":{"type":["boolean","null"],"destination_column_name":"col_bool"},"col_char":{"type":["string","null"],"destination_column_name":"col_char"},"col_character":{"type":["string","null"],"destination_column_name":"col_character"},"col_character_varying":{"type":["null","string"],"destination_column_name":"col_character_varying"},"col_circle":{"type":["string","null"],"destination_column_name":"col_circle"},"col_cursor":{"type":["integer_small","null"],"destination_column_name":"col_cursor"},"col_date":{"type":["timestamp","null"],"destination_column_name":"col_date"},"col_decimal":{"type":["number","null"],"destination_column_name":"col_decimal"},"col_double_precision":{"type":["number","null"],"destination_column_name":"col_double_precision"},"col_float4":{"type":["number_small","null"],"destination_column_name":"col_float4"},"col_int":{"type":["null","integer_small"],"destination_column_name":"col_int"},"col_int2":{"type":["integer_small","null"],"destination_column_name":"col_int2"},"col_integer":{"type":["integer_small","null"],"destination_column_name":"col_integer"},"col_interval":{"type":["string","null"],"destination_column_name":"col_interval"},"col_json":{"type":["null","string"],"destination_column_name":"col_json"},"col_jsonb":{"type":["string","null"],"destination_column_name":"col_jsonb"},"col_name":{"type":["string","null"],"destination_column_name":"col_name"},"col_numeric":{"type":["null","number"],"destination_column_name":"col_numeric"},"col_point":{"type":["string","null"],"destination_column_name":"col_point"},"col_polygon":{"type":["string","null"],"destination_column_name":"col_polygon"},"col_real":{"type":["number_small","null"],"destination_column_name":"col_real"},"col_text":{"type":["string","null"],"destination_column_name":"col_text"},"col_timestamp":{"type":["timestamp","null"],"destination_column_name":"col_timestamp"},"col_timestamptz":{"type":["timestamp","null"],"destination_column_name":"col_timestamptz"},"col_uuid":{"type":["string","null"],"destination_column_name":"col_uuid"},"col_varbit":{"type":["string","null"],"destination_column_name":"col_varbit"},"col_xml":{"type":["string","null"],"destination_column_name":"col_xml"},"excludedcolumn":{"type":["integer_small","null"],"destination_column_name":"excludedcolumn"}}},"supported_sync_modes":["full_refresh","incremental","cdc","strict_cdc"],"source_defined_primary_key":["col_bigserial"],"available_cursor_fields":["col_numeric","col_text","col_uuid","col_circle","col_bool","col_decimal","col_float4","col_json","col_double_precision","col_point","col_date","col_integer","col_real","col_timestamp","col_xml","col_cursor","col_interval","col_jsonb","excludedcolumn","col_char","col_character","col_int","col_timestamptz","col_bigserial","col_character_varying","col_int2","col_varbit","col_polygon","col_bigint","col_name"],"sync_mode":"cdc","destination_database":"postgres_postgres:public","destination_table":"postgres_test_table_olake","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file +{"selected_streams":{"public":[{"partition_regex":"","stream_name":"test_table_olake_${suite}","normalization":true,"use_source_column_names":false,"selected_columns":{"columns":["col_char","col_varbit","_cdc_lsn","col_bigserial","col_polygon","col_interval","col_timestamp","col_circle","col_numeric","_op_type","col_xml","col_name","col_int2","col_uuid","col_decimal","col_double_precision","excludedcolumn","col_cursor","col_int","col_timestamptz","col_integer","col_character","col_float4","col_json","col_character_varying","_cdc_timestamp","_olake_id","col_bigint","col_point","col_date","col_jsonb","col_bool","col_real","_olake_timestamp","col_text"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"test_table_olake_${suite}","namespace":"public","type_schema":{"properties":{"_cdc_lsn":{"type":["string","null"],"destination_column_name":"_cdc_lsn","olake_column":true},"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp","olake_column":true},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["null","timestamp_micro"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"col_bigint":{"type":["integer","null"],"destination_column_name":"col_bigint"},"col_bigserial":{"type":["integer"],"destination_column_name":"col_bigserial"},"col_bool":{"type":["boolean","null"],"destination_column_name":"col_bool"},"col_char":{"type":["string","null"],"destination_column_name":"col_char"},"col_character":{"type":["string","null"],"destination_column_name":"col_character"},"col_character_varying":{"type":["null","string"],"destination_column_name":"col_character_varying"},"col_circle":{"type":["string","null"],"destination_column_name":"col_circle"},"col_cursor":{"type":["integer_small","null"],"destination_column_name":"col_cursor"},"col_date":{"type":["timestamp","null"],"destination_column_name":"col_date"},"col_decimal":{"type":["number","null"],"destination_column_name":"col_decimal"},"col_double_precision":{"type":["number","null"],"destination_column_name":"col_double_precision"},"col_float4":{"type":["number_small","null"],"destination_column_name":"col_float4"},"col_int":{"type":["null","integer_small"],"destination_column_name":"col_int"},"col_int2":{"type":["integer_small","null"],"destination_column_name":"col_int2"},"col_integer":{"type":["integer_small","null"],"destination_column_name":"col_integer"},"col_interval":{"type":["string","null"],"destination_column_name":"col_interval"},"col_json":{"type":["null","string"],"destination_column_name":"col_json"},"col_jsonb":{"type":["string","null"],"destination_column_name":"col_jsonb"},"col_name":{"type":["string","null"],"destination_column_name":"col_name"},"col_numeric":{"type":["null","number"],"destination_column_name":"col_numeric"},"col_point":{"type":["string","null"],"destination_column_name":"col_point"},"col_polygon":{"type":["string","null"],"destination_column_name":"col_polygon"},"col_real":{"type":["number_small","null"],"destination_column_name":"col_real"},"col_text":{"type":["string","null"],"destination_column_name":"col_text"},"col_timestamp":{"type":["timestamp","null"],"destination_column_name":"col_timestamp"},"col_timestamptz":{"type":["timestamp","null"],"destination_column_name":"col_timestamptz"},"col_uuid":{"type":["string","null"],"destination_column_name":"col_uuid"},"col_varbit":{"type":["string","null"],"destination_column_name":"col_varbit"},"col_xml":{"type":["string","null"],"destination_column_name":"col_xml"},"excludedcolumn":{"type":["integer_small","null"],"destination_column_name":"excludedcolumn"}}},"supported_sync_modes":["full_refresh","incremental","cdc","strict_cdc"],"source_defined_primary_key":["col_bigserial"],"available_cursor_fields":["col_numeric","col_text","col_uuid","col_circle","col_bool","col_decimal","col_float4","col_json","col_double_precision","col_point","col_date","col_integer","col_real","col_timestamp","col_xml","col_cursor","col_interval","col_jsonb","excludedcolumn","col_char","col_character","col_int","col_timestamptz","col_bigserial","col_character_varying","col_int2","col_varbit","col_polygon","col_bigint","col_name"],"sync_mode":"cdc","destination_database":"postgres_postgres:public","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":true,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/s3/s3_test.go b/tests/s3/s3_test.go index 7a2fc7480..53641f012 100644 --- a/tests/s3/s3_test.go +++ b/tests/s3/s3_test.go @@ -5,27 +5,30 @@ import ( "github.com/datazip-inc/olake/tests/testutils" "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/require" ) // s3BaseConfig returns an IntegrationTest for one source format variant. Each variant owns a -// testdata// directory, which is what GetTestConfig's third argument selects. -func s3BaseConfig(t *testing.T, variant S3TestVariant) *testutils.IntegrationTest { - filterConfig := S3FilterConfig - if variant.DataFormat == "xml" { - filterConfig = S3XMLFilterConfig - } - return &testutils.IntegrationTest{ - TestConfig: testutils.GetTestConfig(t, string(constants.S3), variant.DataFormat), - Namespace: "s3", - ExpectedData: variant.ExpectedData, +// testdata// directory, which is what DataFormat selects. +func s3BaseConfig(t *testing.T, variant S3TestVariant) *integration.Test { + config, err := testutils.NewTestConfig(t, constants.S3, "s3", S3DestinationDB, nil, + testutils.WithDataFormat(variant.DataFormat)) + require.NoError(t, err, "failed to build the test config") + config.ColumnToExclude = excludedColumn + config.CursorField = S3CursorField + config.PartitionRegex = S3PartitionRegex + config.FilterConfig = variant.FilterConfig + + cfg := &integration.Test{ + TestConfig: config, + ExpectedData: variant.ExpectedRowData(seedValues), + ExpectedUpdatedData: variant.ExpectedRowData(updatedValues), DestinationDataTypeSchema: variant.DestinationSchema, - ExecuteQuery: ExecuteQueryFactory(variant), - ColumnToExclude: excludedColumn, - DestinationDB: S3DestinationDB, - CursorField: S3CursorField, - PartitionRegex: S3PartitionRegex, - FilterConfig: filterConfig, } + // The factory closes over the test it drives, so it is built once that exists. + config.ExecuteQuery = ExecuteQueryFactory(variant, cfg) + return cfg } func TestS3Discover(t *testing.T) { @@ -42,8 +45,6 @@ func TestS3Sync(t *testing.T) { t.Run(variant.Name, func(t *testing.T) { t.Parallel() cfg := s3BaseConfig(t, variant) - cfg.IsolateSuite(t, variant.Name) - cfg.ExpectedUpdatedData = variant.ExpectedUpdatedData // The "evolve-schema" operation ships a file carrying a column discover has not // seen (see S3TestVariant.BuildEvolvedFile), so the update sync must land it in // the destination as a string column. @@ -52,3 +53,46 @@ func TestS3Sync(t *testing.T) { }) } } + +// TestS3Compatibility runs every source format. Each variant owns its testdata directory, source prefix +// and stream name, so the three share one destination namespace without colliding. +// func TestS3Compatibility(t *testing.T) { +// t.Parallel() +// for _, variant := range S3TestVariants { +// t.Run(variant.Name, func(t *testing.T) { +// t.Parallel() +// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { +// base := s3BaseConfig(t, variant) +// cfg := &compatibility.Test{IntegrationTest: base} +// // Same isolation TestS3Sync applies: Parquet and ParquetInMemory share a +// // DataFormat, so without it they share every name the suite derives from it. +// cfg.IntegrationTest.Suite = variant.Name +// // Type tags for compatibility_rules.json's s3 rules; the driver-level _olake_id and +// // _last_modified_time policies are column-keyed there and need no tags. +// switch variant.DataFormat { +// case "json": +// cfg.ColumnTypes = map[string][]string{"mixed_col": {"mixed"}} +// case "csv": +// cfg.ColumnTypes = map[string][]string{evolvedColumn: {"evolved"}} +// case "parquet": +// cfg.ColumnTypes = map[string][]string{ +// "map_col": {"map"}, +// "struct_col": {"struct"}, +// "list_col": {"list"}, +// "int96_col": {"int96"}, +// "ts_col": {"timestamp"}, +// "ts_ms_col": {"timestamp"}, +// "ts_ns_col": {"timestamp"}, +// "ts_far_col": {"timestamp"}, +// "uuid_col": {"uuid"}, +// } +// } +// // The closure reads SeedExcludedColumns at call time; RunBackwardCompatibility fills it +// // in after resolving the rules above against the baseline. +// cfg.SupportsSeedExclusion = true +// base.TestConfig.ExecuteQuery = ExecuteQueryFactoryExcluding(variant, cfg.IntegrationTest, func() []string { return cfg.SeedExcludedColumns }) +// return cfg +// }) +// }) +// } +// } diff --git a/tests/s3/s3_util_test.go b/tests/s3/s3_util_test.go index 01bc4cb8e..21a23c2c0 100644 --- a/tests/s3/s3_util_test.go +++ b/tests/s3/s3_util_test.go @@ -10,22 +10,21 @@ import ( "maps" "math" "math/big" - "net" "net/url" "os" - "path/filepath" "strings" "testing" "time" "github.com/apache/arrow-go/v18/arrow" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/require" "github.com/google/uuid" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" pq "github.com/parquet-go/parquet-go" "github.com/parquet-go/parquet-go/deprecated" - "github.com/stretchr/testify/require" ) // The S3 integration test reuses the MinIO instance from the Iceberg destination stack @@ -229,7 +228,7 @@ var ( // S3CSVToDestinationSchema and S3JSONToDestinationSchema are the expected destination // schemas for the two text variants, whose parsers infer every number as double. They - // share the text columns and differ where the formats do: only CSV can carry an + // share the text columns and differ where the variants do: only CSV can carry an // all-empty column, only JSON can carry missing fields and nested values. S3CSVToDestinationSchema = s3TextDestinationSchema(map[string]string{ "null_col": "string", @@ -516,7 +515,9 @@ func mustJSON(v any) string { } // buildFileFn renders rowsPerFile rows carrying vals, with ids startID..startID+rowsPerFile-1. -type buildFileFn func(t *testing.T, startID int64, vals rowValues) []byte +// buildFileFn renders one source file. excluded names the seed columns the compatibility suite is +// leaving out for this baseline (CompatibilityColumnRule.ExcludeBelow); it is empty everywhere else. +type buildFileFn func(t *testing.T, startID int64, vals rowValues, excluded []string) []byte // S3TestVariant describes one source file format exercised by TestS3Integration. Each // variant owns a testdata// directory holding its committed source.json, @@ -536,10 +537,11 @@ type S3TestVariant struct { // UpdatedDestinationSchema is the destination schema after the "evolve-schema" // operation ran; same as DestinationSchema where BuildEvolvedFile is nil. UpdatedDestinationSchema map[string]string - // ExpectedData and ExpectedUpdatedData are the values every synced row must carry. They - // are per variant because a Parquet file can express types that CSV and JSON cannot. - ExpectedData map[string]interface{} - ExpectedUpdatedData map[string]interface{} + // ExpectedRowData builds the values every synced row must carry -- per variant, because a + // Parquet file can express types CSV and JSON cannot. A builder rather than a map: each + // config needs its own, since applyWriterExpectations rewrites them per destination writer + // and the compatibility suite runs six configs of one variant at once. + ExpectedRowData func(v rowValues) map[string]interface{} // WriterExpectedData returns the expected values for the columns whose synced value // depends on which destination writer ran (see textWriterExpectedData). Merged into // ExpectedData/ExpectedUpdatedData by applyWriterExpectations; nil when every column @@ -548,6 +550,9 @@ type S3TestVariant struct { // ParquetStreaming is the parquet.streaming_enabled value every sync of the variant // runs with; meaningful only when DataFormat is "parquet" (see applyParquetStreamingMode). ParquetStreaming bool + // FilterConfig is the filter every sync of the variant runs with. XML infers every column + // as a string, so its conditions stay string-only or they match no row at all. + FilterConfig string } // S3TestVariants lists every source format covered by TestS3Integration. @@ -561,9 +566,9 @@ var S3TestVariants = []S3TestVariant{ BuildEvolvedFile: buildEvolvedCSVFile, DestinationSchema: S3CSVToDestinationSchema, UpdatedDestinationSchema: S3CSVUpdatedDestinationSchema, - ExpectedData: expectedCSVData(seedValues), - ExpectedUpdatedData: expectedCSVData(updatedValues), + ExpectedRowData: expectedCSVData, WriterExpectedData: textWriterExpectedData, + FilterConfig: S3FilterConfig, }, { Name: "JSON", @@ -574,9 +579,9 @@ var S3TestVariants = []S3TestVariant{ BuildEvolvedFile: buildEvolvedJSONLFile, DestinationSchema: S3JSONToDestinationSchema, UpdatedDestinationSchema: S3JSONUpdatedDestinationSchema, - ExpectedData: expectedJSONData(seedValues), - ExpectedUpdatedData: expectedJSONData(updatedValues), + ExpectedRowData: expectedJSONData, WriterExpectedData: textWriterExpectedData, + FilterConfig: S3FilterConfig, }, { // Identical to Parquet except streaming_enabled=false: every sync loads whole files @@ -588,9 +593,9 @@ var S3TestVariants = []S3TestVariant{ BuildEvolvedFile: buildEvolvedParquetFile, DestinationSchema: S3ParquetToDestinationSchema, UpdatedDestinationSchema: S3ParquetUpdatedDestinationSchema, - ExpectedData: expectedParquetData(seedValues), - ExpectedUpdatedData: expectedParquetData(updatedValues), + ExpectedRowData: expectedParquetData, WriterExpectedData: parquetWriterExpectedData, + FilterConfig: S3FilterConfig, }, { // The driver's file matcher recognizes no gzip variant for Parquet, so this stream @@ -602,10 +607,10 @@ var S3TestVariants = []S3TestVariant{ BuildEvolvedFile: buildEvolvedParquetFile, DestinationSchema: S3ParquetToDestinationSchema, UpdatedDestinationSchema: S3ParquetUpdatedDestinationSchema, - ExpectedData: expectedParquetData(seedValues), - ExpectedUpdatedData: expectedParquetData(updatedValues), + ExpectedRowData: expectedParquetData, WriterExpectedData: parquetWriterExpectedData, ParquetStreaming: true, + FilterConfig: S3FilterConfig, }, { Name: "XML", @@ -616,9 +621,9 @@ var S3TestVariants = []S3TestVariant{ BuildEvolvedFile: buildEvolvedXMLFile, DestinationSchema: S3XMLToDestinationSchema, UpdatedDestinationSchema: S3XMLUpdatedDestinationSchema, - ExpectedData: expectedXMLData(seedValues), - ExpectedUpdatedData: expectedXMLData(updatedValues), + ExpectedRowData: expectedXMLData, WriterExpectedData: textWriterExpectedData, + FilterConfig: S3XMLFilterConfig, }, } @@ -630,13 +635,15 @@ type s3Source struct { prefix string } -func (v S3TestVariant) source(t *testing.T) s3Source { +// source reads the suite's own source config: applySuite rewrites path_prefix there per suite, and +// uploads have to land where discover will look. +func (v S3TestVariant) source(t *testing.T, conf *testutils.TestConfig) s3Source { t.Helper() - config := testutils.ReadSourceConfig(t, filepath.Join("testdata", v.DataFormat, "source.json")) + config := conf.SourceBaseConfig endpoint, err := url.Parse(config.String("endpoint")) require.NoError(t, err, "failed to parse endpoint") - client, err := minio.New(net.JoinHostPort("127.0.0.1", endpoint.Port()), &minio.Options{ + client, err := minio.New(testutils.HostAddress(endpoint.Host), &minio.Options{ Creds: credentials.NewStaticV4(config.String("access_key_id"), config.String("secret_access_key"), ""), }) require.NoError(t, err, "failed to create MinIO client") @@ -674,14 +681,12 @@ const ( writerArrow s3DestinationWriter = "arrow" ) -// currentDestinationWriter reads the live arrow_writes flag from the destination config the -// next sync will run with, and reports which writer that is. -func (v S3TestVariant) currentDestinationWriter(t *testing.T, config *testutils.TestConfig) s3DestinationWriter { +// currentDestinationWriter reads the arrow_writes flag out of destinationFile, the destination +// config the next sync will run with, and reports which writer that is. +func (v S3TestVariant) currentDestinationWriter(t *testing.T, config *testutils.TestConfig, destinationFile string) s3DestinationWriter { t.Helper() - // The harness picks a writer by swapping IcebergDestinationPath between two files in its - // private working directory (see testIcebergWriter) - destPath := filepath.Join(config.HostTestDataPath, filepath.Base(config.IcebergDestinationPath)) + destPath := config.GetFilePath(destinationFile) data, err := os.ReadFile(destPath) require.NoError(t, err, "failed to read %s", destPath) var destConfig struct { @@ -698,19 +703,18 @@ func (v S3TestVariant) currentDestinationWriter(t *testing.T, config *testutils. } // applyWriterExpectations retargets the writer-dependent expected values at the writer the -// next sync will use. The harness toggles arrow_writes in the variant's -// iceberg_destination.json before each Iceberg writer block but asserts every block against -// the same ExpectedData maps, so this hook -- the only variant-owned code that runs between -// the toggle and the verification -- reads the live flag and updates the maps in place. -func (v S3TestVariant) applyWriterExpectations(t *testing.T, config *testutils.TestConfig) { +// next sync will use. Each Iceberg writer block names its own destination config but asserts +// against the same ExpectedData maps, so this hook -- the only variant-owned code that runs +// between the switch and the verification -- reads the live flag and updates the maps in place. +func (v S3TestVariant) applyWriterExpectations(t *testing.T, cfg *integration.Test, config *testutils.TestConfig) { t.Helper() if v.WriterExpectedData == nil { return } - writer := v.currentDestinationWriter(t, config) - maps.Copy(v.ExpectedData, v.WriterExpectedData(seedValues, writer)) - maps.Copy(v.ExpectedUpdatedData, v.WriterExpectedData(updatedValues, writer)) + writer := v.currentDestinationWriter(t, config, cfg.IcebergDestinationFile()) + maps.Copy(cfg.ExpectedData, v.WriterExpectedData(seedValues, writer)) + maps.Copy(cfg.ExpectedUpdatedData, v.WriterExpectedData(updatedValues, writer)) } // applyParquetStreamingMode pins parquet.streaming_enabled in this config's source.json @@ -720,7 +724,7 @@ func (v S3TestVariant) applyParquetStreamingMode(t *testing.T, config *testutils return } - path := config.HostSourcePath + path := config.GetFilePath("source.json") data, err := os.ReadFile(path) require.NoError(t, err, "failed to read %s", path) @@ -740,18 +744,29 @@ func (v S3TestVariant) applyParquetStreamingMode(t *testing.T, config *testutils // the variant's path prefix: "create" ensures the bucket exists, "add" seeds the stream, // "insert"/"update" upload a further file each, and "clean"/"drop" remove everything under // the prefix. -func ExecuteQueryFactory(variant S3TestVariant) func(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { +func ExecuteQueryFactory(variant S3TestVariant, cfg *integration.Test) func(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { + return ExecuteQueryFactoryExcluding(variant, cfg, nil) +} + +// ExecuteQueryFactoryExcluding is ExecuteQueryFactory with the compatibility suite's seed exclusions: +// seedExcluded is read per call, because RunBackwardCompatibility fills the list in after the config is +// built. A nil getter is the plain fixture, every column seeded. +func ExecuteQueryFactoryExcluding(variant S3TestVariant, cfg *integration.Test, seedExcluded func() []string) func(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { return func(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() + var excluded []string + if seedExcluded != nil { + excluded = seedExcluded() + } // Every destination block starts by re-seeding the source through this hook, so // refreshing the expectations here keeps them aligned with whichever writer the - // harness toggled the destination to since the last operation. - variant.applyWriterExpectations(t, conf) + // harness pointed the destination at since the last operation. + variant.applyWriterExpectations(t, cfg, conf) variant.applyParquetStreamingMode(t, conf) - src := variant.source(t) - prefix := src.prefix + "/" + testutils.TestTableName(conf) + "/" + src := variant.source(t, conf) + prefix := src.prefix + "/" + conf.GetTableName() + "/" switch operation { case "drop-all": @@ -773,19 +788,19 @@ func ExecuteQueryFactory(variant S3TestVariant) func(ctx context.Context, t *tes src.removeUnder(ctx, t, prefix) case "add": - // One plain file and, where the format allows it, one gzipped: a single stream + // One plain file and, where the variant allows it, one gzipped: a single stream // mixing both proves compression is detected per file rather than per stream. - variant.putFile(ctx, t, src, prefix, "seed_1", variant.BuildFile, 1, seedValues, false, false) - variant.putFile(ctx, t, src, prefix, "seed_2", variant.BuildFile, 4, seedValues, variant.Gzipped, false) + variant.putFile(ctx, t, src, prefix, "seed_1", variant.BuildFile, 1, seedValues, false, excluded) + variant.putFile(ctx, t, src, prefix, "seed_2", variant.BuildFile, 4, seedValues, variant.Gzipped, excluded) case "insert": // A file the incremental cursor has not seen: it is stamped after the previous // sync, so only its rows are re-read. - variant.putFile(ctx, t, src, prefix, "insert_1", variant.BuildFile, 7, seedValues, false, true) + variant.putFilePastCursor(ctx, t, src, prefix, "insert_1", variant.BuildFile, 7, seedValues, false, excluded) case "update": // Object stores have no in-place update: changed data arrives as another file. - variant.putFile(ctx, t, src, prefix, "update_1", variant.BuildFile, 10, updatedValues, false, true) + variant.putFilePastCursor(ctx, t, src, prefix, "update_1", variant.BuildFile, 10, updatedValues, false, excluded) case "evolve-schema": // An object store's ALTER TABLE: a file whose rows carry a column discover has @@ -795,7 +810,7 @@ func ExecuteQueryFactory(variant S3TestVariant) func(ctx context.Context, t *tes // ExpectedUpdatedData; evolvedColumn itself is asserted through the schema, not // per row, since the "update" file's rows sync a null there. if variant.BuildEvolvedFile != nil { - variant.putFile(ctx, t, src, prefix, "evolve_1", variant.BuildEvolvedFile, 13, updatedValues, false, true) + variant.putFilePastCursor(ctx, t, src, prefix, "evolve_1", variant.BuildEvolvedFile, 13, updatedValues, false, excluded) } default: @@ -807,10 +822,10 @@ func ExecuteQueryFactory(variant S3TestVariant) func(ctx context.Context, t *tes // putFile renders one file with build and uploads it under prefix. Gzipped files get a // ".gz" suffix, which is what both the driver's file matcher and its reader use to detect // compression. -func (v S3TestVariant) putFile(ctx context.Context, t *testing.T, src s3Source, prefix, name string, build buildFileFn, startID int64, vals rowValues, gzipped bool, wait bool) { +func (v S3TestVariant) putFile(ctx context.Context, t *testing.T, src s3Source, prefix, name string, build buildFileFn, startID int64, vals rowValues, gzipped bool, excluded []string) string { t.Helper() - data := build(t, startID, vals) + data := build(t, startID, vals, excluded) ext := v.PlainExt if gzipped { data = gzipBytes(t, data) @@ -818,17 +833,38 @@ func (v S3TestVariant) putFile(ctx context.Context, t *testing.T, src s3Source, } key := prefix + name + ext - - // TODO: the driver should handle same-second arrivals itself (`>=` plus tracking the file keys - // already synced at the cursor's second); this guard papers over real, silent data loss. - if wait { - t.Logf("waiting before putting file to avoid s3 driver skipping the new file...") - time.Sleep(1 * time.Second) - } - _, err := src.client.PutObject(ctx, src.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}) require.NoError(t, err, "failed to upload %s", key) t.Logf("Uploaded s3://%s/%s (%d bytes)", src.bucket, key, len(data)) + return key +} + +// putFilePastCursor is putFile for a file the next incremental sync must pick up: the driver's +// cursor keeps LastModified at whole seconds with a strict >, so a file landing in the same second +// as the previous sync's newest object is silently skipped. Re-upload until the object's second is +// past every object already under the prefix. +// TODO: the driver should handle same-second arrivals itself (`>=` plus tracking the file keys +// already synced at the cursor's second); this guard papers over real, silent data loss. +func (v S3TestVariant) putFilePastCursor(ctx context.Context, t *testing.T, src s3Source, prefix, name string, build buildFileFn, startID int64, vals rowValues, gzipped bool, excluded []string) { + t.Helper() + + var prevMax time.Time + for obj := range src.client.ListObjects(ctx, src.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + require.NoError(t, obj.Err, "failed to list objects under %s", prefix) + if obj.LastModified.After(prevMax) { + prevMax = obj.LastModified + } + } + prevSecond := prevMax.UTC().Truncate(time.Second) + for { + key := v.putFile(ctx, t, src, prefix, name, build, startID, vals, gzipped, excluded) + info, err := src.client.StatObject(ctx, src.bucket, key, minio.StatObjectOptions{}) + require.NoError(t, err, "failed to stat %s", key) + if info.LastModified.UTC().Truncate(time.Second).After(prevSecond) { + return + } + time.Sleep(200 * time.Millisecond) + } } // Sub-second timestamp layouts for the text variants. Fixed-width fractions (not the @@ -854,14 +890,15 @@ func mixedValue(id int64) (csvCell, jsonToken string) { } } -func buildCSVFile(_ *testing.T, startID int64, vals rowValues) []byte { +func buildCSVFile(t *testing.T, startID int64, vals rowValues, excluded []string) []byte { + evolvedColumnExcluded(t, excluded) return csvFile(startID, vals, false) } // buildEvolvedCSVFile is buildCSVFile plus evolvedColumn: a header the discovered schema // lacks, which the parser must stream through for the destination to evolve. -func buildEvolvedCSVFile(_ *testing.T, startID int64, vals rowValues) []byte { - return csvFile(startID, vals, true) +func buildEvolvedCSVFile(t *testing.T, startID int64, vals rowValues, excluded []string) []byte { + return csvFile(startID, vals, !evolvedColumnExcluded(t, excluded)) } func csvFile(startID int64, vals rowValues, evolved bool) []byte { @@ -875,7 +912,7 @@ func csvFile(startID int64, vals rowValues, evolved bool) []byte { id := startID + i mixed, _ := mixedValue(id) // null_col is the empty cell after mixed_col: CSV cannot omit a column, so an - // empty value is how the format spells null. + // empty value is how the variant spells null. b.WriteString(fmt.Sprintf("%d,%s,%t,%v,%d,%s,,%s,%s,%s,%s,%s,%s", id, vals.Str, vals.Bool, vals.Float, vals.Int64, mixed, vals.TS.UTC().Format(time.DateOnly), @@ -892,13 +929,14 @@ func csvFile(startID int64, vals rowValues, evolved bool) []byte { return []byte(b.String()) } -func buildJSONLFile(_ *testing.T, startID int64, vals rowValues) []byte { +func buildJSONLFile(t *testing.T, startID int64, vals rowValues, excluded []string) []byte { + evolvedColumnExcluded(t, excluded) return jsonlFile(startID, vals, false) } // buildEvolvedJSONLFile is buildJSONLFile plus evolvedColumn on every record. -func buildEvolvedJSONLFile(_ *testing.T, startID int64, vals rowValues) []byte { - return jsonlFile(startID, vals, true) +func buildEvolvedJSONLFile(t *testing.T, startID int64, vals rowValues, excluded []string) []byte { + return jsonlFile(startID, vals, !evolvedColumnExcluded(t, excluded)) } func jsonlFile(startID int64, vals rowValues, evolved bool) []byte { @@ -937,12 +975,13 @@ func jsonlFile(startID int64, vals rowValues, evolved bool) []byte { return []byte(b.String()) } -func buildXMLFile(_ *testing.T, startID int64, vals rowValues) []byte { +func buildXMLFile(t *testing.T, startID int64, vals rowValues, excluded []string) []byte { + evolvedColumnExcluded(t, excluded) return xmlFile(startID, vals, false) } -func buildEvolvedXMLFile(_ *testing.T, startID int64, vals rowValues) []byte { - return xmlFile(startID, vals, true) +func buildEvolvedXMLFile(t *testing.T, startID int64, vals rowValues, excluded []string) []byte { + return xmlFile(startID, vals, !evolvedColumnExcluded(t, excluded)) } func xmlFile(startID int64, vals rowValues, evolved bool) []byte { @@ -1108,17 +1147,33 @@ func parquetTestGroup() pq.Group { } } -var ( - parquetTestSchema = pq.NewSchema("s3_parquet_row", parquetTestGroup()) - - // evolvedParquetTestSchema is the base schema plus evolvedColumn, for the file the - // "evolve-schema" operation uploads. - evolvedParquetTestSchema = pq.NewSchema("s3_parquet_row", func() pq.Group { - group := parquetTestGroup() - group[evolvedColumn] = pq.String() - return group - }()) -) +// parquetExcludableColumns are the seed columns the fixture knows how to leave out, all of them +// #1020 (v0.9.1) fixes: the group-typed three panic a older parser on Kind(), and int96 is typed +// timestamptz by discover while the reader hands back a string, which the iceberg flush rejects. +var parquetExcludableColumns = []string{"map_col", "struct_col", "list_col", "int96_col"} + +// parquetGroupExcluding is the seed's column group minus the columns this baseline cannot read. +func parquetGroupExcluding(t *testing.T, excluded []string) pq.Group { + t.Helper() + drop, err := testutils.SeedColumnsExcluded(excluded, parquetExcludableColumns) + require.NoError(t, err, "s3 parquet seed exclusion") + + group := parquetTestGroup() + for column := range drop { + delete(group, column) + } + return group +} + +// evolvedColumnExcluded reports whether the compatibility suite is holding the evolve column back for +// this baseline. It is the one column the text variants know how to leave out, so it doubles as +// their guard: any other name fails loudly rather than seeding what the baseline cannot survive. +func evolvedColumnExcluded(t *testing.T, excluded []string) bool { + t.Helper() + drop, err := testutils.SeedColumnsExcluded(excluded, []string{evolvedColumn}) + require.NoError(t, err, "s3 seed exclusion") + return drop[evolvedColumn] +} func makeParquetRow(id int64, vals rowValues) parquetRow { row := parquetRow{ @@ -1178,14 +1233,14 @@ func makeParquetRow(id int64, vals rowValues) parquetRow { return row } -func buildParquetFile(t *testing.T, startID int64, vals rowValues) []byte { +func buildParquetFile(t *testing.T, startID int64, vals rowValues, excluded []string) []byte { t.Helper() rows := make([]parquetRow, 0, rowsPerFile) for i := int64(0); i < rowsPerFile; i++ { rows = append(rows, makeParquetRow(startID+i, vals)) } - return writeParquetRows(t, parquetTestSchema, rows) + return writeParquetRows(t, pq.NewSchema("s3_parquet_row", parquetGroupExcluding(t, excluded)), rows) } func writeParquetRows(t *testing.T, schema *pq.Schema, rows []parquetRow) []byte { @@ -1206,7 +1261,7 @@ func writeParquetRows(t *testing.T, schema *pq.Schema, rows []parquetRow) []byte // buildEvolvedParquetFile is buildParquetFile plus evolvedColumn on every row: a column // the discovered schema lacks, which the parser hands through for the destination to evolve. -func buildEvolvedParquetFile(t *testing.T, startID int64, vals rowValues) []byte { +func buildEvolvedParquetFile(t *testing.T, startID int64, vals rowValues, excluded []string) []byte { t.Helper() rows := make([]parquetRow, 0, rowsPerFile) for i := int64(0); i < rowsPerFile; i++ { @@ -1215,7 +1270,9 @@ func buildEvolvedParquetFile(t *testing.T, startID int64, vals rowValues) []byte rows = append(rows, row) } - return writeParquetRows(t, evolvedParquetTestSchema, rows) + group := parquetGroupExcluding(t, excluded) + group[evolvedColumn] = pq.String() + return writeParquetRows(t, pq.NewSchema("s3_parquet_row", group), rows) } // decimalToFixedBytes renders the unscaled integer as the 16 byte big-endian two's diff --git a/tests/s3/testdata/csv/source.json b/tests/s3/testdata/csv/source.template.json similarity index 88% rename from tests/s3/testdata/csv/source.json rename to tests/s3/testdata/csv/source.template.json index ca132433c..245389e7b 100644 --- a/tests/s3/testdata/csv/source.json +++ b/tests/s3/testdata/csv/source.template.json @@ -1,7 +1,7 @@ { "bucket_name": "olake-s3-test", "region": "us-east-1", - "path_prefix": "csv", + "path_prefix": "${suite}", "access_key_id": "admin", "secret_access_key": "password", "endpoint": "http://host.docker.internal:9000", diff --git a/tests/s3/testdata/csv/streams.template.json b/tests/s3/testdata/csv/streams.template.json new file mode 100644 index 000000000..54b73a397 --- /dev/null +++ b/tests/s3/testdata/csv/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"s3":[{"partition_regex":"","stream_name":"test_table_olake_${suite}","normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["_olake_id","id","date_col","_last_modified_time","float_col","_op_type","str_col","int_col","mixed_col","ts_milli_col","ts_micro_col","_olake_timestamp","excluded_col","ts_col","null_col","bool_col","ts_nano_col"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"test_table_olake_${suite}","namespace":"s3","type_schema":{"properties":{"_last_modified_time":{"type":["string"],"destination_column_name":"_last_modified_time"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"bool_col":{"type":["boolean","null"],"destination_column_name":"bool_col"},"date_col":{"type":["timestamp","null"],"destination_column_name":"date_col"},"excluded_col":{"type":["string","null"],"destination_column_name":"excluded_col"},"float_col":{"type":["number","null"],"destination_column_name":"float_col"},"id":{"type":["number","null"],"destination_column_name":"id"},"int_col":{"type":["null","number"],"destination_column_name":"int_col"},"mixed_col":{"type":["string","null"],"destination_column_name":"mixed_col"},"null_col":{"type":["string","null"],"destination_column_name":"null_col"},"str_col":{"type":["null","string"],"destination_column_name":"str_col"},"ts_col":{"type":["timestamp","null"],"destination_column_name":"ts_col"},"ts_micro_col":{"type":["timestamp_micro","null"],"destination_column_name":"ts_micro_col"},"ts_milli_col":{"type":["timestamp_milli","null"],"destination_column_name":"ts_milli_col"},"ts_nano_col":{"type":["timestamp_nano","null"],"destination_column_name":"ts_nano_col"}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":[],"available_cursor_fields":["_last_modified_time"],"cursor_field":"_last_modified_time","sync_mode":"incremental","destination_database":"s3_olake_s3_test:s3","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":false,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/s3/testdata/csv/test_streams.json b/tests/s3/testdata/csv/test_streams.json deleted file mode 100644 index 30a88ac0b..000000000 --- a/tests/s3/testdata/csv/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"s3":[{"partition_regex":"","stream_name":"s3_csv_test_table_olake","normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["_olake_id","id","date_col","_last_modified_time","float_col","_op_type","str_col","int_col","mixed_col","ts_milli_col","ts_micro_col","_olake_timestamp","excluded_col","ts_col","null_col","bool_col","ts_nano_col"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"s3_csv_test_table_olake","namespace":"s3","type_schema":{"properties":{"_last_modified_time":{"type":["string"],"destination_column_name":"_last_modified_time"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"bool_col":{"type":["boolean","null"],"destination_column_name":"bool_col"},"date_col":{"type":["timestamp","null"],"destination_column_name":"date_col"},"excluded_col":{"type":["string","null"],"destination_column_name":"excluded_col"},"float_col":{"type":["number","null"],"destination_column_name":"float_col"},"id":{"type":["number","null"],"destination_column_name":"id"},"int_col":{"type":["null","number"],"destination_column_name":"int_col"},"mixed_col":{"type":["string","null"],"destination_column_name":"mixed_col"},"null_col":{"type":["string","null"],"destination_column_name":"null_col"},"str_col":{"type":["null","string"],"destination_column_name":"str_col"},"ts_col":{"type":["timestamp","null"],"destination_column_name":"ts_col"},"ts_micro_col":{"type":["timestamp_micro","null"],"destination_column_name":"ts_micro_col"},"ts_milli_col":{"type":["timestamp_milli","null"],"destination_column_name":"ts_milli_col"},"ts_nano_col":{"type":["timestamp_nano","null"],"destination_column_name":"ts_nano_col"}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":[],"available_cursor_fields":["_last_modified_time"],"cursor_field":"_last_modified_time","sync_mode":"incremental","destination_database":"s3_olake_s3_test:s3","destination_table":"s3_csv_test_table_olake","default_stream_properties":{"normalization":false,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/s3/testdata/json/source.json b/tests/s3/testdata/json/source.template.json similarity index 88% rename from tests/s3/testdata/json/source.json rename to tests/s3/testdata/json/source.template.json index 3bad151d4..0d89b41b2 100644 --- a/tests/s3/testdata/json/source.json +++ b/tests/s3/testdata/json/source.template.json @@ -1,7 +1,7 @@ { "bucket_name": "olake-s3-test", "region": "us-east-1", - "path_prefix": "json", + "path_prefix": "${suite}", "access_key_id": "admin", "secret_access_key": "password", "endpoint": "http://host.docker.internal:9000", diff --git a/tests/s3/testdata/json/streams.template.json b/tests/s3/testdata/json/streams.template.json new file mode 100644 index 000000000..f12b3edf6 --- /dev/null +++ b/tests/s3/testdata/json/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"s3":[{"partition_regex":"","stream_name":"test_table_olake_${suite}","normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["_op_type","ts_milli_col","bool_col","_last_modified_time","ts_micro_col","excluded_col","float_col","mixed_col","date_col","ts_nano_col","str_col","int_col","id","object_col","_olake_timestamp","_olake_id","optional_col","ts_col","array_col"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"test_table_olake_${suite}","namespace":"s3","type_schema":{"properties":{"_last_modified_time":{"type":["string"],"destination_column_name":"_last_modified_time"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"array_col":{"type":["array"],"destination_column_name":"array_col"},"bool_col":{"type":["boolean"],"destination_column_name":"bool_col"},"date_col":{"type":["timestamp"],"destination_column_name":"date_col"},"excluded_col":{"type":["string"],"destination_column_name":"excluded_col"},"float_col":{"type":["number"],"destination_column_name":"float_col"},"id":{"type":["number"],"destination_column_name":"id"},"int_col":{"type":["number"],"destination_column_name":"int_col"},"mixed_col":{"type":["number","string","boolean"],"destination_column_name":"mixed_col"},"object_col":{"type":["object"],"destination_column_name":"object_col"},"optional_col":{"type":["string"],"destination_column_name":"optional_col"},"str_col":{"type":["string"],"destination_column_name":"str_col"},"ts_col":{"type":["timestamp"],"destination_column_name":"ts_col"},"ts_micro_col":{"type":["timestamp_micro"],"destination_column_name":"ts_micro_col"},"ts_milli_col":{"type":["timestamp_milli"],"destination_column_name":"ts_milli_col"},"ts_nano_col":{"type":["timestamp_nano"],"destination_column_name":"ts_nano_col"}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":[],"available_cursor_fields":["_last_modified_time"],"cursor_field":"_last_modified_time","sync_mode":"incremental","destination_database":"s3_olake_s3_test:s3","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":false,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/s3/testdata/json/test_streams.json b/tests/s3/testdata/json/test_streams.json deleted file mode 100644 index c900fb7b4..000000000 --- a/tests/s3/testdata/json/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"s3":[{"partition_regex":"","stream_name":"s3_json_test_table_olake","normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["_op_type","ts_milli_col","bool_col","_last_modified_time","ts_micro_col","excluded_col","float_col","mixed_col","date_col","ts_nano_col","str_col","int_col","id","object_col","_olake_timestamp","_olake_id","optional_col","ts_col","array_col"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"s3_json_test_table_olake","namespace":"s3","type_schema":{"properties":{"_last_modified_time":{"type":["string"],"destination_column_name":"_last_modified_time"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"array_col":{"type":["array"],"destination_column_name":"array_col"},"bool_col":{"type":["boolean"],"destination_column_name":"bool_col"},"date_col":{"type":["timestamp"],"destination_column_name":"date_col"},"excluded_col":{"type":["string"],"destination_column_name":"excluded_col"},"float_col":{"type":["number"],"destination_column_name":"float_col"},"id":{"type":["number"],"destination_column_name":"id"},"int_col":{"type":["number"],"destination_column_name":"int_col"},"mixed_col":{"type":["number","string","boolean"],"destination_column_name":"mixed_col"},"object_col":{"type":["object"],"destination_column_name":"object_col"},"optional_col":{"type":["string"],"destination_column_name":"optional_col"},"str_col":{"type":["string"],"destination_column_name":"str_col"},"ts_col":{"type":["timestamp"],"destination_column_name":"ts_col"},"ts_micro_col":{"type":["timestamp_micro"],"destination_column_name":"ts_micro_col"},"ts_milli_col":{"type":["timestamp_milli"],"destination_column_name":"ts_milli_col"},"ts_nano_col":{"type":["timestamp_nano"],"destination_column_name":"ts_nano_col"}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":[],"available_cursor_fields":["_last_modified_time"],"cursor_field":"_last_modified_time","sync_mode":"incremental","destination_database":"s3_olake_s3_test:s3","destination_table":"s3_json_test_table_olake","default_stream_properties":{"normalization":false,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/s3/testdata/parquet/source.json b/tests/s3/testdata/parquet/source.template.json similarity index 90% rename from tests/s3/testdata/parquet/source.json rename to tests/s3/testdata/parquet/source.template.json index 84039c0bd..dc6d1ca6d 100644 --- a/tests/s3/testdata/parquet/source.json +++ b/tests/s3/testdata/parquet/source.template.json @@ -1,7 +1,7 @@ { "bucket_name": "olake-s3-test", "region": "us-east-1", - "path_prefix": "parquet", + "path_prefix": "${suite}", "access_key_id": "admin", "secret_access_key": "password", "endpoint": "http://host.docker.internal:9000", diff --git a/tests/s3/testdata/parquet/streams.template.json b/tests/s3/testdata/parquet/streams.template.json new file mode 100644 index 000000000..1dbe64307 --- /dev/null +++ b/tests/s3/testdata/parquet/streams.template.json @@ -0,0 +1 @@ +{"selected_streams":{"s3":[{"partition_regex":"","stream_name":"test_table_olake_${suite}","normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["ts_far_col","_op_type","dec64_col","dec_bytes_col","uint16_col","ts_ns_col","_last_modified_time","list_col","uuid_col","_olake_timestamp","_olake_id","uint64_col","ts_ms_col","empty_col","time_ms_col","bytes_col","str_col","unicode_col","int32_col","int16_col","date_col","json_col","float32_col","uint32_col","dec32_col","int8_col","struct_col","bool_col","int64_col","map_col","int96_col","ts_col","enum_col","float_col","id","time_ns_col","time_us_col","excluded_col","uint8_col","null_col"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"test_table_olake_${suite}","namespace":"s3","type_schema":{"properties":{"_last_modified_time":{"type":["string"],"destination_column_name":"_last_modified_time"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["null","timestamp_micro"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"bool_col":{"type":["boolean"],"destination_column_name":"bool_col"},"bytes_col":{"type":["string"],"destination_column_name":"bytes_col"},"date_col":{"type":["timestamp"],"destination_column_name":"date_col"},"dec32_col":{"type":["number"],"destination_column_name":"dec32_col"},"dec64_col":{"type":["number"],"destination_column_name":"dec64_col"},"dec_bytes_col":{"type":["number"],"destination_column_name":"dec_bytes_col"},"empty_col":{"type":["string"],"destination_column_name":"empty_col"},"enum_col":{"type":["string"],"destination_column_name":"enum_col"},"excluded_col":{"type":["string"],"destination_column_name":"excluded_col"},"float32_col":{"type":["number_small"],"destination_column_name":"float32_col"},"float_col":{"type":["number"],"destination_column_name":"float_col"},"id":{"type":["integer"],"destination_column_name":"id"},"int16_col":{"type":["integer_small"],"destination_column_name":"int16_col"},"int32_col":{"type":["integer_small"],"destination_column_name":"int32_col"},"int64_col":{"type":["integer"],"destination_column_name":"int64_col"},"int8_col":{"type":["integer_small"],"destination_column_name":"int8_col"},"int96_col":{"type":["timestamp"],"destination_column_name":"int96_col"},"json_col":{"type":["string"],"destination_column_name":"json_col"},"list_col":{"type":["array"],"destination_column_name":"list_col"},"map_col":{"type":["object"],"destination_column_name":"map_col"},"null_col":{"type":["string","null"],"destination_column_name":"null_col"},"str_col":{"type":["string"],"destination_column_name":"str_col"},"struct_col":{"type":["object"],"destination_column_name":"struct_col"},"time_ms_col":{"type":["integer"],"destination_column_name":"time_ms_col"},"time_ns_col":{"type":["integer"],"destination_column_name":"time_ns_col"},"time_us_col":{"type":["integer"],"destination_column_name":"time_us_col"},"ts_col":{"type":["timestamp_micro"],"destination_column_name":"ts_col"},"ts_far_col":{"type":["timestamp_micro"],"destination_column_name":"ts_far_col"},"ts_ms_col":{"type":["timestamp_milli"],"destination_column_name":"ts_ms_col"},"ts_ns_col":{"type":["timestamp_nano"],"destination_column_name":"ts_ns_col"},"uint16_col":{"type":["integer_small"],"destination_column_name":"uint16_col"},"uint32_col":{"type":["integer"],"destination_column_name":"uint32_col"},"uint64_col":{"type":["integer"],"destination_column_name":"uint64_col"},"uint8_col":{"type":["integer_small"],"destination_column_name":"uint8_col"},"unicode_col":{"type":["string"],"destination_column_name":"unicode_col"},"uuid_col":{"type":["string"],"destination_column_name":"uuid_col"}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":[],"available_cursor_fields":["_last_modified_time"],"cursor_field":"_last_modified_time","sync_mode":"incremental","destination_database":"s3_olake_s3_test:s3","destination_table":"test_table_olake_${suite}","default_stream_properties":{"normalization":false,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/s3/testdata/parquet/test_streams.json b/tests/s3/testdata/parquet/test_streams.json deleted file mode 100644 index 72eaebb58..000000000 --- a/tests/s3/testdata/parquet/test_streams.json +++ /dev/null @@ -1 +0,0 @@ -{"selected_streams":{"s3":[{"partition_regex":"","stream_name":"s3_parquet_test_table_olake","normalization":false,"use_source_column_names":false,"selected_columns":{"columns":["ts_far_col","_op_type","dec64_col","dec_bytes_col","uint16_col","ts_ns_col","_last_modified_time","list_col","uuid_col","_olake_timestamp","_olake_id","uint64_col","ts_ms_col","empty_col","time_ms_col","bytes_col","str_col","unicode_col","int32_col","int16_col","date_col","json_col","float32_col","uint32_col","dec32_col","int8_col","struct_col","bool_col","int64_col","map_col","int96_col","ts_col","enum_col","float_col","id","time_ns_col","time_us_col","excluded_col","uint8_col","null_col"],"sync_new_columns":true}}]},"streams":[{"stream":{"name":"s3_parquet_test_table_olake","namespace":"s3","type_schema":{"properties":{"_last_modified_time":{"type":["string"],"destination_column_name":"_last_modified_time"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id","olake_column":true},"_olake_timestamp":{"type":["null","timestamp_micro"],"destination_column_name":"_olake_timestamp","olake_column":true},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type","olake_column":true},"bool_col":{"type":["boolean"],"destination_column_name":"bool_col"},"bytes_col":{"type":["string"],"destination_column_name":"bytes_col"},"date_col":{"type":["timestamp"],"destination_column_name":"date_col"},"dec32_col":{"type":["number"],"destination_column_name":"dec32_col"},"dec64_col":{"type":["number"],"destination_column_name":"dec64_col"},"dec_bytes_col":{"type":["number"],"destination_column_name":"dec_bytes_col"},"empty_col":{"type":["string"],"destination_column_name":"empty_col"},"enum_col":{"type":["string"],"destination_column_name":"enum_col"},"excluded_col":{"type":["string"],"destination_column_name":"excluded_col"},"float32_col":{"type":["number_small"],"destination_column_name":"float32_col"},"float_col":{"type":["number"],"destination_column_name":"float_col"},"id":{"type":["integer"],"destination_column_name":"id"},"int16_col":{"type":["integer_small"],"destination_column_name":"int16_col"},"int32_col":{"type":["integer_small"],"destination_column_name":"int32_col"},"int64_col":{"type":["integer"],"destination_column_name":"int64_col"},"int8_col":{"type":["integer_small"],"destination_column_name":"int8_col"},"int96_col":{"type":["timestamp"],"destination_column_name":"int96_col"},"json_col":{"type":["string"],"destination_column_name":"json_col"},"list_col":{"type":["array"],"destination_column_name":"list_col"},"map_col":{"type":["object"],"destination_column_name":"map_col"},"null_col":{"type":["string","null"],"destination_column_name":"null_col"},"str_col":{"type":["string"],"destination_column_name":"str_col"},"struct_col":{"type":["object"],"destination_column_name":"struct_col"},"time_ms_col":{"type":["integer"],"destination_column_name":"time_ms_col"},"time_ns_col":{"type":["integer"],"destination_column_name":"time_ns_col"},"time_us_col":{"type":["integer"],"destination_column_name":"time_us_col"},"ts_col":{"type":["timestamp_micro"],"destination_column_name":"ts_col"},"ts_far_col":{"type":["timestamp_micro"],"destination_column_name":"ts_far_col"},"ts_ms_col":{"type":["timestamp_milli"],"destination_column_name":"ts_ms_col"},"ts_ns_col":{"type":["timestamp_nano"],"destination_column_name":"ts_ns_col"},"uint16_col":{"type":["integer_small"],"destination_column_name":"uint16_col"},"uint32_col":{"type":["integer"],"destination_column_name":"uint32_col"},"uint64_col":{"type":["integer"],"destination_column_name":"uint64_col"},"uint8_col":{"type":["integer_small"],"destination_column_name":"uint8_col"},"unicode_col":{"type":["string"],"destination_column_name":"unicode_col"},"uuid_col":{"type":["string"],"destination_column_name":"uuid_col"}}},"supported_sync_modes":["full_refresh","incremental"],"source_defined_primary_key":[],"available_cursor_fields":["_last_modified_time"],"cursor_field":"_last_modified_time","sync_mode":"incremental","destination_database":"s3_olake_s3_test:s3","destination_table":"s3_parquet_test_table_olake","default_stream_properties":{"normalization":false,"append_mode":false}}}]} \ No newline at end of file diff --git a/tests/s3/testdata/xml/source.json b/tests/s3/testdata/xml/source.template.json similarity index 74% rename from tests/s3/testdata/xml/source.json rename to tests/s3/testdata/xml/source.template.json index 7257d6077..099334e10 100644 --- a/tests/s3/testdata/xml/source.json +++ b/tests/s3/testdata/xml/source.template.json @@ -1,11 +1,13 @@ { "bucket_name": "olake-s3-test", "region": "us-east-1", - "path_prefix": "xml", + "path_prefix": "${suite}", "access_key_id": "admin", "secret_access_key": "password", "endpoint": "http://host.docker.internal:9000", "file_format": "xml", - "xml": { "row_identifier": "order" }, + "xml": { + "row_identifier": "order" + }, "compression": "gzip" } diff --git a/tests/s3/testdata/xml/test_streams.json b/tests/s3/testdata/xml/streams.template.json similarity index 97% rename from tests/s3/testdata/xml/test_streams.json rename to tests/s3/testdata/xml/streams.template.json index b4b158c02..123433957 100644 --- a/tests/s3/testdata/xml/test_streams.json +++ b/tests/s3/testdata/xml/streams.template.json @@ -3,7 +3,7 @@ "s3": [ { "partition_regex": "", - "stream_name": "s3_xml_test_table_olake", + "stream_name": "test_table_olake_${suite}", "normalization": false, "use_source_column_names": false, "selected_columns": { @@ -39,7 +39,7 @@ "streams": [ { "stream": { - "name": "s3_xml_test_table_olake", + "name": "test_table_olake_${suite}", "namespace": "s3", "type_schema": { "properties": { @@ -197,7 +197,7 @@ "cursor_field": "_last_modified_time", "sync_mode": "incremental", "destination_database": "s3_olake_s3_test:s3", - "destination_table": "s3_xml_test_table_olake", + "destination_table": "test_table_olake_${suite}", "default_stream_properties": { "normalization": false, "append_mode": false diff --git a/tests/testutils/constants/constants.go b/tests/testutils/constants/constants.go index 68363dd58..2056b30f8 100644 --- a/tests/testutils/constants/constants.go +++ b/tests/testutils/constants/constants.go @@ -28,12 +28,6 @@ const ( // CdcTimestamp is the column name olake writes the CDC event timestamp into. const CdcTimestamp = "_cdc_timestamp" -// LatestStateVersion is the state file format olake writes today. A state file without it reads as -// version 0, which puts the sync on legacy type mapping -- so the harness has to name the version. -// TODO: temporary copy of constants/state_version.go's LatestStateVersion; bump it in lockstep -// until both move to external secrets alongside the source configs. -const LatestStateVersion = 7 - // SkipCDCDrivers are drivers that do not run a CDC-based sync. Unlike the identifiers above this is // test-side policy, not an olake contract -- it decides which suites skip their CDC subtests. var SkipCDCDrivers = []DriverType{Oracle, DB2, S3} diff --git a/tests/testutils/docker.go b/tests/testutils/docker.go index b8b35bf73..1d2a86f83 100644 --- a/tests/testutils/docker.go +++ b/tests/testutils/docker.go @@ -5,12 +5,10 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "sync" "sync/atomic" - "testing" - - "github.com/stretchr/testify/require" ) const ( @@ -18,18 +16,20 @@ const ( // olake input and output lives under it, since the CLI writes next to --config containerTestDataDir = "/testdata" - // driverImageEnvVar pins the image to run instead of olake/source-:local, and means - // the caller already built exactly what it wants tested (see getOrBuildDriverImage) + // driverImageEnvVar pins the image under test, so a caller that has already built or pulled it + // (CI does) is not made to build it again. driverImageEnvVar = "OLAKE_DRIVER_IMAGE" + + // driverVersionEnvVar is used to specify what is the version of the driver to run the test for + // by default the driver version is the current code, which is built as `local` + driverVersionEnvVar = "OLAKE_DRIVER_VERSION" + + // currentDriverVersion refers to the version of driver image of current code + currentDriverVersion = "local" ) -// driverImageRef returns the image the harness runs, `olake/source-:local` as -// built by `make docker..build`; OLAKE_DRIVER_IMAGE overrides it. -func driverImageRef(driver string) string { - if ref := os.Getenv(driverImageEnvVar); ref != "" { - return ref - } - return fmt.Sprintf("olake/source-%s:local", driver) +func getDriverImage(driver, version string) string { + return fmt.Sprintf("olake/source-%s:%s", driver, version) } var ( @@ -38,33 +38,30 @@ var ( containerSeq atomic.Int64 ) -// getOrBuildDriverImage returns the driver image, rebuilding it via `make docker..build` -// so a local run tests current code. OLAKE_DRIVER_IMAGE suppresses the build; sync.Once bounds it. -func getOrBuildDriverImage(t *testing.T, cfg *TestConfig) string { - t.Helper() - ref := driverImageRef(cfg.Driver) - if os.Getenv(driverImageEnvVar) != "" { - return ref - } +// buildDriverImage builds the driver image if needed. +func buildDriverImage(cfg *TestConfig) error { ensureImageOnce.Do(func() { - t.Logf("building driver image %s with `make docker.%s.build` to pick up the latest local changes", ref, cfg.Driver) - defer trackPhaseTiming(t, "driver-image", ref)() - cmd := exec.Command("make", fmt.Sprintf("docker.%s.build", cfg.Driver)) - cmd.Dir = cfg.HostRootPath - if out, err := cmd.CombinedOutput(); err != nil { - ensureImageErr = fmt.Errorf("failed to build driver image %s (the iceberg jar must be built first, see destination/iceberg/olake-iceberg-java-writer): %w\n%s", ref, err, out) + cmd := exec.Command("make", fmt.Sprintf("docker.%s.build", cfg.Driver), fmt.Sprintf("IMAGE_TAG=%s", currentDriverVersion)) + cmd.Dir = cfg.OlakeRootPath + + out, err := cmd.CombinedOutput() + if err != nil { + ensureImageErr = fmt.Errorf("`make docker.%s.build` failed in %s: %s\n%s", cfg.Driver, cfg.OlakeRootPath, err, out) } }) - require.NoError(t, ensureImageErr, "driver image unavailable") - return ref + return ensureImageErr } -// dockerRunArgs builds the `docker run` args that invoke the image's ENTRYPOINT with olakeArgs, -// testdata bind-mounted so host and container share the config/catalog/state files. -func dockerRunArgs(cfg *TestConfig, extraFlags []string, olakeArgs []string) []string { +// DockerRunArgs builds the `docker run` argument list that invokes the driver image exactly +// as a user would: the image's ENTRYPOINT (./olake) runs with olakeArgs appended. The +// driver's testdata directory is mounted at /testdata so the config/catalog/state files are +// shared with the host and the CLI writes its outputs (streams.json, state.json, ...) back +// there. extraFlags carries per-invocation docker flags (host gateway, network, name); image is +// explicit rather than derived so one suite can hand successive syncs to different images. +func DockerRunArgs(cfg *TestConfig, image string, extraFlags []string, olakeArgs []string) []string { args := []string{ "run", "--rm", - "-v", fmt.Sprintf("%s:%s", cfg.HostTestDataPath, containerTestDataDir), + "-v", fmt.Sprintf("%s:%s", cfg.TestWorkingDir, containerTestDataDir), "--tmpfs", fmt.Sprintf("%s/logs", containerTestDataDir), "-e", "TELEMETRY_DISABLED=true", "-e", "OLAKE_TIMING=1", @@ -74,53 +71,75 @@ func dockerRunArgs(cfg *TestConfig, extraFlags []string, olakeArgs []string) []s args = append(args, "--platform", cfg.ImagePlatform) } args = append(args, extraFlags...) - args = append(args, driverImageRef(cfg.Driver)) + args = append(args, image) return append(args, olakeArgs...) } -// runOlake runs the driver image once as a user would and returns the container's exit code and -// combined output. err is non-nil only when docker itself fails to launch. -func runOlake(ctx context.Context, t *testing.T, cfg *TestConfig, olakeArgs ...string) (int, []byte, error) { - t.Helper() - getOrBuildDriverImage(t, cfg) - defer trackPhaseTiming(t, cfg.Driver, olakeArgs[0]+" run")() - - name := fmt.Sprintf("olake-it-%s-%d-%d", cfg.Driver, os.Getpid(), containerSeq.Add(1)) - t.Cleanup(func() { - if exec.Command("docker", "rm", "-f", name).Run() == nil { - t.Logf("reaped leaked container %s", name) - } - }) - args := dockerRunArgs(cfg, []string{"--add-host", "host.docker.internal:host-gateway", "--name", name}, olakeArgs) +func generateUniqueContainerName(cfg *TestConfig) string { + return fmt.Sprintf("olake-it-%s-%s-%d-%d", cfg.Driver, cfg.Suite, os.Getpid(), containerSeq.Add(1)) +} + +// RunOlake runs the driver image once, exactly like a real user would: +// +// docker run --rm -v :/testdata olake/source-:local +func RunOlake(ctx context.Context, cfg *TestConfig, olakeArgs ...string) (int, []byte, error) { + name := generateUniqueContainerName(cfg) + args := DockerRunArgs(cfg, cfg.DriverImage, []string{"--add-host", "host.docker.internal:host-gateway", "--name", name}, olakeArgs) runCtx, cancel := context.WithTimeout(ctx, SyncTimeout) defer cancel() + out, err := exec.CommandContext(runCtx, "docker", args...).CombinedOutput() - logContainerTimings(t, out) if runCtx.Err() == context.DeadlineExceeded { - err = exec.Command("docker", "rm", "-f", name).Run() - if err != nil { - t.Logf("error stopping docker container after timeout: %v", err) + if rmErr := exec.Command("docker", "rm", "-f", name).Run(); rmErr != nil { + return -1, out, fmt.Errorf("olake %s of driver %s timed out after %s, and its container %s could not be removed: %s", + olakeArgs[0], cfg.Driver, SyncTimeout, name, rmErr) } - return -1, out, fmt.Errorf("olake %s run timed out after %s", olakeArgs[0], SyncTimeout) + return -1, out, fmt.Errorf("olake %s of driver %s timed out after %s; container %s was removed", olakeArgs[0], cfg.Driver, SyncTimeout, name) } - return dockerExitResult(out, err, olakeArgs[0]) + + return DockerExitResult(out, err, olakeArgs[0]) +} + +// ensureImagePresent pulls image unless the local daemon already has it. Used for compatibility +// baselines, which come from a registry rather than from a build; platform is passed through so +// an amd64-only baseline can be pulled on an arm64 host. +// +// A pull failure is returned rather than fataled: a baseline tag older than the driver itself +// legitimately has no image, and the caller turns that into a skip, not a failure. +func EnsureImagePresent(image, platform string) error { + if err := exec.Command("docker", "image", "inspect", image).Run(); err == nil { + return nil + } + args := []string{"pull"} + if platform != "" { + args = append(args, "--platform", platform) + } + args = append(args, image) + if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil { + return fmt.Errorf("failed to pull %s: %s\n%s", image, err, out) + } + return nil } -// logContainerTimings re-emits the `[timing]` lines the driver wrote inside the container, which a -// successful `docker run` would otherwise drop, leaving every sync as one opaque span. -func logContainerTimings(t *testing.T, out []byte) { - t.Helper() +// logContainerTimings re-emits the `[timing]` lines the driver wrote inside the container. A +// successful `docker run`'s output is otherwise dropped on the floor, so without this the +// in-container breakdown is invisible and every sync reads as one opaque span. The leading +// log prefix is trimmed so the forwarded lines line up with the harness's own. +func ContainerTimings(out []byte) []string { + var timings []string for _, line := range strings.Split(string(out), "\n") { if idx := strings.Index(line, "[timing]"); idx >= 0 { - t.Logf(" container %s", strings.TrimSpace(line[idx:])) + timings = append(timings, strings.TrimSpace(line[idx:])) } } + return timings } -// dockerExitResult normalizes `docker run`'s outcome: a non-zero container exit is carried in -// exitCode, and only a failure to launch docker itself comes back as err. -func dockerExitResult(out []byte, err error, what string) (int, []byte, error) { +// DockerExitResult normalizes `docker run`'s outcome into (exitCode, output, err): a non-zero +// container exit is a normal result carried in exitCode; only a failure to launch docker +// itself is returned as err. +func DockerExitResult(out []byte, err error, what string) (int, []byte, error) { if err == nil { return 0, out, nil } @@ -130,22 +149,26 @@ func dockerExitResult(out []byte, err error, what string) (int, []byte, error) { return -1, out, fmt.Errorf("docker run (%s) failed to execute: %w", what, err) } -// syncArgs builds the `olake sync ...` argument vector run against the driver image. -func syncArgs(config TestConfig, useState bool, destinationType string, flags ...string) []string { - args := []string{"sync", "--config", config.SourcePath, "--catalog", config.CatalogPath} - switch destinationType { - case "iceberg": - args = append(args, "--destination", config.IcebergDestinationPath) - case "parquet": - args = append(args, "--destination", config.ParquetDestinationPath) - } +func ContainerPath(fileName string) string { + return filepath.Join(containerTestDataDir, fileName) +} + +// SyncArgs builds the `olake sync ...` argument vector run against the driver image. +func SyncArgs(useState bool, destinationFile string, flags ...string) []string { + p := ContainerPath + args := []string{"sync", "--config", p("source.json"), "--catalog", p("streams.json")} + + args = append(args, "--destination", p(destinationFile)) + if useState { - args = append(args, "--state", config.StatePath) + args = append(args, "--state", p("state.json")) } + return append(args, flags...) } -// discoverArgs builds the `olake discover ...` argument vector run against the driver image. -func discoverArgs(config TestConfig, flags ...string) []string { - return append([]string{"discover", "--config", config.SourcePath}, flags...) +// DiscoverArgs builds the `olake discover ...` argument vector run against the driver image. +func DiscoverArgs(flags ...string) []string { + p := ContainerPath + return append([]string{"discover", "--config", p("source.json")}, flags...) } diff --git a/tests/testutils/integration/2pc.go b/tests/testutils/integration/2pc.go new file mode 100644 index 000000000..3bcdb7813 --- /dev/null +++ b/tests/testutils/integration/2pc.go @@ -0,0 +1,272 @@ +package integration + +import ( + "context" + "fmt" + "slices" + "testing" + + "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/constants" +) + +// Test2PCIntegration runs the full Two-Phase Commit (2PC) failure-recovery integration test +// suite against the driver image. It exercises CDC and incremental state-recovery scenarios +// independently of the happy-path integration tests, allowing them to be scheduled and +// reported separately. +func (cfg *Test) Test2PCIntegration(t *testing.T) { + ctx := t.Context() + + currentTestTable := cfg.GetTableName() + + t.Run("Sync", func(t *testing.T) { + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "create") + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "clean") + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "add") + + if err := testutils.UpdateSelectedStreams(cfg.TestConfig, cfg.Namespace, cfg.PartitionRegex, cfg.FilterConfig, []string{currentTestTable}, cfg.ColumnToExclude); err != nil { + t.Fatalf("failed to enable normalization and partition regex in streams.json: %s", err) + } + t.Logf("Enabled normalization and added partition regex in %s", "test_stream.json") + + writerTypes := []struct { + name string + useArrow bool + }{ + {"Legacy", false}, + {"Arrow", true}, + } + + if !slices.Contains(constants.SkipCDCDrivers, constants.DriverType(cfg.TestConfig.Driver)) { + for _, wt := range writerTypes { + t.Run(fmt.Sprintf("Iceberg (%s) 2PC CDC Recovery tests", wt.name), func(t *testing.T) { + if err := cfg.IcebergWriter(ctx, t, currentTestTable, wt.useArrow, cfg.Iceberg2PCCDCRecovery); err != nil { + t.Fatalf("Iceberg (%s) 2PC CDC Recovery tests failed: %v", wt.name, err) + } + }) + } + } + + if cfg.TestConfig.Driver != string(constants.Kafka) { + for _, wt := range writerTypes { + t.Run(fmt.Sprintf("Iceberg (%s) 2PC Incremental Recovery tests", wt.name), func(t *testing.T) { + if err := cfg.IcebergWriter(ctx, t, currentTestTable, wt.useArrow, cfg.Iceberg2PCIncrementalRecovery); err != nil { + t.Fatalf("Iceberg (%s) 2PC Incremental Recovery tests failed: %v", wt.name, err) + } + }) + } + } + + if testutils.KeepTestData() { + t.Logf("keeping %s 2PC sync test data (%s) is set", cfg.TestConfig.Driver, testutils.KeepTestDataEnvVar) + } else { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") + t.Logf("%s 2PC sync test cleanup", cfg.TestConfig.Driver) + } + }) +} + +// Iceberg2PCCDCRecovery tests 2PC (Two-Phase Commit) failure recovery for CDC mode using +// the Iceberg destination. It simulates a state-save failure mid-sync: saves a pre-insert +// checkpoint, performs a CDC insert, then restores to the checkpoint and inserts a second +// record (insert_2pc) to verify the driver correctly recovers without duplicating rows. +func (cfg *Test) Iceberg2PCCDCRecovery( + ctx context.Context, + t *testing.T, + testTable string, +) error { + t.Log("Starting Iceberg 2PC CDC Recovery tests") + + if err := cfg.resetTable(ctx, t); err != nil { + return fmt.Errorf("failed to reset table: %w", err) + } + + // Drop the Iceberg table and reset state before the first sync, so stale rows and the + // olake_2pc table property left by a previous run can't leak into this run's recovery timeline. + DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + if err := testutils.ResetStateFile(cfg.TestConfig); err != nil { + return fmt.Errorf("failed to reset state: %w", err) + } + + twoPCCDCTestCases := []syncTestCase{ + { + name: testutils.Ternary(cfg.TestConfig.Driver == string(constants.Kafka), "CDC - initial load", "Full-Refresh").(string), + operation: "", + useState: false, + opSymbol: testutils.Ternary(cfg.TestConfig.Driver == string(constants.Kafka), "c", "r").(string), + expected: cfg.ExpectedData, + verifyNoDuplicates: true, + expectedRowCountByOpType: 5, + }, + { + name: "CDC - insert", + operation: testutils.Ternary(cfg.TestConfig.Driver == string(constants.Kafka), "add", "insert").(string), + useState: true, + opSymbol: "c", + expected: cfg.ExpectedData, + preSetup: testutils.Ternary(cfg.TestConfig.Driver == string(constants.Kafka), []func(*testutils.TestConfig) error{}, []func(*testutils.TestConfig) error{testutils.SaveStateFile}).([]func(*testutils.TestConfig) error), + verifyNoDuplicates: cfg.TestConfig.Driver == string(constants.Kafka), + expectedRowCountByOpType: 10, + }, + { + // Simulate 2PC failure: restore state to pre-insert checkpoint, insert a + // second record, run sync. The driver recovers: it advances state to the + // committed metadata LSN by making a bounded sync. + // expectedRowCountByOpType=1 because no new data lands in Iceberg here, + // as it just recovers the sync from state -> metadata LSN. + name: "CDC - Recovery Sync", + operation: "insert_2pc", + useState: true, + opSymbol: "c", + expected: cfg.ExpectedData, + verifyNoDuplicates: true, + expectedRowCountByOpType: int64(testutils.Ternary(cfg.TestConfig.Driver == string(constants.Kafka), 11, 1).(int)), + preSetup: testutils.Ternary(cfg.TestConfig.Driver == string(constants.Kafka), []func(*testutils.TestConfig) error{}, []func(*testutils.TestConfig) error{testutils.RestoreStateFile}).([]func(*testutils.TestConfig) error), + }, + { + // After the recovery sync advanced state to the committed metadata LSN, + // a normal sync should see both the original insert and insert_2pc rows. + name: "CDC - Post Recovery Sync", + useState: true, + opSymbol: "c", + expected: cfg.ExpectedData, + verifyNoDuplicates: true, + expectedRowCountByOpType: int64(testutils.Ternary(cfg.TestConfig.Driver == string(constants.Kafka), 12, 2).(int)), + }, + } + + for _, tc := range twoPCCDCTestCases { + t.Run(tc.name, func(t *testing.T) { + for _, preSetup := range tc.preSetup { + if err := preSetup(cfg.TestConfig); err != nil { + t.Fatalf("%s pre-sync setup failed: %v", tc.name, err) + } + } + + if err := cfg.runSyncAndVerify( + ctx, t, testTable, tc.useState, "iceberg", + tc.operation, tc.opSymbol, tc.expected, + tc.name != "Full-Refresh", + ); err != nil { + t.Fatalf("%s test failed: %v", tc.name, err) + } + + if tc.verifyNoDuplicates { + VerifyIcebergNoDuplicates(ctx, t, testTable, cfg.TestConfig.DestinationDB, tc.opSymbol, tc.expectedRowCountByOpType) + } + }) + } + + t.Log("Iceberg 2PC CDC Recovery tests completed successfully") + DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + t.Logf("Dropped Iceberg table after 2PC CDC tests: %s", testTable) + return nil +} + +// Iceberg2PCIncrementalRecovery tests 2PC (Two-Phase Commit) failure recovery for +// incremental mode using the Iceberg destination. It simulates a state-save failure after +// the cursor advances: saves a pre-insert checkpoint, performs an incremental insert, then +// restores to the checkpoint and inserts a second record (insert_2pc) to verify that the +// cursor re-reads the overlapping range, deduplicates the original insert via MERGE INTO, +// and correctly surfaces only the net-new insert_2pc row. +func (cfg *Test) Iceberg2PCIncrementalRecovery( + ctx context.Context, + t *testing.T, + testTable string, +) error { + t.Log("Starting Iceberg 2PC Incremental Recovery tests") + + if err := cfg.resetTable(ctx, t); err != nil { + return fmt.Errorf("failed to reset table: %w", err) + } + + // Drop the Iceberg table before the first sync, so stale rows and the olake_2pc table + // property left by a previous run can't leak into this run's recovery timeline. + DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + + // Patch streams.json: set sync_mode = incremental, cursor_field + if err := updateStreamConfig(cfg.TestConfig, cfg.TestConfig.Namespace, testTable, "incremental", cfg.TestConfig.CursorField); err != nil { + return fmt.Errorf("failed to patch streams.json for incremental: %s", err) + } + + // Reset state so initial incremental behaves like a first full incremental load + if err := testutils.ResetStateFile(cfg.TestConfig); err != nil { + return fmt.Errorf("failed to reset state for incremental: %s", err) + } + + twoPCIncrementalTestCases := []syncTestCase{ + { + name: "Full-Refresh", + operation: "", + useState: false, + opSymbol: "r", + expected: cfg.ExpectedData, + verifyNoDuplicates: true, + expectedRowCountByOpType: 5, + }, + { + name: "Incremental - insert", + operation: "insert", + useState: true, + opSymbol: "u", + expected: cfg.ExpectedData, + preSetup: []func(*testutils.TestConfig) error{ + testutils.SaveStateFile, + }, + }, + { + // Simulate 2PC failure: restore cursor to pre-insert checkpoint, insert a + // second record, run sync. The cursor re-reads the range and deduplicates + // the original insert via MERGE INTO; insert_2pc is net-new. + // expectedRowCountByOpType=1: only insert_2pc is visible (original deduplicated). + name: "Incremental - State Save Failure Sync", + operation: "insert_2pc", + useState: true, + opSymbol: "u", + expected: cfg.ExpectedData, + verifyNoDuplicates: true, + expectedRowCountByOpType: 1, + preSetup: []func(*testutils.TestConfig) error{ + testutils.RestoreStateFile, + }, + }, + { + // After recovery, state is now consistent. A normal sync should see both + // the original insert row and insert_2pc row — 2 distinct records total. + name: "Incremental - Post Recovery Sync", + useState: true, + opSymbol: "u", + expected: cfg.ExpectedData, + verifyNoDuplicates: true, + expectedRowCountByOpType: 2, // insert row + insert_2pc row, both unique by _olake_id + }, + } + + for _, tc := range twoPCIncrementalTestCases { + t.Run(tc.name, func(t *testing.T) { + for _, preSetup := range tc.preSetup { + if err := preSetup(cfg.TestConfig); err != nil { + t.Fatalf("%s pre-sync setup failed: %v", tc.name, err) + } + } + + if err := cfg.runSyncAndVerify( + ctx, t, testTable, tc.useState, "iceberg", + tc.operation, tc.opSymbol, tc.expected, + false, + ); err != nil { + t.Fatalf("Incremental 2PC test %s failed: %v", tc.name, err) + } + + if tc.verifyNoDuplicates { + VerifyIcebergNoDuplicates(ctx, t, testTable, cfg.TestConfig.DestinationDB, tc.opSymbol, tc.expectedRowCountByOpType) + } + }) + } + + t.Log("Iceberg 2PC Incremental Recovery tests completed successfully") + DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + t.Logf("Dropped Iceberg table after 2PC Incremental tests: %s", testTable) + return nil +} diff --git a/tests/testutils/integration/discover.go b/tests/testutils/integration/discover.go new file mode 100644 index 000000000..e15582db5 --- /dev/null +++ b/tests/testutils/integration/discover.go @@ -0,0 +1,122 @@ +package integration + +import ( + "encoding/json" + "fmt" + "maps" + "os" + "slices" + "testing" + + "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/require" +) + +// Its caller must not be parallel. The compare is an equality one, so it only holds while this +// table is the only thing in the source -- and every other suite seeds one of its own. Leaving the +// test serial is what orders it ahead of them: Go resumes parallel tests only once the serial ones +// in the package are done. +func (cfg *Test) TestDiscover(t *testing.T) { + ctx := t.Context() + + // 1. Empty the source, then seed just this table. drop-all is what makes the compare below an + // equality one: discover enumerates everything, so anything an aborted run (or a perf seed) + // left behind would show up as an extra stream. Safe only here -- the discover suite runs + // alone, while every parallel suite owns a table drop-all would take with it. + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "drop-all") + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "create") + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "add") + // Deferred, so a failed discover still hands the parallel suites behind it a clean source. + defer func() { + if testutils.KeepTestData() { + t.Logf("keeping %s discover data in source as (%s) is set", cfg.TestConfig.Driver, testutils.KeepTestDataEnvVar) + } else { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") + } + }() + + // 2. Stage what discover has to reproduce before it writes its own streams.json next to the config + generateExpectedStreams(t, cfg.TestConfig) + + // 3. Run discover against the driver image + code, out, err := testutils.RunOlake(ctx, cfg.TestConfig, testutils.DiscoverArgs()...) + if err != nil || code != 0 { + t.Fatal(testutils.RenderOlakeFailure(code, err, out)) + } + + verifyDiscoveredStreams(t, cfg.GetFilePath("expected_discover_streams.json"), cfg.GetFilePath("streams.json")) +} + +// generateExpectedStreams writes the catalog discover has to return: the one applySuite already +// rendered from streams.template.json, under a name discover's own output cannot overwrite. +func generateExpectedStreams(t *testing.T, config *testutils.TestConfig) { + t.Helper() + require.NoError(t, testutils.CopyFile(config.GetFilePath("streams.json"), config.GetFilePath("expected_discover_streams.json")), + "failed to generate the expected discover catalog") +} + +// verifyDiscoveredStreams asserts the discovered catalog holds exactly the streams expected_streams.json +func verifyDiscoveredStreams(t *testing.T, expectedPath, actualPath string) { + t.Helper() + + load := func(path, what string) map[string]interface{} { + data, err := os.ReadFile(path) + require.NoError(t, err, "failed to read %s streams JSON (%s)", what, path) + var doc map[string]interface{} + require.NoError(t, json.Unmarshal(data, &doc), "failed to parse %s streams JSON (%s)", what, path) + return doc + } + expected := load(expectedPath, "expected") + actual := load(actualPath, "discovered") + + // streams[]: keyed by namespace.name, which is what makes a stream unique in a catalog. + indexStreams := func(doc map[string]interface{}) map[string]interface{} { + out := map[string]interface{}{} + entries, _ := doc["streams"].([]interface{}) + for _, raw := range entries { + wrapper, ok := raw.(map[string]interface{}) + if !ok { + continue + } + stream, ok := wrapper["stream"].(map[string]interface{}) + if !ok { + continue + } + out[fmt.Sprintf("%v.%v", stream["namespace"], stream["name"])] = wrapper + } + return out + } + // selected_streams: a map of namespace -> []{stream_name, ...}; key the same way. + indexSelected := func(doc map[string]interface{}) map[string]interface{} { + out := map[string]interface{}{} + byNamespace, _ := doc["selected_streams"].(map[string]interface{}) + for namespace, raw := range byNamespace { + entries, _ := raw.([]interface{}) + for _, entry := range entries { + selected, ok := entry.(map[string]interface{}) + if !ok { + continue + } + out[fmt.Sprintf("%v.%v", namespace, selected["stream_name"])] = selected + } + } + return out + } + + compare := func(section string, want, got map[string]interface{}) { + require.Equal(t, slices.Sorted(maps.Keys(want)), slices.Sorted(maps.Keys(got)), + "%s: discover returned a different set of streams than expected_streams.json", section) + for key, wantEntry := range want { + wantJSON, err := json.Marshal(wantEntry) + require.NoError(t, err) + gotJSON, err := json.Marshal(got[key]) + require.NoError(t, err) + require.Truef(t, testutils.NormalizedEqual(string(wantJSON), string(gotJSON)), + "%s: discovered %q does not match expected_streams.json\nExpected:\n%s\nGot:\n%s", section, key, wantJSON, gotJSON) + } + } + compare("streams", indexStreams(expected), indexStreams(actual)) + compare("selected_streams", indexSelected(expected), indexSelected(actual)) + + t.Logf("Generated streams validated with test streams") +} diff --git a/tests/testutils/integration/iceberg.go b/tests/testutils/integration/iceberg.go new file mode 100644 index 000000000..f27efff12 --- /dev/null +++ b/tests/testutils/integration/iceberg.go @@ -0,0 +1,80 @@ +package integration + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/apache/spark-connect-go/v35/spark/sql" + "github.com/datazip-inc/olake/tests/testutils" +) + +const ( + IcebergCatalog = "olake_iceberg" + // IP literal, not "localhost": a hostname sends grpc-go through a DNS resolver that stalls + // every new connection ~20s when the DNS servers are slow (measured 20.22s vs 42ms). + sparkConnectAddress = "sc://127.0.0.1:15002" +) + +// The Spark Connect session is shared: each one costs ~175ms to build and every verification needs it. +var ( + sharedSparkOnce sync.Once + sharedSpark sql.SparkSession + sharedSparkErr error +) + +// sparkSession returns the shared Spark Connect session, building it on first use and warming it +// so the one-off server bootstrap is timed here instead of inflating whichever verify runs first. +func SparkSession(ctx context.Context, t *testing.T) (sql.SparkSession, error) { + sharedSparkOnce.Do(func() { + // The shared session outlives whichever test builds it, so its construction must not be + // tied to that test's context (t.Context cancels when the test ends). + ctx := context.WithoutCancel(ctx) + defer testutils.TrackPhaseTiming(t, "spark", "session build")() + for attempt := 1; ; attempt++ { + sharedSpark, sharedSparkErr = sql.NewSessionBuilder().Remote(sparkConnectAddress).Build(ctx) + if sharedSparkErr == nil || attempt == 3 { + break + } + t.Logf("Attempt %d/3: Failed to connect to Spark, retrying in 2s: %v", attempt, sharedSparkErr) + time.Sleep(2 * time.Second) + } + if sharedSparkErr != nil { + return + } + // Spark's vectorized parquet reader mis-decodes DELTA_LENGTH_BYTE_ARRAY columns that hold + // nulls, reading every value after a null back as "" -- which reads as a data bug in a file + // the writer got right. Session-scoped, so every query below sees what was actually written. + if _, err := sharedSpark.Sql(ctx, "SET spark.sql.parquet.enableVectorizedReader=false"); err != nil { + t.Logf("WARNING: could not disable Spark's vectorized parquet reader, so parquet assertions may report spurious empty strings for nullable byte-array columns: %v", err) + } + if _, err := sharedSpark.Sql(ctx, "SELECT 1"); err != nil { + t.Logf("Spark session warm-up query failed (non-fatal): %v", err) + } + }) + return sharedSpark, sharedSparkErr +} + +// dropIcebergTable drops an Iceberg table using Spark SQL +func DropIcebergTable(t *testing.T, tableName, icebergDB string) { + t.Helper() + ctx := t.Context() + spark, err := SparkSession(ctx, t) + if err != nil { + t.Logf("Failed to connect to Spark Connect server for dropping table: %v", err) + return + } + + fullTableName := fmt.Sprintf("%s.%s.%s", IcebergCatalog, icebergDB, tableName) + dropQuery := fmt.Sprintf("DROP TABLE IF EXISTS %s", fullTableName) + t.Logf("Dropping Iceberg table: %s", dropQuery) + + _, err = spark.Sql(ctx, dropQuery) + if err != nil { + t.Logf("Failed to drop Iceberg table %s: %v", fullTableName, err) + return + } + t.Logf("Successfully dropped Iceberg table: %s", fullTableName) +} diff --git a/tests/testutils/integration/integration.go b/tests/testutils/integration/integration.go new file mode 100644 index 000000000..45eb0bf23 --- /dev/null +++ b/tests/testutils/integration/integration.go @@ -0,0 +1,179 @@ +package integration + +import ( + "context" + "testing" + + "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/constants" +) + +const ( + icebergDestinationFile = "iceberg_destination.json" + icebergArrowDestinationFile = "iceberg_destination_arrow.json" + parquetDestinationFile = "parquet_destination.json" +) + +type Test struct { + *testutils.TestConfig + + // icebergDestination is the destination config the next iceberg sync runs against, which is how + // IcebergWriter selects between the legacy and the arrow writer. + icebergDestination string + ExpectedData map[string]interface{} + ExpectedUpdatedData map[string]interface{} + DestinationDataTypeSchema map[string]string + UpdatedDestinationDataTypeSchema map[string]string + DefaultCDCColumnsSchema map[string]string + + // The fields below exist for the backward-compatibility suite (compatibility.go) and are zero for + // every other suite, which keeps their behavior identical to before they existed. + +} + +// reset table and add back data to the table +func (cfg *Test) resetTable(ctx context.Context, t *testing.T) error { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "create") + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "add") + if cfg.TestConfig.Driver == string(constants.DB2) { + // to populate stats for DB2 + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "populate-stats") + } + return nil +} + +// runSyncAndVerify executes a sync command and verifies the results in Iceberg +func (cfg *Test) runSyncAndVerify( + ctx context.Context, + t *testing.T, + testTable string, + useState bool, + destinationType string, + operation string, + opSymbol string, + schema map[string]interface{}, + isCDC bool, +) error { + cmd := testutils.SyncArgs(useState, cfg.destinationFile(destinationType), "--destination-database-prefix", cfg.UniqueID()) + + // Execute operation before sync if needed + if useState && operation != "" { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, operation) + // SQL Server CDC is asynchronous: the capture job only picks up the DML above on its next + // transaction-log scan, and the sync's change window ends at the job's processed max LSN + // (sys.fn_cdc_get_max_lsn), so syncing too early would see no changes. Wait for the capture + // job to advance past the DML. Incremental runs read the table directly and need no wait. + if isCDC && cfg.TestConfig.Driver == "mssql" { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "wait-cdc-catchup") + } + } + + // Run sync against the driver image + code, out, err := testutils.RunOlake(ctx, cfg.TestConfig, cmd...) + if err != nil || code != 0 { + return testutils.RenderOlakeFailure(code, err, out) + } + + t.Logf("Sync successful for %s driver", cfg.TestConfig.Driver) + + // Use evolved schema only for CDC "update" operation (where schema evolution is expected) + // Incremental "insert" uses opSymbol "u" but doesn't have schema evolution + evolvedSchema := operation == "update" + + // Verification reads the destination back through Spark Connect (with retries), a real slice of + // sync wall-clock; time it as its own phase. + defer testutils.TrackPhaseTiming(t, cfg.TestConfig.Driver, destinationType+" verify")() + + switch destinationType { + case "iceberg": + { + if evolvedSchema { + VerifyIcebergSync(t, testTable, cfg.TestConfig.DestinationDB, cfg.UpdatedDestinationDataTypeSchema, cfg.DefaultCDCColumnsSchema, schema, opSymbol, cfg.TestConfig.PartitionRegex, cfg.TestConfig.Driver, isCDC, cfg.TestConfig.ColumnToExclude) + } else { + VerifyIcebergSync(t, testTable, cfg.TestConfig.DestinationDB, cfg.DestinationDataTypeSchema, cfg.DefaultCDCColumnsSchema, schema, opSymbol, cfg.TestConfig.PartitionRegex, cfg.TestConfig.Driver, isCDC, cfg.TestConfig.ColumnToExclude) + } + } + case "parquet": + { + if evolvedSchema { + VerifyParquetSync(t, testTable, cfg.TestConfig.DestinationDB, cfg.UpdatedDestinationDataTypeSchema, cfg.DefaultCDCColumnsSchema, schema, opSymbol, cfg.TestConfig.Driver, isCDC, cfg.TestConfig.ColumnToExclude) + } else { + VerifyParquetSync(t, testTable, cfg.TestConfig.DestinationDB, cfg.DestinationDataTypeSchema, cfg.DefaultCDCColumnsSchema, schema, opSymbol, cfg.TestConfig.Driver, isCDC, cfg.TestConfig.ColumnToExclude) + } + } + } + + return nil +} + +// destinationFile names the destination config a sync of this kind runs against. The iceberg one +// is whichever writer variant IcebergWriter selected, defaulting to the committed base config. +func (cfg *Test) destinationFile(destinationType string) string { + if destinationType == "parquet" { + return parquetDestinationFile + } + if cfg.icebergDestination == "" { + return icebergDestinationFile + } + return cfg.icebergDestination +} + +// IcebergDestinationFile names the iceberg destination config the next sync runs against, for +// suites whose expectations depend on which writer that config selects. +func (cfg *Test) IcebergDestinationFile() string { + return cfg.destinationFile("iceberg") +} + +func (cfg *Test) IcebergWriter( + ctx context.Context, + t *testing.T, + testTable string, + useArrowWriter bool, + testFunc func(context.Context, *testing.T, string) error, +) error { + // Writer variants are separate config files, so no suite ever edits one in place; SyncArgs + // hands whichever is named here to --destination. + cfg.icebergDestination = icebergDestinationFile + if useArrowWriter { + cfg.icebergDestination = icebergArrowDestinationFile + } + + return testFunc(ctx, t, testTable) +} + +type syncTestCase struct { + name string + operation string + useState bool + opSymbol string + expected map[string]interface{} + preSetup []func(*testutils.TestConfig) error // host-side actions executed before the sync + verifyNoDuplicates bool // if true, assert COUNT(*) == COUNT(DISTINCT _olake_id) after sync + expectedRowCountByOpType int64 // when > 0, assert COUNT(DISTINCT _olake_id) == this value (catches over-sync and under-sync) +} + +// updateStreamConfig sets sync_mode and cursor_field on the stream identified by +// namespace+name in streams[]. +func updateStreamConfig(config *testutils.TestConfig, namespace, streamName, syncMode, cursorField string) error { + // in case of Oracle, the stream names are in uppercase in streams.json + streamName = testutils.NormalizeStreamName(config.Driver, streamName) + return testutils.EditJSONFile(config.GetFilePath("streams.json"), func(doc map[string]interface{}) error { + streams, _ := doc["streams"].([]interface{}) + for _, raw := range streams { + wrapper, ok := raw.(map[string]interface{}) + if !ok { + continue + } + stream, ok := wrapper["stream"].(map[string]interface{}) + if !ok { + continue + } + if stream["namespace"] == namespace && stream["name"] == streamName { + stream["sync_mode"] = syncMode + stream["cursor_field"] = cursorField + } + } + return nil + }) +} diff --git a/tests/testutils/integration/parquet.go b/tests/testutils/integration/parquet.go new file mode 100644 index 000000000..cd46bb209 --- /dev/null +++ b/tests/testutils/integration/parquet.go @@ -0,0 +1,108 @@ +package integration + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const parquetTestBucket = "warehouse" + +// newMinIOClient returns a client for the MinIO instance backing the parquet destination in tests. +func newMinIOClient() (*minio.Client, error) { + client, err := minio.New("localhost:9000", &minio.Options{ + Creds: credentials.NewStaticV4("admin", "password", ""), + Secure: false, + }) + if err != nil { + return nil, fmt.Errorf("failed to create MinIO client: %s", err) + } + return client, nil +} + +// listParquetObjects lists the .parquet objects lying directly in a table's folder in MinIO. +func listParquetObjects(ctx context.Context, client *minio.Client, parquetDB, tableName string) ([]minio.ObjectInfo, error) { + objects := []minio.ObjectInfo{} + for object := range client.ListObjects(ctx, parquetTestBucket, minio.ListObjectsOptions{ + Prefix: parquetTablePath(parquetDB, tableName), + Recursive: false, + }) { + if object.Err != nil { + return nil, fmt.Errorf("error listing objects: %s", object.Err) + } + if strings.HasSuffix(object.Key, ".parquet") { + objects = append(objects, object) + } + } + return objects, nil +} + +// parquetTablePath is the MinIO key prefix a stream's parquet files are written under. +func parquetTablePath(parquetDB, tableName string) string { + return fmt.Sprintf("%s/%s/", parquetDB, tableName) +} + +// DeleteParquetFiles deletes only .parquet files directly in the table folder in MinIO +func DeleteParquetFiles(t *testing.T, parquetDB, tableName string) error { + t.Helper() + parquetPath := parquetTablePath(parquetDB, tableName) + + t.Logf("Cleaning up .parquet files in: s3a://%s/%s", parquetTestBucket, parquetPath) + + minioClient, err := newMinIOClient() + if err != nil { + return err + } + + ctx := t.Context() + + objects, err := listParquetObjects(ctx, minioClient, parquetDB, tableName) + if err != nil { + return err + } + + for _, object := range objects { + t.Logf("Deleting: %s", strings.TrimPrefix(object.Key, parquetPath)) + + if err := minioClient.RemoveObject(ctx, parquetTestBucket, object.Key, minio.RemoveObjectOptions{}); err != nil { + return fmt.Errorf("failed to delete %s: %s", object.Key, err) + } + } + + t.Logf("--- Cleanup Complete: Deleted %d files ---", len(objects)) + return nil +} + +// deleteParquetTable wipes a table's prefix recursively, unlike DeleteParquetFiles: it takes the +// destination metadata with it, so the next sync starts as a genuinely initial one. +func deleteParquetTable(t *testing.T, parquetDB, tableName string) error { + t.Helper() + parquetPath := parquetTablePath(parquetDB, tableName) + + minioClient, err := newMinIOClient() + if err != nil { + return err + } + + ctx := context.Background() + deletedCount := 0 + for object := range minioClient.ListObjects(ctx, parquetTestBucket, minio.ListObjectsOptions{ + Prefix: parquetPath, + Recursive: true, + }) { + if object.Err != nil { + return fmt.Errorf("error listing objects: %s", object.Err) + } + if err := minioClient.RemoveObject(ctx, parquetTestBucket, object.Key, minio.RemoveObjectOptions{}); err != nil { + return fmt.Errorf("failed to delete %s: %s", object.Key, err) + } + deletedCount++ + } + + t.Logf("--- Parquet Table Cleanup Complete: Deleted %d objects ---", deletedCount) + return nil +} diff --git a/tests/testutils/parquet_rolling.go b/tests/testutils/integration/parquet_rolling.go similarity index 83% rename from tests/testutils/parquet_rolling.go rename to tests/testutils/integration/parquet_rolling.go index 362ae4229..9d11be2a1 100644 --- a/tests/testutils/parquet_rolling.go +++ b/tests/testutils/integration/parquet_rolling.go @@ -1,4 +1,4 @@ -package testutils +package integration import ( "bytes" @@ -8,10 +8,11 @@ import ( "slices" "testing" + "github.com/datazip-inc/olake/tests/testutils" "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/require" "github.com/minio/minio-go/v7" pqgo "github.com/parquet-go/parquet-go" - "github.com/stretchr/testify/require" ) const ( @@ -44,33 +45,33 @@ func hasParquetRollingTest(driver string) bool { // The seeded data (~8MB) is far smaller than the backfill chunk size (EffectiveParquetSize, // 2GB), so the whole table is one chunk handled by one writer — meaning multiple bounded // files can only come from rolling, not from chunk fan-out. -func (cfg *IntegrationTest) testParquetRolling(ctx context.Context, t *testing.T, testTable string) error { +func (cfg *Test) testParquetRolling(ctx context.Context, t *testing.T, testTable string) error { // Rolling needs the whole table to land in a single writer, so drop the partition regex // (identity partitioning would fan the rows out into a file per value) and the filter config // (the bulk rows are not shaped to satisfy it). Normalization stays on, as in the rest of the // Sync flow. Safe to leave patched: this is the flow's last sub-test and streams.json is // regenerated by Discover on every run. - if err := updateSelectedStreams(cfg.TestConfig, cfg.Namespace, "", "", []string{testTable}, cfg.ColumnToExclude); err != nil { + if err := testutils.UpdateSelectedStreams(cfg.TestConfig, cfg.TestConfig.Namespace, "", "", []string{testTable}, cfg.TestConfig.ColumnToExclude); err != nil { return fmt.Errorf("failed to clear partition regex and filter config in streams.json: %s", err) } // Swap the handful of datatype rows for a bulk payload. - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "clean") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "rolling_seed") + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "clean") + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "rolling_seed") // Start from an empty destination folder — the earlier parquet sub-tests leave files and // destination metadata behind, while this is an independent initial sync. - if err := deleteParquetTable(t, cfg.DestinationDB, testTable); err != nil { + if err := deleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { return fmt.Errorf("failed to reset parquet table before rolling sync: %s", err) } defer func() { - if err := deleteParquetTable(t, cfg.DestinationDB, testTable); err != nil { + if err := deleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { t.Logf("cleanup: failed to reset parquet table: %v", err) } }() // Full load (no --state) with a small buffer so rolled files hug the threshold. - if code, out, err := runOlake(ctx, t, cfg.TestConfig, syncArgs(*cfg.TestConfig, false, "parquet")...); err != nil || code != 0 { + if code, out, err := testutils.RunOlake(ctx, cfg.TestConfig, testutils.SyncArgs(false, parquetDestinationFile)...); err != nil || code != 0 { return fmt.Errorf("rolling sync failed (%d): %v\n%s", code, err, out) } @@ -81,14 +82,14 @@ func (cfg *IntegrationTest) testParquetRolling(ctx context.Context, t *testing.T // verifyParquetRolling reads the rolled objects from MinIO and asserts the writer // (1) produced multiple files, (2) bounded every file near the roll threshold (the core // proof — a broken roll would leave one oversized file), and (3) preserved every row. -func (cfg *IntegrationTest) verifyParquetRolling(t *testing.T, table string) { +func (cfg *Test) verifyParquetRolling(t *testing.T, table string) { t.Helper() ctx := t.Context() client, err := newMinIOClient() require.NoError(t, err) - objects, err := listParquetObjects(ctx, client, cfg.DestinationDB, table) + objects, err := listParquetObjects(ctx, client, cfg.TestConfig.DestinationDB, table) require.NoError(t, err, "failed to list rolled parquet files") var totalSize int64 diff --git a/tests/testutils/integration/sync.go b/tests/testutils/integration/sync.go new file mode 100644 index 000000000..cd1ef51a2 --- /dev/null +++ b/tests/testutils/integration/sync.go @@ -0,0 +1,507 @@ +package integration + +import ( + "context" + "fmt" + "slices" + "testing" + + "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/constants" +) + +// TestSync runs the happy-path sync suite: full load, CDC and incremental, over both Iceberg writers +// and Parquet. It seeds its catalog from streams.template.json instead of discovering one, the way the +// 2PC and rebalance suites do -- TestDiscover already proves the two are identical. +func (cfg *Test) TestSync(t *testing.T) { + ctx := t.Context() + testTable := cfg.GetTableName() + + // 1. Query on test table; drop first so an aborted run's leftovers cannot survive + // the CREATE IF NOT EXISTS + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "create") + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "clean") + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "add") + + // 2. Enable normalization, partition regex, filter and column exclusion in streams.json + if err := testutils.UpdateSelectedStreams(cfg.TestConfig, cfg.Namespace, cfg.PartitionRegex, cfg.FilterConfig, []string{testTable}, cfg.ColumnToExclude); err != nil { + t.Fatalf("failed to enable normalization and partition regex in streams.json: %s", err) + } + t.Logf("Enabled normalization and added partition regex in %s", cfg.GetFilePath("streams.json")) + + writerTypes := []struct { + name string + useArrow bool + }{ + {"Legacy", false}, + {"Arrow", true}, + } + + // Skip cdc tests for drivers not supporting cdc mode + if !slices.Contains(constants.SkipCDCDrivers, constants.DriverType(cfg.Driver)) { + for _, wt := range writerTypes { + t.Run(fmt.Sprintf("Iceberg (%s) Full load + CDC tests", wt.name), func(t *testing.T) { + if err := cfg.IcebergWriter(ctx, t, testTable, wt.useArrow, cfg.IcebergFullLoadAndCDC); err != nil { + t.Fatalf("Iceberg (%s) Full load + CDC tests failed: %v", wt.name, err) + } + }) + } + + t.Run("Parquet Full load + CDC tests", func(t *testing.T) { + if err := cfg.ParquetFullLoadAndCDC(ctx, t, testTable); err != nil { + t.Fatalf("Parquet Full load + CDC tests failed: %v", err) + } + }) + } + + // Skip incremental tests for drivers not supporting incremental mode + if cfg.Driver != string(constants.Kafka) { + for _, wt := range writerTypes { + t.Run(fmt.Sprintf("Iceberg (%s) Full load + Incremental tests", wt.name), func(t *testing.T) { + if err := cfg.IcebergWriter(ctx, t, testTable, wt.useArrow, cfg.IcebergFullLoadAndIncremental); err != nil { + t.Fatalf("Iceberg (%s) Full load + Incremental tests failed: %v", wt.name, err) + } + }) + } + + t.Run("Parquet Full load + Incremental tests", func(t *testing.T) { + if err := cfg.ParquetFullLoadAndIncremental(ctx, t, testTable); err != nil { + t.Fatalf("Parquet Full load + Incremental tests failed: %v", err) + } + }) + } + + // Asserts the writer splits bulk output into size-bounded files without losing rows. Runs + // last: it replaces the table contents and clears streams.json's regex/filter config. + if hasParquetRollingTest(cfg.Driver) { + t.Run("Parquet Rolling", func(t *testing.T) { + if err := cfg.testParquetRolling(ctx, t, testTable); err != nil { + t.Fatalf("Parquet Rolling test failed: %v", err) + } + }) + } + + // 3. Clean up + if testutils.KeepTestData() { + t.Logf("keeping %s source data for Sync as (%s) is set", cfg.Driver, testutils.KeepTestDataEnvVar) + } else { + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") + t.Logf("%s sync test cleanup", cfg.Driver) + } +} + +// IcebergFullLoadAndCDC tests Full load and CDC operations +func (cfg *Test) IcebergFullLoadAndCDC( + ctx context.Context, + t *testing.T, + testTable string, +) error { + t.Log("Starting Iceberg Full load + CDC tests") + + if err := cfg.resetTable(ctx, t); err != nil { + return fmt.Errorf("failed to reset table: %w", err) + } + + // The seed rows sit in the CDC log, and before #843 (v0.5.1) the mssql driver captured its + // initial LSN without waiting for the async capture agent -- a lagging agent puts that LSN + // before the seed, and the first stateful sync replays the seed rows as CDC inserts + // (relabeling r to c through the upsert). Wait here so every binary snapshots past the seed. + if cfg.TestConfig.Driver == "mssql" { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "wait-cdc-catchup") + } + + dbTestCases := []syncTestCase{ + { + name: "Full-Refresh", + operation: "", + useState: false, + opSymbol: "r", + expected: cfg.ExpectedData, + }, + { + name: "CDC - insert", + operation: "insert", + useState: true, + opSymbol: "c", + expected: cfg.ExpectedData, + }, + { + name: "CDC - update", + operation: "update", + useState: true, + opSymbol: "u", + expected: cfg.ExpectedUpdatedData, + }, + { + name: "CDC - delete", + operation: "delete", + useState: true, + opSymbol: "d", + expected: nil, + }, + } + + kafkaTestCases := []syncTestCase{ + { + name: "CDC - strict - insert", + operation: "", + useState: false, + opSymbol: "c", + expected: cfg.ExpectedData, + }, + { + name: "CDC - strict - update", + operation: "update", + useState: true, + opSymbol: "c", + expected: cfg.ExpectedUpdatedData, + }, + } + + testCases := testutils.Ternary(cfg.TestConfig.Driver == string(constants.Kafka), kafkaTestCases, dbTestCases).([]syncTestCase) + + // Run each test case. t.Fatalf below ends only its own subtest, so stop the loop explicitly: + // every case after the first failure is a stateful sync built on state the failed one never + // wrote, and it costs a full sync each to learn nothing. + for _, tc := range testCases { + if !t.Run(tc.name, func(t *testing.T) { + // schema evolution + if tc.operation == "update" { + if cfg.TestConfig.Driver != "mongodb" && cfg.TestConfig.Driver != "mssql" && cfg.TestConfig.Driver != "kafka" { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "evolve-schema") + } + } + + if err := cfg.runSyncAndVerify( + ctx, + t, + testTable, + tc.useState, + "iceberg", + tc.operation, + tc.opSymbol, + tc.expected, + tc.name != "Full-Refresh", + ); err != nil { + t.Fatalf("%s test failed: %v", tc.name, err) + } + }) { + t.Logf("stopping this scenario after %q failed; the remaining cases depend on the state it did not write", tc.name) + break + } + } + + t.Log("Iceberg Full load + CDC tests completed successfully") + + if testutils.KeepTestData() { + t.Logf("keeping %s source data (%s) is set", cfg.TestConfig.Driver, testutils.KeepTestDataEnvVar) + } else { + // Drop the Iceberg table after all tests are finished + DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + t.Logf("Dropped Iceberg table: %s", testTable) + } + + return nil +} + +// IcebergFullLoadAndCDC tests Full load and CDC operations +func (cfg *Test) ParquetFullLoadAndCDC( + ctx context.Context, + t *testing.T, + testTable string, +) error { + t.Log("Starting Parquet Full load + CDC tests") + + if err := cfg.resetTable(ctx, t); err != nil { + return fmt.Errorf("failed to reset table: %s", err) + } + if err := deleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + return fmt.Errorf("failed to reset parquet table: %s", err) + } + + // The seed rows sit in the CDC log, and before #843 (v0.5.1) the mssql driver captured its + // initial LSN without waiting for the async capture agent -- a lagging agent puts that LSN + // before the seed, and the first stateful sync replays the seed rows as CDC inserts + // (relabeling r to c through the upsert). Wait here so every binary snapshots past the seed. + if cfg.TestConfig.Driver == "mssql" { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "wait-cdc-catchup") + } + + dbTestCases := []syncTestCase{ + { + name: "Full-Refresh", + operation: "", + useState: false, + opSymbol: "r", + expected: cfg.ExpectedData, + }, + { + name: "CDC - insert", + operation: "insert", + useState: true, + opSymbol: "c", + expected: cfg.ExpectedData, + }, + { + name: "CDC - update", + operation: "update", + useState: true, + opSymbol: "u", + expected: cfg.ExpectedUpdatedData, + }, + { + name: "CDC - delete", + operation: "delete", + useState: true, + opSymbol: "d", + expected: nil, + }, + } + + kafkaTestCases := []syncTestCase{ + { + name: "CDC - strict - insert", + operation: "", + useState: false, + opSymbol: "c", + expected: cfg.ExpectedData, + }, + { + name: "CDC - strict - update", + operation: "update", + useState: true, + opSymbol: "c", + expected: cfg.ExpectedUpdatedData, + }, + } + + testCases := testutils.Ternary(cfg.TestConfig.Driver == string(constants.Kafka), kafkaTestCases, dbTestCases).([]syncTestCase) + + // Run each test case, stopping at the first failure -- see the same loop in + // IcebergFullLoadAndCDC for why continuing only burns syncs. + for _, tc := range testCases { + if !t.Run(tc.name, func(t *testing.T) { + // schema evolution + if tc.operation == "update" { + if cfg.TestConfig.Driver != "mongodb" && cfg.TestConfig.Driver != "mssql" && cfg.TestConfig.Driver != "kafka" { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "evolve-schema") + } + } + + // Delete parquet files before next operation to avoid error due to schema changes. + // Kept even for the compatibility suite (unlike the Iceberg drops), because the files are + // genuinely unreadable together: successive syncs write the same column with different + // types -- measured on postgres, col_float4 FLOAT then DOUBLE and col_int INT then + // BIGINT -- so Spark rejects the directory with CANNOT_MERGE_SCHEMAS, mergeSchema or + // not. That is F2 in docs/backward-compatibility.md (parquet has no schema evolution; + // the break surfaces in the reader). The consequence for compatibility is that a parquet + // variant compares only its LAST case's output; see compareVariant. + if err := DeleteParquetFiles(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + t.Fatalf("Failed to delete parquet files before %s: %v", tc.name, err) + } + + if err := cfg.runSyncAndVerify( + ctx, + t, + testTable, + tc.useState, + "parquet", + tc.operation, + tc.opSymbol, + tc.expected, + tc.name != "Full-Refresh", + ); err != nil { + t.Fatalf("%s test failed: %v", tc.name, err) + } + }) { + t.Logf("stopping this scenario after %q failed; the remaining cases depend on the state it did not write", tc.name) + break + } + } + + t.Log("Parquet Full load + CDC tests completed successfully") + return nil +} + +// TODO: add incremntal test for string time, timestamp with timezone, datetime, float, int as cursor field +// IcebergFullLoadAndIncremental tests Full load and Incremental operations +func (cfg *Test) IcebergFullLoadAndIncremental( + ctx context.Context, + t *testing.T, + testTable string, +) error { + t.Log("Starting Iceberg Full load + Incremental tests") + + if err := cfg.resetTable(ctx, t); err != nil { + return fmt.Errorf("failed to reset table: %s", err) + } + + // Patch streams.json: set sync_mode = incremental, cursor_field = "id" + if err := updateStreamConfig(cfg.TestConfig, cfg.TestConfig.Namespace, testTable, "incremental", cfg.TestConfig.CursorField); err != nil { + return fmt.Errorf("failed to patch streams.json for incremental: %s", err) + } + + // Reset state so initial incremental behaves like a first full incremental load + if err := testutils.ResetStateFile(cfg.TestConfig); err != nil { + return fmt.Errorf("failed to reset state for incremental: %s", err) + } + + // Test cases for incremental sync + incrementalTestCases := []syncTestCase{ + { + name: "Full-Refresh", + operation: "", + useState: false, + opSymbol: "r", + expected: cfg.ExpectedData, + }, + { + name: "Incremental - insert", + operation: "insert", + useState: true, + opSymbol: "u", + expected: cfg.ExpectedData, + }, + { + name: "Incremental - update", + operation: "update", + useState: true, + opSymbol: "u", + expected: cfg.ExpectedUpdatedData, + }, + } + + // Run each incremental test case + for _, tc := range incrementalTestCases { + t.Run(tc.name, func(t *testing.T) { + // schema evolution + if tc.operation == "update" { + if cfg.TestConfig.Driver != string(constants.MongoDB) && cfg.TestConfig.Driver != "mssql" { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "evolve-schema") + } + } + + // drop iceberg table before sync + DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + t.Logf("Dropped Iceberg table: %s", testTable) + + if err := cfg.runSyncAndVerify( + ctx, + t, + testTable, + tc.useState, + "iceberg", + tc.operation, + tc.opSymbol, + tc.expected, + false, + ); err != nil { + t.Fatalf("Incremental test %s failed: %v", tc.name, err) + } + }) + } + + t.Log("Iceberg Full load + Incremental tests completed successfully") + + if testutils.KeepTestData() { + t.Logf("keeping %s source data (%s) is set", cfg.TestConfig.Driver, testutils.KeepTestDataEnvVar) + } else { + DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + t.Logf("Dropped Iceberg table: %s", testTable) + } + + return nil +} + +// ParquetFullLoadAndIncremental tests Full load and Incremental operations for Parquet +func (cfg *Test) ParquetFullLoadAndIncremental( + ctx context.Context, + t *testing.T, + testTable string, +) error { + t.Log("Starting Parquet Full load + Incremental tests") + + if err := cfg.resetTable(ctx, t); err != nil { + return fmt.Errorf("failed to reset table: %s", err) + } + if err := deleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + return fmt.Errorf("failed to reset parquet table: %s", err) + } + + // Patch streams.json: set sync_mode = incremental, cursor_field = "id" + if err := updateStreamConfig(cfg.TestConfig, cfg.TestConfig.Namespace, testTable, "incremental", cfg.TestConfig.CursorField); err != nil { + return fmt.Errorf("failed to patch streams.json for incremental: %s", err) + } + + // Reset state so initial incremental behaves like a first full incremental load + if err := testutils.ResetStateFile(cfg.TestConfig); err != nil { + return fmt.Errorf("failed to reset state for incremental: %s", err) + } + + // Test cases for incremental sync + incrementalTestCases := []syncTestCase{ + { + name: "Full-Refresh", + operation: "", + useState: false, + opSymbol: "r", + expected: cfg.ExpectedData, + }, + { + name: "Incremental - insert", + operation: "insert", + useState: true, + opSymbol: "u", + expected: cfg.ExpectedData, + }, + { + name: "Incremental - update", + operation: "update", + useState: true, + opSymbol: "u", + expected: cfg.ExpectedUpdatedData, + }, + } + + // Run each incremental test case + for _, tc := range incrementalTestCases { + t.Run(tc.name, func(t *testing.T) { + // schema evolution + if tc.operation == "update" { + if cfg.TestConfig.Driver != string(constants.MongoDB) && cfg.TestConfig.Driver != "mssql" { + cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "evolve-schema") + } + } + + // Delete parquet files before next operation to avoid error due to schema changes. + // Kept even for the compatibility suite (unlike the Iceberg drops), because the files are + // genuinely unreadable together: successive syncs write the same column with different + // types -- measured on postgres, col_float4 FLOAT then DOUBLE and col_int INT then + // BIGINT -- so Spark rejects the directory with CANNOT_MERGE_SCHEMAS, mergeSchema or + // not. That is F2 in docs/backward-compatibility.md (parquet has no schema evolution; + // the break surfaces in the reader). The consequence for compatibility is that a parquet + // variant compares only its LAST case's output; see compareVariant. + if err := DeleteParquetFiles(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + t.Fatalf("Failed to delete parquet files before %s: %v", tc.name, err) + } + + if err := cfg.runSyncAndVerify( + ctx, + t, + testTable, + tc.useState, + "parquet", + tc.operation, + tc.opSymbol, + tc.expected, + false, + ); err != nil { + t.Fatalf("Incremental test %s failed: %v", tc.name, err) + } + }) + } + + t.Log("Parquet Full load + Incremental tests completed successfully") + return nil +} diff --git a/tests/testutils/integration/verify.go b/tests/testutils/integration/verify.go new file mode 100644 index 000000000..4806e4437 --- /dev/null +++ b/tests/testutils/integration/verify.go @@ -0,0 +1,468 @@ +package integration + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/spark-connect-go/v35/spark/sql" + "github.com/apache/spark-connect-go/v35/spark/sql/types" + "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/require" +) + +// TODO: Refactor parsing logic into a reusable utility functions +// verifyIcebergSync verifies that data was correctly synchronized to Iceberg +func VerifyIcebergSync(t *testing.T, tableName, icebergDB string, datatypeSchema map[string]string, defaultCDCColumnsSchema map[string]string, schema map[string]interface{}, opSymbol, partitionRegex, driver string, isCDC bool, excludedColumn string) { + t.Helper() + ctx := t.Context() + spark, err := SparkSession(ctx, t) + require.NoError(t, err, "Failed to connect to Spark Connect server") + + fullTableName := fmt.Sprintf("%s.%s.%s", IcebergCatalog, icebergDB, tableName) + // The shared session caches table snapshots, so refresh to see the rows the sync just committed. + // Non-fatal: on a first sync the table may not exist yet, which the retry loop below handles. + if _, refreshErr := spark.Sql(ctx, fmt.Sprintf("REFRESH TABLE %s", fullTableName)); refreshErr != nil { + t.Logf("REFRESH TABLE before verify (non-fatal): %v", refreshErr) + } + selectQuery := fmt.Sprintf( + "SELECT * FROM %s WHERE _op_type = '%s'", + fullTableName, opSymbol, + ) + // In kafka, _op_type is always 'c' and col_included appears only in new rows. + // To check new record, col_included is used. + if driver == string(constants.Kafka) { + if _, ok := schema["col_included"]; ok { + selectQuery += " AND col_included IS NOT NULL" + } + } + t.Logf("Executing query: %s", selectQuery) + + var selectRows []types.Row + var queryErr error + maxRetries := 20 + retryDelay := 5 * time.Second + + for attempt := 0; attempt < maxRetries; attempt++ { + if attempt > 0 { + time.Sleep(retryDelay) + } + var selectQueryDf sql.DataFrame + // This is to check if the table exists in destination, as race condition might cause table to not be created yet + selectQueryDf, queryErr = spark.Sql(ctx, selectQuery) + if queryErr != nil { + t.Logf("Query attempt %d failed: %v", attempt+1, queryErr) + continue + } + + // To ensure stale data is not being used for verification + selectRows, queryErr = selectQueryDf.Collect(ctx) + if queryErr != nil { + t.Logf("Query attempt %d failed (Collect error): %v", attempt+1, queryErr) + continue + } + if len(selectRows) > 0 { + queryErr = nil + break + } + + // For delete operations, 0 rows is acceptable - exit immediately without retrying + if opSymbol == "d" { + queryErr = nil + t.Logf("Delete verification passed: found 0 rows for _op_type = 'd' (acceptable)") + break + } + + // for every type of operation, op symbol will be different, using that to ensure data is not stale + queryErr = fmt.Errorf("stale data: query succeeded but returned 0 rows for _op_type = '%s'", opSymbol) + t.Logf("Query attempt %d/%d failed: %v", attempt+1, maxRetries, queryErr) + + // Force Spark to refresh the table metadata from the Iceberg catalog. + refreshQuery := fmt.Sprintf("REFRESH TABLE %s", fullTableName) + if _, refreshErr := spark.Sql(ctx, refreshQuery); refreshErr != nil { + t.Logf("REFRESH TABLE attempt %d failed (non-fatal): %v", attempt+1, refreshErr) + } + } + + // For delete operations, accept both 0 and 1 row (both are valid outcomes) + if opSymbol == "d" { + if len(selectRows) > 0 { + deletedID := selectRows[0].Value("_olake_id") + require.NotEmpty(t, deletedID, "Delete verification failed: _olake_id should not be empty") + } + t.Logf("Delete verification passed: found %d row(s) for _op_type = 'd'", len(selectRows)) + return + } + require.NoError(t, queryErr, "Failed to collect data rows from Iceberg after %d attempts: %v", maxRetries, queryErr) + require.NotEmpty(t, selectRows, "No rows returned for _op_type = '%s'", opSymbol) + + for rowIdx, row := range selectRows { + icebergMap := make(map[string]interface{}, len(schema)+1) + for _, col := range row.FieldNames() { + icebergMap[col] = row.Value(col) + } + for key, expected := range schema { + icebergValue, ok := icebergMap[key] + require.Truef(t, ok, "Row %d: missing column %q in Iceberg result", rowIdx, key) + require.Equal(t, expected, icebergValue, "Row %d: mismatch on %q: Iceberg has %#v, expected %#v", rowIdx, key, icebergValue, expected) + } + if isCDC { + for key := range defaultCDCColumnsSchema { + icebergValue, ok := icebergMap[key] + require.Truef(t, ok, "Row %d: missing column %q in Iceberg result", rowIdx, key) + // Kafka offset, partition can be 0, NotEmpty fails for 0 so we check for NotNil instead. + if key == "_kafka_offset" || key == "_kafka_partition" { + require.NotNil(t, icebergValue, "Row %d: expected column %q to be non-empty, got %#v", rowIdx, key, icebergValue) + } else { + require.NotEmpty(t, icebergValue, "Row %d: expected column %q to be non-empty, got %#v", rowIdx, key, icebergValue) + } + if key == constants.CdcTimestamp { + ts, ok := normalizeToTime(icebergValue) + require.Truef(t, ok, "Row %d: expected %q to be a timestamp, got %T (%#v)", rowIdx, key, icebergValue, icebergValue) + minAllowed := time.Now().Add(-1 * time.Hour) + require.Falsef(t, ts.Before(time.Now().Add(-1*time.Hour)), "Row %d: %q is too old: %v, should not be earlier than %v", rowIdx, key, ts, minAllowed) + } + } + } + if !isCDC && icebergMap[constants.CdcTimestamp] != nil { + ts, ok := normalizeToTime(icebergMap[constants.CdcTimestamp]) + require.Truef(t, ok, "expected %q to be a timestamp, got %T", constants.CdcTimestamp, icebergMap[constants.CdcTimestamp]) + // Normalize to UTC to keep tests stable across environments (Local vs UTC). + require.Equal(t, time.Unix(0, 0).UTC(), ts.UTC()) + } + } + t.Logf("Verified Iceberg synced data with respect to data synced from source[%s] found equal", driver) + + describeQuery := fmt.Sprintf("DESCRIBE TABLE %s", fullTableName) + describeDf, err := spark.Sql(ctx, describeQuery) + require.NoError(t, err, "Failed to describe Iceberg table") + + describeRows, err := describeDf.Collect(ctx) + require.NoError(t, err, "Failed to collect describe data from Iceberg") + icebergSchema := make(map[string]string) + for _, row := range describeRows { + colName := row.Value("col_name").(string) + dataType := row.Value("data_type").(string) + if !strings.HasPrefix(colName, "#") { + icebergSchema[colName] = dataType + } + } + + if excludedColumn != "" { + _, ok := icebergSchema[testutils.Reformat(excludedColumn)] + require.Falsef(t, ok, "Excluded column %q should not exist in Iceberg schema", excludedColumn) + } + + for col, dbType := range datatypeSchema { + iceType, found := icebergSchema[col] + require.True(t, found, "Column %s not found in Iceberg schema", col) + + expectedIceType, mapped := testutils.GlobalTypeMapping[dbType] + if !mapped { + t.Errorf("No mapping defined for driver type %s (column %s)", dbType, col) + } + require.Equal(t, expectedIceType, iceType, + "Data type mismatch for column %s: expected %s, got %s", col, expectedIceType, iceType) + } + t.Logf("Verified datatypes in Iceberg after sync") + // Verify datatypes for CDC/default columns as well + if isCDC { + for col, expectedIceType := range defaultCDCColumnsSchema { + iceType, found := icebergSchema[col] + require.True(t, found, "CDC column %s not found in Iceberg schema", col) + + require.Equal(t, expectedIceType, iceType, + "CDC data type mismatch for column %s: expected %s, got %s", col, expectedIceType, iceType) + } + t.Logf("Verified datatypes for CDC columns in Iceberg after sync") + } + + // Partition verification using only metadata tables + if partitionRegex == "" { + t.Log("No partitionRegex provided, skipping partition verification") + return + } + // Extract partition columns from describe rows + partitionCols := extractFirstPartitionColFromRows(describeRows) + require.NotEmpty(t, partitionCols, "Partition columns not found in Iceberg metadata") + + // Parse expected partition columns from pattern like "/{col,identity}" + // Supports multiple entries like "/{col1,identity}" by taking the first token as the source column + clean := strings.TrimPrefix(partitionRegex, "/{") + clean = strings.TrimSuffix(clean, "}") + toks := strings.Split(clean, ",") + expectedCol := strings.TrimSpace(toks[0]) + require.Equal(t, expectedCol, partitionCols, "Partition column does not match expected '%s'", expectedCol) + t.Logf("Verified partition column: %s", expectedCol) +} + +// VerifyIcebergNoDuplicates asserts that no duplicate _olake_id values exist for the given +// _op_type in the Iceberg table. +func VerifyIcebergNoDuplicates(ctx context.Context, t *testing.T, tableName, icebergDB, opSymbol string, expectedRowCountByOpType int64) { + t.Helper() + + spark, err := SparkSession(ctx, t) + require.NoError(t, err, "Failed to connect to Spark Connect server for duplicate check") + + fullTableName := fmt.Sprintf("%s.%s.%s", IcebergCatalog, icebergDB, tableName) + + // Refresh to get the latest committed Iceberg snapshot. + refreshQuery := fmt.Sprintf("REFRESH TABLE %s", fullTableName) + if _, refreshErr := spark.Sql(ctx, refreshQuery); refreshErr != nil { + t.Logf("REFRESH TABLE (non-fatal): %v", refreshErr) + } + + countQuery := fmt.Sprintf( + "SELECT COUNT(*) AS total, COUNT(DISTINCT _olake_id) AS distinct_count FROM %s WHERE _op_type = '%s'", + fullTableName, opSymbol, + ) + t.Logf("Executing duplicate-check query: %s", countQuery) + + df, err := spark.Sql(ctx, countQuery) + require.NoError(t, err, "Failed to run duplicate-check COUNT query") + + rows, err := df.Collect(ctx) + require.NoError(t, err, "Failed to collect duplicate-check COUNT results") + require.Len(t, rows, 1, "COUNT query must return exactly one row") + + total, ok := rows[0].Value("total").(int64) + require.True(t, ok, "COUNT(*) value is not int64: %T", rows[0].Value("total")) + + distinct, ok2 := rows[0].Value("distinct_count").(int64) + require.True(t, ok2, "COUNT(DISTINCT) value is not int64: %T", rows[0].Value("distinct_count")) + + // 1. No duplicates: every row must have a unique _olake_id. + require.Equal(t, total, distinct, + "Duplicate rows detected for _op_type='%s': total=%d, distinct=%d. "+ + "Iceberg MERGE INTO did not deduplicate re-synced records.", + opSymbol, total, distinct) + + // 2. Exact count: when caller specifies an expected row count, enforce it so that both + // over-sync (old rows re-processed and inserted again) and under-sync (new rows missed) + // are caught. + if expectedRowCountByOpType > 0 { + require.Equal(t, expectedRowCountByOpType, distinct, + "Row count mismatch for _op_type='%s': expected %d distinct rows, got %d. "+ + "Either old rows were re-synced (over-sync) or new rows were missed (under-sync).", + opSymbol, expectedRowCountByOpType, distinct) + } + + t.Logf("Duplicate check passed for _op_type='%s': %d rows, all unique by _olake_id (expected %d)", + opSymbol, distinct, expectedRowCountByOpType) +} + +// VerifyParquetSync verifies that data was correctly synchronized to Parquet files in MinIO +func VerifyParquetSync(t *testing.T, tableName, parquetDB string, datatypeSchema map[string]string, defaultCDCColumnsSchema map[string]string, schema map[string]interface{}, opSymbol, driver string, isCDC bool, excludedColumn string) { + t.Helper() + ctx := t.Context() + + spark, err := SparkSession(ctx, t) + require.NoError(t, err, "Failed to connect to Spark Connect server") + + parquetPath := fmt.Sprintf("s3a://warehouse/%s/%s", parquetDB, tableName) + viewName := fmt.Sprintf("`%s_view_%d`", tableName, time.Now().UnixNano()) + + // create a temporary view for parquet files, allows to run describe query + createViewQuery := fmt.Sprintf( + "CREATE OR REPLACE TEMP VIEW %s AS SELECT * FROM parquet.`%s/*.parquet`", + viewName, parquetPath, + ) + + // Retry logic for transient Spark connection issues (e.g., catalog connection pool exhaustion) + const maxRetries = 3 + for attempt := 1; attempt <= maxRetries; attempt++ { + _, err = spark.Sql(ctx, createViewQuery) + if err == nil { + break + } + // For delete operations, if path doesn't exist that's acceptable (no data written) + if opSymbol == "d" && strings.Contains(err.Error(), "PATH_NOT_FOUND") { + t.Logf("Delete verification passed: Parquet path does not exist (no data written)") + return + } + if attempt < maxRetries { + t.Logf("Attempt %d/%d: Failed to create view, retrying in 2s: %v", attempt, maxRetries, err) + time.Sleep(2 * time.Second) + } + } + require.NoError(t, err, "Failed to create temporary view for Parquet files") + + defer func() { + dropViewQuery := fmt.Sprintf("DROP VIEW IF EXISTS %s", viewName) + t.Logf("Dropping temporary view: %s", dropViewQuery) + _, _ = spark.Sql(ctx, dropViewQuery) + }() + + selectQuery := fmt.Sprintf( + "SELECT * FROM %s WHERE `_op_type` = '%s'", + viewName, opSymbol, + ) + // In kafka, _op_type is always 'c' and col_included appears only in new rows. + // To check new record, col_included is used. + if driver == string(constants.Kafka) { + if _, ok := schema["col_included"]; ok { + selectQuery += " AND `col_included` IS NOT NULL" + } + } + t.Logf("Executing Parquet query: %s", selectQuery) + + df, err := spark.Sql(ctx, selectQuery) + require.NoError(t, err, "Failed to run select query on Parquet files") + + rows, err := df.Collect(ctx) + require.NoError(t, err, "Failed to collect rows from Parquet query") + + // For delete operations, accept both 0 and 1 row (both are valid outcomes) + if opSymbol == "d" { + if len(rows) > 0 { + deletedID := rows[0].Value("_olake_id") + require.NotEmpty(t, deletedID, "Delete verification failed: _olake_id should not be empty") + } + t.Logf("Delete verification passed: found %d row(s) for _op_type = 'd'", len(rows)) + return + } + + // For non-delete operations, require at least one row + require.NotEmpty(t, rows, "No rows returned for _op_type = '%s'", opSymbol) + + for rowIdx, row := range rows { + parquetMap := make(map[string]interface{}, len(schema)+1) + for _, col := range row.FieldNames() { + parquetMap[col] = row.Value(col) + } + for key, expected := range schema { + val, ok := parquetMap[key] + require.Truef(t, ok, "Row %d: missing column %q in Parquet result", rowIdx, key) + require.Equal(t, expected, val, + "Row %d: mismatch on %q: Parquet has %#v, expected %#v", rowIdx, key, val, expected) + } + if isCDC { + for key := range defaultCDCColumnsSchema { + val, ok := parquetMap[key] + require.Truef(t, ok, "Row %d: missing column %q in Parquet result", rowIdx, key) + // Kafka offset, partition can be 0, NotEmpty fails for 0 so we check for NotNil instead. + if key == "_kafka_offset" || key == "_kafka_partition" { + require.NotNil(t, val, "Row %d: expected column %q to be non-empty, got %#v", rowIdx, key, val) + } else { + require.NotEmpty(t, val, "Row %d: expected column %q to be non-empty, got %#v", rowIdx, key, val) + } + if key == constants.CdcTimestamp { + ts, ok := normalizeToTime(val) + require.Truef(t, ok, "Row %d: expected %q to be a timestamp, got %T (%#v)", rowIdx, key, val, val) + minAllowed := time.Now().Add(-1 * time.Hour) + require.Falsef(t, ts.Before(time.Now().Add(-1*time.Hour)), "Row %d: %q is too old: %v, should not be earlier than %v", rowIdx, key, ts, minAllowed) + } + } + } + if !isCDC && parquetMap[constants.CdcTimestamp] != nil { + ts, ok := normalizeToTime(parquetMap[constants.CdcTimestamp]) + require.Truef(t, ok, "expected %q to be a timestamp, got %T", constants.CdcTimestamp, parquetMap[constants.CdcTimestamp]) + // Normalize to UTC to keep tests stable across environments (Local vs UTC). + require.Equal(t, time.Unix(0, 0).UTC(), ts.UTC()) + } + } + + t.Logf("Verified Parquet synced data with respect to data synced from source[%s] found equal", driver) + + describeQuery := fmt.Sprintf("DESCRIBE TABLE %s", viewName) + descDF, err := spark.Sql(ctx, describeQuery) + require.NoError(t, err, "Failed to describe Parquet view") + + descRows, err := descDF.Collect(ctx) + require.NoError(t, err, "Failed to collect schema info from Parquet view") + + parquetSchema := make(map[string]string) + for _, row := range descRows { + colName := row.Value("col_name").(string) + dataType := row.Value("data_type").(string) + if !strings.HasPrefix(colName, "#") { + parquetSchema[colName] = dataType + } + } + if excludedColumn != "" { + _, ok := parquetSchema[testutils.Reformat(excludedColumn)] + require.Falsef(t, ok, "Excluded column %q should not exist in Parquet schema", excludedColumn) + } + + for col, dbType := range datatypeSchema { + pqType, found := parquetSchema[col] + require.True(t, found, "Column %s not found in Parquet schema", col) + + expectedType, mapped := testutils.GlobalTypeMapping[dbType] + if !mapped { + t.Errorf("No mapping defined for driver type %s (column %s)", dbType, col) + } + require.Equal(t, expectedType, pqType, + "Data type mismatch for column %s: expected %s, got %s", col, expectedType, pqType) + } + t.Logf("Verified datatypes in Parquet after sync") + // Verify datatypes for CDC/default columns as well + if isCDC { + for col, expectedPqType := range defaultCDCColumnsSchema { + pqType, found := parquetSchema[col] + require.True(t, found, "CDC column %s not found in Parquet schema", col) + require.Equal(t, expectedPqType, pqType, + "CDC data type mismatch for column %s: expected %s, got %s", col, expectedPqType, pqType) + } + } + t.Logf("Verified datatypes for CDC columns in Parquet after sync") +} + +// extractFirstPartitionColFromRows extracts the first partition column from DESCRIBE EXTENDED rows +func extractFirstPartitionColFromRows(rows []types.Row) string { + inPartitionSection := false + + for _, row := range rows { + // Convert []any -> []string + vals := row.Values() + parts := make([]string, len(vals)) + for i, v := range vals { + if v == nil { + parts[i] = "" + } else { + parts[i] = fmt.Sprint(v) // safe string conversion + } + } + line := strings.TrimSpace(strings.Join(parts, " ")) + if line == "" { + continue + } + + if strings.HasPrefix(line, "# Partition Information") { + inPartitionSection = true + continue + } + + if inPartitionSection { + if strings.HasPrefix(line, "# col_name") { + continue + } + + if strings.HasPrefix(line, "#") { + break + } + + fields := strings.Fields(line) + if len(fields) > 0 { + return fields[0] // return the first partition col + } + } + } + + return "" +} + +func normalizeToTime(v interface{}) (time.Time, bool) { + switch ts := v.(type) { + case time.Time: + return ts, true + case arrow.Timestamp: + return time.Unix(0, int64(ts)*int64(time.Microsecond)).UTC(), true + default: + return time.Time{}, false + } +} diff --git a/tests/testutils/performance/benchmarks.go b/tests/testutils/performance/benchmarks.go new file mode 100644 index 000000000..26ab93e7c --- /dev/null +++ b/tests/testutils/performance/benchmarks.go @@ -0,0 +1,113 @@ +package performance + +import ( + "fmt" + "os" + "time" + + "github.com/datazip-inc/olake/tests/testutils" +) + +const ( + // BenchmarkThreshold is the share of the recorded average RPS a run must reach to pass. + BenchmarkThreshold = 0.9 + maxRPSHistorySize = 5 +) + +// SyncSpeed is the shape of the stats.json a sync writes; its Speed reads " rps". +type SyncSpeed struct { + Speed string `json:"Speed"` +} + +// history stores the RPS values and the last updated time for a given mode. +type history struct { + RPS []float64 `json:"rps"` + UpdatedAt time.Time `json:"updated_at"` +} + +// benchmarkStore stores the benchmark RPS history for backfill and CDC modes. +type benchmarkStore struct { + Backfill history `json:"backfill"` + CDC history `json:"cdc"` + FilePath string `json:"-"` +} + +// initializes the benchmark store with the given path and loads the stored benchmarks data from the file. +func loadBenchmarks(path string) (*benchmarkStore, error) { + store := &benchmarkStore{ + Backfill: history{ + RPS: make([]float64, 0, maxRPSHistorySize), + UpdatedAt: time.Now().UTC(), + }, + CDC: history{ + RPS: make([]float64, 0, maxRPSHistorySize), + UpdatedAt: time.Now().UTC(), + }, + FilePath: path, + } + if err := store.load(); err != nil { + return nil, err + } + return store, nil +} + +// load loads the stored benchmarks data from the file. +func (s *benchmarkStore) load() error { + if err := testutils.UnmarshalFile(s.FilePath, s, false); err != nil { + if _, statErr := os.Stat(s.FilePath); os.IsNotExist(statErr) { + // Missing file is acceptable, it will be created when the first RPS is recorded. + return nil + } + return fmt.Errorf("failed to load rps benchmarks from file %s: %s", s.FilePath, err) + } + + return nil +} + +// record records a new benchmark RPS value for the given driver and mode, and persists it to the file. +func (s *benchmarkStore) record( + isBackfill bool, + rps float64, +) error { + rpsValues := testutils.Ternary( + isBackfill, + s.Backfill.RPS, + s.CDC.RPS, + ).([]float64) + + rpsValues = append(rpsValues, rps) + + // Truncate history to maintain a rolling window of the last maxRPSHistorySize values. + if len(rpsValues) > maxRPSHistorySize { + rpsValues = rpsValues[1:] + } + + if isBackfill { + s.Backfill.RPS = rpsValues + s.Backfill.UpdatedAt = time.Now().UTC() + } else { + s.CDC.RPS = rpsValues + s.CDC.UpdatedAt = time.Now().UTC() + } + + return testutils.FileLoggerWithPath(s, s.FilePath) +} + +// stats returns the average RPS and count of past RPS values for the given driver and mode. +// The count cannot exceed maxRPSHistorySize. +func (s *benchmarkStore) stats( + isBackfill bool, +) (averageRPS float64, observations int) { + rpsValues := testutils.Ternary( + isBackfill, + s.Backfill.RPS, + s.CDC.RPS, + ).([]float64) + + if len(rpsValues) == 0 { + // No benchmarks recorded for this mode yet. + return 0, 0 + } + + return testutils.Average(rpsValues), len(rpsValues) +} diff --git a/tests/testutils/performance/performance.go b/tests/testutils/performance/performance.go new file mode 100644 index 000000000..21b7577c6 --- /dev/null +++ b/tests/testutils/performance/performance.go @@ -0,0 +1,221 @@ +package performance + +import ( + "context" + "fmt" + "os/exec" + "strings" + "testing" + + "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/require" +) + +const icebergDestinationFile = "iceberg_destination.json" + +// Test is one driver's benchmark: the streams it reads and the config that names where they live. +type Test struct { + *testutils.TestConfig + BackfillStreams []string + CDCStreams []string +} + +// validate checks the fields the benchmark itself needs; NewTestConfig has already validated the +// TestConfig by the time one reaches here. +func (cfg *Test) validate(t *testing.T) { + t.Helper() + require.NotNil(t, cfg.TestConfig, "performance.Test.TestConfig is not set") + // A benchmark with nothing to read still reports a rate, and it is the rate of doing nothing. + require.Falsef(t, len(cfg.BackfillStreams) == 0 && len(cfg.CDCStreams) == 0, + "performance.Test declares neither BackfillStreams nor CDCStreams") + // TODO: assert BackfillStreams and CDCStreams are disjoint. GetBackfillStreamsFromCDC derives + // one from the other by trimming "_cdc", so a CDC stream without that suffix passes through + // unchanged and is counted on both sides of the ratio. +} + +// GetBackfillStreamsFromCDC derives the backfill stream names from the CDC ones, +// e.g. "demo_cdc" -> "demo". +func GetBackfillStreamsFromCDC(cdcStreams []string) []string { + backfillStreams := []string{} + for _, stream := range cdcStreams { + backfillStreams = append(backfillStreams, strings.TrimSuffix(stream, "_cdc")) + } + return backfillStreams +} + +// TestPerformance benchmarks the driver against the instances its source config names: a backfill +// sync first, then a CDC one for the drivers that declare CDC streams. Each phase is asserted +// against the RPS history committed for the driver, then appended to it. +// +// The phases run in sequence rather than parallel: they share the source, the state file and the +// destination, and a benchmark that races another sync measures the contention, not the driver. +func (cfg *Test) TestPerformance(t *testing.T) { + cfg.validate(t) + ctx := t.Context() + + // The CDC configuration a previous run left behind (a slot holding its own WAL, a binlog + // position) is what the next backfill would have to read past, so start from a clean one. + if cfg.Driver == string(constants.Postgres) || cfg.Driver == string(constants.MySQL) { + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "reset_cdc_config") + t.Log("CDC config reset completed") + } + + if !t.Run("Backfill", func(t *testing.T) { + if err := cfg.runBackfill(ctx, t); err != nil { + t.Fatalf("backfill benchmark failed: %s", err) + } + }) { + t.Log("stopping after the backfill phase failed; the CDC phase reads the state it did not write") + return + } + + if len(cfg.CDCStreams) == 0 { + return + } + t.Run("CDC", func(t *testing.T) { + if err := cfg.runCDC(ctx, t); err != nil { + t.Fatalf("cdc benchmark failed: %s", err) + } + }) +} + +// runBackfill measures a full read of BackfillStreams. +func (cfg *Test) runBackfill(ctx context.Context, t *testing.T) error { + if err := cfg.discoverStreams(ctx, cfg.BackfillStreams); err != nil { + return err + } + + // MySQL derives its chunk plan from InnoDB statistics, which drift between runs; seed the + // committed plan instead so every benchmark measures the same split. + usePreChunkedState := cfg.Driver == string(constants.MySQL) + if usePreChunkedState { + if err := testutils.CopyFile(cfg.GetFilePath("performance_state.json"), cfg.GetFilePath("state.json")); err != nil { + return fmt.Errorf("failed to seed the pre-chunked state: %s", err) + } + } + + defer testutils.TrackPhaseTiming(t, cfg.Driver, "backfill sync")() + if out, err := cfg.timedSync(ctx, usePreChunkedState); err != nil { + return fmt.Errorf("backfill sync failed: %s\n%s", err, out) + } + + return cfg.recordRPS(t, true) +} + +// runCDC measures a read of the changes bulk_cdc_data_insert leaves behind. The stateless sync +// before it is what puts the driver's CDC cursor ahead of them. +func (cfg *Test) runCDC(ctx context.Context, t *testing.T) error { + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "setup_cdc") + + if err := cfg.discoverStreams(ctx, cfg.CDCStreams); err != nil { + return err + } + + if code, out, err := cfg.runOlake(ctx, testutils.SyncArgs(false, icebergDestinationFile, cfg.destinationPrefix()...)...); err != nil || code != 0 { + return fmt.Errorf("failed to write the initial CDC state: %s\n%s", err, out) + } + + cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "bulk_cdc_data_insert") + + defer testutils.TrackPhaseTiming(t, cfg.Driver, "cdc sync")() + if out, err := cfg.timedSync(ctx, true); err != nil { + return fmt.Errorf("cdc sync failed: %s\n%s", err, out) + } + + return cfg.recordRPS(t, false) +} + +// discoverStreams runs discover and selects the streams the phase measures, so the benchmark reads +// the same catalog a deployed sync would build for itself. +func (cfg *Test) discoverStreams(ctx context.Context, streams []string) error { + code, out, err := cfg.runOlake(ctx, testutils.DiscoverArgs(cfg.destinationPrefix()...)...) + if err != nil || code != 0 { + return fmt.Errorf("discover failed: %s\n%s", err, out) + } + if err := testutils.UpdateSelectedStreams(cfg.TestConfig, cfg.Namespace, "", "", streams, ""); err != nil { + return fmt.Errorf("failed to select %s: %s", strings.Join(streams, ", "), err) + } + return nil +} + +// destinationPrefix names the destination database every phase writes into. +func (cfg *Test) destinationPrefix() []string { + return []string{"--destination-database-prefix", fmt.Sprintf("performance_%s", cfg.Driver)} +} + +// runOlake runs the driver image with host networking so the benchmark reaches the external +// instances directly, exactly as a deployed sync would. +func (cfg *Test) runOlake(ctx context.Context, olakeArgs ...string) (int, []byte, error) { + args := testutils.DockerRunArgs(cfg.TestConfig, cfg.DriverImage, []string{"--network", "host"}, olakeArgs) + out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput() + return testutils.DockerExitResult(out, err, olakeArgs[0]) +} + +// timedSync runs a sync bounded by SyncTimeout. Hitting the window is expected -- this is a bounded +// throughput measurement, not a completeness one -- so the still-running container is stopped and +// whatever it managed reads out of stats.json. +func (cfg *Test) timedSync(ctx context.Context, useState bool) ([]byte, error) { + // Named, so a sync that outlives its window can be stopped rather than hunted for. + name := fmt.Sprintf("olake-perf-%s", cfg.Driver) + _ = exec.Command("docker", "rm", "-f", name).Run() // drop any stale container from a previous run + + timedCtx, cancel := context.WithTimeout(ctx, testutils.SyncTimeout) + defer cancel() + + olakeArgs := testutils.SyncArgs(useState, icebergDestinationFile, cfg.destinationPrefix()...) + args := testutils.DockerRunArgs(cfg.TestConfig, cfg.DriverImage, []string{"--network", "host", "--name", name}, olakeArgs) + out, err := exec.CommandContext(timedCtx, "docker", args...).CombinedOutput() + if timedCtx.Err() == context.DeadlineExceeded { + _ = exec.Command("docker", "kill", name).Run() + return out, nil + } + + code, out, err := testutils.DockerExitResult(out, err, "sync") + if err != nil { + return out, err + } + if code != 0 { + return out, testutils.RenderOlakeFailure(code, nil, nil) + } + return out, nil +} + +// recordRPS asserts the rate the phase just wrote to stats.json against the driver's history, then +// appends it. A driver with no history yet passes and seeds it, which is how a new one is onboarded. +func (cfg *Test) recordRPS(t *testing.T, isBackfill bool) error { + rps, err := cfg.syncedRPS() + if err != nil { + return err + } + + benchmarks, err := loadBenchmarks(cfg.GetFixturePath("benchmarks.json")) + if err != nil { + return err + } + averageRPS, observations := benchmarks.stats(isBackfill) + mode := testutils.Ternary(isBackfill, "backfill", "cdc").(string) + t.Logf("%s %s: currentRPS %.2f, averageRPS %.2f, observations %d", cfg.Driver, mode, rps, averageRPS, observations) + + if observations == 0 { + t.Logf("no benchmarks recorded for %s %s yet, seeding the history with this run", cfg.Driver, mode) + } else { + require.GreaterOrEqualf(t, rps, BenchmarkThreshold*averageRPS, + "%s %s performance below benchmark: %.2f rps against an average of %.2f", cfg.Driver, mode, rps, averageRPS) + } + + return benchmarks.record(isBackfill, rps) +} + +// syncedRPS reads the rate the last sync reported, which it writes to stats.json as " rps". +func (cfg *Test) syncedRPS() (float64, error) { + var stats SyncSpeed + if err := testutils.UnmarshalFile(cfg.GetFilePath("stats.json"), &stats, false); err != nil { + return 0, err + } + rps, err := testutils.ParseFloat64(strings.Split(stats.Speed, " ")[0]) + if err != nil { + return 0, fmt.Errorf("failed to read the RPS out of %q: %s", stats.Speed, err) + } + return rps, nil +} diff --git a/tests/testutils/require/require.go b/tests/testutils/require/require.go new file mode 100644 index 000000000..24c712d44 --- /dev/null +++ b/tests/testutils/require/require.go @@ -0,0 +1,144 @@ +// Package require is the harness's drop-in for testify's require: same names, same arguments, +// same semantics, with every failure report re-emitted red and bold and attributed to the caller. +package require + +import ( + "fmt" + "testing" + "time" + + trequire "github.com/stretchr/testify/require" +) + +const ( + red = "\033[31m" + reset = "\033[0m" +) + +// failT collects what testify would have reported so run can replay it on the real testing.T -- +// through the caller's frame, which is what keeps the file:line prefix on the test that failed. +type failT struct { + messages []string + failed bool +} + +func (c *failT) Errorf(format string, args ...any) { + c.messages = append(c.messages, fmt.Sprintf(format, args...)) +} + +func (c *failT) FailNow() { + c.failed = true +} + +// run replays a captured failure on t: the report in color, then the FailNow testify requested. +func run(t *testing.T, check func(c *failT)) { + t.Helper() + c := &failT{} + check(c) + for _, message := range c.messages { + t.Errorf(red+"%s"+reset, message) + } + if c.failed { + t.FailNow() + } +} + +func Contains(t *testing.T, s, contains any, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Contains(c, s, contains, msgAndArgs...) }) +} + +func Containsf(t *testing.T, s, contains any, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Containsf(c, s, contains, msg, msgAndArgs...) }) +} + +func Empty(t *testing.T, object any, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Empty(c, object, msgAndArgs...) }) +} + +func Equal(t *testing.T, expected, actual any, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Equal(c, expected, actual, msgAndArgs...) }) +} + +func Equalf(t *testing.T, expected, actual any, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Equalf(c, expected, actual, msg, msgAndArgs...) }) +} + +func Falsef(t *testing.T, value bool, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Falsef(c, value, msg, msgAndArgs...) }) +} + +func GreaterOrEqualf(t *testing.T, e1, e2 any, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.GreaterOrEqualf(c, e1, e2, msg, msgAndArgs...) }) +} + +func Greaterf(t *testing.T, e1, e2 any, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Greaterf(c, e1, e2, msg, msgAndArgs...) }) +} + +func Len(t *testing.T, object any, length int, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Len(c, object, length, msgAndArgs...) }) +} + +func LessOrEqualf(t *testing.T, e1, e2 any, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.LessOrEqualf(c, e1, e2, msg, msgAndArgs...) }) +} + +func NoError(t *testing.T, err error, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.NoError(c, err, msgAndArgs...) }) +} + +func NoErrorf(t *testing.T, err error, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.NoErrorf(c, err, msg, msgAndArgs...) }) +} + +func NotContainsf(t *testing.T, s, contains any, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.NotContainsf(c, s, contains, msg, msgAndArgs...) }) +} + +func NotEmpty(t *testing.T, object any, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.NotEmpty(c, object, msgAndArgs...) }) +} + +func NotEqualf(t *testing.T, expected, actual any, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.NotEqualf(c, expected, actual, msg, msgAndArgs...) }) +} + +func NotNil(t *testing.T, object any, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.NotNil(c, object, msgAndArgs...) }) +} + +func True(t *testing.T, value bool, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.True(c, value, msgAndArgs...) }) +} + +func Truef(t *testing.T, value bool, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Truef(c, value, msg, msgAndArgs...) }) +} + +func Zerof(t *testing.T, i any, msg string, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Zerof(c, i, msg, msgAndArgs...) }) +} + +func Eventually(t *testing.T, condition func() bool, waitFor, tick time.Duration, msgAndArgs ...any) { + t.Helper() + run(t, func(c *failT) { trequire.Eventually(c, condition, waitFor, tick, msgAndArgs...) }) +} diff --git a/tests/testutils/source_config.go b/tests/testutils/source_config.go index a92f11130..4bc4fbc0d 100644 --- a/tests/testutils/source_config.go +++ b/tests/testutils/source_config.go @@ -1,9 +1,9 @@ package testutils import ( - "testing" - - "github.com/stretchr/testify/require" + "fmt" + "net" + "strings" ) // SourceConfig is a driver's source.json read as an untyped map. @@ -20,11 +20,44 @@ import ( type SourceConfig map[string]any // ReadSourceConfig loads path as an untyped source config. -func ReadSourceConfig(t *testing.T, path string) SourceConfig { - t.Helper() +func ReadSourceConfig(path string) (SourceConfig, error) { config := SourceConfig{} - require.NoError(t, UnmarshalFile(path, &config, false), "read source config %s", path) - return config + if err := UnmarshalFile(path, &config, false); err != nil { + return nil, fmt.Errorf("failed to read the source config at %s: %s", path, err) + } + return config, nil +} + +// containerHost is how the driver container reaches the host's published ports. The harness runs +// on the host itself, where that name does not resolve. +const containerHost = "host.docker.internal" + +// HostAddress rewrites an address the driver container uses into one the harness can dial. Host +// and "host:port" forms are both accepted; anything already reachable is returned unchanged. +func HostAddress(address string) string { + if !strings.Contains(address, containerHost) { + return address + } + if host, port, err := net.SplitHostPort(address); err == nil && host == containerHost { + return net.JoinHostPort("127.0.0.1", port) + } + return strings.ReplaceAll(address, containerHost, "127.0.0.1") +} + +// Host returns key as an address the harness can dial, translated out of the container's view. +func (c SourceConfig) Host(key string) string { + return HostAddress(c.String(key)) +} + +// Hosts returns key as addresses the harness can dial, for the drivers that spell their host list +// as an array. +func (c SourceConfig) Hosts(key string) []string { + raw := c.Strings(key) + hosts := make([]string, 0, len(raw)) + for _, host := range raw { + hosts = append(hosts, HostAddress(host)) + } + return hosts } // String returns key as a string. diff --git a/tests/testutils/state_version.go b/tests/testutils/state_version.go new file mode 100644 index 000000000..cdf6deedb --- /dev/null +++ b/tests/testutils/state_version.go @@ -0,0 +1,70 @@ +package testutils + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" +) + +// The product's constants/state-versions.json is the single source of truth for state-file +// semantics: the version the build writes today, and the release history behind every bump. The +// harness holds no copy of its own -- go:embed cannot reach the file from another module, so it is +// read at runtime, once per process. + +// StateVersionBaseline is one entry of the manifest's release history: the release that introduced +// a state version, which drivers it gated, and why. The compatibility suite sweeps these tags. +type StateVersionBaseline struct { + StateVersion int `json:"state_version"` + ReleaseTag string `json:"release_tag"` + Drivers string `json:"drivers"` + Note string `json:"note"` +} + +type stateVersionManifest struct { + LatestStateVersion int `json:"latest_state_version"` + Baselines []StateVersionBaseline `json:"baselines"` +} + +var ( + stateVersionOnce sync.Once + stateVersionValue stateVersionManifest + stateVersionErr error +) + +func readStateVersionManifest(rootPath string) (stateVersionManifest, error) { + stateVersionOnce.Do(func() { + path := filepath.Join(rootPath, "constants", "state-versions.json") + data, err := os.ReadFile(path) + if err != nil { + stateVersionErr = fmt.Errorf("failed to read the product state versions at %s: %w", path, err) + return + } + if err := json.Unmarshal(data, &stateVersionValue); err != nil { + stateVersionErr = fmt.Errorf("failed to parse %s: %w", path, err) + return + } + if stateVersionValue.LatestStateVersion <= 0 { + stateVersionErr = fmt.Errorf("%s does not set latest_state_version to a positive integer", path) + return + } + if len(stateVersionValue.Baselines) == 0 { + stateVersionErr = fmt.Errorf("%s carries no baselines; the compatibility sweep would silently shrink", path) + return + } + }) + return stateVersionValue, stateVersionErr +} + +// ProductStateVersion is the state-file version the product writes today. +func ProductStateVersion(rootPath string) (int, error) { + manifest, err := readStateVersionManifest(rootPath) + return manifest.LatestStateVersion, err +} + +// StateVersionBaselines is the manifest's release history, in file order. +func StateVersionBaselines(rootPath string) ([]StateVersionBaseline, error) { + manifest, err := readStateVersionManifest(rootPath) + return manifest.Baselines, err +} diff --git a/tests/testutils/test_utils.go b/tests/testutils/test_utils.go index a5c3757cf..81798e265 100644 --- a/tests/testutils/test_utils.go +++ b/tests/testutils/test_utils.go @@ -1,535 +1,365 @@ package testutils import ( - "bytes" "context" "encoding/json" + "errors" "fmt" - "maps" "os" - "os/exec" - "path" "path/filepath" + "regexp" "slices" "strings" - "sync" "testing" "time" - "github.com/apache/arrow-go/v18/arrow" - "github.com/apache/spark-connect-go/v35/spark/sql" - "github.com/apache/spark-connect-go/v35/spark/sql/types" "github.com/datazip-inc/olake/tests/testutils/constants" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/stretchr/testify/require" ) const ( - icebergCatalog = "olake_iceberg" - parquetTestBucket = "warehouse" - // IP literal, not "localhost": a hostname sends grpc-go through a DNS resolver that stalls - // every new connection ~20s when the DNS servers are slow (measured 20.22s vs 42ms). - sparkConnectAddress = "sc://127.0.0.1:15002" - SyncTimeout = 10 * time.Minute - BenchmarkThreshold = 0.9 - maxRPSHistorySize = 5 - kafkaRebalanceBulkMessageCount = int64(100_000) -) + SyncTimeout = 10 * time.Minute -type IntegrationTest struct { - TestConfig *TestConfig - ExpectedData map[string]interface{} - ExpectedUpdatedData map[string]interface{} - DestinationDataTypeSchema map[string]string - UpdatedDestinationDataTypeSchema map[string]string - DefaultCDCColumnsSchema map[string]string - Namespace string - ExecuteQuery func(ctx context.Context, t *testing.T, conf *TestConfig, operation string) - DestinationDB string - CursorField string - PartitionRegex string - FilterConfig string - ColumnToExclude string -} + KeepTestDataEnvVar = "OLAKE_TEST_KEEP_DATA" +) -type PerformanceTest struct { - TestConfig *TestConfig - Namespace string - BackfillStreams []string - CDCStreams []string - ExecuteQuery func(ctx context.Context, t *testing.T, conf *TestConfig, operation string) -} +// ExecuteQueryFn drives the driver's source: the suites call it with an operation name, and the +// driver's own implementation knows what that means for its source. +type ExecuteQueryFn func(ctx context.Context, t *testing.T, cfg *TestConfig, operation string) -type SyncSpeed struct { - Speed string `json:"Speed"` -} +// TestConfig holds the configuration for a single test suite run per driver type TestConfig struct { Driver string DataFormat string - // Suite names one driver's concurrent suite ("2pc", "rebalance"; empty for integration) so - // destination namespaces and CDC readers don't collide (see applySuite). + + // Suite is the unique identifier for the test suite running. + // This is used to isolate test data and resources for concurrent test runs. Suite string + // ImagePlatform overrides the platform the driver image runs under, for images that exist // only for amd64 and run emulated elsewhere. ImagePlatform string - // SourceBaseConfig, when set, is the parsed source config ExecuteQuery implementations connect - // with; nil means the integration containers' fixed local credentials. - SourceBaseConfig SourceConfig - - // Host-side paths, read and written directly by the harness. - HostRootPath string // repo root, used to `make docker..build` when the image is missing - HostTestDataPath string // this config's private /tmp working dir, mounted at /testdata in the driver container - HostTestCatalogPath string // expected test_streams.json (committed fixture) - HostCatalogPath string // streams.json generated by discover / consumed by sync - HostStatePath string - HostStateCheckpointPath string // backup of state.json used in 2PC recovery tests - HostPerformanceStatePath string // committed pre-chunked seed state, copied over state.json by the performance suite - HostSourcePath string // working copy of source.json; variantSourceOverride edits it per suite - HostStatsPath string - BenchmarksPath string - - // Container-side paths, passed as arguments to the olake binary. - SourcePath string - CatalogPath string - IcebergDestinationPath string - ParquetDestinationPath string - StatePath string -} -// applySuite names the suite and rewires source.json where concurrent suites contend on a CDC -// reader; file isolation itself comes from every TestConfig owning a private working dir. -func applySuite(t *testing.T, c *TestConfig, suite string) { - t.Helper() - if suite == "" { - return - } - c.Suite = strings.ToLower(suite) - if edit := variantSourceOverride(c); edit != nil { - require.NoError(t, editJSONFile(c.HostSourcePath, edit), - "failed to derive the source config for suite %q", suite) - } + // DriverImage the test is intedned to run under. Defaults to the building the image of the current codebase + DriverImage string + + // OlakeRootPath is the repo the tests run from, the directory `make docker..build` + // runs in and the committed fixtures are read from. Resolved by setupWorkingDir. + OlakeRootPath string + + // TestWorkingDir is this config's private /tmp working dir, the folder where we run olake + // commands and expect the generated files. Every file the suite reads or writes lives in it and + // is addressed by name through GetFilePath, so there is no path to keep a field for. + TestWorkingDir string + + // SourceBaseConfig is the working copy of source.json, parsed: the suite's own credentials, + // database and prefixes, after applySuite renamed what it isolates. ExecuteQuery connects with + // it, so the harness and olake always drive the same source. + SourceBaseConfig SourceConfig `json:"-"` + + // Driver shape: the same for every suite this driver runs, so it is declared once here + // rather than per test. + Namespace string + ExecuteQuery ExecuteQueryFn `json:"-"` + DestinationDB string + CursorField string + PartitionRegex string + FilterConfig string + ColumnToExclude string + + // sourceEdit and streamEdit are the driver's own per-suite isolation: what a driver must rename + // so its concurrent suites do not contend, and whatever that rename implies for the catalog. + sourceEdit ConfigEditFn + streamEdit ConfigEditFn } -// IsolateSuite scopes a whole test to a concurrent suite: its table and namespace names, and the -// DestinationDB verify and drop target, all pick up the suite the sync will write under. -func (cfg *IntegrationTest) IsolateSuite(t *testing.T, suite string) { +type TestConfigOption func(*TestConfig) + +// ConfigEditFn edits one of the suite's working-copy JSON files. It receives the config so a +// driver can name what it isolates after the suite. +type ConfigEditFn func(cfg *TestConfig, doc map[string]interface{}) error + +// NewTestConfig builds a driver's config from what a suite cannot derive for itself: the source it +// reads, the namespace it drives and the destination olake derives from them. dataFormat names the +// driver's testdata subdirectory, for the drivers that have one. +func NewTestConfig(t *testing.T, driver constants.DriverType, namespace, destinationDB string, executeQuery ExecuteQueryFn, opts ...TestConfigOption) (*TestConfig, error) { t.Helper() - if suite == "" { - return + cfg := &TestConfig{ + Driver: string(driver), + Namespace: namespace, + DestinationDB: destinationDB, + ExecuteQuery: executeQuery, } - applySuite(t, cfg.TestConfig, suite) - cfg.DestinationDB = withSuite(cfg.DestinationDB, cfg.TestConfig.Suite) -} -// copyJSONWithEdit reads the JSON at srcHost, applies edit, and writes the result to dstHost -- -// used to derive a per-suite config from a shared base file without touching the base. -func copyJSONWithEdit(srcHost, dstHost string, edit func(map[string]interface{}) error) error { - raw, err := os.ReadFile(srcHost) - if err != nil { - return fmt.Errorf("failed to read %s: %s", srcHost, err) - } - doc, err := parseJSONDoc(raw) - if err != nil { - return fmt.Errorf("failed to parse %s: %s", srcHost, err) + for _, opt := range opts { + opt(cfg) } - if err := edit(doc); err != nil { - return err - } - out, err := json.MarshalIndent(doc, "", " ") + + err := cfg.setup(t) if err != nil { - return fmt.Errorf("failed to marshal %s: %s", dstHost, err) + return nil, err } - return writeHostFile(dstHost, out) + + return cfg, nil } -// variantSourceOverride gives a suite its own CDC reader where concurrent ones contend: a postgres -// replication slot, a kafka consumer group, a whole mssql database. nil for drivers that can share -// source.json. Whatever it renames here, SuiteDatabase and the driver's own connection must match. -func variantSourceOverride(c *TestConfig) func(map[string]interface{}) error { - switch c.Driver { - case string(constants.MSSQL): - // Table separation alone races: DROP/CREATE TABLE modify database-scoped shared metadata - // (system catalog, cdc schema) even for separate tables, and the loser transaction fails - // as the deadlock victim (error 1205) -- so each suite owns a whole database. - return func(doc map[string]interface{}) error { - base, ok := doc["database"].(string) - if !ok { - return fmt.Errorf("no database in source config") - } - doc["database"] = SuiteDatabase(base, c.Suite) - return nil - } - case string(constants.Postgres): - return func(doc map[string]interface{}) error { - updateMethod, ok := doc["update_method"].(map[string]interface{}) - if !ok { - return fmt.Errorf("no update_method object in source config") - } - updateMethod["replication_slot"] = TestTableName(c) - return nil - } - case string(constants.Kafka): - return func(doc map[string]interface{}) error { - base, ok := doc["consumer_group_id"].(string) - if !ok || base == "" { - return fmt.Errorf("no consumer_group_id in source config") - } - doc["consumer_group_id"] = withSuite(base, c.Suite) - return nil - } +func WithImagePlatform(platform string) TestConfigOption { + return func(c *TestConfig) { + c.ImagePlatform = platform } - return nil } -// TestTableName is the source table a suite drives. The suite suffix is what keeps concurrent -// suites off each other's table -- without it they race the same DROP/CREATE. -func TestTableName(c *TestConfig) string { - name := Ternary(c.DataFormat == "", - fmt.Sprintf("%s_test_table_olake", c.Driver), - fmt.Sprintf("%s_%s_test_table_olake", c.Driver, c.DataFormat)).(string) - return withSuite(name, c.Suite) +func WithDataFormat(dataFormat string) TestConfigOption { + return func(c *TestConfig) { + c.DataFormat = dataFormat + } } -// withSuite names a suite's own copy of a shared resource (table, database, consumer group, namespace). -func withSuite(base, suite string) string { - return Ternary(suite == "", base, fmt.Sprintf("%s_%s", base, suite)).(string) +func WithSourceEdit(edit ConfigEditFn) TestConfigOption { + return func(c *TestConfig) { + c.sourceEdit = edit + } } -// SuiteDatabase names the source database a suite owns, for drivers whose CDC is database-scoped. -// Both sides must agree: variantSourceOverride rewrites source.json for olake, and the driver's -// own ExecuteQuery connection calls this with the same suite. -func SuiteDatabase(base, suite string) string { - return withSuite(base, suite) +func WithStreamEdit(edit ConfigEditFn) TestConfigOption { + return func(c *TestConfig) { + c.streamEdit = edit + } } -// destinationDBPrefix is passed as --destination-database-prefix. It carries the suite because -// Iceberg.Check probes _test_olake on every sync, and suites would race that CREATE TABLE. -func destinationDBPrefix(c *TestConfig) string { - prefix := Ternary(c.DataFormat == "", - fmt.Sprintf("integration_%s", c.Driver), - fmt.Sprintf("integration_%s_%s", c.Driver, c.DataFormat)).(string) - return withSuite(prefix, c.Suite) +func (c *TestConfig) generateSuiteName(t *testing.T) { + t.Helper() + nonSuiteChars := regexp.MustCompile(`[^a-z0-9]+`) + suite := strings.ToLower(t.Name()) + suite = strings.TrimPrefix(suite, "test") + suite = strings.TrimPrefix(suite, strings.ToLower(string(c.Driver))) + c.Suite = strings.Trim(nonSuiteChars.ReplaceAllString(suite, "_"), "_") } -// verifyDiscoveredStreams asserts the discovered catalog holds exactly the streams test_streams.json -func verifyDiscoveredStreams(t *testing.T, expectedPath, actualPath string) { +// setup initializes the derived fields of the TestConfig and does the steup for configuring the test isolation like isolated configs +func (c *TestConfig) setup(t *testing.T) error { t.Helper() - load := func(path, what string) map[string]interface{} { - data, err := os.ReadFile(path) - require.NoError(t, err, "failed to read %s streams JSON (%s)", what, path) - var doc map[string]interface{} - require.NoError(t, json.Unmarshal(data, &doc), "failed to parse %s streams JSON (%s)", what, path) - return doc + c.generateSuiteName(t) + if err := c.setupWorkingDir(t); err != nil { + return err } - expected := load(expectedPath, "expected") - actual := load(actualPath, "discovered") - - // streams[]: keyed by namespace.name, which is what makes a stream unique in a catalog. - indexStreams := func(doc map[string]interface{}) map[string]interface{} { - out := map[string]interface{}{} - entries, _ := doc["streams"].([]interface{}) - for _, raw := range entries { - wrapper, ok := raw.(map[string]interface{}) - if !ok { - continue - } - stream, ok := wrapper["stream"].(map[string]interface{}) - if !ok { - continue - } - out[fmt.Sprintf("%v.%v", stream["namespace"], stream["name"])] = wrapper - } - return out + if err := c.getOrBuildDriverImage(); err != nil { + return err } - // selected_streams: a map of namespace -> []{stream_name, ...}; key the same way. - indexSelected := func(doc map[string]interface{}) map[string]interface{} { - out := map[string]interface{}{} - byNamespace, _ := doc["selected_streams"].(map[string]interface{}) - for namespace, raw := range byNamespace { - entries, _ := raw.([]interface{}) - for _, entry := range entries { - selected, ok := entry.(map[string]interface{}) - if !ok { - continue - } - out[fmt.Sprintf("%v.%v", namespace, selected["stream_name"])] = selected - } - } - return out + c.addTimingLogsMiddleware() + + if err := c.applySuite(); err != nil { + return err + } + + sourceConfig, err := ReadSourceConfig(c.GetFilePath("source.json")) + if err != nil { + return fmt.Errorf("failed to read the source config of driver %q suite %q: %s", c.Driver, c.Suite, err) + } + c.SourceBaseConfig = sourceConfig + + return nil +} + +func (c *TestConfig) String() string { + config, _ := json.MarshalIndent(c, "", " ") + return string(config) +} + +// getOrBuildDriverImage just sets the driver image in case builds the driver image against current codebase +func (c *TestConfig) getOrBuildDriverImage() error { + if c.DriverImage != "" { + return nil + } + if image := os.Getenv(driverImageEnvVar); image != "" { + c.DriverImage = image + return nil } - compare := func(section string, want, got map[string]interface{}) { - require.Equal(t, slices.Sorted(maps.Keys(want)), slices.Sorted(maps.Keys(got)), - "%s: discover returned a different set of streams than test_streams.json", section) - for key, wantEntry := range want { - wantJSON, err := json.Marshal(wantEntry) - require.NoError(t, err) - gotJSON, err := json.Marshal(got[key]) - require.NoError(t, err) - require.Truef(t, NormalizedEqual(string(wantJSON), string(gotJSON)), - "%s: discovered %q does not match test_streams.json\nExpected:\n%s\nGot:\n%s", section, key, wantJSON, gotJSON) + driverVersion := os.Getenv(driverVersionEnvVar) + if driverVersion == "" { + if err := buildDriverImage(c); err != nil { + return fmt.Errorf("failed to build the %s driver image from the current codebase: %s", c.Driver, err) } + driverVersion = currentDriverVersion } - compare("streams", indexStreams(expected), indexStreams(actual)) - compare("selected_streams", indexSelected(expected), indexSelected(actual)) - t.Logf("Generated streams validated with test streams") + c.DriverImage = getDriverImage(c.Driver, driverVersion) + return nil } -// seedCatalogFromTestStreams writes test_streams.json out as the suite's catalog, retargeting it at -// the suite's table and namespace. Field-by-field: stream names carry the source's casing, tables -// the destination's, so one text substitution would rename only one of them. -func seedCatalogFromTestStreams(t *testing.T, c *TestConfig, testTable string) { - t.Helper() - if c.Suite == "" { - data, err := os.ReadFile(c.HostTestCatalogPath) - require.NoError(t, err, "failed to read test_streams.json") - require.NoError(t, writeHostFile(c.HostCatalogPath, data), "failed to write %s", c.HostCatalogPath) - return +func (c *TestConfig) addTimingLogsMiddleware() { + executeFunc := c.ExecuteQuery + c.ExecuteQuery = func(ctx context.Context, t *testing.T, cfg *TestConfig, operation string) { + defer TrackPhaseTiming(t, c.Driver, fmt.Sprintf("query %q", operation))() + executeFunc(ctx, t, cfg, operation) } +} - base := strings.TrimSuffix(testTable, "_"+c.Suite) - fromStream, toStream := normalizeStreamName(c.Driver, base), normalizeStreamName(c.Driver, testTable) +// UniqueID identifies this run among every suite that can be running beside it: the driver and the +// suite itself. Everything a suite must not share is named after it. +func (c *TestConfig) UniqueID() string { + return Combine(c.withSuite(c.Driver)) +} - require.NoError(t, copyJSONWithEdit(c.HostTestCatalogPath, c.HostCatalogPath, func(doc map[string]interface{}) error { - entries, _ := doc["streams"].([]interface{}) - for _, raw := range entries { - wrapper, ok := raw.(map[string]interface{}) - if !ok { - continue - } - stream, ok := wrapper["stream"].(map[string]interface{}) - if !ok { - continue - } - if stream["name"] == fromStream { - stream["name"] = toStream - } - if stream["destination_table"] == base { - stream["destination_table"] = testTable - } - // A baked destination_database is used verbatim (it overrides the prefix flag), so suffix - // it or concurrent suites race the CREATE on one shared namespace. - if ddb, ok := stream["destination_database"].(string); ok && ddb != "" { - stream["destination_database"] = withSuite(ddb, c.Suite) - } - } - byNamespace, _ := doc["selected_streams"].(map[string]interface{}) - for _, raw := range byNamespace { - selected, _ := raw.([]interface{}) - for _, entry := range selected { - stream, ok := entry.(map[string]interface{}) - if !ok { - continue - } - if stream["stream_name"] == fromStream { - stream["stream_name"] = toStream - } - } - } - return nil - }), "failed to seed the %q catalog", c.Suite) +func (c *TestConfig) withSuite(base string) string { + return Combine(base, c.Suite) } -// WithImagePlatform is used to override the platform for the test container image. -// This is useful for testing on different architectures (e.g., arm64 vs amd64). -func (t *TestConfig) WithImagePlatform(platform string) *TestConfig { - t.ImagePlatform = platform - return t +// TestTableName is the source table a suite drives. The suite suffix is what keeps concurrent +// suites off each other's table -- without it they race the same DROP/CREATE. +func (c *TestConfig) GetTableName() string { + return Combine("test_table_olake", c.Suite) } -// history stores the RPS values and the last updated time for a given mode. -type history struct { - RPS []float64 `json:"rps"` - UpdatedAt time.Time `json:"updated_at"` +// GetFilePath addresses a file in the suite's working directory by name -- the configs, the +// catalog, state and stats all live there, and the container reads them under the same names. +func (c *TestConfig) GetFilePath(fileName string) string { + return filepath.Join(c.TestWorkingDir, fileName) } -// benchmarkStore stores the benchmark RPS history for backfill and CDC modes. -type benchmarkStore struct { - Backfill history `json:"backfill"` - CDC history `json:"cdc"` - FilePath string `json:"-"` +// GetFixturePath addresses a committed fixture in the driver's testdata directory, for the one +// thing a run must outlive its working directory: the benchmark history the perf suite appends to. +// Everything else a suite reads is the working copy setupWorkingDir made, via GetFilePath. +func (c *TestConfig) GetFixturePath(fileName string, dataFormat ...string) string { + return filepath.Join(c.OlakeRootPath, "tests", c.Driver, "testdata", filepath.Join(dataFormat...), fileName) } -// initializes the benchmark store with the given path and loads the stored benchmarks data from the file. -func loadBenchmarks(path string) (*benchmarkStore, error) { - store := &benchmarkStore{ - Backfill: history{ - RPS: make([]float64, 0, maxRPSHistorySize), - UpdatedAt: time.Now().UTC(), - }, - CDC: history{ - RPS: make([]float64, 0, maxRPSHistorySize), - UpdatedAt: time.Now().UTC(), - }, - FilePath: path, - } - if err := store.load(); err != nil { - return nil, err +// setupWorkingDir gives the suite a private working directory holding its own copy of every config +// the driver container reads, so the repo fixtures stay read-only and concurrent suites never share +// a writable file. The shared fixtures land first and the driver's own overwrite them by name, so a +// driver overrides a common config just by committing a file of the same name. +func (c *TestConfig) setupWorkingDir(t *testing.T) (err error) { + c.TestWorkingDir = t.TempDir() + + c.OlakeRootPath, err = repoRoot() + if err != nil { + return fmt.Errorf("failed to determine the repo root; the tests run from a git checkout: %s", err) } - return store, nil -} -// load loads the stored benchmarks data from the file. -func (s *benchmarkStore) load() error { - if err := UnmarshalFile(s.FilePath, s, false); err != nil { - if _, statErr := os.Stat(s.FilePath); os.IsNotExist(statErr) { - // Missing file is acceptable, it will be created when the first RPS is recorded. - return nil + commonFixturesDir := filepath.Join(c.OlakeRootPath, "tests/testdata") + driverFixuresDir := filepath.Join(c.OlakeRootPath, "tests", c.Driver, "testdata", c.DataFormat) + for _, fixtures := range []string{commonFixturesDir, driverFixuresDir} { + if err := copyDirFiles(fixtures, c.TestWorkingDir); err != nil { + return fmt.Errorf("failed to copy the fixtures of %s into %s: %s", fixtures, c.TestWorkingDir, err) } - return fmt.Errorf("failed to load rps benchmarks from file %s: %s", s.FilePath, err) } - return nil } -// record records a new benchmark RPS value for the given driver and mode, and persists it to the file. -func (s *benchmarkStore) record( - isBackfill bool, - rps float64, -) error { - rpsValues := Ternary( - isBackfill, - s.Backfill.RPS, - s.CDC.RPS, - ).([]float64) - - rpsValues = append(rpsValues, rps) - - // Truncate history to maintain a rolling window of the last maxRPSHistorySize values. - if len(rpsValues) > maxRPSHistorySize { - rpsValues = rpsValues[1:] +// applySuite derives every config the driver container reads from its committed base, so the base +// files stay untouched, and retargets the copies at the names this suite owns. +func (c *TestConfig) applySuite() error { + c.DestinationDB = c.withSuite(c.DestinationDB) + + enableArrowWrites := func(destinationConf map[string]interface{}) error { + writer, ok := destinationConf["writer"].(map[string]interface{}) + if !ok { + return fmt.Errorf("no writer object in iceberg_destination.json") + } + writer["arrow_writes"] = true + + return nil + } + + err := CopyJSONWithEdit(c.GetFilePath("iceberg_destination.json"), c.GetFilePath("iceberg_destination_arrow.json"), enableArrowWrites) + if err != nil { + return fmt.Errorf("failed to derive the arrow destination config of driver %q suite %q: %s", c.Driver, c.Suite, err) } - if isBackfill { - s.Backfill.RPS = rpsValues - s.Backfill.UpdatedAt = time.Now().UTC() - } else { - s.CDC.RPS = rpsValues - s.CDC.UpdatedAt = time.Now().UTC() + isolateSource := func(source map[string]interface{}) error { + if c.sourceEdit == nil { + return nil + } + return c.sourceEdit(c, source) + } + if err := c.getOrRenderConfig("source.template.json", "source.json", isolateSource); err != nil { + return fmt.Errorf("failed to isolate the source config of driver %q for suite %q: %s", c.Driver, c.Suite, err) } - return FileLoggerWithPath(s, s.FilePath) + isolateCatalog := func(catalog map[string]interface{}) error { + if c.streamEdit == nil { + return nil + } + return c.streamEdit(c, catalog) + } + if err := c.getOrRenderConfig("streams.template.json", "streams.json", isolateCatalog); err != nil { + return fmt.Errorf("failed to retarget the catalog of driver %q at suite %q table %s: %s", c.Driver, c.Suite, c.GetTableName(), err) + } + return nil } -// stats returns the average RPS and count of past RPS values for the given driver and mode. -// The count cannot exceed maxRPSHistorySize. -func (s *benchmarkStore) stats( - isBackfill bool, -) (averageRPS float64, observations int) { - rpsValues := Ternary( - isBackfill, - s.Backfill.RPS, - s.CDC.RPS, - ).([]float64) - - if len(rpsValues) == 0 { - // No benchmarks recorded for this mode yet. - return 0, 0 +func (c *TestConfig) getOrRenderConfig(template, configPath string, edit editFunc) error { + _, err := os.Stat(c.GetFilePath(configPath)) + if errors.Is(err, os.ErrNotExist) { + return c.renderConfig(template, configPath, edit) + } else if err != nil { + return err } - return Average(rpsValues), len(rpsValues) + return nil } -// driverOrCommonConfig returns the driver's own fixture for file when present, else the shared -// one under tests/testdata; source.json never falls back. -func driverOrCommonConfig(fixturesPath, testsDir, file string) string { - p := filepath.Join(fixturesPath, file) - if file == "source.json" { - return p +// renderConfig expands the placeholders of the committed template in base into the working copy the +// container reads at out, and applies edit to the result. +func (c *TestConfig) renderConfig(base, out string, edit editFunc) error { + raw, err := os.ReadFile(c.GetFilePath(base)) + if err != nil { + return fmt.Errorf("failed to read %s: %s", base, err) } - if _, err := os.Stat(p); os.IsNotExist(err) { - return filepath.Join(testsDir, "..", "testdata", file) + expanded, err := c.expandPlaceholders(raw) + if err != nil { + return fmt.Errorf("failed to expand %s: %s", base, err) + } + doc, err := ParseJSONDoc(expanded) + if err != nil { + return fmt.Errorf("failed to parse %s: %s", base, err) + } + if err := edit(doc); err != nil { + return err + } + data, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal %s: %s", out, err) } - return p + return WriteHostFile(c.GetFilePath(out), data) } -// GetTestConfig returns the test config for a driver, called from a test in tests/. -// extraParams[0] optionally selects a testdata sub-directory (kafka's "json"/"avro" formats). -func GetTestConfig(t *testing.T, driver string, extraParams ...string) *TestConfig { - t.Helper() - // pwd is olake/tests/(driver) - pwd, err := os.Getwd() - require.NoError(t, err, "failed to determine the test directory") - // root path is olake's root path - rootPath := filepath.Join(pwd, "../..") - dataFormat := "" - if len(extraParams) > 0 { - dataFormat = extraParams[0] - } - // Every config owns a fresh /tmp working dir seeded from the committed testdata, so the repo - // files are read-only fixtures and concurrent suites never share a writable file - fixturesPath := filepath.Join(pwd, "testdata", dataFormat) - workDir, err := os.MkdirTemp("/tmp", fmt.Sprintf("olake-it-%s-", driver)) - require.NoError(t, err, "failed to create the test working directory") - // Removed once the test passes, kept on failure for inspection. Logged, not fatal - t.Cleanup(func() { - if !t.Failed() { - if err := os.RemoveAll(workDir); err != nil { - t.Logf("failed to remove the test working directory %s: %s", workDir, err) - } +// placeholder matches the ${name} form alone: the source configs carry credentials, and a secret +// holding a bare $ has to survive rendering untouched. +var placeholder = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}`) + +// expandPlaceholders substitutes the ${suite} a committed config spells its per-suite names with -- +// ${SUITE} for the drivers whose identifiers are uppercase -- so the file reads as what it renders. +func (c *TestConfig) expandPlaceholders(raw []byte) ([]byte, error) { + var unknown []string + expanded := placeholder.ReplaceAllFunc(raw, func(match []byte) []byte { + switch name := string(placeholder.FindSubmatch(match)[1]); name { + case "suite": + return []byte(c.Suite) + case "SUITE": + return []byte(strings.ToUpper(c.Suite)) + default: + unknown = append(unknown, name) + return match } }) - fixturePath := func(file string) string { return filepath.Join(fixturesPath, file) } - hostPath := func(file string) string { return filepath.Join(workDir, file) } - containerPath := func(file string) string { return path.Join(containerTestDataDir, file) } - for _, file := range []string{"source.json", "iceberg_destination.json", "parquet_destination.json"} { - require.NoError(t, copyFile(driverOrCommonConfig(fixturesPath, pwd, file), hostPath(file)), "failed to seed the test working directory") - } - // The arrow writer variant is derived, never committed: the base config stays the single - // source of truth, and writer variants become a pure file choice (see testIcebergWriter). - require.NoError(t, copyJSONWithEdit(hostPath("iceberg_destination.json"), hostPath("iceberg_destination_arrow.json"), - func(doc map[string]interface{}) error { - writer, ok := doc["writer"].(map[string]interface{}) - if !ok { - return fmt.Errorf("no writer object in iceberg_destination.json") - } - writer["arrow_writes"] = true - return nil - }), "failed to derive the arrow destination config") - return &TestConfig{ - Driver: driver, - DataFormat: dataFormat, - HostRootPath: rootPath, - HostTestDataPath: workDir, - HostTestCatalogPath: fixturePath("test_streams.json"), - HostCatalogPath: hostPath("streams.json"), - HostStatePath: hostPath("state.json"), - HostStateCheckpointPath: hostPath("state_checkpoint.json"), - HostPerformanceStatePath: fixturePath("performance_state.json"), - HostSourcePath: hostPath("source.json"), - HostStatsPath: hostPath("stats.json"), - BenchmarksPath: fixturePath("benchmarks.json"), - SourcePath: containerPath("source.json"), - CatalogPath: containerPath("streams.json"), - IcebergDestinationPath: containerPath("iceberg_destination.json"), - ParquetDestinationPath: containerPath("parquet_destination.json"), - StatePath: containerPath("state.json"), + if len(unknown) > 0 { + return nil, fmt.Errorf("undefined placeholder(s) %s: a config can only carry ${suite} and ${SUITE}", strings.Join(unknown, ", ")) } + return expanded, nil } // The helpers below edit the driver's config/catalog files on the host; the container sees the // changes through the /testdata mount. -// parseJSONDoc decodes a JSON object keeping numbers as json.Number, so values the edit does not -// touch round-trip as their original literals instead of through float64 (which corrupts int64s -// beyond 2^53 and renders large values in scientific notation). -func parseJSONDoc(raw []byte) (map[string]interface{}, error) { - dec := json.NewDecoder(bytes.NewReader(raw)) - dec.UseNumber() - var doc map[string]interface{} - return doc, dec.Decode(&doc) -} - -// editJSONFile reads path, applies edit to the decoded document, and writes it back. -func editJSONFile(path string, edit func(doc map[string]interface{}) error) error { +// EditJSONFile reads path, applies edit to the decoded document, and writes it back. +func EditJSONFile(path string, edit func(doc map[string]interface{}) error) error { raw, err := os.ReadFile(path) if err != nil { return fmt.Errorf("failed to read %s: %s", path, err) } - doc, err := parseJSONDoc(raw) + doc, err := ParseJSONDoc(raw) if err != nil { return fmt.Errorf("failed to parse %s: %s", path, err) } @@ -540,31 +370,31 @@ func editJSONFile(path string, edit func(doc map[string]interface{}) error) erro if err != nil { return fmt.Errorf("failed to marshal %s: %s", path, err) } - return writeHostFile(path, out) + return WriteHostFile(path, out) } -// writeHostFile writes to the shared /testdata mount, unlinking first: the container runs as root, +// WriteHostFile writes to the shared /testdata mount, unlinking first: the container runs as root, // so on Linux CI the test user cannot truncate a file a previous run left behind, only replace it. -func writeHostFile(path string, data []byte) error { +func WriteHostFile(path string, data []byte) error { _ = os.Remove(path) return os.WriteFile(path, data, 0600) } -// normalizeStreamName uppercases the stream name for drivers whose catalogs store +// NormalizeStreamName uppercases the stream name for drivers whose catalogs store // uppercase identifiers (e.g. Oracle). -func normalizeStreamName(driver, streamName string) string { +func NormalizeStreamName(driver, streamName string) string { return Ternary(slices.Contains(constants.UppercaseStreamDrivers, constants.DriverType(driver)), strings.ToUpper(streamName), streamName).(string) } -// updateSelectedStreams rewrites selected_streams so only the given streams stay selected, with +// UpdateSelectedStreams rewrites selected_streams so only the given streams stay selected, with // normalization enabled and the partition regex, filter config and excluded column applied. -func updateSelectedStreams(config *TestConfig, namespace, partitionRegex, filterConfig string, streams []string, columnToExclude string) error { +func UpdateSelectedStreams(config *TestConfig, namespace, partitionRegex, filterConfig string, streams []string, columnToExclude string, extraExcluded ...string) error { if len(streams) == 0 { return nil } selectedNames := make(map[string]bool, len(streams)) for _, s := range streams { - selectedNames[normalizeStreamName(config.Driver, s)] = true + selectedNames[NormalizeStreamName(config.Driver, s)] = true } var filter interface{} = map[string]interface{}{} @@ -574,7 +404,7 @@ func updateSelectedStreams(config *TestConfig, namespace, partitionRegex, filter } } - return editJSONFile(config.HostCatalogPath, func(doc map[string]interface{}) error { + return EditJSONFile(config.GetFilePath("streams.json"), func(doc map[string]interface{}) error { selected, _ := doc["selected_streams"].(map[string]interface{}) nsStreams, _ := selected[namespace].([]interface{}) kept := make([]interface{}, 0, len(nsStreams)) @@ -586,35 +416,32 @@ func updateSelectedStreams(config *TestConfig, namespace, partitionRegex, filter stream["normalization"] = true stream["partition_regex"] = partitionRegex stream["filter_config"] = filter - if columnToExclude != "" { - if selectedColumns, ok := stream["selected_columns"].(map[string]interface{}); ok { - if columns, ok := selectedColumns["columns"].([]interface{}); ok { - remaining := make([]interface{}, 0, len(columns)) - for _, col := range columns { - if fmt.Sprint(col) != columnToExclude { - remaining = append(remaining, col) - } - } - selectedColumns["columns"] = remaining + for _, excluded := range append([]string{columnToExclude}, extraExcluded...) { + if excluded == "" { + continue + } + selectedColumns, ok := stream["selected_columns"].(map[string]interface{}) + if !ok { + continue + } + columns, ok := selectedColumns["columns"].([]interface{}) + if !ok { + continue + } + remaining := make([]interface{}, 0, len(columns)) + for _, col := range columns { + if fmt.Sprint(col) != excluded { + remaining = append(remaining, col) } } + selectedColumns["columns"] = remaining } kept = append(kept, stream) } doc["selected_streams"] = map[string]interface{}{namespace: kept} - return nil - }) -} -// updateStreamConfig sets sync_mode and cursor_field on the stream identified by -// namespace+name in streams[]. -func updateStreamConfig(config *TestConfig, namespace, streamName, syncMode, cursorField string) error { - // in case of Oracle, the stream names are in uppercase in stream.json - streamName = normalizeStreamName(config.Driver, streamName) - return editJSONFile(config.HostCatalogPath, func(doc map[string]interface{}) error { - streams, _ := doc["streams"].([]interface{}) - for _, raw := range streams { - wrapper, ok := raw.(map[string]interface{}) + for _, entry := range doc["streams"].([]interface{}) { + wrapper, ok := entry.(map[string]interface{}) if !ok { continue } @@ -622,1836 +449,75 @@ func updateStreamConfig(config *TestConfig, namespace, streamName, syncMode, cur if !ok { continue } - if stream["namespace"] == namespace && stream["name"] == streamName { - stream["sync_mode"] = syncMode - stream["cursor_field"] = cursorField + destinationDB, ok := stream["destination_database"].(string) + if !ok || destinationDB == "" { + continue + } + if !strings.HasSuffix(destinationDB, config.Suite) { + destinationDB = config.withSuite(destinationDB) + stream["destination_database"] = destinationDB } + config.DestinationDB = strings.ReplaceAll(destinationDB, ":", "_") } return nil }) } -// resetStateFile clears state.json so incremental can perform its initial load +// ResetStateFile clears state.json so incremental can perform its initial load // (equivalent to a full load on first run), irrespective of any previous CDC run. -func resetStateFile(config *TestConfig) error { - return writeHostFile(config.HostStatePath, fmt.Appendf(nil, `{"version": %d}`, constants.LatestStateVersion)) -} - -func copyFile(src, dst string) error { - data, err := os.ReadFile(src) +// +// Every call site must keep this BEFORE a stateless (useState=false) sync, which is where they +// all sit today. The version written here is the product's current one (ProductStateVersion), and +// the stateless load that follows overwrites the file with whatever version the binary that ran +// it stamps (protocol/root.go writes state next to --config even with no --state flag). +// The compatibility suite depends on that overwrite: it is how a baseline image's own state version ends +// up pinning the candidate's syncs. Call this after a compatibility run's initial load instead and the +// pipeline is silently promoted to latest semantics -- the suite would pass while testing nothing. +func ResetStateFile(config *TestConfig) error { + version, err := ProductStateVersion(config.OlakeRootPath) if err != nil { - return fmt.Errorf("failed to read %s: %s", src, err) - } - return writeHostFile(dst, data) -} - -// saveStateFile copies state.json to the checkpoint state file. -func saveStateFile(config *TestConfig) error { - return copyFile(config.HostStatePath, config.HostStateCheckpointPath) -} - -// restoreStateFile replaces state.json with the previously saved checkpoint backup. -func restoreStateFile(config *TestConfig) error { - return copyFile(config.HostStateCheckpointPath, config.HostStatePath) -} - -// seedPerformanceState primes state.json from the committed pre-chunked seed. The seed is -// copied rather than passed to sync directly because sync writes the running state back over -// --state, which would rewrite the fixture on every benchmark run. -func seedPerformanceState(config *TestConfig) error { - return copyFile(config.HostPerformanceStatePath, config.HostStatePath) -} - -// to get backfill streams from cdc streams e.g. "demo_cdc" -> "demo" -func GetBackfillStreamsFromCDC(cdcStreams []string) []string { - backfillStreams := []string{} - for _, stream := range cdcStreams { - backfillStreams = append(backfillStreams, strings.TrimSuffix(stream, "_cdc")) - } - return backfillStreams -} - -// reset table and add back data to the table -func (cfg *IntegrationTest) resetTable(ctx context.Context, t *testing.T) error { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "create") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "add") - if cfg.TestConfig.Driver == string(constants.DB2) { - // to populate stats for DB2 - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "populate-stats") + return err } - return nil + return WriteHostFile(config.GetFilePath("state.json"), fmt.Appendf(nil, `{"version": %d}`, version)) } -// newMinIOClient returns a client for the MinIO instance backing the parquet destination in tests. -func newMinIOClient() (*minio.Client, error) { - client, err := minio.New("localhost:9000", &minio.Options{ - Creds: credentials.NewStaticV4("admin", "password", ""), - Secure: false, - }) +func CopyFile(src, dst string) error { + data, err := os.ReadFile(src) if err != nil { - return nil, fmt.Errorf("failed to create MinIO client: %s", err) - } - return client, nil -} - -// listParquetObjects lists the .parquet objects lying directly in a table's folder in MinIO. -func listParquetObjects(ctx context.Context, client *minio.Client, parquetDB, tableName string) ([]minio.ObjectInfo, error) { - objects := []minio.ObjectInfo{} - for object := range client.ListObjects(ctx, parquetTestBucket, minio.ListObjectsOptions{ - Prefix: parquetTablePath(parquetDB, tableName), - Recursive: false, - }) { - if object.Err != nil { - return nil, fmt.Errorf("error listing objects: %s", object.Err) - } - if strings.HasSuffix(object.Key, ".parquet") { - objects = append(objects, object) - } + return fmt.Errorf("failed to read %s: %s", src, err) } - return objects, nil -} - -// parquetTablePath is the MinIO key prefix a stream's parquet files are written under. -func parquetTablePath(parquetDB, tableName string) string { - return fmt.Sprintf("%s/%s/", parquetDB, tableName) + return WriteHostFile(dst, data) } -// DeleteParquetFiles deletes only .parquet files directly in the table folder in MinIO -func DeleteParquetFiles(t *testing.T, parquetDB, tableName string) error { - t.Helper() - parquetPath := parquetTablePath(parquetDB, tableName) - - t.Logf("Cleaning up .parquet files in: s3a://%s/%s", parquetTestBucket, parquetPath) - - minioClient, err := newMinIOClient() - if err != nil { - return err - } - - ctx := t.Context() - - objects, err := listParquetObjects(ctx, minioClient, parquetDB, tableName) - if err != nil { - return err - } - - for _, object := range objects { - t.Logf("Deleting: %s", strings.TrimPrefix(object.Key, parquetPath)) - - if err := minioClient.RemoveObject(ctx, parquetTestBucket, object.Key, minio.RemoveObjectOptions{}); err != nil { - return fmt.Errorf("failed to delete %s: %s", object.Key, err) - } - } - - t.Logf("--- Cleanup Complete: Deleted %d files ---", len(objects)) - return nil +// SaveStateFile copies state.json to the checkpoint state file. +func SaveStateFile(config *TestConfig) error { + return CopyFile(config.GetFilePath("state.json"), config.GetFilePath("state_checkpoint.json")) } -func deleteParquetTable(t *testing.T, parquetDB, tableName string) error { - t.Helper() - parquetPath := parquetTablePath(parquetDB, tableName) - - minioClient, err := newMinIOClient() - if err != nil { - return err - } - - ctx := context.Background() - deletedCount := 0 - for object := range minioClient.ListObjects(ctx, parquetTestBucket, minio.ListObjectsOptions{ - Prefix: parquetPath, - Recursive: true, - }) { - if object.Err != nil { - return fmt.Errorf("error listing objects: %s", object.Err) - } - if err := minioClient.RemoveObject(ctx, parquetTestBucket, object.Key, minio.RemoveObjectOptions{}); err != nil { - return fmt.Errorf("failed to delete %s: %s", object.Key, err) - } - deletedCount++ - } - - t.Logf("--- Parquet Table Cleanup Complete: Deleted %d objects ---", deletedCount) - return nil +// RestoreStateFile replaces state.json with the previously saved checkpoint backup. +func RestoreStateFile(config *TestConfig) error { + return CopyFile(config.GetFilePath("state_checkpoint.json"), config.GetFilePath("state.json")) } // syncTestCase represents a test case for sync operations -type syncTestCase struct { - name string - operation string - useState bool - opSymbol string - expected map[string]interface{} - preSetup []func(*TestConfig) error // host-side actions executed before the sync - verifyNoDuplicates bool // if true, assert COUNT(*) == COUNT(DISTINCT _olake_id) after sync - expectedRowCountByOpType int64 // when > 0, assert COUNT(DISTINCT _olake_id) == this value (catches over-sync and under-sync) -} - -// runSyncAndVerify executes a sync command and verifies the results in Iceberg -func (cfg *IntegrationTest) runSyncAndVerify( - ctx context.Context, - t *testing.T, - testTable string, - useState bool, - destinationType string, - operation string, - opSymbol string, - schema map[string]interface{}, - isCDC bool, -) error { - destDBPrefix := destinationDBPrefix(cfg.TestConfig) - cmd := syncArgs(*cfg.TestConfig, useState, destinationType, "--destination-database-prefix", destDBPrefix) - - // Execute operation before sync if needed - if useState && operation != "" { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, operation) - // SQL Server CDC is asynchronous: the capture job only picks up the DML above on its next - // transaction-log scan, and the sync's change window ends at the job's processed max LSN - // (sys.fn_cdc_get_max_lsn), so syncing too early would see no changes. Wait for the capture - // job to advance past the DML. Incremental runs read the table directly and need no wait. - if isCDC && cfg.TestConfig.Driver == "mssql" { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "wait-cdc-catchup") - } - } - - // Run sync against the driver image - code, out, err := runOlake(ctx, t, cfg.TestConfig, cmd...) - if err != nil || code != 0 { - return fmt.Errorf("sync failed (%d): %s\n%s", code, err, out) - } - - t.Logf("Sync successful for %s driver", cfg.TestConfig.Driver) - - // Use evolved schema only for CDC "update" operation (where schema evolution is expected) - // Incremental "insert" uses opSymbol "u" but doesn't have schema evolution - evolvedSchema := operation == "update" - - // Verification reads the destination back through Spark Connect (with retries), a real slice of - // sync wall-clock; time it as its own phase. - defer trackPhaseTiming(t, cfg.TestConfig.Driver, destinationType+" verify")() - - switch destinationType { - case "iceberg": - { - if evolvedSchema { - VerifyIcebergSync(t, testTable, cfg.DestinationDB, cfg.UpdatedDestinationDataTypeSchema, cfg.DefaultCDCColumnsSchema, schema, opSymbol, cfg.PartitionRegex, cfg.TestConfig.Driver, isCDC, cfg.ColumnToExclude) - } else { - VerifyIcebergSync(t, testTable, cfg.DestinationDB, cfg.DestinationDataTypeSchema, cfg.DefaultCDCColumnsSchema, schema, opSymbol, cfg.PartitionRegex, cfg.TestConfig.Driver, isCDC, cfg.ColumnToExclude) - } - } - case "parquet": - { - if evolvedSchema { - VerifyParquetSync(t, testTable, cfg.DestinationDB, cfg.UpdatedDestinationDataTypeSchema, cfg.DefaultCDCColumnsSchema, schema, opSymbol, cfg.TestConfig.Driver, isCDC, cfg.ColumnToExclude) - } else { - VerifyParquetSync(t, testTable, cfg.DestinationDB, cfg.DestinationDataTypeSchema, cfg.DefaultCDCColumnsSchema, schema, opSymbol, cfg.TestConfig.Driver, isCDC, cfg.ColumnToExclude) - } - } +// RenderOlakeFailure formats a failed sync's exit, translating the one code worth translating: 137 is +// SIGKILL, which in this harness almost always means the --memory cap or the docker VM OOM-killed +func RenderOlakeFailure(code int, err error, out []byte) error { + hint := "" + if code == 137 { + hint = " [exit 137 = SIGKILL: the container was OOM-killed -- see the --memory cap and the docker VM's total memory]" } - - return nil -} - -func (cfg *IntegrationTest) testIcebergWriter( - ctx context.Context, - t *testing.T, - testTable string, - useArrowWriter bool, - testFunc func(context.Context, *testing.T, string) error, -) error { - // Writer variants are separate config files, so no suite ever edits one in place. - file := "iceberg_destination.json" - if useArrowWriter { - file = "iceberg_destination_arrow.json" + if err != nil { + return fmt.Errorf("sync failed (%d)%s: %s\n%s", code, hint, err, out) } - cfg.TestConfig.IcebergDestinationPath = path.Join(containerTestDataDir, file) - - return testFunc(ctx, t, testTable) + return fmt.Errorf("sync failed (%d)%s\n%s", code, hint, out) } -// testIcebergFullLoadAndCDC tests Full load and CDC operations -func (cfg *IntegrationTest) testIcebergFullLoadAndCDC( - ctx context.Context, - t *testing.T, - testTable string, -) error { - t.Log("Starting Iceberg Full load + CDC tests") - - if err := cfg.resetTable(ctx, t); err != nil { - return fmt.Errorf("failed to reset table: %w", err) - } - - dbTestCases := []syncTestCase{ - { - name: "Full-Refresh", - operation: "", - useState: false, - opSymbol: "r", - expected: cfg.ExpectedData, - }, - { - name: "CDC - insert", - operation: "insert", - useState: true, - opSymbol: "c", - expected: cfg.ExpectedData, - }, - { - name: "CDC - update", - operation: "update", - useState: true, - opSymbol: "u", - expected: cfg.ExpectedUpdatedData, - }, - { - name: "CDC - delete", - operation: "delete", - useState: true, - opSymbol: "d", - expected: nil, - }, - } - - kafkaTestCases := []syncTestCase{ - { - name: "CDC - strict - insert", - operation: "", - useState: false, - opSymbol: "c", - expected: cfg.ExpectedData, - }, - { - name: "CDC - strict - update", - operation: "update", - useState: true, - opSymbol: "c", - expected: cfg.ExpectedUpdatedData, - }, - } - - testCases := Ternary(cfg.TestConfig.Driver == string(constants.Kafka), kafkaTestCases, dbTestCases).([]syncTestCase) - - // Run each test case - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // schema evolution - if tc.operation == "update" { - if cfg.TestConfig.Driver != "mongodb" && cfg.TestConfig.Driver != "mssql" && cfg.TestConfig.Driver != "kafka" { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "evolve-schema") - } - } - - if err := cfg.runSyncAndVerify( - ctx, - t, - testTable, - tc.useState, - "iceberg", - tc.operation, - tc.opSymbol, - tc.expected, - tc.name != "Full-Refresh", - ); err != nil { - t.Fatalf("%s test failed: %v", tc.name, err) - } - }) - } - - t.Log("Iceberg Full load + CDC tests completed successfully") - - // Drop the Iceberg table after all tests are finished - dropIcebergTable(t, testTable, cfg.DestinationDB) - t.Logf("Dropped Iceberg table: %s", testTable) - - return nil -} - -// testIcebergFullLoadAndCDC tests Full load and CDC operations -func (cfg *IntegrationTest) testParquetFullLoadAndCDC( - ctx context.Context, - t *testing.T, - testTable string, -) error { - t.Log("Starting Parquet Full load + CDC tests") - - if err := cfg.resetTable(ctx, t); err != nil { - return fmt.Errorf("failed to reset table: %s", err) - } - if err := deleteParquetTable(t, cfg.DestinationDB, testTable); err != nil { - return fmt.Errorf("failed to reset parquet table: %s", err) - } - - dbTestCases := []syncTestCase{ - { - name: "Full-Refresh", - operation: "", - useState: false, - opSymbol: "r", - expected: cfg.ExpectedData, - }, - { - name: "CDC - insert", - operation: "insert", - useState: true, - opSymbol: "c", - expected: cfg.ExpectedData, - }, - { - name: "CDC - update", - operation: "update", - useState: true, - opSymbol: "u", - expected: cfg.ExpectedUpdatedData, - }, - { - name: "CDC - delete", - operation: "delete", - useState: true, - opSymbol: "d", - expected: nil, - }, - } - - kafkaTestCases := []syncTestCase{ - { - name: "CDC - strict - insert", - operation: "", - useState: false, - opSymbol: "c", - expected: cfg.ExpectedData, - }, - { - name: "CDC - strict - update", - operation: "update", - useState: true, - opSymbol: "c", - expected: cfg.ExpectedUpdatedData, - }, - } - - testCases := Ternary(cfg.TestConfig.Driver == string(constants.Kafka), kafkaTestCases, dbTestCases).([]syncTestCase) - - // Run each test case - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // schema evolution - if tc.operation == "update" { - if cfg.TestConfig.Driver != "mongodb" && cfg.TestConfig.Driver != "mssql" && cfg.TestConfig.Driver != "kafka" { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "evolve-schema") - } - } - - // Delete parquet files before next operation to avoid error due to schema changes - if err := DeleteParquetFiles(t, cfg.DestinationDB, testTable); err != nil { - t.Fatalf("Failed to delete parquet files before %s: %v", tc.name, err) - } - - if err := cfg.runSyncAndVerify( - ctx, - t, - testTable, - tc.useState, - "parquet", - tc.operation, - tc.opSymbol, - tc.expected, - tc.name != "Full-Refresh", - ); err != nil { - t.Fatalf("%s test failed: %v", tc.name, err) - } - }) - } - - t.Log("Parquet Full load + CDC tests completed successfully") - return nil -} - -// TODO: add incremntal test for string time, timestamp with timezone, datetime, float, int as cursor field -// testIcebergFullLoadAndIncremental tests Full load and Incremental operations -func (cfg *IntegrationTest) testIcebergFullLoadAndIncremental( - ctx context.Context, - t *testing.T, - testTable string, -) error { - t.Log("Starting Iceberg Full load + Incremental tests") - - if err := cfg.resetTable(ctx, t); err != nil { - return fmt.Errorf("failed to reset table: %s", err) - } - - // Patch streams.json: set sync_mode = incremental, cursor_field = "id" - if err := updateStreamConfig(cfg.TestConfig, cfg.Namespace, testTable, "incremental", cfg.CursorField); err != nil { - return fmt.Errorf("failed to patch streams.json for incremental: %s", err) - } - - // Reset state so initial incremental behaves like a first full incremental load - if err := resetStateFile(cfg.TestConfig); err != nil { - return fmt.Errorf("failed to reset state for incremental: %s", err) - } - - // Test cases for incremental sync - incrementalTestCases := []syncTestCase{ - { - name: "Full-Refresh", - operation: "", - useState: false, - opSymbol: "r", - expected: cfg.ExpectedData, - }, - { - name: "Incremental - insert", - operation: "insert", - useState: true, - opSymbol: "u", - expected: cfg.ExpectedData, - }, - { - name: "Incremental - update", - operation: "update", - useState: true, - opSymbol: "u", - expected: cfg.ExpectedUpdatedData, - }, - } - - // Run each incremental test case - for _, tc := range incrementalTestCases { - t.Run(tc.name, func(t *testing.T) { - // schema evolution - if tc.operation == "update" { - if cfg.TestConfig.Driver != string(constants.MongoDB) && cfg.TestConfig.Driver != "mssql" { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "evolve-schema") - } - } - - // drop iceberg table before sync - dropIcebergTable(t, testTable, cfg.DestinationDB) - t.Logf("Dropped Iceberg table: %s", testTable) - - if err := cfg.runSyncAndVerify( - ctx, - t, - testTable, - tc.useState, - "iceberg", - tc.operation, - tc.opSymbol, - tc.expected, - false, - ); err != nil { - t.Fatalf("Incremental test %s failed: %v", tc.name, err) - } - }) - } - - t.Log("Iceberg Full load + Incremental tests completed successfully") - - // Drop the Iceberg table after all tests are finished, so the incremental - // cursor state left in the table's olake_2pc property is not read back as - // CDC state by a later run. - dropIcebergTable(t, testTable, cfg.DestinationDB) - t.Logf("Dropped Iceberg table: %s", testTable) - - return nil -} - -// testParquetFullLoadAndIncremental tests Full load and Incremental operations for Parquet -func (cfg *IntegrationTest) testParquetFullLoadAndIncremental( - ctx context.Context, - t *testing.T, - testTable string, -) error { - t.Log("Starting Parquet Full load + Incremental tests") - - if err := cfg.resetTable(ctx, t); err != nil { - return fmt.Errorf("failed to reset table: %s", err) - } - if err := deleteParquetTable(t, cfg.DestinationDB, testTable); err != nil { - return fmt.Errorf("failed to reset parquet table: %s", err) - } - - // Patch streams.json: set sync_mode = incremental, cursor_field = "id" - if err := updateStreamConfig(cfg.TestConfig, cfg.Namespace, testTable, "incremental", cfg.CursorField); err != nil { - return fmt.Errorf("failed to patch streams.json for incremental: %s", err) - } - - // Reset state so initial incremental behaves like a first full incremental load - if err := resetStateFile(cfg.TestConfig); err != nil { - return fmt.Errorf("failed to reset state for incremental: %s", err) - } - - // Test cases for incremental sync - incrementalTestCases := []syncTestCase{ - { - name: "Full-Refresh", - operation: "", - useState: false, - opSymbol: "r", - expected: cfg.ExpectedData, - }, - { - name: "Incremental - insert", - operation: "insert", - useState: true, - opSymbol: "u", - expected: cfg.ExpectedData, - }, - { - name: "Incremental - update", - operation: "update", - useState: true, - opSymbol: "u", - expected: cfg.ExpectedUpdatedData, - }, - } - - // Run each incremental test case - for _, tc := range incrementalTestCases { - t.Run(tc.name, func(t *testing.T) { - // schema evolution - if tc.operation == "update" { - if cfg.TestConfig.Driver != string(constants.MongoDB) && cfg.TestConfig.Driver != "mssql" { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "evolve-schema") - } - } - - // Delete parquet files before next operation to avoid error due to schema changes - if err := DeleteParquetFiles(t, cfg.DestinationDB, testTable); err != nil { - t.Fatalf("Failed to delete parquet files before %s: %v", tc.name, err) - } - - if err := cfg.runSyncAndVerify( - ctx, - t, - testTable, - tc.useState, - "parquet", - tc.operation, - tc.opSymbol, - tc.expected, - false, - ); err != nil { - t.Fatalf("Incremental test %s failed: %v", tc.name, err) - } - }) - } - - t.Log("Parquet Full load + Incremental tests completed successfully") - return nil -} - -// testIceberg2PCCDCRecovery tests 2PC (Two-Phase Commit) failure recovery for CDC mode using -// the Iceberg destination. It simulates a state-save failure mid-sync: saves a pre-insert -// checkpoint, performs a CDC insert, then restores to the checkpoint and inserts a second -// record (insert_2pc) to verify the driver correctly recovers without duplicating rows. -func (cfg *IntegrationTest) testIceberg2PCCDCRecovery( - ctx context.Context, - t *testing.T, - testTable string, -) error { - t.Log("Starting Iceberg 2PC CDC Recovery tests") - - if err := cfg.resetTable(ctx, t); err != nil { - return fmt.Errorf("failed to reset table: %w", err) - } - - // Drop the Iceberg table and reset state before the first sync, so stale rows and the - // olake_2pc table property left by a previous run can't leak into this run's recovery timeline. - dropIcebergTable(t, testTable, cfg.DestinationDB) - if err := resetStateFile(cfg.TestConfig); err != nil { - return fmt.Errorf("failed to reset state: %w", err) - } - - twoPCCDCTestCases := []syncTestCase{ - { - name: Ternary(cfg.TestConfig.Driver == string(constants.Kafka), "CDC - initial load", "Full-Refresh").(string), - operation: "", - useState: false, - opSymbol: Ternary(cfg.TestConfig.Driver == string(constants.Kafka), "c", "r").(string), - expected: cfg.ExpectedData, - verifyNoDuplicates: true, - expectedRowCountByOpType: 5, - }, - { - name: "CDC - insert", - operation: Ternary(cfg.TestConfig.Driver == string(constants.Kafka), "add", "insert").(string), - useState: true, - opSymbol: "c", - expected: cfg.ExpectedData, - preSetup: Ternary(cfg.TestConfig.Driver == string(constants.Kafka), []func(*TestConfig) error{}, []func(*TestConfig) error{saveStateFile}).([]func(*TestConfig) error), - verifyNoDuplicates: cfg.TestConfig.Driver == string(constants.Kafka), - expectedRowCountByOpType: 10, - }, - { - // Simulate 2PC failure: restore state to pre-insert checkpoint, insert a - // second record, run sync. The driver recovers: it advances state to the - // committed metadata LSN by making a bounded sync. - // expectedRowCountByOpType=1 because no new data lands in Iceberg here, - // as it just recovers the sync from state -> metadata LSN. - name: "CDC - Recovery Sync", - operation: "insert_2pc", - useState: true, - opSymbol: "c", - expected: cfg.ExpectedData, - verifyNoDuplicates: true, - expectedRowCountByOpType: int64(Ternary(cfg.TestConfig.Driver == string(constants.Kafka), 11, 1).(int)), - preSetup: Ternary(cfg.TestConfig.Driver == string(constants.Kafka), []func(*TestConfig) error{}, []func(*TestConfig) error{restoreStateFile}).([]func(*TestConfig) error), - }, - { - // After the recovery sync advanced state to the committed metadata LSN, - // a normal sync should see both the original insert and insert_2pc rows. - name: "CDC - Post Recovery Sync", - useState: true, - opSymbol: "c", - expected: cfg.ExpectedData, - verifyNoDuplicates: true, - expectedRowCountByOpType: int64(Ternary(cfg.TestConfig.Driver == string(constants.Kafka), 12, 2).(int)), - }, - } - - for _, tc := range twoPCCDCTestCases { - t.Run(tc.name, func(t *testing.T) { - for _, preSetup := range tc.preSetup { - if err := preSetup(cfg.TestConfig); err != nil { - t.Fatalf("%s pre-sync setup failed: %v", tc.name, err) - } - } - - if err := cfg.runSyncAndVerify( - ctx, t, testTable, tc.useState, "iceberg", - tc.operation, tc.opSymbol, tc.expected, - tc.name != "Full-Refresh", - ); err != nil { - t.Fatalf("%s test failed: %v", tc.name, err) - } - - if tc.verifyNoDuplicates { - VerifyIcebergNoDuplicates(ctx, t, testTable, cfg.DestinationDB, tc.opSymbol, tc.expectedRowCountByOpType) - } - }) - } - - t.Log("Iceberg 2PC CDC Recovery tests completed successfully") - dropIcebergTable(t, testTable, cfg.DestinationDB) - t.Logf("Dropped Iceberg table after 2PC CDC tests: %s", testTable) - return nil -} - -// testIceberg2PCIncrementalRecovery tests 2PC (Two-Phase Commit) failure recovery for -// incremental mode using the Iceberg destination. It simulates a state-save failure after -// the cursor advances: saves a pre-insert checkpoint, performs an incremental insert, then -// restores to the checkpoint and inserts a second record (insert_2pc) to verify that the -// cursor re-reads the overlapping range, deduplicates the original insert via MERGE INTO, -// and correctly surfaces only the net-new insert_2pc row. -func (cfg *IntegrationTest) testIceberg2PCIncrementalRecovery( - ctx context.Context, - t *testing.T, - testTable string, -) error { - t.Log("Starting Iceberg 2PC Incremental Recovery tests") - - if err := cfg.resetTable(ctx, t); err != nil { - return fmt.Errorf("failed to reset table: %w", err) - } - - // Drop the Iceberg table before the first sync, so stale rows and the olake_2pc table - // property left by a previous run can't leak into this run's recovery timeline. - dropIcebergTable(t, testTable, cfg.DestinationDB) - - // Patch streams.json: set sync_mode = incremental, cursor_field - if err := updateStreamConfig(cfg.TestConfig, cfg.Namespace, testTable, "incremental", cfg.CursorField); err != nil { - return fmt.Errorf("failed to patch streams.json for incremental: %s", err) - } - - // Reset state so initial incremental behaves like a first full incremental load - if err := resetStateFile(cfg.TestConfig); err != nil { - return fmt.Errorf("failed to reset state for incremental: %s", err) - } - - twoPCIncrementalTestCases := []syncTestCase{ - { - name: "Full-Refresh", - operation: "", - useState: false, - opSymbol: "r", - expected: cfg.ExpectedData, - verifyNoDuplicates: true, - expectedRowCountByOpType: 5, - }, - { - name: "Incremental - insert", - operation: "insert", - useState: true, - opSymbol: "u", - expected: cfg.ExpectedData, - preSetup: []func(*TestConfig) error{ - saveStateFile, - }, - }, - { - // Simulate 2PC failure: restore cursor to pre-insert checkpoint, insert a - // second record, run sync. The cursor re-reads the range and deduplicates - // the original insert via MERGE INTO; insert_2pc is net-new. - // expectedRowCountByOpType=1: only insert_2pc is visible (original deduplicated). - name: "Incremental - State Save Failure Sync", - operation: "insert_2pc", - useState: true, - opSymbol: "u", - expected: cfg.ExpectedData, - verifyNoDuplicates: true, - expectedRowCountByOpType: 1, - preSetup: []func(*TestConfig) error{ - restoreStateFile, - }, - }, - { - // After recovery, state is now consistent. A normal sync should see both - // the original insert row and insert_2pc row — 2 distinct records total. - name: "Incremental - Post Recovery Sync", - useState: true, - opSymbol: "u", - expected: cfg.ExpectedData, - verifyNoDuplicates: true, - expectedRowCountByOpType: 2, // insert row + insert_2pc row, both unique by _olake_id - }, - } - - for _, tc := range twoPCIncrementalTestCases { - t.Run(tc.name, func(t *testing.T) { - for _, preSetup := range tc.preSetup { - if err := preSetup(cfg.TestConfig); err != nil { - t.Fatalf("%s pre-sync setup failed: %v", tc.name, err) - } - } - - if err := cfg.runSyncAndVerify( - ctx, t, testTable, tc.useState, "iceberg", - tc.operation, tc.opSymbol, tc.expected, - false, - ); err != nil { - t.Fatalf("Incremental 2PC test %s failed: %v", tc.name, err) - } - - if tc.verifyNoDuplicates { - VerifyIcebergNoDuplicates(ctx, t, testTable, cfg.DestinationDB, tc.opSymbol, tc.expectedRowCountByOpType) - } - }) - } - - t.Log("Iceberg 2PC Incremental Recovery tests completed successfully") - dropIcebergTable(t, testTable, cfg.DestinationDB) - t.Logf("Dropped Iceberg table after 2PC Incremental tests: %s", testTable) - return nil -} - -// keepTestData reports whether OLAKE_TEST_KEEP_DATA=true, the dev switch that skips the -// final source-data drop so the seeded tables/files survive the run for inspection. -// Every pre-test drop/clean still runs, so the next run starts from a clean slate -// regardless of the switch. -func keepTestData() bool { - return os.Getenv("OLAKE_TEST_KEEP_DATA") == "true" -} - -// Test2PCIntegration runs the full Two-Phase Commit (2PC) failure-recovery integration test -// suite against the driver image. It exercises CDC and incremental state-recovery scenarios -// independently of the happy-path integration tests, allowing them to be scheduled and -// reported separately. -func (cfg *IntegrationTest) Test2PCIntegration(t *testing.T) { - cfg.IsolateSuite(t, "2pc") - ctx := t.Context() - cfg.ExecuteQuery = timedExecuteQuery(cfg.TestConfig.Driver, cfg.ExecuteQuery) - - t.Logf("Root Project directory: %s", cfg.TestConfig.HostRootPath) - t.Logf("Test data directory: %s", cfg.TestConfig.HostTestDataPath) - currentTestTable := TestTableName(cfg.TestConfig) - - // Postgres only, and for the whole suite: olake validates the CDC config on EVERY sync, so a - // slot scoped to just the CDC tests fails the incremental ones whose config still names it. - if cfg.TestConfig.Driver == string(constants.Postgres) { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "create-slot") - defer cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop-slot") - } - - // 2PC tests don't need schema discovery — the schema is already validated by the regular integration test. - seedCatalogFromTestStreams(t, cfg.TestConfig, currentTestTable) - - t.Run("Sync", func(t *testing.T) { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "create") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "clean") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "add") - - if err := updateSelectedStreams(cfg.TestConfig, cfg.Namespace, cfg.PartitionRegex, cfg.FilterConfig, []string{currentTestTable}, cfg.ColumnToExclude); err != nil { - t.Fatalf("failed to enable normalization and partition regex in streams.json: %s", err) - } - t.Logf("Enabled normalization and added partition regex in %s", cfg.TestConfig.HostCatalogPath) - - writerTypes := []struct { - name string - useArrow bool - }{ - {"Legacy", false}, - {"Arrow", true}, - } - - if !slices.Contains(constants.SkipCDCDrivers, constants.DriverType(cfg.TestConfig.Driver)) { - for _, wt := range writerTypes { - t.Run(fmt.Sprintf("Iceberg (%s) 2PC CDC Recovery tests", wt.name), func(t *testing.T) { - if err := cfg.testIcebergWriter(ctx, t, currentTestTable, wt.useArrow, cfg.testIceberg2PCCDCRecovery); err != nil { - t.Fatalf("Iceberg (%s) 2PC CDC Recovery tests failed: %v", wt.name, err) - } - }) - } - } - - if cfg.TestConfig.Driver != string(constants.Kafka) { - for _, wt := range writerTypes { - t.Run(fmt.Sprintf("Iceberg (%s) 2PC Incremental Recovery tests", wt.name), func(t *testing.T) { - if err := cfg.testIcebergWriter(ctx, t, currentTestTable, wt.useArrow, cfg.testIceberg2PCIncrementalRecovery); err != nil { - t.Fatalf("Iceberg (%s) 2PC Incremental Recovery tests failed: %v", wt.name, err) - } - }) - } - } - - if keepTestData() { - t.Logf("keeping %s source data (OLAKE_TEST_KEEP_DATA=true)", cfg.TestConfig.Driver) - } else { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") - t.Logf("%s 2PC sync test cleanup", cfg.TestConfig.Driver) - } - }) -} - -// WaitForSyncProgress blocks until the running sync has reported its first records in stats.json. -// A driver uses it to time an event at a point where the sync is demonstrably mid-flight, rather -// than guessing with a sleep. -func WaitForSyncProgress(ctx context.Context, t *testing.T, statsPath string) { - t.Helper() - - require.Eventually(t, func() bool { - if ctx.Err() != nil { - return true - } - - var stats struct { - SyncedRecords int64 `json:"Synced Records"` - } - if err := UnmarshalFile(statsPath, &stats, false); err != nil { - return false - } - if stats.SyncedRecords > 0 { - t.Logf("sync started: %d records synced", stats.SyncedRecords) - return true - } - return false - }, SyncTimeout, time.Second) -} - -// runRebalanceSync runs a sync command for the rebalance test. -func (cfg *IntegrationTest) runRebalanceSync( - ctx context.Context, - t *testing.T, - useState bool, -) error { - t.Helper() - - destDBPrefix := destinationDBPrefix(cfg.TestConfig) - cmd := syncArgs(*cfg.TestConfig, useState, "iceberg", "--destination-database-prefix", destDBPrefix) - - code, out, err := runOlake(ctx, t, cfg.TestConfig, cmd...) - if err != nil { - return fmt.Errorf("sync exec error: %w\n%s", err, out) - } - if code != 0 { - return fmt.Errorf("sync failed (%d): %s", code, out) - } - t.Logf("sync completed successfully") - return nil -} - -// testKafkaRebalance exercises consumer-group rebalance recovery while syncing a large bulk of messages. -func (cfg *IntegrationTest) testKafkaRebalance( - ctx context.Context, - t *testing.T, - testTable string, -) error { - t.Log("Starting Kafka rebalance recovery test") - - dropIcebergTable(t, testTable, cfg.DestinationDB) - if err := resetStateFile(cfg.TestConfig); err != nil { - return fmt.Errorf("failed to reset state file: %s", err) - } - - rebalanceTestCases := []syncTestCase{ - { - name: "CDC - first rebalance sync", - operation: "insert_rebalance", - useState: true, - }, - { - // Stop the trigger consumer before resuming so it cannot hold partition assignments. - name: "CDC - second rebalance sync", - operation: "stop_rebalance", - useState: true, - }, - } - - for _, tc := range rebalanceTestCases { - t.Run(tc.name, func(t *testing.T) { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, tc.operation) - - if err := cfg.runRebalanceSync(ctx, t, tc.useState); err != nil { - t.Fatalf("%s failed: %v", tc.name, err) - } - }) - } - - VerifyIcebergNoDuplicates(ctx, t, testTable, cfg.DestinationDB, "c", kafkaRebalanceBulkMessageCount) - - t.Log("Kafka rebalance recovery test completed successfully") - - dropIcebergTable(t, testTable, cfg.DestinationDB) - t.Logf("Dropped Iceberg table: %s", testTable) - - return nil -} - -// TestRebalance runs the Kafka consumer-group rebalance recovery integration test in an isolated container. -func (cfg *IntegrationTest) TestRebalance(t *testing.T) { - cfg.IsolateSuite(t, "rebalance") - ctx := t.Context() - - t.Logf("Root Project directory: %s", cfg.TestConfig.HostRootPath) - t.Logf("Test data directory: %s", cfg.TestConfig.HostTestDataPath) - currentTestTable := TestTableName(cfg.TestConfig) - - seedCatalogFromTestStreams(t, cfg.TestConfig, currentTestTable) - - t.Run("Sync", func(t *testing.T) { - // 1. Query on test table - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "create") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "clean") - - // 2. Enable normalization and partition regex in streams.json - if err := updateSelectedStreams(cfg.TestConfig, cfg.Namespace, cfg.PartitionRegex, cfg.FilterConfig, []string{currentTestTable}, cfg.ColumnToExclude); err != nil { - t.Fatalf("failed to enable normalization and partition regex in streams.json: %s", err) - } - t.Logf("Enabled normalization and added partition regex in %s", cfg.TestConfig.HostCatalogPath) - - // 3. Run Kafka rebalance recovery test (legacy Iceberg writer) - if err := cfg.testIcebergWriter(ctx, t, currentTestTable, false, cfg.testKafkaRebalance); err != nil { - t.Fatalf("Kafka rebalance test failed: %v", err) - } - - // 4. Clean up - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") - t.Logf("%s rebalance test cleanup", cfg.TestConfig.Driver) - }) +// KeepTestData reports whether the test suite should keep the source data after a run, for debugging. +func KeepTestData() bool { + return strings.EqualFold(os.Getenv(KeepTestDataEnvVar), "true") } // TestDiscover seeds the source with this driver's test table, runs discover against the driver -// image and asserts the catalog it writes matches test_streams.json exactly. +// image and asserts the catalog it writes matches the one rendered from streams.template.json exactly. // -// Its caller must not be parallel. The compare is an equality one, so it only holds while this -// table is the only thing in the source -- and every other suite seeds one of its own. Leaving the -// test serial is what orders it ahead of them: Go resumes parallel tests only once the serial ones -// in the package are done. -func (cfg *IntegrationTest) TestDiscover(t *testing.T) { - ctx := t.Context() - cfg.ExecuteQuery = timedExecuteQuery(cfg.TestConfig.Driver, cfg.ExecuteQuery) - - t.Logf("Root Project directory: %s", cfg.TestConfig.HostRootPath) - t.Logf("Test data directory: %s", cfg.TestConfig.HostTestDataPath) - - // 1. Empty the source, then seed just this table. drop-all is what makes the compare below an - // equality one: discover enumerates everything, so anything an aborted run (or a perf seed) - // left behind would show up as an extra stream. Safe only here -- the discover suite runs - // alone, while every parallel suite owns a table drop-all would take with it. - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop-all") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "create") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "add") - // Deferred, so a failed discover still hands the parallel suites behind it a clean source. - defer func() { - if keepTestData() { - t.Logf("keeping %s source data (OLAKE_TEST_KEEP_DATA=true)", cfg.TestConfig.Driver) - return - } - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") - }() - - // 2. Run discover against the driver image - code, out, err := runOlake(ctx, t, cfg.TestConfig, discoverArgs(*cfg.TestConfig)...) - if err != nil || code != 0 { - t.Fatalf("discover failed (%d): %s\n%s", code, err, string(out)) - } - - // 3. Verify streams.json describes exactly the streams we expect - verifyDiscoveredStreams(t, cfg.TestConfig.HostTestCatalogPath, cfg.TestConfig.HostCatalogPath) -} - -// TestSync runs the happy-path sync suite: full load, CDC and incremental, over both Iceberg writers -// and Parquet. It seeds its catalog from test_streams.json instead of discovering one, the way the -// 2PC and rebalance suites do -- TestDiscover already proves the two are identical. -func (cfg *IntegrationTest) TestSync(t *testing.T) { - ctx := t.Context() - cfg.ExecuteQuery = timedExecuteQuery(cfg.TestConfig.Driver, cfg.ExecuteQuery) - - t.Logf("Root Project directory: %s", cfg.TestConfig.HostRootPath) - t.Logf("Test data directory: %s", cfg.TestConfig.HostTestDataPath) - currentTestTable := TestTableName(cfg.TestConfig) - - seedCatalogFromTestStreams(t, cfg.TestConfig, currentTestTable) - - // 1. Query on test table; drop first so an aborted run's leftovers cannot survive - // the CREATE IF NOT EXISTS - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "create") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "clean") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "add") - - // 2. Enable normalization, partition regex, filter and column exclusion in streams.json - if err := updateSelectedStreams(cfg.TestConfig, cfg.Namespace, cfg.PartitionRegex, cfg.FilterConfig, []string{currentTestTable}, cfg.ColumnToExclude); err != nil { - t.Fatalf("failed to enable normalization and partition regex in streams.json: %s", err) - } - t.Logf("Enabled normalization and added partition regex in %s", cfg.TestConfig.HostCatalogPath) - - writerTypes := []struct { - name string - useArrow bool - }{ - {"Legacy", false}, - {"Arrow", true}, - } - - // Skip cdc tests for drivers not supporting cdc mode - if !slices.Contains(constants.SkipCDCDrivers, constants.DriverType(cfg.TestConfig.Driver)) { - for _, wt := range writerTypes { - t.Run(fmt.Sprintf("Iceberg (%s) Full load + CDC tests", wt.name), func(t *testing.T) { - if err := cfg.testIcebergWriter(ctx, t, currentTestTable, wt.useArrow, cfg.testIcebergFullLoadAndCDC); err != nil { - t.Fatalf("Iceberg (%s) Full load + CDC tests failed: %v", wt.name, err) - } - }) - } - - t.Run("Parquet Full load + CDC tests", func(t *testing.T) { - if err := cfg.testParquetFullLoadAndCDC(ctx, t, currentTestTable); err != nil { - t.Fatalf("Parquet Full load + CDC tests failed: %v", err) - } - }) - } - - // Skip incremental tests for drivers not supporting incremental mode - if cfg.TestConfig.Driver != string(constants.Kafka) { - for _, wt := range writerTypes { - t.Run(fmt.Sprintf("Iceberg (%s) Full load + Incremental tests", wt.name), func(t *testing.T) { - if err := cfg.testIcebergWriter(ctx, t, currentTestTable, wt.useArrow, cfg.testIcebergFullLoadAndIncremental); err != nil { - t.Fatalf("Iceberg (%s) Full load + Incremental tests failed: %v", wt.name, err) - } - }) - } - - t.Run("Parquet Full load + Incremental tests", func(t *testing.T) { - if err := cfg.testParquetFullLoadAndIncremental(ctx, t, currentTestTable); err != nil { - t.Fatalf("Parquet Full load + Incremental tests failed: %v", err) - } - }) - } - - // Asserts the writer splits bulk output into size-bounded files without losing rows. Runs - // last: it replaces the table contents and clears streams.json's regex/filter config. - if hasParquetRollingTest(cfg.TestConfig.Driver) { - t.Run("Parquet Rolling", func(t *testing.T) { - if err := cfg.testParquetRolling(ctx, t, currentTestTable); err != nil { - t.Fatalf("Parquet Rolling test failed: %v", err) - } - }) - } - - // 3. Clean up - if keepTestData() { - t.Logf("keeping %s source data (OLAKE_TEST_KEEP_DATA=true)", cfg.TestConfig.Driver) - return - } - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") - t.Logf("%s sync test cleanup", cfg.TestConfig.Driver) -} - -var ( - sharedSparkOnce sync.Once - sharedSpark sql.SparkSession - sharedSparkErr error -) - -// sparkSession returns the shared Spark Connect session, building it on first use and warming it -// so the one-off server bootstrap is timed here instead of inflating whichever verify runs first. -func sparkSession(ctx context.Context, t *testing.T) (sql.SparkSession, error) { - sharedSparkOnce.Do(func() { - // The shared session outlives whichever test builds it, so its construction must not be - // tied to that test's context (t.Context cancels when the test ends). - ctx := context.WithoutCancel(ctx) - defer trackPhaseTiming(t, "spark", "session build")() - for attempt := 1; ; attempt++ { - sharedSpark, sharedSparkErr = sql.NewSessionBuilder().Remote(sparkConnectAddress).Build(ctx) - if sharedSparkErr == nil || attempt == 3 { - break - } - t.Logf("Attempt %d/3: Failed to connect to Spark, retrying in 2s: %v", attempt, sharedSparkErr) - time.Sleep(2 * time.Second) - } - if sharedSparkErr != nil { - return - } - if _, err := sharedSpark.Sql(ctx, "SELECT 1"); err != nil { - t.Logf("Spark session warm-up query failed (non-fatal): %v", err) - } - }) - return sharedSpark, sharedSparkErr -} - -// dropIcebergTable drops an Iceberg table using Spark SQL -func dropIcebergTable(t *testing.T, tableName, icebergDB string) { - t.Helper() - ctx := t.Context() - spark, err := sparkSession(ctx, t) - if err != nil { - t.Logf("Failed to connect to Spark Connect server for dropping table: %v", err) - return - } - - fullTableName := fmt.Sprintf("%s.%s.%s", icebergCatalog, icebergDB, tableName) - dropQuery := fmt.Sprintf("DROP TABLE IF EXISTS %s", fullTableName) - t.Logf("Dropping Iceberg table: %s", dropQuery) - - _, err = spark.Sql(ctx, dropQuery) - if err != nil { - t.Logf("Failed to drop Iceberg table %s: %v", fullTableName, err) - return - } - t.Logf("Successfully dropped Iceberg table: %s", fullTableName) -} - -// TODO: Refactor parsing logic into a reusable utility functions -// verifyIcebergSync verifies that data was correctly synchronized to Iceberg -func VerifyIcebergSync(t *testing.T, tableName, icebergDB string, datatypeSchema map[string]string, defaultCDCColumnsSchema map[string]string, schema map[string]interface{}, opSymbol, partitionRegex, driver string, isCDC bool, excludedColumn string) { - t.Helper() - ctx := t.Context() - spark, err := sparkSession(ctx, t) - require.NoError(t, err, "Failed to connect to Spark Connect server") - - fullTableName := fmt.Sprintf("%s.%s.%s", icebergCatalog, icebergDB, tableName) - // The shared session caches table snapshots, so refresh to see the rows the sync just committed. - // Non-fatal: on a first sync the table may not exist yet, which the retry loop below handles. - if _, refreshErr := spark.Sql(ctx, fmt.Sprintf("REFRESH TABLE %s", fullTableName)); refreshErr != nil { - t.Logf("REFRESH TABLE before verify (non-fatal): %v", refreshErr) - } - selectQuery := fmt.Sprintf( - "SELECT * FROM %s WHERE _op_type = '%s'", - fullTableName, opSymbol, - ) - // In kafka, _op_type is always 'c' and col_included appears only in new rows. - // To check new record, col_included is used. - if driver == string(constants.Kafka) { - if _, ok := schema["col_included"]; ok { - selectQuery += " AND col_included IS NOT NULL" - } - } - t.Logf("Executing query: %s", selectQuery) - - var selectRows []types.Row - var queryErr error - maxRetries := 20 - retryDelay := 5 * time.Second - - for attempt := 0; attempt < maxRetries; attempt++ { - if attempt > 0 { - time.Sleep(retryDelay) - } - var selectQueryDf sql.DataFrame - // This is to check if the table exists in destination, as race condition might cause table to not be created yet - selectQueryDf, queryErr = spark.Sql(ctx, selectQuery) - if queryErr != nil { - t.Logf("Query attempt %d failed: %v", attempt+1, queryErr) - continue - } - - // To ensure stale data is not being used for verification - selectRows, queryErr = selectQueryDf.Collect(ctx) - if queryErr != nil { - t.Logf("Query attempt %d failed (Collect error): %v", attempt+1, queryErr) - continue - } - if len(selectRows) > 0 { - queryErr = nil - break - } - - // For delete operations, 0 rows is acceptable - exit immediately without retrying - if opSymbol == "d" { - queryErr = nil - t.Logf("Delete verification passed: found 0 rows for _op_type = 'd' (acceptable)") - break - } - - // for every type of operation, op symbol will be different, using that to ensure data is not stale - queryErr = fmt.Errorf("stale data: query succeeded but returned 0 rows for _op_type = '%s'", opSymbol) - t.Logf("Query attempt %d/%d failed: %v", attempt+1, maxRetries, queryErr) - - // Force Spark to refresh the table metadata from the Iceberg catalog. - refreshQuery := fmt.Sprintf("REFRESH TABLE %s", fullTableName) - if _, refreshErr := spark.Sql(ctx, refreshQuery); refreshErr != nil { - t.Logf("REFRESH TABLE attempt %d failed (non-fatal): %v", attempt+1, refreshErr) - } - } - - // For delete operations, accept both 0 and 1 row (both are valid outcomes) - if opSymbol == "d" { - if len(selectRows) > 0 { - deletedID := selectRows[0].Value("_olake_id") - require.NotEmpty(t, deletedID, "Delete verification failed: _olake_id should not be empty") - } - t.Logf("Delete verification passed: found %d row(s) for _op_type = 'd'", len(selectRows)) - return - } - require.NoError(t, queryErr, "Failed to collect data rows from Iceberg after %d attempts: %v", maxRetries, queryErr) - require.NotEmpty(t, selectRows, "No rows returned for _op_type = '%s'", opSymbol) - - for rowIdx, row := range selectRows { - icebergMap := make(map[string]interface{}, len(schema)+1) - for _, col := range row.FieldNames() { - icebergMap[col] = row.Value(col) - } - for key, expected := range schema { - icebergValue, ok := icebergMap[key] - require.Truef(t, ok, "Row %d: missing column %q in Iceberg result", rowIdx, key) - require.Equal(t, expected, icebergValue, "Row %d: mismatch on %q: Iceberg has %#v, expected %#v", rowIdx, key, icebergValue, expected) - } - if isCDC { - for key := range defaultCDCColumnsSchema { - icebergValue, ok := icebergMap[key] - require.Truef(t, ok, "Row %d: missing column %q in Iceberg result", rowIdx, key) - // Kafka offset, partition can be 0, NotEmpty fails for 0 so we check for NotNil instead. - if key == "_kafka_offset" || key == "_kafka_partition" { - require.NotNil(t, icebergValue, "Row %d: expected column %q to be non-empty, got %#v", rowIdx, key, icebergValue) - } else { - require.NotEmpty(t, icebergValue, "Row %d: expected column %q to be non-empty, got %#v", rowIdx, key, icebergValue) - } - if key == constants.CdcTimestamp { - ts, ok := normalizeToTime(icebergValue) - require.Truef(t, ok, "Row %d: expected %q to be a timestamp, got %T (%#v)", rowIdx, key, icebergValue, icebergValue) - minAllowed := time.Now().Add(-1 * time.Hour) - require.Falsef(t, ts.Before(time.Now().Add(-1*time.Hour)), "Row %d: %q is too old: %v, should not be earlier than %v", rowIdx, key, ts, minAllowed) - } - } - } - if !isCDC && icebergMap[constants.CdcTimestamp] != nil { - ts, ok := normalizeToTime(icebergMap[constants.CdcTimestamp]) - require.Truef(t, ok, "expected %q to be a timestamp, got %T", constants.CdcTimestamp, icebergMap[constants.CdcTimestamp]) - // Normalize to UTC to keep tests stable across environments (Local vs UTC). - require.Equal(t, time.Unix(0, 0).UTC(), ts.UTC()) - } - } - t.Logf("Verified Iceberg synced data with respect to data synced from source[%s] found equal", driver) - - describeQuery := fmt.Sprintf("DESCRIBE TABLE %s", fullTableName) - describeDf, err := spark.Sql(ctx, describeQuery) - require.NoError(t, err, "Failed to describe Iceberg table") - - describeRows, err := describeDf.Collect(ctx) - require.NoError(t, err, "Failed to collect describe data from Iceberg") - icebergSchema := make(map[string]string) - for _, row := range describeRows { - colName := row.Value("col_name").(string) - dataType := row.Value("data_type").(string) - if !strings.HasPrefix(colName, "#") { - icebergSchema[colName] = dataType - } - } - - if excludedColumn != "" { - _, ok := icebergSchema[Reformat(excludedColumn)] - require.Falsef(t, ok, "Excluded column %q should not exist in Iceberg schema", excludedColumn) - } - - for col, dbType := range datatypeSchema { - iceType, found := icebergSchema[col] - require.True(t, found, "Column %s not found in Iceberg schema", col) - - expectedIceType, mapped := GlobalTypeMapping[dbType] - if !mapped { - t.Errorf("No mapping defined for driver type %s (column %s)", dbType, col) - } - require.Equal(t, expectedIceType, iceType, - "Data type mismatch for column %s: expected %s, got %s", col, expectedIceType, iceType) - } - t.Logf("Verified datatypes in Iceberg after sync") - // Verify datatypes for CDC/default columns as well - if isCDC { - for col, expectedIceType := range defaultCDCColumnsSchema { - iceType, found := icebergSchema[col] - require.True(t, found, "CDC column %s not found in Iceberg schema", col) - - require.Equal(t, expectedIceType, iceType, - "CDC data type mismatch for column %s: expected %s, got %s", col, expectedIceType, iceType) - } - t.Logf("Verified datatypes for CDC columns in Iceberg after sync") - } - - // Partition verification using only metadata tables - if partitionRegex == "" { - t.Log("No partitionRegex provided, skipping partition verification") - return - } - // Extract partition columns from describe rows - partitionCols := extractFirstPartitionColFromRows(describeRows) - require.NotEmpty(t, partitionCols, "Partition columns not found in Iceberg metadata") - - // Parse expected partition columns from pattern like "/{col,identity}" - // Supports multiple entries like "/{col1,identity}" by taking the first token as the source column - clean := strings.TrimPrefix(partitionRegex, "/{") - clean = strings.TrimSuffix(clean, "}") - toks := strings.Split(clean, ",") - expectedCol := strings.TrimSpace(toks[0]) - require.Equal(t, expectedCol, partitionCols, "Partition column does not match expected '%s'", expectedCol) - t.Logf("Verified partition column: %s", expectedCol) -} - -// VerifyIcebergNoDuplicates asserts that no duplicate _olake_id values exist for the given -// _op_type in the Iceberg table. -func VerifyIcebergNoDuplicates(ctx context.Context, t *testing.T, tableName, icebergDB, opSymbol string, expectedRowCountByOpType int64) { - t.Helper() - - spark, err := sparkSession(ctx, t) - require.NoError(t, err, "Failed to connect to Spark Connect server for duplicate check") - - fullTableName := fmt.Sprintf("%s.%s.%s", icebergCatalog, icebergDB, tableName) - - // Refresh to get the latest committed Iceberg snapshot. - refreshQuery := fmt.Sprintf("REFRESH TABLE %s", fullTableName) - if _, refreshErr := spark.Sql(ctx, refreshQuery); refreshErr != nil { - t.Logf("REFRESH TABLE (non-fatal): %v", refreshErr) - } - - countQuery := fmt.Sprintf( - "SELECT COUNT(*) AS total, COUNT(DISTINCT _olake_id) AS distinct_count FROM %s WHERE _op_type = '%s'", - fullTableName, opSymbol, - ) - t.Logf("Executing duplicate-check query: %s", countQuery) - - df, err := spark.Sql(ctx, countQuery) - require.NoError(t, err, "Failed to run duplicate-check COUNT query") - - rows, err := df.Collect(ctx) - require.NoError(t, err, "Failed to collect duplicate-check COUNT results") - require.Len(t, rows, 1, "COUNT query must return exactly one row") - - total, ok := rows[0].Value("total").(int64) - require.True(t, ok, "COUNT(*) value is not int64: %T", rows[0].Value("total")) - - distinct, ok2 := rows[0].Value("distinct_count").(int64) - require.True(t, ok2, "COUNT(DISTINCT) value is not int64: %T", rows[0].Value("distinct_count")) - - // 1. No duplicates: every row must have a unique _olake_id. - require.Equal(t, total, distinct, - "Duplicate rows detected for _op_type='%s': total=%d, distinct=%d. "+ - "Iceberg MERGE INTO did not deduplicate re-synced records.", - opSymbol, total, distinct) - - // 2. Exact count: when caller specifies an expected row count, enforce it so that both - // over-sync (old rows re-processed and inserted again) and under-sync (new rows missed) - // are caught. - if expectedRowCountByOpType > 0 { - require.Equal(t, expectedRowCountByOpType, distinct, - "Row count mismatch for _op_type='%s': expected %d distinct rows, got %d. "+ - "Either old rows were re-synced (over-sync) or new rows were missed (under-sync).", - opSymbol, expectedRowCountByOpType, distinct) - } - - t.Logf("Duplicate check passed for _op_type='%s': %d rows, all unique by _olake_id (expected %d)", - opSymbol, distinct, expectedRowCountByOpType) -} - -// VerifyParquetSync verifies that data was correctly synchronized to Parquet files in MinIO -func VerifyParquetSync(t *testing.T, tableName, parquetDB string, datatypeSchema map[string]string, defaultCDCColumnsSchema map[string]string, schema map[string]interface{}, opSymbol, driver string, isCDC bool, excludedColumn string) { - t.Helper() - ctx := t.Context() - - spark, err := sparkSession(ctx, t) - require.NoError(t, err, "Failed to connect to Spark Connect server") - - parquetPath := fmt.Sprintf("s3a://warehouse/%s/%s", parquetDB, tableName) - viewName := fmt.Sprintf("`%s_view_%d`", tableName, time.Now().UnixNano()) - - // create a temporary view for parquet files, allows to run describe query - createViewQuery := fmt.Sprintf( - "CREATE OR REPLACE TEMP VIEW %s AS SELECT * FROM parquet.`%s/*.parquet`", - viewName, parquetPath, - ) - - // Retry logic for transient Spark connection issues (e.g., catalog connection pool exhaustion) - const maxRetries = 3 - for attempt := 1; attempt <= maxRetries; attempt++ { - _, err = spark.Sql(ctx, createViewQuery) - if err == nil { - break - } - // For delete operations, if path doesn't exist that's acceptable (no data written) - if opSymbol == "d" && strings.Contains(err.Error(), "PATH_NOT_FOUND") { - t.Logf("Delete verification passed: Parquet path does not exist (no data written)") - return - } - if attempt < maxRetries { - t.Logf("Attempt %d/%d: Failed to create view, retrying in 2s: %v", attempt, maxRetries, err) - time.Sleep(2 * time.Second) - } - } - require.NoError(t, err, "Failed to create temporary view for Parquet files") - - defer func() { - dropViewQuery := fmt.Sprintf("DROP VIEW IF EXISTS %s", viewName) - t.Logf("Dropping temporary view: %s", dropViewQuery) - _, _ = spark.Sql(ctx, dropViewQuery) - }() - - selectQuery := fmt.Sprintf( - "SELECT * FROM %s WHERE `_op_type` = '%s'", - viewName, opSymbol, - ) - // In kafka, _op_type is always 'c' and col_included appears only in new rows. - // To check new record, col_included is used. - if driver == string(constants.Kafka) { - if _, ok := schema["col_included"]; ok { - selectQuery += " AND `col_included` IS NOT NULL" - } - } - t.Logf("Executing Parquet query: %s", selectQuery) - - df, err := spark.Sql(ctx, selectQuery) - require.NoError(t, err, "Failed to run select query on Parquet files") - - rows, err := df.Collect(ctx) - require.NoError(t, err, "Failed to collect rows from Parquet query") - - // For delete operations, accept both 0 and 1 row (both are valid outcomes) - if opSymbol == "d" { - if len(rows) > 0 { - deletedID := rows[0].Value("_olake_id") - require.NotEmpty(t, deletedID, "Delete verification failed: _olake_id should not be empty") - } - t.Logf("Delete verification passed: found %d row(s) for _op_type = 'd'", len(rows)) - return - } - - // For non-delete operations, require at least one row - require.NotEmpty(t, rows, "No rows returned for _op_type = '%s'", opSymbol) - - for rowIdx, row := range rows { - parquetMap := make(map[string]interface{}, len(schema)+1) - for _, col := range row.FieldNames() { - parquetMap[col] = row.Value(col) - } - for key, expected := range schema { - val, ok := parquetMap[key] - require.Truef(t, ok, "Row %d: missing column %q in Parquet result", rowIdx, key) - require.Equal(t, expected, val, - "Row %d: mismatch on %q: Parquet has %#v, expected %#v", rowIdx, key, val, expected) - } - if isCDC { - for key := range defaultCDCColumnsSchema { - val, ok := parquetMap[key] - require.Truef(t, ok, "Row %d: missing column %q in Parquet result", rowIdx, key) - // Kafka offset, partition can be 0, NotEmpty fails for 0 so we check for NotNil instead. - if key == "_kafka_offset" || key == "_kafka_partition" { - require.NotNil(t, val, "Row %d: expected column %q to be non-empty, got %#v", rowIdx, key, val) - } else { - require.NotEmpty(t, val, "Row %d: expected column %q to be non-empty, got %#v", rowIdx, key, val) - } - if key == constants.CdcTimestamp { - ts, ok := normalizeToTime(val) - require.Truef(t, ok, "Row %d: expected %q to be a timestamp, got %T (%#v)", rowIdx, key, val, val) - minAllowed := time.Now().Add(-1 * time.Hour) - require.Falsef(t, ts.Before(time.Now().Add(-1*time.Hour)), "Row %d: %q is too old: %v, should not be earlier than %v", rowIdx, key, ts, minAllowed) - } - } - } - if !isCDC && parquetMap[constants.CdcTimestamp] != nil { - ts, ok := normalizeToTime(parquetMap[constants.CdcTimestamp]) - require.Truef(t, ok, "expected %q to be a timestamp, got %T", constants.CdcTimestamp, parquetMap[constants.CdcTimestamp]) - // Normalize to UTC to keep tests stable across environments (Local vs UTC). - require.Equal(t, time.Unix(0, 0).UTC(), ts.UTC()) - } - } - - t.Logf("Verified Parquet synced data with respect to data synced from source[%s] found equal", driver) - - describeQuery := fmt.Sprintf("DESCRIBE TABLE %s", viewName) - descDF, err := spark.Sql(ctx, describeQuery) - require.NoError(t, err, "Failed to describe Parquet view") - - descRows, err := descDF.Collect(ctx) - require.NoError(t, err, "Failed to collect schema info from Parquet view") - - parquetSchema := make(map[string]string) - for _, row := range descRows { - colName := row.Value("col_name").(string) - dataType := row.Value("data_type").(string) - if !strings.HasPrefix(colName, "#") { - parquetSchema[colName] = dataType - } - } - if excludedColumn != "" { - _, ok := parquetSchema[Reformat(excludedColumn)] - require.Falsef(t, ok, "Excluded column %q should not exist in Parquet schema", excludedColumn) - } - - for col, dbType := range datatypeSchema { - pqType, found := parquetSchema[col] - require.True(t, found, "Column %s not found in Parquet schema", col) - - expectedType, mapped := GlobalTypeMapping[dbType] - if !mapped { - t.Errorf("No mapping defined for driver type %s (column %s)", dbType, col) - } - require.Equal(t, expectedType, pqType, - "Data type mismatch for column %s: expected %s, got %s", col, expectedType, pqType) - } - t.Logf("Verified datatypes in Parquet after sync") - // Verify datatypes for CDC/default columns as well - if isCDC { - for col, expectedPqType := range defaultCDCColumnsSchema { - pqType, found := parquetSchema[col] - require.True(t, found, "CDC column %s not found in Parquet schema", col) - require.Equal(t, expectedPqType, pqType, - "CDC data type mismatch for column %s: expected %s, got %s", col, expectedPqType, pqType) - } - } - t.Logf("Verified datatypes for CDC columns in Parquet after sync") -} - -func (cfg *PerformanceTest) TestPerformance(t *testing.T) { - ctx := t.Context() - // The perf suite runs against the external DBs source.json describes, so hand every - // ExecuteQuery their credentials; the integration suites leave this nil (local containers). - cfg.TestConfig.SourceBaseConfig = ReadSourceConfig(t, cfg.TestConfig.HostSourcePath) - - // checks if the current rps (from stats.json) is at least 90% of the benchmark rps - checkBenchmarkRPS := func(config TestConfig, isBackfill bool) (bool, float64, error) { - // get current RPS - var stats SyncSpeed - if err := UnmarshalFile(config.HostStatsPath, &stats, false); err != nil { - return false, 0, err - } - rps, err := ParseFloat64(strings.Split(stats.Speed, " ")[0]) - if err != nil { - return false, 0, fmt.Errorf("failed to get RPS from stats: %s", err) - } - - // Get past benchmark RPS stats - benchmarks, err := loadBenchmarks(config.BenchmarksPath) - if err != nil { - return false, 0, err - } - - averageRPS, observations := benchmarks.stats(isBackfill) - t.Logf("currentRPS: %.2f, averageRPS: %.2f, observations: %d", rps, averageRPS, observations) - - // No benchmarks exist yet for this driver/mode - // Skip validation to allow initial benchmarking. - if observations == 0 { - t.Logf("No benchmarks exist yet for %s %s mode, skipping validation", config.Driver, Ternary(isBackfill, "backfill", "cdc").(string)) - return true, rps, nil - } - if rps < BenchmarkThreshold*averageRPS { - return false, rps, nil - } - return true, rps, nil - } - - recordBenchmark := func(config TestConfig, isBackfill bool, rps float64) error { - benchmarks, err := loadBenchmarks(config.BenchmarksPath) - if err != nil { - return err - } - return benchmarks.record(isBackfill, rps) - } - - // runPerfOlake runs the driver image with host networking so the perf run reaches the - // external benchmark databases directly, exactly as a deployed sync would. - runPerfOlake := func(olakeArgs ...string) (int, []byte, error) { - getOrBuildDriverImage(t, cfg.TestConfig) - args := dockerRunArgs(cfg.TestConfig, []string{"--network", "host"}, olakeArgs) - out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput() - return dockerExitResult(out, err, olakeArgs[0]) - } - - // syncWithTimeout runs a sync bounded by SyncTimeout. Hitting the window is expected (it is - // a bounded throughput measurement, not a failure), so the still-running container is stopped. - syncWithTimeout := func(olakeArgs ...string) ([]byte, error) { - name := fmt.Sprintf("olake-perf-%s", cfg.TestConfig.Driver) - _ = exec.Command("docker", "rm", "-f", name).Run() // drop any stale container from a previous run - timedCtx, cancel := context.WithTimeout(ctx, SyncTimeout) - defer cancel() - args := dockerRunArgs(cfg.TestConfig, []string{"--network", "host", "--name", name}, olakeArgs) - out, err := exec.CommandContext(timedCtx, "docker", args...).CombinedOutput() - if timedCtx.Err() == context.DeadlineExceeded { - _ = exec.Command("docker", "kill", name).Run() - return out, nil - } - code, out, derr := dockerExitResult(out, err, "sync") - if derr != nil { - return out, derr - } - if code != 0 { - return out, fmt.Errorf("sync failed (%d)", code) - } - return out, nil - } - - t.Run("performance", func(t *testing.T) { - // reset CDC config - if cfg.TestConfig.Driver == string(constants.Postgres) || cfg.TestConfig.Driver == string(constants.MySQL) { - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "reset_cdc_config") - t.Log("CDC config reset completed") - } - - t.Logf("(backfill) running performance test for %s", cfg.TestConfig.Driver) - - destDBPrefix := fmt.Sprintf("performance_%s", cfg.TestConfig.Driver) - - t.Log("(backfill) discover started") - if code, output, err := runPerfOlake(discoverArgs(*cfg.TestConfig, "--destination-database-prefix", destDBPrefix)...); err != nil || code != 0 { - t.Fatalf("failed to perform discover:\n%s", string(output)) - } - t.Log("(backfill) discover completed") - - if err := updateSelectedStreams(cfg.TestConfig, cfg.Namespace, "", "", cfg.BackfillStreams, ""); err != nil { - t.Fatalf("failed to update streams: %s", err) - } - - t.Log("(backfill) sync started") - // MySQL derives its chunk plan from InnoDB statistics, which drift between runs; seed the - // committed plan instead so every benchmark measures the same split. - usePreChunkedState := cfg.TestConfig.Driver == string(constants.MySQL) - if usePreChunkedState { - if err := seedPerformanceState(cfg.TestConfig); err != nil { - t.Fatalf("failed to seed pre-chunked state from %s: %s", cfg.TestConfig.HostPerformanceStatePath, err) - } - } - if output, err := syncWithTimeout(syncArgs(*cfg.TestConfig, usePreChunkedState, "iceberg", "--destination-database-prefix", destDBPrefix)...); err != nil { - t.Fatalf("failed to perform sync:\n%s", string(output)) - } - t.Log("(backfill) sync completed") - - checkRPS, currentRPS, err := checkBenchmarkRPS(*cfg.TestConfig, true) - if err != nil { - t.Fatalf("failed to check RPS: %s", err) - } - require.True(t, checkRPS, fmt.Sprintf("%s backfill performance below benchmark", cfg.TestConfig.Driver)) - - if err := recordBenchmark(*cfg.TestConfig, true, currentRPS); err != nil { - t.Fatalf("failed to write RPS history: %s", err) - } - t.Logf("✅ SUCCESS: %s backfill", cfg.TestConfig.Driver) - - if len(cfg.CDCStreams) > 0 { - t.Logf("(cdc) running performance test for %s", cfg.TestConfig.Driver) - - t.Log("(cdc) setup cdc started") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "setup_cdc") - t.Log("(cdc) setup cdc completed") - - t.Log("(cdc) discover started") - if code, output, err := runPerfOlake(discoverArgs(*cfg.TestConfig, "--destination-database-prefix", destDBPrefix)...); err != nil || code != 0 { - t.Fatalf("failed to perform discover:\n%s", string(output)) - } - t.Log("(cdc) discover completed") - - if err := updateSelectedStreams(cfg.TestConfig, cfg.Namespace, "", "", cfg.CDCStreams, ""); err != nil { - t.Fatalf("failed to update streams: %s", err) - } - - t.Log("(cdc) state creation started") - if code, output, err := runPerfOlake(syncArgs(*cfg.TestConfig, false, "iceberg", "--destination-database-prefix", destDBPrefix)...); err != nil || code != 0 { - t.Fatalf("failed to perform initial sync:\n%s", string(output)) - } - t.Log("(cdc) state creation completed") - - t.Log("(cdc) trigger cdc started") - cfg.ExecuteQuery(ctx, t, cfg.TestConfig, "bulk_cdc_data_insert") - t.Log("(cdc) trigger cdc completed") - - t.Log("(cdc) sync started") - if output, err := syncWithTimeout(syncArgs(*cfg.TestConfig, true, "iceberg", "--destination-database-prefix", destDBPrefix)...); err != nil { - t.Fatalf("failed to perform CDC sync:\n%s", string(output)) - } - t.Log("(cdc) sync completed") - - checkRPS, currentRPS, err := checkBenchmarkRPS(*cfg.TestConfig, false) - if err != nil { - t.Fatalf("failed to check RPS: %s", err) - } - require.True(t, checkRPS, fmt.Sprintf("%s CDC performance below benchmark", cfg.TestConfig.Driver)) - - if err := recordBenchmark(*cfg.TestConfig, false, currentRPS); err != nil { - t.Fatalf("failed to write RPS history: %s", err) - } - t.Logf("✅ SUCCESS: %s cdc", cfg.TestConfig.Driver) - } - }) -} - -// extractFirstPartitionColFromRows extracts the first partition column from DESCRIBE EXTENDED rows -func extractFirstPartitionColFromRows(rows []types.Row) string { - inPartitionSection := false - - for _, row := range rows { - // Convert []any -> []string - vals := row.Values() - parts := make([]string, len(vals)) - for i, v := range vals { - if v == nil { - parts[i] = "" - } else { - parts[i] = fmt.Sprint(v) // safe string conversion - } - } - line := strings.TrimSpace(strings.Join(parts, " ")) - if line == "" { - continue - } - - if strings.HasPrefix(line, "# Partition Information") { - inPartitionSection = true - continue - } - - if inPartitionSection { - if strings.HasPrefix(line, "# col_name") { - continue - } - - if strings.HasPrefix(line, "#") { - break - } - - fields := strings.Fields(line) - if len(fields) > 0 { - return fields[0] // return the first partition col - } - } - } - - return "" -} - -func normalizeToTime(v interface{}) (time.Time, bool) { - switch ts := v.(type) { - case time.Time: - return ts, true - case arrow.Timestamp: - return time.Unix(0, int64(ts)*int64(time.Microsecond)).UTC(), true - default: - return time.Time{}, false - } -} diff --git a/tests/testutils/timing.go b/tests/testutils/timing.go index 105a0d4fb..cb57159f1 100644 --- a/tests/testutils/timing.go +++ b/tests/testutils/timing.go @@ -1,8 +1,6 @@ package testutils import ( - "context" - "fmt" "testing" "time" ) @@ -29,21 +27,7 @@ func logPhaseTiming(t *testing.T, scope, phase string, d time.Duration) { // trackPhaseTiming starts a wall-clock timer and returns a stop func that logs the elapsed span. // Call stop() when the phase ends, or `defer trackPhaseTiming(t, scope, phase)()` to time a scope // (the deferred form also captures the duration when the body t.Fatal/Goexits). -func trackPhaseTiming(t *testing.T, scope, phase string) (stop func()) { +func TrackPhaseTiming(t *testing.T, scope, phase string) (stop func()) { start := time.Now() return func() { logPhaseTiming(t, scope, phase, time.Since(start)) } } - -// timedExecuteQuery wraps IntegrationTest.ExecuteQuery so every source-DB operation -// (create/clean/add/drop/evolve-schema/...) is timed by name. These run against the source and, -// for CDC engines like MSSQL (capture-instance enablement + readiness polling), are a real slice -// of wall-clock that would otherwise fall through the cracks between the other phase timings. -func timedExecuteQuery( - driver string, - executeQuery func(context.Context, *testing.T, *TestConfig, string), -) func(context.Context, *testing.T, *TestConfig, string) { - return func(ctx context.Context, t *testing.T, conf *TestConfig, operation string) { - defer trackPhaseTiming(t, driver, fmt.Sprintf("query %q", operation))() - executeQuery(ctx, t, conf, operation) - } -} diff --git a/tests/testutils/utils.go b/tests/testutils/utils.go index 3d33b7e99..537f0cd03 100644 --- a/tests/testutils/utils.go +++ b/tests/testutils/utils.go @@ -4,10 +4,14 @@ package testutils import ( + "bytes" "context" "encoding/json" "fmt" "os" + "os/exec" + "path/filepath" + "slices" "sort" "strconv" "strings" @@ -170,3 +174,89 @@ func RetryOnBackoff(ctx context.Context, attempts int, sleep time.Duration, f fu } return err } + +func Combine(components ...string) string { + parts := make([]string, 0, len(components)) + for _, str := range components { + if str != "" { + parts = append(parts, str) + } + } + return strings.Join(parts, "_") +} + +type editFunc func(map[string]interface{}) error + +// CopyJSONWithEdit reads the JSON at src, applies edit, and writes the result to dst -- +// used to derive a per-suite config from a shared base file without touching the base. +func CopyJSONWithEdit(src, dst string, edit editFunc) error { + raw, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("failed to read %s: %s", src, err) + } + doc, err := ParseJSONDoc(raw) + if err != nil { + return fmt.Errorf("failed to parse %s: %s", src, err) + } + if err := edit(doc); err != nil { + return err + } + out, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal %s: %s", dst, err) + } + return WriteHostFile(dst, out) +} + +// ParseJSONDoc decodes a JSON object keeping numbers as json.Number, so values the edit does not +// touch round-trip as their original literals instead of through float64 (which corrupts int64s +// beyond 2^53 and renders large values in scientific notation). +func ParseJSONDoc(raw []byte) (map[string]interface{}, error) { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var doc map[string]interface{} + return doc, dec.Decode(&doc) +} + +// SeedColumnsExcluded is the fixture-side guard for seed exclusion: it verifies every requested +// column is one the fixture knows how to leave out, so an unknown name fails loudly instead of +// silently seeding a column the baseline cannot survive. +func SeedColumnsExcluded(excluded, supported []string) (map[string]bool, error) { + drop := make(map[string]bool, len(excluded)) + for _, col := range excluded { + if !slices.Contains(supported, col) { + return nil, fmt.Errorf("column %q cannot be excluded from the seed data; the fixture supports excluding only %s", + col, strings.Join(supported, ", ")) + } + drop[col] = true + } + return drop, nil +} + +// copyDirFiles copies every file in src into dst, replacing what is already there. Files only: +// a driver's data-format fixtures are a directory of their own, copied as their own source. +func copyDirFiles(src, dst string) error { + entries, err := os.ReadDir(src) + if err != nil { + return fmt.Errorf("failed to read %s: %s", src, err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + if err := CopyFile(filepath.Join(src, entry.Name()), filepath.Join(dst, entry.Name())); err != nil { + return err + } + } + return nil +} + +// repoRoot is the git checkout the tests run from. Read straight from git rather than derived from +// the running test's directory, which is the one thing here the repo layout does not fix. +func repoRoot() (string, error) { + root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(root)), nil +} From addfa0d5e688bd9dbe8eccbe92ef8e69d3f99fc7 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Thu, 27 Aug 2026 15:16:34 +0530 Subject: [PATCH 04/20] refactor: compat tests --- .github/workflows/performance-test.yml | 181 ------ ...tegration-tests.yml => test-preflight.yml} | 165 ++---- .github/workflows/tests.yml | 398 +++++++++++++ Makefile | 43 +- tests/db2/db2_test.go | 28 +- tests/db2/db2_util_test.go | 57 +- tests/kafka/kafka_test.go | 34 +- tests/kafka/rebalance_test.go | 4 +- tests/mongodb/mongodb_test.go | 30 +- tests/mongodb/mongodb_util_test.go | 55 +- tests/mssql/mssql_test.go | 28 +- tests/mysql/mysql_test.go | 47 +- tests/mysql/mysql_util_test.go | 186 +++--- tests/oracle/oracle_test.go | 27 +- tests/postgres/postgres_test.go | 29 +- tests/s3/s3_test.go | 61 +- tests/s3/s3_util_test.go | 72 ++- .../testutils/compatibility/compatibility.go | 555 ++++++++++++++++++ .../compatibility/compatibility_columns.go | 173 ++++++ .../compatibility/compatibility_rules.go | 395 +++++++++++++ .../compatibility/compatibility_rules.json | 101 ++++ tests/testutils/compatibility/scenarios.go | 232 ++++++++ tests/testutils/ddl.go | 34 ++ tests/testutils/docker.go | 145 +++-- tests/testutils/{integration => }/iceberg.go | 5 +- tests/testutils/integration/2pc.go | 8 +- .../testutils/integration/parquet_rolling.go | 10 +- tests/testutils/integration/sync.go | 14 +- tests/testutils/integration/verify.go | 10 +- tests/testutils/{integration => }/parquet.go | 32 +- tests/testutils/performance/performance.go | 4 +- tests/testutils/require/require.go | 13 +- tests/testutils/state_version.go | 19 +- tests/testutils/test_utils.go | 78 ++- tests/testutils/utils.go | 28 +- 35 files changed, 2557 insertions(+), 744 deletions(-) delete mode 100644 .github/workflows/performance-test.yml rename .github/workflows/{integration-tests.yml => test-preflight.yml} (55%) create mode 100644 .github/workflows/tests.yml create mode 100644 tests/testutils/compatibility/compatibility.go create mode 100644 tests/testutils/compatibility/compatibility_columns.go create mode 100644 tests/testutils/compatibility/compatibility_rules.go create mode 100644 tests/testutils/compatibility/compatibility_rules.json create mode 100644 tests/testutils/compatibility/scenarios.go create mode 100644 tests/testutils/ddl.go rename tests/testutils/{integration => }/iceberg.go (95%) rename tests/testutils/{integration => }/parquet.go (67%) diff --git a/.github/workflows/performance-test.yml b/.github/workflows/performance-test.yml deleted file mode 100644 index b7bbb3c99..000000000 --- a/.github/workflows/performance-test.yml +++ /dev/null @@ -1,181 +0,0 @@ -name: Performance Tests - -on: - push: - branches: - - "staging" - paths: - - '**/*.go' - - '**/*.java' - -permissions: - actions: read - id-token: write - contents: read - -jobs: - performance-tests: - environment: Performance Testing - runs-on: ubuntu-latest - strategy: - matrix: - include: - - driver: mysql - driver_upper: MYSQL - instance_id: 8 - - driver: postgres - driver_upper: POSTGRES - instance_id: 9 - # TODO: add benchmark tests for the below databases - # - driver: mongodb - # driver_upper: MONGODB - # instance_id: - # - driver: oracle - # driver_upper: ORACLE - # instance_id: - fail-fast: false - - steps: - - name: Checkout code - uses: actions/checkout@v7 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version-file: "go.mod" - - - name: Restore driver external dependencies - uses: ./.github/actions/driver-deps - - - name: Set up Java for Maven - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '17' - - - name: Setup VPN Client - if: matrix.driver != 'oracle' - run: | - sudo apt-get update -qq && sudo apt-get install -y openvpn iproute2 iputils-ping netcat-openbsd telnet - sudo mkdir -p /etc/openvpn/client && sudo chmod 700 /etc/openvpn/client - echo "${{ secrets[format('{0}_OPENVPN_CONFIG', matrix.driver_upper)] }}" | base64 --decode | sudo tee /etc/openvpn/client/client.ovpn > /dev/null - echo "${{ secrets.OPENVPN_USERNAME }}" | sudo tee /etc/openvpn/client/auth.txt > /dev/null - sudo chmod 600 /etc/openvpn/client/client.ovpn /etc/openvpn/client/auth.txt - sudo chown root:root /etc/openvpn/client/client.ovpn /etc/openvpn/client/auth.txt - sudo openvpn --config /etc/openvpn/client/client.ovpn --daemon ovpn-client --log /var/log/openvpn-client.log --verb 3 - echo "Establishing VPN connection..." - for i in {1..30}; do - if ip addr show | grep -q "tun0\|tap0"; then echo "✅ VPN connected"; break; fi - [ $i -eq 30 ] && { echo "❌ VPN timeout"; sudo cat /var/log/openvpn-client.log; exit 1; } - sleep 2 - done - sudo resolvectl dns tun0 ${{ secrets.VPN_DNS_SERVER }} - sudo resolvectl domain tun0 ~private.postgres.database.azure.com - - - name: Wake up server - if: matrix.driver != 'oracle' - run: | - mkdir -p ~/.ssh - echo "${{ secrets.TESTING_SERVER_SSH_KEY }}" | base64 --decode > ~/.ssh/key.pem - chmod 600 ~/.ssh/key.pem - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -i ~/.ssh/key.pem ${{ secrets.TESTING_SERVER_SSH_USER_HOST }} "/usr/local/bin/wake-up ${{ matrix.instance_id }}" - echo "Waiting 2 minutes for database to be ready..." - sleep 120 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-session-name: performance_test_gh - role-to-assume: ${{ secrets.AWS_GITHUB_ROLE }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Install Go Dependencies - working-directory: ./tests/testutils - run: go mod download - - - name: Build Iceberg Sink - working-directory: ./destination/iceberg/olake-iceberg-java-writer - run: mvn clean package -DskipTests - - # The image the suite runs against. Built here rather than left to the harness, which would - # otherwise build it inside `go test` where it is invisible in the step timings. - - name: Build Driver Image - run: make docker.${{ matrix.driver }}.build - - - name: Create source - run: | - echo '${{ secrets[format('{0}_SOURCE_JSON', matrix.driver_upper)] }}' | base64 --decode > ./tests/${{ matrix.driver }}/testdata/source.json - - - name: Create destination - run: | - echo '${{ secrets[format('{0}_DESTINATION_JSON', matrix.driver_upper)] }}' | base64 --decode > ./tests/${{ matrix.driver }}/testdata/iceberg_destination.json - - - name: Get Last Successful Run ID - id: last_run - env: - GH_TOKEN: ${{ github.token }} - run: | - PREVIOUS_ID=$(gh run list --workflow ".github/workflows/performance-test.yml" --branch "staging" --status success --limit 1 --json databaseId --jq '.[0].databaseId') - - if [ -n "$PREVIOUS_ID" ]; then - echo "id=$PREVIOUS_ID" >> $GITHUB_OUTPUT - fi - - - name: Download Benchmarks History - if: steps.last_run.outputs.id != '' - uses: actions/download-artifact@v8 - continue-on-error: true - with: - name: benchmarks_${{ matrix.driver }} - path: ./tests/${{ matrix.driver }}/testdata/ - run-id: ${{ steps.last_run.outputs.id }} - github-token: ${{ github.token }} - - # Points the harness at the image built above; without it getOrBuildDriverImage rebuilds it. - - name: Run Performance Tests - env: - OLAKE_DRIVER_IMAGE: olake/source-${{ matrix.driver }}:local - run: make test.performance.${{ matrix.driver }} - - - name: Upload Benchmarks History - uses: actions/upload-artifact@v7 - if: success() - continue-on-error: true - with: - name: benchmarks_${{ matrix.driver }} - path: ./tests/${{ matrix.driver }}/testdata/benchmarks.json - retention-days: 90 - - - name: Sleep server - if: always() && matrix.driver != 'oracle' - run: | - mkdir -p ~/.ssh - echo "${{ secrets.TESTING_SERVER_SSH_KEY }}" | base64 --decode > ~/.ssh/key.pem - chmod 600 ~/.ssh/key.pem - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -i ~/.ssh/key.pem ${{ secrets.TESTING_SERVER_SSH_USER_HOST }} "/usr/local/bin/sleep-off ${{ matrix.instance_id }}" - rm -f ~/.ssh/key.pem - - - name: Cleanup - if: always() - run: | - # Delete all Glue databases matching prefix - for db in $(aws glue get-databases \ - --query "DatabaseList[?starts_with(Name, 'performance_${{ matrix.driver }}')].Name" \ - --output text); do - echo "Deleting Glue database: $db" - aws glue delete-database --name "$db" \ - || echo "Failed to delete Glue database: $db" - done - - # Delete corresponding S3 path - aws s3 rm s3://dz-stag-github-actions/performance_${{ matrix.driver }}/ --recursive \ - || { echo "Failed to delete S3 bucket: performance_${{ matrix.driver }}"; \ - aws s3 ls s3://dz-stag-github-actions/performance_${{ matrix.driver }} || true; } - - echo "Catalog cleanup completed" - - if [[ "${{ matrix.driver }}" != "oracle" ]]; then - sudo pkill -f "openvpn.*client.ovpn" || true - sudo rm -rf /etc/openvpn/client/ /var/log/openvpn-client.log || true - echo "VPN cleanup completed" - fi diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/test-preflight.yml similarity index 55% rename from .github/workflows/integration-tests.yml rename to .github/workflows/test-preflight.yml index 28ab4d4ef..bda1c0c3d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/test-preflight.yml @@ -1,32 +1,29 @@ -name: Integration Tests +name: Test Preflight on: - # A push runs the suite post-merge and publishes the caches every pull request restores from. - push: - branches: - - "master" - - "staging" - pull_request: - branches: - - "*" - paths: &paths - - '**/*.go' - - '**/*.java' - - '**/go.mod' - - '**/go.work' - - '**/pom.xml' - - 'Dockerfile' - - '.dockerignore' - - 'Makefile' - - 'drivers/*/driver.mk' - - 'drivers/*/docker-compose.yml' - - 'drivers/**.conf' - - 'destination/iceberg/local-test/**' - - 'tests/**' - - '.golangci.yml' - - '.github/actions/**' - - '.github/scripts/**' - - '.github/workflows/integration-tests.yml' + workflow_call: + inputs: + suite: + description: 'Suite being gated, for the approval job name.' + required: true + type: string + environment: + description: 'Approval environment. Empty runs ungated, which is what a push to a protected branch does.' + required: false + default: '' + type: string + go-checks: + description: 'Run lint + gosec with the Go cache job. The suite that owns the checks sets it; the others just consume the cache.' + required: false + default: false + type: boolean + outputs: + drivers: + description: 'JSON array of drivers the caller should fan its matrix out over; [] means nothing to test.' + value: ${{ jobs.preflight.outputs.drivers }} + attempt: + description: 'The run attempt the approval was given on, so a caller can re-prompt on a re-run.' + value: ${{ jobs.approve.outputs.attempt }} jobs: # Ungated, like the three cache jobs below it: the environment prompts once per wave of jobs that @@ -48,7 +45,7 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - # Shared with any workflow that wants to skip untouched drivers; the matrix below fans out + # Shared with any workflow that wants to skip untouched drivers; the caller's matrix fans out # over whatever it returns, and an empty array skips the driver jobs entirely. - name: Resolve driver matrix id: drivers @@ -66,7 +63,8 @@ jobs: Makefile drivers/*/driver.mk .golangci.yml - .github/workflows/integration-tests.yml + .github/workflows/test-preflight.yml + .github/workflows/tests.yml run: | changed=$(.github/scripts/changed-in-paths.sh <<<"$GO_CHECK_PATHS") echo "changed=$changed" >> "$GITHUB_OUTPUT" @@ -97,23 +95,23 @@ jobs: # The run's single approval, last before the matrix so a pre-job failure fails or skips it too -- # then any re-run re-executes it, and every attempt that reaches the drivers prompts exactly once. approve: - name: Approve integration tests + name: Approve ${{ inputs.suite }} tests needs: [preflight, build-jar, apt-warm, go-cache] - if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.build-jar.result != 'failure' && needs.apt-warm.result != 'failure' && needs.go-cache.result != 'failure' }} + if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.preflight.outputs.drivers != '[]' && needs.build-jar.result != 'failure' && needs.apt-warm.result != 'failure' && needs.go-cache.result != 'failure' }} runs-on: ubuntu-latest timeout-minutes: 5 - environment: ${{ github.event_name == 'pull_request' && 'integration_tests' || '' }} + environment: ${{ inputs.environment }} outputs: attempt: ${{ github.run_attempt }} steps: - - run: echo "Approved -- running the driver matrix." + - run: echo "Approved -- running the ${{ inputs.suite }} matrix." # Only when the jar is missing: preflight already looked up the cache, so an unchanged writer # skips Maven and this whole job. build-jar: name: Build Iceberg writer jar needs: preflight - if: needs.preflight.outputs.jar-cached != 'true' + if: needs.preflight.outputs.drivers != '[]' && needs.preflight.outputs.jar-cached != 'true' runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -140,7 +138,7 @@ jobs: apt-warm: name: Warm base docker layers needs: preflight - if: needs.preflight.outputs.apt-warmed != 'true' + if: needs.preflight.outputs.drivers != '[]' && needs.preflight.outputs.apt-warmed != 'true' runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -172,7 +170,7 @@ jobs: go-cache: name: Go checks + cache needs: preflight - if: needs.preflight.outputs.go-cached != 'true' || needs.preflight.outputs.go-checks-changed == 'true' + if: needs.preflight.outputs.drivers != '[]' && (needs.preflight.outputs.go-cached != 'true' || needs.preflight.outputs.go-checks-changed == 'true') runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -207,105 +205,16 @@ jobs: # Installs golangci-lint as a side effect (the Makefile guard skips it when the restored # cache already carries the binary), so it rides into the post-job save with gosec below. - name: golangci-lint tests modules + if: inputs.go-checks run: make test.lint - name: install gosec + if: inputs.go-checks env: GOSEC_VERSION: v2.22.11 run: $(go env GOPATH)/bin/gosec --version 2>/dev/null | grep -q "${GOSEC_VERSION#v}" || curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b $(go env GOPATH)/bin $GOSEC_VERSION - name: Run Gosec on tests modules + if: inputs.go-checks working-directory: tests run: $(go env GOPATH)/bin/gosec -exclude=G115 -tests -severity=high -confidence=medium ./... - - # One job per driver, each on its own VM with its own Docker daemon: it brings up its own source - # plus destination stack, builds its own image and runs every suite for that driver. - integration-tests: - name: Test ${{ matrix.driver }} - needs: [preflight, approve] - if: ${{ !cancelled() && needs.approve.result == 'success' }} - runs-on: 16gb-runner - environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - driver: ${{ fromJSON(needs.preflight.outputs.drivers) }} - steps: - - name: Checkout code - uses: actions/checkout@v7 - - # The iceberg catalog's postgres runs as uid 999 and has to own its bind mount. - - name: Set up Data Directories - run: | - sudo mkdir -p destination/iceberg/local-test/data/{postgres-data,minio-data,ivy-cache} - sudo chown -R 999:999 destination/iceberg/local-test/data - sudo chmod -R 777 destination/iceberg/local-test/data - - - name: Restore driver external dependencies - uses: ./.github/actions/driver-deps - - # Bounded so a wedged pull is a killed step rather than the job's whole budget. - - name: Start containers - id: containers - background: true - timeout-minutes: 15 - run: make -j2 --output-sync=target olake.${{ matrix.driver }}.up olake.destination.all.up - - - name: Set up Docker Buildx - uses: ./.github/actions/buildx - - - name: Restore Iceberg writer jar - uses: ./.github/actions/iceberg-jar - - # Background: the ~600MB Go cache restore below overlaps it instead of delaying it. No - # --cache-to -- exporting this build measured ~119s against the ~85s build it would save. - - name: Build driver image - id: image - background: true - timeout-minutes: 20 - run: make docker.${{ matrix.driver }}.build DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --cache-from type=gha,scope=olake-${{ matrix.driver }} --load" - - # setup-go's cache keys on go.sum, which this repo does not track, so it would cache nothing. - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version-file: "go.mod" - cache: false - - # Restore-only: the modules and compiled shared deps are identical for every driver, and seven - # jobs saving a ~350MB copy each would evict the jar from the cache. - - name: Restore shared Go caches - uses: ./.github/actions/go-caches - - # Compiles the test binary while the container pull and the image build are still in flight. - - name: Install Dependencies - id: go-build - timeout-minutes: 15 - run: make test.build.${{ matrix.driver }} - - - name: Wait for background jobs - wait: [containers, image, go-build] - - - name: Wait for source + destination readiness - timeout-minutes: 3 - run: make -j2 --output-sync=target olake.${{ matrix.driver }}.wait olake.destination.all.wait - - # Serial, once: olake processes reaching a fresh catalog race each other's CREATE TABLE on - # iceberg_tables, and the harness has no runtime fallback for that. - - name: Check destination (bootstrap Iceberg catalog) - run: | - set -euo pipefail - echo "Bootstrapping Iceberg catalog via '${{ matrix.driver }}'..." - docker run --rm \ - -v "$PWD/tests/testdata:/testdata" \ - --add-host host.docker.internal:host-gateway \ - -e TELEMETRY_DISABLED=true \ - "olake/source-${{ matrix.driver }}:local" \ - check --destination /testdata/iceberg_destination.json - - # Integration + 2PC + (kafka) Rebalance in one go test, against the image built above. - - name: Run tests - env: - OLAKE_DRIVER_IMAGE: olake/source-${{ matrix.driver }}:local - run: make test.integration.${{ matrix.driver }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..79ab75dbf --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,398 @@ +name: Tests + +on: + workflow_dispatch: + inputs: + baseline: + description: 'Compat baseline: a release tag (v0.6.5), a full image ref, or a commit sha. Empty = the full manifest sweep.' + required: false + default: '' + type: string + # A push runs the suite post-merge and publishes the caches every pull request restores from. + push: + branches: + - "master" + - "staging" + paths: &paths + # olake itself: change any of it and the binary the suites run changes. + - '**/*.go' + - '**/go.mod' + - '**/go.work' + - '**/*.java' + - '**/pom.xml' + - 'Dockerfile' + - '.dockerignore' + - 'Makefile' + - 'drivers/*/driver.mk' + - 'drivers/**.conf' + - 'constants/state-versions.json' + # what the suites are run with: the harness, the stacks they need, and the CI itself. + - 'tests/**' + - 'drivers/*/docker-compose.yml' + - 'destination/iceberg/local-test/**' + - '.golangci.yml' + - '.github/actions/**' + - '.github/scripts/**' + - '.github/workflows/tests.yml' + - '.github/workflows/test-preflight.yml' + pull_request: + branches: + - "*" + paths: *paths + +jobs: + # Detect changes, publish the caches, build the jar, take the run's single approval -- once for + # both matrices below, and shared with performance-test.yml (test-preflight.yml). + # go-checks runs here: this workflow owns the tests modules' lint and gosec. + preflight: + # The called workflow's jobs render as " / ", so this is what keeps + # the internal name out of the run's job list. The job id stays preflight for `needs:`. + name: Tests + uses: ./.github/workflows/test-preflight.yml + permissions: + contents: read + pull-requests: read + with: + suite: integration + compatibility + environment: ${{ github.event_name == 'pull_request' && 'integration_tests' || '' }} + go-checks: true + secrets: inherit + + # One job per driver, each on its own VM with its own Docker daemon: it brings up its own source + # plus destination stack, builds its own image and runs the driver's suites against it. Every step + # up to the last is anchored here and aliased by the compatibility job, which needs the same setup. + # + # No `if:` needed: `needs` already requires preflight to have succeeded, and a matrix over an + # empty driver array is skipped on its own. + integration-tests: + name: Integration Test ${{ matrix.driver }} + needs: preflight + if: needs.preflight.outputs.drivers != '[]' + runs-on: 16gb-runner + environment: ${{ github.event_name == 'pull_request' && needs.preflight.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + driver: ${{ fromJSON(needs.preflight.outputs.drivers) }} + steps: + # Full history so the compatibility job can build the PR's base commit; the extra objects cost this + # job a few seconds and keep one checkout definition for both. + - &checkout + name: Checkout code + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + # The iceberg catalog's postgres runs as uid 999 and has to own its bind mount. + - &data-dirs + name: Set up Data Directories + run: | + sudo mkdir -p destination/iceberg/local-test/data/{postgres-data,minio-data,ivy-cache} + sudo chown -R 999:999 destination/iceberg/local-test/data + sudo chmod -R 777 destination/iceberg/local-test/data + + - &driver-deps + name: Restore driver external dependencies + uses: ./.github/actions/driver-deps + + # Bounded so a wedged pull is a killed step rather than the job's whole budget. + - &containers + name: Start containers + id: containers + background: true + timeout-minutes: 15 + run: make -j2 --output-sync=target olake.${{ matrix.driver }}.up olake.destination.all.up + + - &buildx + name: Set up Docker Buildx + uses: ./.github/actions/buildx + + - &jar + name: Restore Iceberg writer jar + uses: ./.github/actions/iceberg-jar + + # Background: the ~600MB Go cache restore below overlaps it instead of delaying it. No + # --cache-to -- exporting this build measured ~119s against the ~85s build it would save. + - &driver-image + name: Build driver image + id: image + background: true + timeout-minutes: 20 + run: make docker.${{ matrix.driver }}.build DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --cache-from type=gha,scope=olake-${{ matrix.driver }} --load" + + # setup-go's cache keys on go.sum, which this repo does not track, so it would cache nothing. + - &go + name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: "go.mod" + cache: false + + # Restore-only: the modules and compiled shared deps are identical for every driver, and seven + # jobs saving a ~350MB copy each would evict the jar from the cache. + - &go-caches + name: Restore shared Go caches + uses: ./.github/actions/go-caches + + # Compiles the test binary while the container pull and the image build are still in flight. + - &test-build + name: Install Dependencies + id: go-build + timeout-minutes: 15 + run: make test.build.${{ matrix.driver }} + + - &wait-background + name: Wait for background jobs + wait: [containers, image, go-build] + + - &wait-ready + name: Wait for source + destination readiness + timeout-minutes: 3 + run: make -j2 --output-sync=target olake.${{ matrix.driver }}.wait olake.destination.all.wait + + # Serial, once: olake processes reaching a fresh catalog race each other's CREATE TABLE on + # iceberg_tables, and the harness has no runtime fallback for that. + - &catalog + name: Check destination (bootstrap Iceberg catalog) + run: | + set -euo pipefail + echo "Bootstrapping Iceberg catalog via '${{ matrix.driver }}'..." + docker run --rm \ + -v "$PWD/tests/testdata:/testdata" \ + --add-host host.docker.internal:host-gateway \ + -e TELEMETRY_DISABLED=true \ + "olakego/source-${{ matrix.driver }}:local" \ + check --destination /testdata/iceberg_destination.json + + # Integration + 2PC + (kafka) Rebalance in one go test, against the image built above. + # The pin is what stops the harness rebuilding that image: it rebuilds by default so a local + # run tests current code, and only a caller that has already built it says otherwise. + - name: Run tests + env: + OLAKE_PRE_BUILT_IMAGE: olakego/source-${{ matrix.driver }}:local + run: make test.integration.${{ matrix.driver }} + + # The same drivers against a baseline image: see the header for what "baseline" means per event. + # Its own job, on its own machine, so a compatibility failure is re-run without re-running integration -- + # the setup is the integration job's, aliased rather than repeated. + compatibility-tests: + name: Backward Compatibility ${{ matrix.driver }} + needs: preflight + # Skipped on a staging push: the sweep is release-gating, and the PR into staging already ran it. + # The drivers check is for visibility, as on the integration job above. + if: (github.event_name != 'push' || github.ref == 'refs/heads/master') && needs.preflight.outputs.drivers != '[]' + runs-on: 16gb-runner + environment: ${{ github.event_name == 'pull_request' && needs.preflight.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} + # Longer than the integration job's 45: a sweep runs one pipeline pair per baseline, in series. + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + driver: ${{ fromJSON(needs.preflight.outputs.drivers) }} + env: + # Empty is the sweep; a PR pins the baseline to its own base commit. + COMPATIBILITY_BASELINE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.baseline }} + steps: + - *checkout + - *data-dirs + - *driver-deps + - *containers + + # The sweep pulls one image per baseline from a shared-egress runner pool, which is exactly + # where Docker Hub's anonymous rate limit bites. + - name: Log in to Docker Hub + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + if [ -n "$DOCKER_USERNAME" ]; then + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + else + echo "::notice::no Docker Hub credentials configured; pulling anonymously" + fi + + - *buildx + - *jar + - *driver-image + + - *go + - *go-caches + - *test-build + - name: Wait for background jobs + wait: [containers, image, go-build] + + - *wait-ready + - *catalog + + # Pinned like the integration job: the candidate side runs the image built above, and only + # the baseline image is resolved (pulled or built) by the harness. + - name: Run compatibility tests + env: + OLAKE_PRE_BUILT_IMAGE: olakego/source-${{ matrix.driver }}:local + run: make test.compatibility.${{ matrix.driver }} COMPATIBILITY_BASELINE="$COMPATIBILITY_BASELINE" + + # Benchmarks against the remote instances, so it shares this run's preflight but none of its + # stack: no local containers, and the approval is its own environment (which is also where its AWS + # and VPN secrets live). Staging pushes only, which is where the benchmark history is kept. + performance-tests: + name: Performance ${{ matrix.driver }} + needs: preflight + if: needs.preflight.outputs.drivers != '[]' && github.event_name == 'push' && github.ref == 'refs/heads/staging' + environment: Performance Testing + permissions: + actions: read + id-token: write + contents: read + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - driver: mysql + driver_upper: MYSQL + instance_id: 8 + - driver: postgres + driver_upper: POSTGRES + instance_id: 9 + # TODO: add benchmark tests for the below databases + # - driver: mongodb + # driver_upper: MONGODB + # instance_id: + # - driver: oracle + # driver_upper: ORACLE + # instance_id: + + steps: + - *checkout + - *go + - *driver-deps + + - name: Set up Java for Maven + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup VPN Client + if: matrix.driver != 'oracle' + run: | + sudo apt-get update -qq && sudo apt-get install -y openvpn iproute2 iputils-ping netcat-openbsd telnet + sudo mkdir -p /etc/openvpn/client && sudo chmod 700 /etc/openvpn/client + echo "${{ secrets[format('{0}_OPENVPN_CONFIG', matrix.driver_upper)] }}" | base64 --decode | sudo tee /etc/openvpn/client/client.ovpn > /dev/null + echo "${{ secrets.OPENVPN_USERNAME }}" | sudo tee /etc/openvpn/client/auth.txt > /dev/null + sudo chmod 600 /etc/openvpn/client/client.ovpn /etc/openvpn/client/auth.txt + sudo chown root:root /etc/openvpn/client/client.ovpn /etc/openvpn/client/auth.txt + sudo openvpn --config /etc/openvpn/client/client.ovpn --daemon ovpn-client --log /var/log/openvpn-client.log --verb 3 + echo "Establishing VPN connection..." + for i in {1..30}; do + if ip addr show | grep -q "tun0\|tap0"; then echo "✅ VPN connected"; break; fi + [ $i -eq 30 ] && { echo "❌ VPN timeout"; sudo cat /var/log/openvpn-client.log; exit 1; } + sleep 2 + done + sudo resolvectl dns tun0 ${{ secrets.VPN_DNS_SERVER }} + sudo resolvectl domain tun0 ~private.postgres.database.azure.com + + - name: Wake up server + if: matrix.driver != 'oracle' + run: | + mkdir -p ~/.ssh + echo "${{ secrets.TESTING_SERVER_SSH_KEY }}" | base64 --decode > ~/.ssh/key.pem + chmod 600 ~/.ssh/key.pem + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -i ~/.ssh/key.pem ${{ secrets.TESTING_SERVER_SSH_USER_HOST }} "/usr/local/bin/wake-up ${{ matrix.instance_id }}" + echo "Waiting 2 minutes for database to be ready..." + sleep 120 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-session-name: performance_test_gh + role-to-assume: ${{ secrets.AWS_GITHUB_ROLE }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Install Go Dependencies + working-directory: ./tests/testutils + run: go mod download + + - *jar + + # The image the suite runs against. Built here rather than left to the harness, which would + # otherwise build it inside `go test` where it is invisible in the step timings. + - name: Build Driver Image + run: make docker.${{ matrix.driver }}.build + + - name: Create source + run: | + echo '${{ secrets[format('{0}_SOURCE_JSON', matrix.driver_upper)] }}' | base64 --decode > ./tests/${{ matrix.driver }}/testdata/source.json + + - name: Create destination + run: | + echo '${{ secrets[format('{0}_DESTINATION_JSON', matrix.driver_upper)] }}' | base64 --decode > ./tests/${{ matrix.driver }}/testdata/iceberg_destination.json + + - name: Get Last Successful Run ID + id: last_run + env: + GH_TOKEN: ${{ github.token }} + run: | + PREVIOUS_ID=$(gh run list --workflow ".github/workflows/tests.yml" --branch "staging" --status success --limit 1 --json databaseId --jq '.[0].databaseId') + + if [ -n "$PREVIOUS_ID" ]; then + echo "id=$PREVIOUS_ID" >> $GITHUB_OUTPUT + fi + + - name: Download Benchmarks History + if: steps.last_run.outputs.id != '' + uses: actions/download-artifact@v8 + continue-on-error: true + with: + name: benchmarks_${{ matrix.driver }} + path: ./tests/${{ matrix.driver }}/testdata/ + run-id: ${{ steps.last_run.outputs.id }} + github-token: ${{ github.token }} + + # Runs against the image built above: the harness resolves "local" to it and skips the build. + - name: Run Performance Tests + run: make test.performance.${{ matrix.driver }} + + - name: Upload Benchmarks History + uses: actions/upload-artifact@v7 + if: success() + continue-on-error: true + with: + name: benchmarks_${{ matrix.driver }} + path: ./tests/${{ matrix.driver }}/testdata/benchmarks.json + retention-days: 90 + + - name: Sleep server + if: always() && matrix.driver != 'oracle' + run: | + mkdir -p ~/.ssh + echo "${{ secrets.TESTING_SERVER_SSH_KEY }}" | base64 --decode > ~/.ssh/key.pem + chmod 600 ~/.ssh/key.pem + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -i ~/.ssh/key.pem ${{ secrets.TESTING_SERVER_SSH_USER_HOST }} "/usr/local/bin/sleep-off ${{ matrix.instance_id }}" + rm -f ~/.ssh/key.pem + + - name: Cleanup + if: always() + run: | + # Delete all Glue databases matching prefix + for db in $(aws glue get-databases \ + --query "DatabaseList[?starts_with(Name, 'performance_${{ matrix.driver }}')].Name" \ + --output text); do + echo "Deleting Glue database: $db" + aws glue delete-database --name "$db" \ + || echo "Failed to delete Glue database: $db" + done + + # Delete corresponding S3 path + aws s3 rm s3://dz-stag-github-actions/performance_${{ matrix.driver }}/ --recursive \ + || { echo "Failed to delete S3 bucket: performance_${{ matrix.driver }}"; \ + aws s3 ls s3://dz-stag-github-actions/performance_${{ matrix.driver }} || true; } + + echo "Catalog cleanup completed" + + if [[ "${{ matrix.driver }}" != "oracle" ]]; then + sudo pkill -f "openvpn.*client.ovpn" || true + sudo rm -rf /etc/openvpn/client/ /var/log/openvpn-client.log || true + echo "VPN cleanup completed" + fi diff --git a/Makefile b/Makefile index c1dbab2f7..f4d679117 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ DOCKER_BUILD ?= docker build $(addsuffix .build,$(addprefix docker.,$(DRIVERS))): docker.%.build: $(DOCKER_BUILD) $(addprefix --platform ,$(call local_driver_platforms,$*)) \ --build-arg DRIVER_NAME=$* \ - -t olake/source-$*:$(IMAGE_TAG) . + -t olakego/source-$*:$(IMAGE_TAG) . gomod: find . -name go.mod -execdir go mod tidy \; @@ -97,8 +97,6 @@ ICEBERG_JAR := $(ICEBERG_WRITER_DIR)/target/olake-iceberg-java-writer-0.0.1-SNAP ICEBERG_JAR_SRCS := $(ICEBERG_WRITER_DIR)/pom.xml $(shell find $(ICEBERG_WRITER_DIR)/src -type f 2>/dev/null) ROOT_JAR := olake-iceberg-java-writer.jar -IMAGE_JAR_DEP = $(if $(OLAKE_DRIVER_IMAGE),,$(ICEBERG_JAR)) - # --- readiness probes (polled by olake..wait / olake.destination.all.wait, incl. in CI) # Defaults only; per-driver probes and overrides live in drivers//driver.mk. WAIT_RETRIES := 30 @@ -167,7 +165,7 @@ CDC_DRIVERS := $(filter-out $(NON_CDC_DRIVERS),$(SOURCE_DRIVERS)) SOURCE_PKGS := $(addsuffix /...,$(addprefix ./,$(SOURCE_DRIVERS))) CDC_PKGS := $(addsuffix /...,$(addprefix ./,$(CDC_DRIVERS))) -# The drivers the integration suites cover, queried by CI (integration-tests.yml) so the list +# The drivers the end-to-end suites cover, queried by CI (end-to-end-tests.yml) so the list # lives in this file only: it is what the driver matrix fans out to on a push to master. .PHONY: print.source-drivers print.source-drivers: @@ -243,7 +241,7 @@ olake.source.all.refresh: # --- destination stack -------------------------------------------------------- olake.destination.all.up: - mkdir -p $(DEST_DATA_DIR)/minio-data $(DEST_DATA_DIR)/postgres-data $(DEST_DATA_DIR)/ivy-cache + mkdir -p $(DEST_DATA_DIR)/postgres-data $(DEST_DATA_DIR)/ivy-cache $(COMPOSE) -f $(DEST_COMPOSE) up -d $(DEST_SERVICES) olake.destination.all.wait: @@ -260,7 +258,7 @@ olake.destination.all.stop: olake.destination.all.teardown: $(COMPOSE) -f $(DEST_COMPOSE) down --volumes --remove-orphans @rm -rf $(DEST_DATA_DIR) || { echo "Could not remove $(DEST_DATA_DIR) (root-owned files on Linux?). Try: sudo rm -rf $(DEST_DATA_DIR)"; exit 1; } - @echo "Removed docker volumes and $(DEST_DATA_DIR) (minio/postgres data and the hive-metastore ivy cache)" + @echo "Removed docker volumes and $(DEST_DATA_DIR) (postgres data and the hive-metastore ivy cache)" olake.destination.all.restart: @$(MAKE) --no-print-directory olake.destination.all.stop @$(MAKE) --no-print-directory olake.destination.all.start @@ -316,7 +314,7 @@ $(foreach d,$(DRIVERS),$(eval $(call DEV_BUILD_template,$(d)))) # --- tests -------------------------------------------------------------------- # Everything one driver's suites need, brought up concurrently. A recursive -j sub-make, since plain # prerequisites only run in parallel when the caller passes -j; every goal is idempotent. -driver_test_setup = $(MAKE) --no-print-directory -j3 olake.$(1).start olake.destination.all.start $(IMAGE_JAR_DEP) +driver_test_setup = $(MAKE) --no-print-directory -j3 olake.$(1).start olake.destination.all.start $(ICEBERG_JAR) # Compile the driver's test binary without running it, so CI pays the cold build while its container # pull and image build are still in flight. Through make, for db2's clidriver and cgo env. @@ -338,7 +336,7 @@ define DRIVER_TEST_template .PHONY: test.integration.$(1) test.integration.$(1): prepare.$(1) @$$(call driver_test_setup,$(1)) - $$(GO_ENV.$(1)) cd tests && go test -v ./$(1)/... -timeout 0 -count=1 -skip 'Performance' + $$(GO_ENV.$(1)) cd tests && go test -v ./$(1)/... -timeout 0 -count=1 -skip 'Performance|Compatibility' endef $(foreach d,$(SOURCE_DRIVERS),$(eval $(call DRIVER_TEST_template,$(d)))) @@ -362,21 +360,36 @@ test.performance.$(1): prepare.$(1) $$(ICEBERG_JAR) endef $(foreach d,$(SOURCE_DRIVERS),$(eval $(call PERFORMANCE_TEST_template,$(d)))) -test.discover: $(addprefix prepare.,$(SOURCE_DRIVERS)) olake.all.start $(IMAGE_JAR_DEP) +test.discover: $(addprefix prepare.,$(SOURCE_DRIVERS)) olake.all.start $(ICEBERG_JAR) $(foreach d,$(SOURCE_DRIVERS),$(GO_ENV.$(d))) cd tests && go test -v -p $(words $(SOURCE_DRIVERS)) $(SOURCE_PKGS) -timeout 0 -count=1 -run 'Discover' -test.sync: $(addprefix prepare.,$(SOURCE_DRIVERS)) olake.all.start $(IMAGE_JAR_DEP) +test.sync: $(addprefix prepare.,$(SOURCE_DRIVERS)) olake.all.start $(ICEBERG_JAR) $(foreach d,$(SOURCE_DRIVERS),$(GO_ENV.$(d))) cd tests && go test -v -p $(words $(SOURCE_DRIVERS)) $(SOURCE_PKGS) -timeout 0 -count=1 -run 'Sync' -test.2pc: $(addprefix prepare.,$(CDC_DRIVERS)) $(addprefix olake.,$(addsuffix .start,$(CDC_DRIVERS))) olake.destination.all.start $(IMAGE_JAR_DEP) +test.2pc: $(addprefix prepare.,$(CDC_DRIVERS)) $(addprefix olake.,$(addsuffix .start,$(CDC_DRIVERS))) olake.destination.all.start $(ICEBERG_JAR) $(foreach d,$(CDC_DRIVERS),$(GO_ENV.$(d))) cd tests && go test -v -p $(words $(CDC_DRIVERS)) $(CDC_PKGS) -timeout 0 -count=1 -run '2PC' +COMPATIBILITY_BASELINE ?= + +define COMPATIBILITY_TEST_template +.PHONY: test.compatibility.$(1) +test.compatibility.$(1): prepare.$(1) + @$$(call driver_test_setup,$(1)) + $$(GO_ENV.$(1)) cd tests && \ + OLAKE_COMPATIBILITY_TEST_BASELINE=$$(COMPATIBILITY_BASELINE) \ + go test -v ./$(1)/... -timeout 0 -count=1 -parallel 8 -run 'Compatibility' +endef +$(foreach d,$(SOURCE_DRIVERS),$(eval $(call COMPATIBILITY_TEST_template,$(d)))) + +.PHONY: test.compatibility +test.compatibility: $(addprefix test.compatibility.,$(SOURCE_DRIVERS)) + # Unit tests across every module in the go.work workspace. Directory patterns # ({{.Dir}}/...), not module-path patterns: in a go.work workspace a path pattern # like /... prefix-matches into sibling modules. test.unit: $(addprefix prepare.,$(DRIVERS)) - $(foreach d,$(DRIVERS),$(GO_ENV.$(d))) go list -m -f '{{.Dir}}/...' | xargs go test -v -count=1 -skip '^Test.*(Discover|Sync|2PC|Performance|Rebalance)$$' + $(foreach d,$(DRIVERS),$(GO_ENV.$(d))) go list -m -f '{{.Dir}}/...' | xargs go test -v -count=1 define print_help_targets $(foreach t,$(HELP_TARGETS), \ @@ -416,7 +429,7 @@ help: @printf " %-44s %s\n" "prepare. | prepare.all" "provision host build deps (db2: IBM clidriver; else no-op)" @echo "" @echo "Docker images:" - @$(foreach d,$(DRIVERS),printf " %-44s %s\n" "docker.$(d).build" "build the $(d) driver image (olake/source-$(d):$(IMAGE_TAG))";) + @$(foreach d,$(DRIVERS),printf " %-44s %s\n" "docker.$(d).build" "build the $(d) driver image (olakego/source-$(d):$(IMAGE_TAG))";) @echo "" @echo "Tests (auto-provision the stacks they need):" @printf " %-44s %s\n" "iceberg.jar" "build the Iceberg writer JAR (skips maven when up to date)" @@ -425,7 +438,9 @@ help: @$(foreach d,$(SOURCE_DRIVERS),printf " %-44s %s\n" "test.sync.$(d)" "sync suite for $(d) (full load, CDC, incremental)";) @$(foreach d,$(CDC_DRIVERS),printf " %-44s %s\n" "test.2pc.$(d)" "2PC recovery suite for $(d)";) @$(foreach d,$(SOURCE_DRIVERS),printf " %-44s %s\n" "test.performance.$(d)" "benchmark suite for $(d) (remote instances, no local stack)";) + @$(foreach d,$(SOURCE_DRIVERS),printf " %-44s %s\n" "test.compatibility.$(d)" "backward-compatibility for upgrading from baseline to latest $(d): COMPATIBILITY_BASELINE=, empty = sweep every baseline in state-versions.json";) @printf " %-44s %s\n" "test.discover | test.sync | test.2pc | test.unit" "aggregate runs (all drivers at once)" + @printf " %-44s %s\n" "test.compatibility" "backward-compatibility for every driver, sequentially (COMPATIBILITY_BASELINE as above)" @printf " %-44s %s\n" "test.build.all" "compile every driver's test binary (CI cache warm)" @if [ -n "$(strip $(HELP_TARGETS))" ]; then \ echo ""; \ @@ -433,7 +448,7 @@ help: $(call print_help_targets) \ fi @echo "" - @echo "Overridables: SOURCE_DRIVERS COMPOSE WAIT_RETRIES WAIT_SLEEP IMAGE_TAG" + @echo "Overridables: SOURCE_DRIVERS COMPOSE WAIT_RETRIES WAIT_SLEEP IMAGE_TAG COMPATIBILITY_BASELINE" .PHONY: lint olake.lint test.lint build \ olake.source.all.start olake.source.all.stop olake.source.all.teardown olake.source.all.restart olake.source.all.refresh \ diff --git a/tests/db2/db2_test.go b/tests/db2/db2_test.go index 4199c1b61..4081359ab 100644 --- a/tests/db2/db2_test.go +++ b/tests/db2/db2_test.go @@ -4,15 +4,16 @@ import ( "testing" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/compatibility" "github.com/datazip-inc/olake/tests/testutils/constants" "github.com/datazip-inc/olake/tests/testutils/integration" "github.com/datazip-inc/olake/tests/testutils/require" ) // db2BaseConfig returns an IntegrationTest pre-populated with all fields shared -func db2BaseConfig(t *testing.T) *integration.Test { +func db2BaseConfig(t *testing.T, opts ...testutils.TestConfigOption) *integration.Test { cfg, err := testutils.NewTestConfig(t, constants.DB2, "DB2INST1", "db2_testdb_db2inst1", ExecuteQuery, - testutils.WithImagePlatform("linux/amd64")) + append([]testutils.TestConfigOption{testutils.WithImagePlatform("linux/amd64")}, opts...)...) require.NoError(t, err, "failed to build the test config") cfg.CursorField = "COL_CURSOR:COL_TIMESTAMP" cfg.PartitionRegex = "/{id, identity}" @@ -60,15 +61,14 @@ func TestDB22PC(t *testing.T) { // TestDB2Compatibility pins the backward-compatibility contract: the same scenarios run on a released // baseline image and on this build after the initial load, and the destinations must match. // See tests/testutils/compatibility.go. -// func TestDB2Compatibility(t *testing.T) { -// t.Parallel() -// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { -// base := db2BaseConfig(t) -// base.ExpectedUpdatedData = ExpectedUpdatedDB2Data -// base.UpdatedDestinationDataTypeSchema = UpdatedDB2ToDestinationSchema -// cfg := &compatibility.Test{IntegrationTest: base} -// // Type tags for compatibility_rules.json's db2 rules; floor and descriptions live there too. -// cfg.ColumnTypes = map[string][]string{"col_decfloat": {"decfloat"}} -// return cfg -// }) -// } +func TestDB2Compatibility(t *testing.T) { + t.Parallel() + fixture := &compatibility.Test{ + NewConfig: func(t *testing.T, version string) *testutils.TestConfig { + return db2BaseConfig(t, testutils.WithDriverVersion(version)).TestConfig + }, + DeclaredSchema: DB2ToDestinationSchema, + ColumnTypes: seedColumnTypes(), + } + fixture.RunBackwardCompatibility(t) +} diff --git a/tests/db2/db2_util_test.go b/tests/db2/db2_util_test.go index 2fe7e0aa5..465a7e020 100644 --- a/tests/db2/db2_util_test.go +++ b/tests/db2/db2_util_test.go @@ -15,6 +15,38 @@ import ( "github.com/jmoiron/sqlx" ) +// seedTableDDL is the seed table's column list, the one place the fixture's schema lives: create +// renders it and seedColumnTypes reads it. +const seedTableDDL = ` + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + col_cursor BIGINT, + col_bigint BIGINT, + col_char CHAR(1), + col_character CHAR(10), + col_varchar VARCHAR(50), + col_date DATE, + col_decimal DECIMAL(10, 2), + col_decfloat DECFLOAT, + col_double DOUBLE, + col_real REAL, + col_int INTEGER, + col_smallint SMALLINT, + col_bool BOOLEAN, + col_clob CLOB(1M), + col_blob BLOB(1M), + col_timestamp TIMESTAMP, + col_time TIME, + col_graphic GRAPHIC(11), + col_vargraphic VARGRAPHIC(14), + excludedColumn INT NULL + ` + +// seedColumnTypes derives every seed column's type tags from the DDL, so a data_types rule in +// compatibility_rules.json follows a seed edit with nothing to declare. +func seedColumnTypes() map[string][]string { + return testutils.DDLColumnTypes(seedTableDDL) +} + var ( dbConnsMu sync.Mutex dbConns = map[string]*sqlx.DB{} @@ -98,30 +130,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, END` case "create": - query = fmt.Sprintf(` - CREATE TABLE %s ( - id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, - col_cursor BIGINT, - col_bigint BIGINT, - col_char CHAR(1), - col_character CHAR(10), - col_varchar VARCHAR(50), - col_date DATE, - col_decimal DECIMAL(10, 2), - col_decfloat DECFLOAT, - col_double DOUBLE, - col_real REAL, - col_int INTEGER, - col_smallint SMALLINT, - col_bool BOOLEAN, - col_clob CLOB(1M), - col_blob BLOB(1M), - col_timestamp TIMESTAMP, - col_time TIME, - col_graphic GRAPHIC(11), - col_vargraphic VARGRAPHIC(14), - excludedColumn INT NULL - )`, integrationTestTable) + query = fmt.Sprintf("CREATE TABLE %s (%s)", integrationTestTable, seedTableDDL) // DB2 has no CREATE TABLE IF NOT EXISTS; tolerate an existing table (SQL0601N, // SQLSTATE 42710) to match the other drivers' create semantics. if cerr := exec(ctx, db, query); cerr != nil && !strings.Contains(cerr.Error(), "SQL0601N") { diff --git a/tests/kafka/kafka_test.go b/tests/kafka/kafka_test.go index 6d343ad9c..2940950f6 100644 --- a/tests/kafka/kafka_test.go +++ b/tests/kafka/kafka_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/compatibility" "github.com/datazip-inc/olake/tests/testutils/constants" "github.com/datazip-inc/olake/tests/testutils/integration" "github.com/datazip-inc/olake/tests/testutils/require" @@ -13,7 +14,7 @@ type kafkaFormat struct { name string // build runs inside the subtest, not beside it: every name a suite owns is derived from // t.Name(), so both formats built against the parent would answer to the same one. - build func(t *testing.T) *integration.Test + build func(t *testing.T, opts ...testutils.TestConfigOption) *integration.Test } var kafkaFormats = []kafkaFormat{ @@ -21,9 +22,9 @@ var kafkaFormats = []kafkaFormat{ {name: "AVRO-Format", build: kafkaAvroBaseConfig}, } -func kafkaJSONBaseConfig(t *testing.T) *integration.Test { +func kafkaJSONBaseConfig(t *testing.T, opts ...testutils.TestConfigOption) *integration.Test { cfg, err := testutils.NewTestConfig(t, constants.Kafka, "topics", "kafka_topics", ExecuteQueryJSON, - testutils.WithDataFormat("json")) + append([]testutils.TestConfigOption{testutils.WithDataFormat("json")}, opts...)...) require.NoError(t, err, "failed to build the test config") cfg.PartitionRegex = "/{int_value,identity}" cfg.ColumnToExclude = "col_excluded" @@ -53,9 +54,9 @@ func kafkaJSONBaseConfig(t *testing.T) *integration.Test { } } -func kafkaAvroBaseConfig(t *testing.T) *integration.Test { +func kafkaAvroBaseConfig(t *testing.T, opts ...testutils.TestConfigOption) *integration.Test { cfg, err := testutils.NewTestConfig(t, constants.Kafka, "topics", "kafka_topics", ExecuteQueryAvro, - testutils.WithDataFormat("avro")) + append([]testutils.TestConfigOption{testutils.WithDataFormat("avro")}, opts...)...) require.NoError(t, err, "failed to build the test config") cfg.PartitionRegex = "/{int64_value,identity}" cfg.ColumnToExclude = "col_excluded" @@ -116,15 +117,14 @@ func TestKafkaRebalance(t *testing.T) { // TestKafkaCompatibility pins the backward-compatibility contract on the JSON format, the same single // format Test2PCIntegration uses: the suite varies only the binary, and avro would add a // schema-registry axis to the comparison. See tests/testutils/compatibility.go. -// func TestKafkaCompatibility(t *testing.T) { -// t.Parallel() -// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { -// base := kafkaJSONBaseConfig(t) -// cfg := &compatibility.Test{IntegrationTest: base} -// // The compatibility floor and its story live in compatibility_rules.json's kafka block. -// // Kafka pipelines interfere across groups: discover enumerates the whole broker, so -// // concurrent groups scan (and race the deletion of) each other's topics. -// cfg.SerialGroups = true -// return cfg -// }) -// } +func TestKafkaCompatibility(t *testing.T) { + t.Parallel() + fixture := &compatibility.Test{ + NewConfig: func(t *testing.T, version string) *testutils.TestConfig { + return kafkaJSONBaseConfig(t, testutils.WithDriverVersion(version)).TestConfig + }, + DeclaredSchema: KafkaToDestinationJSONSchema, + CDCColumnsSchema: ExpectedKafkaDefaultCDCColumnsSchema, + } + fixture.RunBackwardCompatibility(t) +} diff --git a/tests/kafka/rebalance_test.go b/tests/kafka/rebalance_test.go index d5843fa69..acd783999 100644 --- a/tests/kafka/rebalance_test.go +++ b/tests/kafka/rebalance_test.go @@ -73,7 +73,7 @@ func runRebalanceSuite(t *testing.T, cfg *integration.Test) { func rebalanceRecovery(ctx context.Context, t *testing.T, cfg *integration.Test, testTable string) error { t.Log("Starting Kafka rebalance recovery test") - integration.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + testutils.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) if err := testutils.ResetStateFile(cfg.TestConfig); err != nil { return fmt.Errorf("failed to reset state file: %s", err) } @@ -101,7 +101,7 @@ func rebalanceRecovery(ctx context.Context, t *testing.T, cfg *integration.Test, t.Log("Kafka rebalance recovery test completed successfully") - integration.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + testutils.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) t.Logf("Dropped Iceberg table: %s", testTable) return nil diff --git a/tests/mongodb/mongodb_test.go b/tests/mongodb/mongodb_test.go index 22b853f03..95407d808 100644 --- a/tests/mongodb/mongodb_test.go +++ b/tests/mongodb/mongodb_test.go @@ -4,14 +4,15 @@ import ( "testing" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/compatibility" "github.com/datazip-inc/olake/tests/testutils/constants" "github.com/datazip-inc/olake/tests/testutils/integration" "github.com/datazip-inc/olake/tests/testutils/require" ) // mongodbBaseConfig returns an IntegrationTest pre-populated with all fields shared -func mongodbBaseConfig(t *testing.T) *integration.Test { - cfg, err := testutils.NewTestConfig(t, constants.MongoDB, "olake_mongodb_test", "mongodb_olake_mongodb_test", ExecuteQuery) +func mongodbBaseConfig(t *testing.T, opts ...testutils.TestConfigOption) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.MongoDB, "olake_mongodb_test", "mongodb_olake_mongodb_test", ExecuteQuery, opts...) require.NoError(t, err, "failed to build the test config") cfg.CursorField = "id_cursor:id_int" cfg.PartitionRegex = "/{_id,identity}" @@ -77,16 +78,15 @@ func TestMongodb2PC(t *testing.T) { // _id and _olake_id are volatile here, unlike every other driver: the seed inserts documents // without an _id, so the server generates a fresh ObjectID per run and _olake_id, which hashes the // primary key, follows it. Both are still compared by TYPE -- only their values are exempt. -// func TestMongodbCompatibility(t *testing.T) { -// t.Parallel() -// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { -// base := mongodbBaseConfig(t) -// base.ExpectedUpdatedData = ExpectedUpdatedData -// base.UpdatedDestinationDataTypeSchema = UpdatedMongoToDestinationSchema -// cfg := &compatibility.Test{IntegrationTest: base} -// cfg.ExtraVolatileColumns = []string{"_id", "_olake_id"} -// // Type tags for compatibility_rules.json's mongodb rules (G1: id_regex value change at #657). -// cfg.ColumnTypes = map[string][]string{"id_regex": {"regex"}} -// return cfg -// }) -// } +func TestMongodbCompatibility(t *testing.T) { + t.Parallel() + fixture := &compatibility.Test{ + NewConfig: func(t *testing.T, version string) *testutils.TestConfig { + return mongodbBaseConfig(t, testutils.WithDriverVersion(version)).TestConfig + }, + DeclaredSchema: MongoToDestinationSchema, + ColumnTypes: seedColumnTypes(), + CDCColumnsSchema: ExpectedMongoDBDefaultCDCColumnsSchema, + } + fixture.RunBackwardCompatibility(t) +} diff --git a/tests/mongodb/mongodb_util_test.go b/tests/mongodb/mongodb_util_test.go index 0e24359b3..80277b1df 100644 --- a/tests/mongodb/mongodb_util_test.go +++ b/tests/mongodb/mongodb_util_test.go @@ -214,26 +214,47 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, } } +// seedDocument is the document every seeded row starts from; callers set the per-row fields, and +// seedColumnTypes reads the types off it. +func seedDocument() bson.M { + return bson.M{ + "id_bigint": int64(123456789012345), + "id_int": int32(100), + "id_timestamp": time.Date(2023, 1, 1, 12, 0, 0, 0, time.UTC), + "id_double": float64(123.456), + "id_bool": true, + "created_timestamp": primitive.Timestamp{T: uint32(1754905992), I: 1}, + "id_nil": nil, + "id_regex": primitive.Regex{Pattern: "test.*", Options: "i"}, + "id_nested": nestedDoc, + "id_minkey": primitive.MinKey{}, + "id_maxkey": primitive.MaxKey{}, + "name_varchar": "varchar_val", + "excludedColumn": 100, + } +} + +// seedColumnTypes tags every seed field with its BSON type name (regex, date, ...), read off the +// seed document itself, so a data_types rule in compatibility_rules.json follows a seed edit with +// nothing to declare. +func seedColumnTypes() map[string][]string { + types := map[string][]string{} + for field, value := range seedDocument() { + bsonType, _, err := bson.MarshalValue(value) + if err != nil { + continue + } + types[field] = []string{strings.ToLower(bsonType.String())} + } + return types +} + func insertTestData(ctx context.Context, t *testing.T, collection *mongo.Collection) { t.Helper() for i := 1; i <= 5; i++ { - doc := bson.M{ - "id": i, - "id_cursor": i, - "id_bigint": int64(123456789012345), - "id_int": int32(100), - "id_timestamp": time.Date(2023, 1, 1, 12, 0, 0, 0, time.UTC), - "id_double": float64(123.456), - "id_bool": true, - "created_timestamp": primitive.Timestamp{T: uint32(1754905992), I: 1}, - "id_nil": nil, - "id_regex": primitive.Regex{Pattern: "test.*", Options: "i"}, - "id_nested": nestedDoc, - "id_minkey": primitive.MinKey{}, - "id_maxkey": primitive.MaxKey{}, - "name_varchar": "varchar_val", - "excludedColumn": 100, - } + doc := seedDocument() + doc["id"] = i + doc["id_cursor"] = i _, err := collection.InsertOne(ctx, doc) require.NoError(t, err, "Failed to insert test data row %d", i) diff --git a/tests/mssql/mssql_test.go b/tests/mssql/mssql_test.go index a78025935..b8c31ee95 100644 --- a/tests/mssql/mssql_test.go +++ b/tests/mssql/mssql_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/compatibility" "github.com/datazip-inc/olake/tests/testutils/constants" "github.com/datazip-inc/olake/tests/testutils/integration" "github.com/datazip-inc/olake/tests/testutils/require" @@ -11,8 +12,8 @@ import ( // mssqlBaseConfig returns an IntegrationTest pre-populated with all fields shared // by the mssql suites. -func mssqlBaseConfig(t *testing.T) *integration.Test { - cfg, err := testutils.NewTestConfig(t, constants.MSSQL, "dbo", "mssql_olake_mssql_test_dbo", ExecuteQuery) +func mssqlBaseConfig(t *testing.T, opts ...testutils.TestConfigOption) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.MSSQL, "dbo", "mssql_olake_mssql_test_dbo", ExecuteQuery, opts...) require.NoError(t, err, "failed to build the test config") cfg.ColumnToExclude = "excludedColumn" cfg.CursorField = "id_cursor:col_int" @@ -61,13 +62,16 @@ func TestMSSQL2PC(t *testing.T) { // TestMSSQLCompatibility pins the backward-compatibility contract: the same scenarios run on a released // baseline image and on this build after the initial load, and the destinations must match. // See tests/testutils/compatibility.go. -// func TestMSSQLCompatibility(t *testing.T) { -// t.Parallel() -// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { -// base := mssqlBaseConfig(t) -// base.ExpectedUpdatedData = ExpectedUpdatedMSSQLData -// base.UpdatedDestinationDataTypeSchema = MSSQLToDestinationSchema -// cfg := &compatibility.Test{IntegrationTest: base} -// return cfg -// }) -// } +func TestMSSQLCompatibility(t *testing.T) { + t.Parallel() + + fixture := &compatibility.Test{ + NewConfig: func(t *testing.T, version string) *testutils.TestConfig { + return mssqlBaseConfig(t, testutils.WithDriverVersion(version)).TestConfig + }, + DeclaredSchema: MSSQLToDestinationSchema, + CDCColumnsSchema: ExpectedMSSQLDefaultCDCColumnsSchema, + } + + fixture.RunBackwardCompatibility(t) +} diff --git a/tests/mysql/mysql_test.go b/tests/mysql/mysql_test.go index 898ffa7e7..997a3da0f 100644 --- a/tests/mysql/mysql_test.go +++ b/tests/mysql/mysql_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/compatibility" "github.com/datazip-inc/olake/tests/testutils/constants" "github.com/datazip-inc/olake/tests/testutils/integration" "github.com/datazip-inc/olake/tests/testutils/performance" @@ -12,8 +13,8 @@ import ( // mysqlBaseConfig returns an IntegrationTest pre-populated with all fields shared // by the mysql suites. -func mysqlBaseConfig(t *testing.T) *integration.Test { - cfg, err := testutils.NewTestConfig(t, constants.MySQL, "olake_mysql_test", "mysql_olake_mysql_test", ExecuteQuery) +func mysqlBaseConfig(t *testing.T, opts ...testutils.TestConfigOption) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.MySQL, "olake_mysql_test", "mysql_olake_mysql_test", ExecuteQuery, opts...) require.NoError(t, err, "failed to build the test config") cfg.CursorField = "id_cursor:id_smallint" cfg.PartitionRegex = "/{id,identity}" @@ -81,33 +82,15 @@ func TestMySQLPerformance(t *testing.T) { // Baseline defaults to the newest release; OLAKE_COMPATIBILITY_BASELINE picks another tag, image or // commit. v0.4.0 is the newest release still on state version 3, so it is the one that exercises // the UNSIGNED gate. -// func TestMySQLCompatibility(t *testing.T) { -// t.Parallel() -// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { -// base := mysqlBaseConfig(t) -// base.ExpectedUpdatedData = ExpectedUpdatedData -// base.UpdatedDestinationDataTypeSchema = EvolvedMySQLToDestinationSchema -// cfg := &compatibility.Test{IntegrationTest: base} -// // Every known mysql finding, as data (COMPAT_RESULTS_v2.md). The ExcludeBelow columns are -// // the ones #940 ("fix CDC charset corruption for utf16/ucs2/latin1 columns", v0.7.2) added -// // as its own regression test: an older baseline hands their raw bytes to the Iceberg -// // writer as invalid UTF-8, the gRPC marshal fails, and the driver retries on a doubling -// // backoff that looks like a hang -- a hard fail, so they stay out of the seed data -// // entirely. The AssertValueFrom columns synced fine all along but changed value form at -// // the named release, so below it they are compared by type only: SET columns emitted the -// // numeric bitmask on the binlog path before #940 (M1), ENUMs serialized differently before -// // v0.3.9 (M2), and DECIMAL/NUMERIC round-tripped through float32 before v0.3.7 (M3). -// // The closure reads SeedExcludedColumns at call time; RunBackwardCompatibility fills it in after -// // resolving the rules above against the baseline. -// cfg.SupportsSeedExclusion = true -// base.TestConfig.ExecuteQuery = func(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { -// ExecuteQueryExcluding(ctx, t, conf, operation, cfg.SeedExcludedColumns) -// } -// // The filter stays on. It used to be cleared here because v0.4.0 synced the id=999 row -// // that HEAD filtered away -- an 8-vs-7 row count that masked everything behind it. That -// // was the input format, not the binary: filter_config arrived in v0.6.0, so v0.4.0 never -// // saw the key. RunBackwardCompatibility now writes the baseline's own input generation, which -// // hands a pre-v0.6.0 baseline the legacy `filter` string both binaries honor identically. -// return cfg -// }) -// } +func TestMySQLCompatibility(t *testing.T) { + t.Parallel() + fixture := &compatibility.Test{ + DeclaredSchema: MySQLToDestinationSchema, + CDCColumnsSchema: ExpectedMySQLDefaultCDCColumnsSchema, + ColumnTypes: seedColumnTypes(), + } + fixture.NewConfig = func(t *testing.T, version string) *testutils.TestConfig { + return mysqlBaseConfig(t, testutils.WithDriverVersion(version)).TestConfig + } + fixture.RunBackwardCompatibility(t) +} diff --git a/tests/mysql/mysql_util_test.go b/tests/mysql/mysql_util_test.go index 495814b83..02a9a39a4 100644 --- a/tests/mysql/mysql_util_test.go +++ b/tests/mysql/mysql_util_test.go @@ -20,20 +20,76 @@ import ( // PerformanceTest config and the perf operations below. var performanceCDCStreams = []string{"trips_cdc", "fhv_trips_cdc"} -// ExecuteQuery executes MySQL queries for testing based on the operation type -func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { - t.Helper() - ExecuteQueryExcluding(ctx, t, conf, operation, nil) -} - -// versionedSeedColumns are the columns TestMySQLCompatibility can leave out of the seed data for old -// baselines (CompatibilityColumnRule.ExcludeBelow); every other suite seeds all of them. +// versionedSeedColumns are the columns a suite can leave out of the seed data through +// TestConfig.SeedExcludedColumns -- the backward-compatibility suite drops the ones an old +// baseline cannot sync; every other suite leaves the list empty and seeds all of them. var versionedSeedColumns = []struct { name, ddl, value, filteredValue, updateExpr string }{ {"name_ucs2", "name_ucs2 VARCHAR(100) CHARACTER SET ucs2", "'ucs2_val'", "'filtered ucs2'", "name_ucs2 = 'updated ucs2'"}, {"name_utf16le", "name_utf16le VARCHAR(100) CHARACTER SET utf16le", "'utf16le_val'", "'filtered utf16le'", "name_utf16le = 'updated utf16le'"}, {"grade", "grade ENUM('naïve','café','résumé') CHARACTER SET latin1", "'naïve'", "'naïve'", "grade = 'café'"}, + {"name_latin1", "name_latin1 VARCHAR(100) CHARACTER SET latin1", "'latin1_val'", "'filtered latin1'", "name_latin1 = 'updated latin1'"}, + {"permissions", "permissions SET('read','write','execute') CHARACTER SET latin1 DEFAULT NULL", "'read,write'", "'execute'", "permissions = 'read,write,execute'"}, + {"id_bigint_unsigned", "id_bigint_unsigned BIGINT UNSIGNED", "5003", "0", "id_bigint_unsigned = 6003"}, + {"id_bigint_unsigned_signbit", "id_bigint_unsigned_signbit BIGINT UNSIGNED", "9223372036854775808", "0", "id_bigint_unsigned_signbit = 9223372036854775809"}, + {"id_bigint_unsigned_max", "id_bigint_unsigned_max BIGINT UNSIGNED", "18446744073709551615", "0", "id_bigint_unsigned_max = 18446744073709551614"}, +} + +// seedTableDDL is the seed table's column list, the one place the fixture's schema lives: create +// renders it and seedColumnTypes reads it; the versioned columns splice in at %s. +const seedTableDDL = ` + id INT UNSIGNED NOT NULL AUTO_INCREMENT, + id_bigint BIGINT, + id_int INT, + id_cursor INT, + id_int_unsigned INT UNSIGNED, + id_integer INT, + id_integer_unsigned INT UNSIGNED, + id_mediumint MEDIUMINT, + id_mediumint_unsigned MEDIUMINT UNSIGNED, + id_smallint SMALLINT, + id_smallint_unsigned SMALLINT UNSIGNED, + id_tinyint TINYINT, + id_tinyint_unsigned TINYINT UNSIGNED, + id_tinyint_unsigned_max TINYINT UNSIGNED, + id_smallint_unsigned_max SMALLINT UNSIGNED, + id_mediumint_unsigned_max MEDIUMINT UNSIGNED, + id_mediumint_unsigned_signbit MEDIUMINT UNSIGNED, + id_int_unsigned_max INT UNSIGNED, + price_decimal DECIMAL(10,2), + amount_decimal_9_2 DECIMAL(9,2), + price_double DOUBLE, + price_double_precision DOUBLE, + price_float FLOAT, + price_numeric DECIMAL(10,2), + price_real DOUBLE, + name_char CHAR(50), + name_varchar VARCHAR(100), + name_text TEXT, + name_tinytext TINYTEXT, + name_mediumtext MEDIUMTEXT, + name_longtext LONGTEXT, + created_date DATETIME, + created_timestamp TIMESTAMP NULL, + is_active TINYINT(1), + long_varchar MEDIUMTEXT, + name_bool TINYINT(1) DEFAULT '1', + status ENUM('active','inactive','pending') DEFAULT NULL, + priority ENUM('low','medium','high') DEFAULT 'low',%s + tags SET('sports','music','gaming','reading') DEFAULT NULL, + PRIMARY KEY (id), + excludedColumn INT + ` + +// seedColumnTypes derives every seed column's type tags from the DDL, versioned columns included, +// so a data_types rule in compatibility_rules.json follows a seed edit with nothing to declare. +func seedColumnTypes() map[string][]string { + ddl := fmt.Sprintf(seedTableDDL, "") + for _, col := range versionedSeedColumns { + ddl += "\n" + col.ddl + } + return testutils.DDLColumnTypes(ddl) } // seedColumnFragments renders the versioned columns NOT being excluded as the fragments each seed @@ -65,11 +121,12 @@ func seedColumnFragments(t *testing.T, excluded []string) (ddl, cols, vals, filt return ddl, join(names), join(values), join(filtered), join(sets) } -// ExecuteQueryExcluding is ExecuteQuery with columns left out of the seed DDL and DML entirely -- -// the compatibility suite's seed exclusion for columns an old baseline cannot sync at any price. -func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string, excludedColumns []string) { +// ExecuteQuery executes MySQL queries for testing based on the operation type. Columns named in +// conf.SeedExcludedColumns are left out of the seed DDL and DML entirely. +func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() + excludedColumns := conf.SeedExcludedColumns seedDDL, seedCols, seedVals, seedFilteredVals, seedUpdates := seedColumnFragments(t, excludedColumns) var connStr, database string @@ -94,55 +151,7 @@ func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.Te switch operation { case "create": - query = fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s ( - id INT UNSIGNED NOT NULL AUTO_INCREMENT, - id_bigint BIGINT, - id_int INT, - id_cursor INT, - id_int_unsigned INT UNSIGNED, - id_integer INT, - id_integer_unsigned INT UNSIGNED, - id_mediumint MEDIUMINT, - id_mediumint_unsigned MEDIUMINT UNSIGNED, - id_smallint SMALLINT, - id_smallint_unsigned SMALLINT UNSIGNED, - id_tinyint TINYINT, - id_tinyint_unsigned TINYINT UNSIGNED, - id_tinyint_unsigned_max TINYINT UNSIGNED, - id_smallint_unsigned_max SMALLINT UNSIGNED, - id_mediumint_unsigned_max MEDIUMINT UNSIGNED, - id_mediumint_unsigned_signbit MEDIUMINT UNSIGNED, - id_int_unsigned_max INT UNSIGNED, - id_bigint_unsigned BIGINT UNSIGNED, - id_bigint_unsigned_signbit BIGINT UNSIGNED, - id_bigint_unsigned_max BIGINT UNSIGNED, - price_decimal DECIMAL(10,2), - amount_decimal_9_2 DECIMAL(9,2), - price_double DOUBLE, - price_double_precision DOUBLE, - price_float FLOAT, - price_numeric DECIMAL(10,2), - price_real DOUBLE, - name_char CHAR(50), - name_varchar VARCHAR(100), - name_text TEXT, - name_tinytext TINYTEXT, - name_mediumtext MEDIUMTEXT, - name_longtext LONGTEXT, - created_date DATETIME, - created_timestamp TIMESTAMP NULL, - is_active TINYINT(1), - long_varchar MEDIUMTEXT, - name_bool TINYINT(1) DEFAULT '1', - status ENUM('active','inactive','pending') DEFAULT NULL, - priority ENUM('low','medium','high') DEFAULT 'low', - name_latin1 VARCHAR(100) CHARACTER SET latin1,%s - tags SET('sports','music','gaming','reading') DEFAULT NULL, - permissions SET('read','write','execute') CHARACTER SET latin1 DEFAULT NULL, - PRIMARY KEY (id), - excludedColumn INT - )`, integrationTestTable, seedDDL) + query = fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", integrationTestTable, fmt.Sprintf(seedTableDDL, seedDDL)) case "drop": query = fmt.Sprintf("DROP TABLE IF EXISTS %s", integrationTestTable) @@ -168,15 +177,14 @@ func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.Te id_tinyint, id_tinyint_unsigned, id_tinyint_unsigned_max, id_smallint_unsigned_max, id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - id_bigint_unsigned, id_bigint_unsigned_signbit, id_bigint_unsigned_max, price_decimal, amount_decimal_9_2, price_double, price_double_precision, price_float, price_numeric, price_real, name_char, name_varchar, name_text, name_tinytext, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1,%s - tags, permissions, + %s + tags, excludedColumn ) VALUES ( 6, 6, 123456789012345, @@ -185,15 +193,14 @@ func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.Te 50, 51, 255, 65535, 16777215, 8388608, 4294967295, - 5003, 9223372036854775808, 18446744073709551615, 123.45, 5330197.27, 123.456, 123.456, 123.45, 123.45, 123.456, 'c', 'varchar_val', 'text_val', 'tinytext_val', 'mediumtext_val', 'longtext_val', '2023-01-01 12:00:00', '2023-01-01 12:00:00', 1, 'long_varchar_val', 1, 'active', 'high', - 'latin1_val',%s - 'sports,reading', 'read,write', + %s + 'sports,reading', 101 )`, integrationTestTable, seedCols, seedVals) _, err = db.ExecContext(ctx, query) @@ -207,15 +214,14 @@ func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.Te id_tinyint, id_tinyint_unsigned, id_tinyint_unsigned_max, id_smallint_unsigned_max, id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - id_bigint_unsigned, id_bigint_unsigned_signbit, id_bigint_unsigned_max, price_decimal, amount_decimal_9_2, price_double, price_double_precision, price_float, price_numeric, price_real, name_char, name_varchar, name_text, name_tinytext, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1,%s - tags, permissions, + %s + tags, excludedColumn ) VALUES ( -1, 999, 111111111111111, @@ -224,15 +230,14 @@ func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.Te 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 50.123, 50.12, 50.123, 50.123, 50.0, 50.123, 50.123, 'x', 'filtered_val', 'filtered text', 'filtered tiny', 'filtered medium', 'filtered long', '2022-06-15 10:00:00', '2021-06-15 10:00:00', 0, 'filtered long varchar', 0, 'inactive', 'low', - 'filtered latin1',%s - 'music', 'execute', + %s + 'music', 200 )`, integrationTestTable, seedCols, seedFilteredVals) _, err = db.ExecContext(ctx, filteredQuery) @@ -248,15 +253,14 @@ func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.Te id_tinyint, id_tinyint_unsigned, id_tinyint_unsigned_max, id_smallint_unsigned_max, id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - id_bigint_unsigned, id_bigint_unsigned_signbit, id_bigint_unsigned_max, price_decimal, amount_decimal_9_2, price_double, price_double_precision, price_float, price_numeric, price_real, name_char, name_varchar, name_text, name_tinytext, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1,%s - tags, permissions + %s + tags ) VALUES ( 7, 7, 123456789012345, 100, 4294967295, 102, 4294967294, @@ -264,15 +268,14 @@ func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.Te 50, 51, 255, 65535, 16777215, 8388608, 4294967295, - 5003, 9223372036854775808, 18446744073709551615, 123.45, 5330197.27, 123.456, 123.456, 123.45, 123.45, 123.456, 'c', 'varchar_val', 'text_val', 'tinytext_val', 'mediumtext_val', 'longtext_val', '2023-01-01 12:00:00', '2023-01-01 12:00:00', 1, 'long_varchar_val', 1, 'active', 'high', - 'latin1_val',%s - 'sports,reading', 'read,write' + %s + 'sports,reading' )`, integrationTestTable, seedCols, seedVals) case "update": @@ -288,9 +291,6 @@ func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.Te id_tinyint_unsigned_max = 254, id_smallint_unsigned_max = 65534, id_mediumint_unsigned_max = 16777214, id_mediumint_unsigned_signbit = 8388609, id_int_unsigned_max = 4294967294, - id_bigint_unsigned = 6003, - id_bigint_unsigned_signbit = 9223372036854775809, - id_bigint_unsigned_max = 18446744073709551614, price_decimal = 543.21, amount_decimal_9_2 = 1234567.89, price_double = 654.321, price_double_precision = 654.321, price_float = 543.21, price_numeric = 543.21, price_real = 654.321, @@ -301,8 +301,8 @@ func ExecuteQueryExcluding(ctx context.Context, t *testing.T, conf *testutils.Te created_timestamp = '2024-07-01 15:30:00', is_active = 0, long_varchar = 'updated long...', name_bool = 0, status = 'pending', priority = 'low', - name_latin1 = 'updated latin1',%s - tags = 'gaming,reading', permissions = 'read,write,execute', + %s + tags = 'gaming,reading', excludedColumn = 102, includedColumn = 202 WHERE id = 1`, integrationTestTable, seedUpdates) @@ -369,14 +369,13 @@ func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName st id_tinyint, id_tinyint_unsigned, id_tinyint_unsigned_max, id_smallint_unsigned_max, id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - id_bigint_unsigned, id_bigint_unsigned_signbit, id_bigint_unsigned_max, price_decimal, amount_decimal_9_2, price_double, price_double_precision, price_float, price_numeric, price_real, name_char, name_varchar, name_text, name_tinytext, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1,%s - tags, permissions, + %s + tags, excludedColumn ) VALUES ( %d, %d, 123456789012345, @@ -385,14 +384,13 @@ func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName st 50, 51, 255, 65535, 16777215, 8388608, 4294967295, - 5003, 9223372036854775808, 18446744073709551615, 123.45, 5330197.27, 123.456, 123.456, 123.45, 123.45, 123.456, 'c', 'varchar_val', 'text_val', 'tinytext_val', 'mediumtext_val', 'longtext_val', '2023-01-01 12:00:00', '2023-01-01 12:00:00', 1, 'long_varchar_val', 1, 'active', 'high', - 'latin1_val',%s - 'sports,reading', 'read,write', + %s + 'sports,reading', 100 )`, tableName, seedCols, i, i, seedVals) @@ -408,14 +406,13 @@ func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName st id_tinyint, id_tinyint_unsigned, id_tinyint_unsigned_max, id_smallint_unsigned_max, id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - id_bigint_unsigned, id_bigint_unsigned_signbit, id_bigint_unsigned_max, price_decimal, amount_decimal_9_2, price_double, price_double_precision, price_float, price_numeric, price_real, name_char, name_varchar, name_text, name_tinytext, name_mediumtext, name_longtext, created_date, created_timestamp, is_active, long_varchar, name_bool, status, priority, - name_latin1,%s - tags, permissions, + %s + tags, excludedColumn ) VALUES ( -1, 998, 111111111111111, @@ -424,14 +421,13 @@ func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName st 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 500234.123, 500234.12, 500234.123, 500234.123, 500234.0, 500234.123, 500234.123, 'x', 'filtered_val', 'filtered text', 'filtered tiny', 'filtered medium', 'filtered long', '2021-06-15 10:00:00', '2021-06-15 10:00:00', 0, 'filtered long varchar', 0, 'inactive', 'low', - 'filtered latin1',%s - 'music', 'execute', + %s + 'music', 200 )`, tableName, seedCols, seedFilteredVals) _, err := db.ExecContext(ctx, filteredQuery) diff --git a/tests/oracle/oracle_test.go b/tests/oracle/oracle_test.go index dc3259a7c..51db70e01 100644 --- a/tests/oracle/oracle_test.go +++ b/tests/oracle/oracle_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/compatibility" "github.com/datazip-inc/olake/tests/testutils/constants" "github.com/datazip-inc/olake/tests/testutils/integration" "github.com/datazip-inc/olake/tests/testutils/require" @@ -11,8 +12,8 @@ import ( // oracleBaseConfig returns an IntegrationTest pre-populated with all fields shared // by the oracle suites. -func oracleBaseConfig(t *testing.T) *integration.Test { - cfg, err := testutils.NewTestConfig(t, constants.Oracle, "MYUSER", "oracle_myuser", ExecuteQuery) +func oracleBaseConfig(t *testing.T, opts ...testutils.TestConfigOption) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.Oracle, "MYUSER", "oracle_myuser", ExecuteQuery, opts...) require.NoError(t, err, "failed to build the test config") cfg.CursorField = "COL_CURSOR:COL_SMALLINT" cfg.PartitionRegex = "/{id, identity}" @@ -60,15 +61,13 @@ func TestOracle2PC(t *testing.T) { // TestOracleCompatibility pins the backward-compatibility contract: the same scenarios run on a released // baseline image and on this build after the initial load, and the destinations must match. // See tests/testutils/compatibility.go. -// func TestOracleCompatibility(t *testing.T) { -// t.Parallel() -// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { -// base := oracleBaseConfig(t) -// base.ExpectedUpdatedData = ExpectedUpdatedOracleData -// base.UpdatedDestinationDataTypeSchema = UpdatedOracleToDestinationSchema -// cfg := &compatibility.Test{IntegrationTest: base} -// // No floor declared: oracle images exist for every sweep baseline. If a first sweep finds -// // an unrunnable band, declare it here with its reason. -// return cfg -// }) -// } +func TestOracleCompatibility(t *testing.T) { + t.Parallel() + fixture := &compatibility.Test{ + NewConfig: func(t *testing.T, version string) *testutils.TestConfig { + return oracleBaseConfig(t, testutils.WithDriverVersion(version)).TestConfig + }, + DeclaredSchema: OracleToDestinationSchema, + } + fixture.RunBackwardCompatibility(t) +} diff --git a/tests/postgres/postgres_test.go b/tests/postgres/postgres_test.go index c8994089d..a7518aea3 100644 --- a/tests/postgres/postgres_test.go +++ b/tests/postgres/postgres_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/compatibility" "github.com/datazip-inc/olake/tests/testutils/constants" "github.com/datazip-inc/olake/tests/testutils/integration" "github.com/datazip-inc/olake/tests/testutils/performance" @@ -13,8 +14,8 @@ import ( // postgresBaseConfig returns an IntegrationTest pre-populated with all fields shared // by the postgres suites. -func postgresBaseConfig(t *testing.T) *integration.Test { - cfg, err := testutils.NewTestConfig(t, constants.Postgres, "public", "postgres_postgres_public", ExecuteQuery) +func postgresBaseConfig(t *testing.T, opts ...testutils.TestConfigOption) *integration.Test { + cfg, err := testutils.NewTestConfig(t, constants.Postgres, "public", "postgres_postgres_public", ExecuteQuery, opts...) require.NoError(t, err, "failed to build the test config") cfg.CursorField = "col_cursor:col_int" cfg.PartitionRegex = "/{col_bigserial,identity}" @@ -77,15 +78,15 @@ func TestPostgresPerformance(t *testing.T) { // parallel -- once entirely on a released baseline image, once handing off to this build after the // initial load -- and the two destinations must match. The baseline defaults to the newest // release; OLAKE_COMPATIBILITY_BASELINE picks another tag, image or commit. See tests/testutils/compatibility.go. -// func TestPostgresCompatibility(t *testing.T) { -// t.Parallel() -// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { -// base := postgresBaseConfig(t) -// base.ExpectedUpdatedData = ExpectedUpdatedData -// base.UpdatedDestinationDataTypeSchema = UpdatedPostgresToDestinationSchema -// cfg := &compatibility.Test{IntegrationTest: base} -// // No column rules: postgres compares clean on every reachable baseline (COMPAT_RESULTS_v2.md). -// // The OLAKE_COMPATIBILITY_EXCLUDE_COLUMNS sweep hook lives in RunBackwardCompatibility now. -// return cfg -// }) -// } +func TestPostgresCompatibility(t *testing.T) { + t.Parallel() + // No column rules: postgres compares clean on every reachable baseline (COMPAT_RESULTS_v2.md). + fixture := &compatibility.Test{ + NewConfig: func(t *testing.T, version string) *testutils.TestConfig { + return postgresBaseConfig(t, testutils.WithDriverVersion(version)).TestConfig + }, + DeclaredSchema: PostgresToDestinationSchema, + CDCColumnsSchema: ExpectedPostgresDefaultCDCColumnsSchema, + } + fixture.RunBackwardCompatibility(t) +} diff --git a/tests/s3/s3_test.go b/tests/s3/s3_test.go index 53641f012..18401d532 100644 --- a/tests/s3/s3_test.go +++ b/tests/s3/s3_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/compatibility" "github.com/datazip-inc/olake/tests/testutils/constants" "github.com/datazip-inc/olake/tests/testutils/integration" "github.com/datazip-inc/olake/tests/testutils/require" @@ -11,9 +12,9 @@ import ( // s3BaseConfig returns an IntegrationTest for one source format variant. Each variant owns a // testdata// directory, which is what DataFormat selects. -func s3BaseConfig(t *testing.T, variant S3TestVariant) *integration.Test { +func s3BaseConfig(t *testing.T, variant S3TestVariant, opts ...testutils.TestConfigOption) *integration.Test { config, err := testutils.NewTestConfig(t, constants.S3, "s3", S3DestinationDB, nil, - testutils.WithDataFormat(variant.DataFormat)) + append([]testutils.TestConfigOption{testutils.WithDataFormat(variant.DataFormat)}, opts...)...) require.NoError(t, err, "failed to build the test config") config.ColumnToExclude = excludedColumn config.CursorField = S3CursorField @@ -56,43 +57,19 @@ func TestS3Sync(t *testing.T) { // TestS3Compatibility runs every source format. Each variant owns its testdata directory, source prefix // and stream name, so the three share one destination namespace without colliding. -// func TestS3Compatibility(t *testing.T) { -// t.Parallel() -// for _, variant := range S3TestVariants { -// t.Run(variant.Name, func(t *testing.T) { -// t.Parallel() -// compatibility.RunBackwardCompatibility(t, func() *compatibility.Test { -// base := s3BaseConfig(t, variant) -// cfg := &compatibility.Test{IntegrationTest: base} -// // Same isolation TestS3Sync applies: Parquet and ParquetInMemory share a -// // DataFormat, so without it they share every name the suite derives from it. -// cfg.IntegrationTest.Suite = variant.Name -// // Type tags for compatibility_rules.json's s3 rules; the driver-level _olake_id and -// // _last_modified_time policies are column-keyed there and need no tags. -// switch variant.DataFormat { -// case "json": -// cfg.ColumnTypes = map[string][]string{"mixed_col": {"mixed"}} -// case "csv": -// cfg.ColumnTypes = map[string][]string{evolvedColumn: {"evolved"}} -// case "parquet": -// cfg.ColumnTypes = map[string][]string{ -// "map_col": {"map"}, -// "struct_col": {"struct"}, -// "list_col": {"list"}, -// "int96_col": {"int96"}, -// "ts_col": {"timestamp"}, -// "ts_ms_col": {"timestamp"}, -// "ts_ns_col": {"timestamp"}, -// "ts_far_col": {"timestamp"}, -// "uuid_col": {"uuid"}, -// } -// } -// // The closure reads SeedExcludedColumns at call time; RunBackwardCompatibility fills it -// // in after resolving the rules above against the baseline. -// cfg.SupportsSeedExclusion = true -// base.TestConfig.ExecuteQuery = ExecuteQueryFactoryExcluding(variant, cfg.IntegrationTest, func() []string { return cfg.SeedExcludedColumns }) -// return cfg -// }) -// }) -// } -// } +func TestS3Compatibility(t *testing.T) { + t.Parallel() + for _, variant := range S3TestVariants { + t.Run(variant.Name, func(t *testing.T) { + t.Parallel() + fixture := &compatibility.Test{ + DeclaredSchema: variant.DestinationSchema, + ColumnTypes: variant.ColumnTypes(), + } + fixture.NewConfig = func(t *testing.T, version string) *testutils.TestConfig { + return s3BaseConfig(t, variant, testutils.WithDriverVersion(version)).TestConfig + } + fixture.RunBackwardCompatibility(t) + }) + } +} diff --git a/tests/s3/s3_util_test.go b/tests/s3/s3_util_test.go index 21a23c2c0..0b4321dd9 100644 --- a/tests/s3/s3_util_test.go +++ b/tests/s3/s3_util_test.go @@ -744,20 +744,11 @@ func (v S3TestVariant) applyParquetStreamingMode(t *testing.T, config *testutils // the variant's path prefix: "create" ensures the bucket exists, "add" seeds the stream, // "insert"/"update" upload a further file each, and "clean"/"drop" remove everything under // the prefix. +// Columns named in conf.SeedExcludedColumns are left out of the files this uploads entirely. func ExecuteQueryFactory(variant S3TestVariant, cfg *integration.Test) func(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { - return ExecuteQueryFactoryExcluding(variant, cfg, nil) -} - -// ExecuteQueryFactoryExcluding is ExecuteQueryFactory with the compatibility suite's seed exclusions: -// seedExcluded is read per call, because RunBackwardCompatibility fills the list in after the config is -// built. A nil getter is the plain fixture, every column seeded. -func ExecuteQueryFactoryExcluding(variant S3TestVariant, cfg *integration.Test, seedExcluded func() []string) func(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { return func(ctx context.Context, t *testing.T, conf *testutils.TestConfig, operation string) { t.Helper() - var excluded []string - if seedExcluded != nil { - excluded = seedExcluded() - } + excluded := conf.SeedExcludedColumns // Every destination block starts by re-seeding the source through this hook, so // refreshing the expectations here keeps them aligned with whichever writer the @@ -1306,3 +1297,62 @@ func gzipBytes(t *testing.T, data []byte) []byte { require.NoError(t, writer.Close(), "failed to close gzip writer") return buf.Bytes() } + +// ColumnTypes derives type tags for the variant's seed columns, which a data_types rule in +// compatibility_rules.json resolves against. Parquet is the one format with a declared schema, +// parquetTestGroup, so its tags are read off that; a text file's only source type is what the +// driver infers, which DeclaredSchema already carries. +func (v S3TestVariant) ColumnTypes() map[string][]string { + if v.DataFormat != "parquet" { + return nil + } + types := map[string][]string{} + for column, node := range parquetTestGroup() { + types[column] = parquetNodeTags(node) + } + return types +} + +// parquetNodeTags names a leaf by its physical kind and logical type, with the width, sign or time +// unit that tells one apart (uint32, timestamp(nanos)); a group by its shape. +func parquetNodeTags(node pq.Node) []string { + logical := node.Type().LogicalType() + if !node.Leaf() { + switch { + case logical != nil && logical.Map != nil: + return []string{"map"} + case logical != nil && logical.List != nil: + return []string{"list"} + } + return []string{"struct"} + } + tags := []string{strings.ToLower(node.Type().Kind().String())} + if logical == nil { + return tags + } + switch { + case logical.UTF8 != nil: + tags = append(tags, "string") + case logical.Enum != nil: + tags = append(tags, "enum") + case logical.Decimal != nil: + tags = append(tags, "decimal") + case logical.Date != nil: + tags = append(tags, "date") + case logical.Time != nil: + tags = append(tags, "time", "time("+strings.ToLower(logical.Time.Unit.String())+")") + case logical.Timestamp != nil: + tags = append(tags, "timestamp", "timestamp("+strings.ToLower(logical.Timestamp.Unit.String())+")") + case logical.Integer != nil: + sign := "" + if !logical.Integer.IsSigned { + sign = "u" + } + tags = append(tags, fmt.Sprintf("%sint%d", sign, logical.Integer.BitWidth)) + case logical.Json != nil: + tags = append(tags, "json") + case logical.UUID != nil: + tags = append(tags, "uuid") + } + return tags +} diff --git a/tests/testutils/compatibility/compatibility.go b/tests/testutils/compatibility/compatibility.go new file mode 100644 index 000000000..e6eba01e0 --- /dev/null +++ b/tests/testutils/compatibility/compatibility.go @@ -0,0 +1,555 @@ +package compatibility + +// Backward-compatibility suite. +// +// The contract being tested is docs/backward-compatibility.md: upgrading the OLake binary must not +// change the records or the column types an existing pipeline produces. The state file's `version` +// pins that, so a candidate binary reading a state file an older binary wrote must keep the older +// binary's semantics. +// +// Rather than encode per-version expectations -- which rot, and which nobody remembers to add when +// the latest state version is bumped -- this suite runs the same scenario twice, concurrently: +// +// reference : every sync on the BASELINE image +// upgrade : the stateless initial load on the BASELINE image, every --state sync after it on +// the CANDIDATE image +// +// and then asserts the two destinations are indistinguishable. The reference run IS the +// expectation. A gate that stopped firing, a type map that shifted, a state key that got renamed: +// each shows up as a diff between two tables, with no expectation file to maintain. +// +// What this does NOT cover, deliberately: discover output (both runs are seeded from the same +// frozen test_streams.json, so they differ only in the binary -- and discover is ungated by design, +// see A4 in the doc); the reverse direction (a new state file fed to an old image is not a +// supported operation); and any gate older than the baseline being tested. + +import ( + "context" + "fmt" + "maps" + "os" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/apache/spark-connect-go/v35/spark/sql" + "github.com/apache/spark-connect-go/v35/spark/sql/types" + "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/require" +) + +const ( + // compatibilityBaselineEnvVar names the baseline to test the local build against, replacing the + // manifest sweep with a single run. Three forms are accepted, see resolveBaselineImage; + // per-driver overrides use the suffixed form, OLAKE_COMPATIBILITY_BASELINE_POSTGRES. + compatibilityBaselineEnvVar = "OLAKE_COMPATIBILITY_TEST_BASELINE" + + // compatibilityExcludeColumnsEnvVar appends catalog-level column exclusions to every compatibility run, a + // sweep affordance for probing a baseline without editing the driver's rules. + compatibilityExcludeColumnsEnvVar = "OLAKE_COMPATIBILITY_EXCLUDE_COLUMNS" +) + +// Test is the basic compatibility check: one scenario run on two images, whose destinations must +// be indistinguishable. A driver declares one -- NewConfig plus its two-image vocabulary -- and +// the runner fills Reference and Upgrade per variant: the reference config runs the baseline end +// to end, the upgrade config writes its stateless load on the baseline and every stateful sync +// after it on the candidate. +type Test struct { + // NewConfig builds one side's TestConfig from the subtest it runs in; the suite derived from + // t.Name() is what isolates the sides ((baseline x group x variant x side)). + NewConfig func(t *testing.T, DriverVersion string) *testutils.TestConfig + + // DeclaredSchema is the driver's column -> destination-type map, what a data_types rule in + // compatibility_rules.json resolves against. The sync suite asserts the same map, so it cannot + // drift from the fixture. + DeclaredSchema map[string]string + + // ColumnTypes tags columns with what DeclaredSchema cannot express (a charset, a modifier), + // derived by the fixture from its own seed DDL; a data_types rule selects on both. + ColumnTypes map[string][]string + + // CDCColumnsSchema names the driver's CDC metadata columns, which carry source-log coordinates + // and so are compared by type but never by value (see volatileColumns). + CDCColumnsSchema map[string]string +} + +// Validate checks the fixture wired everything a compatibility run reads. +func (f *Test) Validate(t *testing.T) { + t.Helper() + require.NotNil(t, f.NewConfig, "compatibility.Test.NewConfig is not set") + require.NotEmpty(t, f.DeclaredSchema, "compatibility.Test.DeclaredSchema is not set; type-keyed rules would resolve against nothing") +} + +// RunBackwardCompatibility runs one driver's scenarios twice -- a reference run entirely on the baseline +// image and an upgrade run that hands off to the candidate after the initial load -- then asserts +// the two destinations match. Both sides of all three writer groups (iceberg legacy, iceberg +// arrow, parquet) run in parallel, six isolated pipelines at once. +// +// newConfig MUST build a fresh Test from the t it is handed: the suite -- and so every path and +// name the side owns -- derives from that subtest's name. +// RunBackwardCompatibility runs the compatibility scenarios against every baseline the manifest lists, +// oldest first, stopping at the first that fails -- later baselines are newer code and would only +// repeat it. A single explicit baseline runs on its own, without the extra subtest level. +func (f *Test) RunBackwardCompatibility(t *testing.T) { + f.Validate(t) + + currentConf := f.NewConfig(t, testutils.CurrentDriverVersion) + baselineVersions, err := getCompatibilityBaselines(t, currentConf.OlakeRootPath, currentConf.Driver) + require.NoError(t, err) + + for _, version := range baselineVersions { + baselineConf := f.NewConfig(t, version) + if !t.Run(baselineConf.DriverVersion, func(t *testing.T) { f.runCompatibilityBaseline(t, baselineConf, currentConf) }) { + t.Logf("compatibility: stopping the sweep at %s; the later baselines carry newer code and would repeat it", version) + return + } + } +} + +// runCompatibilityBaseline runs every writer group's variants against one baseline: the reference +// side on baseline's image throughout, the upgrade side handing its stateful syncs to upgrade's. +func (f *Test) runCompatibilityBaseline(t *testing.T, baseline, upgrade *testutils.TestConfig) { + spec := baseline.DriverVersion + driver, dataFormat := baseline.Driver, baseline.DataFormat + + // The driver's own floor. A skip, not a failure: the driver declares it cannot run against + // releases this old (the why lives next to the declaration in compatibility_rules.json), and + // that limitation is data, not a regression. + baselineVersion, baselineDated := parseReleaseTag(spec) + floorTag, err := compatibilityGlobalFloor(baseline.OlakeRootPath) + require.NoError(t, err) + globalFloor, _ := parseReleaseTag(floorTag) + if baselineDated && compareRelease(baselineVersion, globalFloor) < 0 { + t.Skipf("baseline %s predates %s, the oldest state-version baseline; the compatibility suite does not run below it", + spec, floorTag) + } + driverRules := compatibilityRules.Drivers[driver] + variantRules := driverRules.Variants[dataFormat] + for _, scoped := range []struct { + scope string + gate compatibilityGate + }{ + {driver, driverRules.compatibilityGate}, + {driver + "/" + dataFormat, variantRules.compatibilityGate}, + } { + if reason := scoped.gate.skipReason(baselineVersion, baselineDated); reason != "" { + t.Skipf("%s cannot run baseline %s: %s (compatibility_rules.json: %s)", + scoped.scope, spec, reason, scoped.gate.Note) + } + } + + // Both images were pulled or built when the caller constructed the two configs, serially, + // before any parallel child starts; the sides below only re-derive the same refs. + baselineImage, candidateImage := baseline.GetDriverImage(), upgrade.GetDriverImage() + require.NotEqualf(t, baselineImage, candidateImage, + "the compatibility baseline and the candidate resolve to the same image (%s); the run would compare it with itself and pass", baselineImage) + t.Logf("compatibility: baseline %s -> candidate %s", baselineImage, candidateImage) + + // Column policies: the baseline's era decides what each column can be asserted on. Applied to + // both sides, so a diff is always the binary and never the fixture. + if declared := driverRules.Variants; len(declared) > 0 { + formats := slices.Sorted(maps.Keys(declared)) + if !slices.Contains(formats, dataFormat) { + t.Logf("NOTE: %s runs data format %q, which compatibility_rules.json does not declare (declared: %v); no variant rule or gate applies to this run.", + driver, dataFormat, formats) + } else { + t.Logf("compatibility: %s declares data formats %v; this run is %q", driver, formats, dataFormat) + } + } + + // Every rule -- type-keyed, column-keyed, dated, unconditional -- resolves here into the one + // policy set the run applies: seeding, catalog and comparison all read it, nothing re-derives. + policies, err := resolveAssertionPolicies(f, spec, floorTag, globalFloor, driverRules, variantRules) + require.NoError(t, err) + for _, note := range policies.notes { + t.Logf("compatibility: %s", note) + } + + // Writer-level gates: a group whose writer has a known bounded regression against this + // baseline is left out, and says so -- the other writers keep their coverage instead of the + // whole baseline being dropped. + var groups []compatibilityGroup + for _, g := range compatibilityVariantGroups(driver) { + if reason := g.gate.skipReason(baselineVersion, baselineDated); reason != "" { + t.Logf("compatibility: writer group %s not run against this baseline: %s", g.name, reason) + continue + } + groups = append(groups, g) + } + require.NotEmpty(t, groups, "no compatibility scenarios for driver %s against this baseline", driver) + + // Each variant runs as its own pair of parallel subtests -- reference entirely on the + // baseline, upgrade handing off to the candidate after the stateless load -- and is compared + // as soon as both sides finish. Every side builds its config inside its own subtest, so the + // suite t.Name() derives is what isolates the pipelines: (baseline x group x variant x side). + referencePick := func(bool) string { return baseline.DriverVersion } + upgradePick := func(useState bool) string { + // useState is the upgrade boundary: the stateless initial load writes the state file on + // the baseline binary, and every sync after it reads that file on the candidate. + return testutils.Ternary(useState, upgrade.DriverVersion, baseline.DriverVersion).(string) + } + // Whichever side fails first stops every group at its next variant boundary: the comparison + // is skipped either way, so the remaining syncs would be minutes of output nothing reads. + aborted := &atomic.Bool{} + // Subtest names double as suite segments, so they stay terse: the table and (postgres) slot + // names built from the suite must clear a 63-byte identifier limit on sweep runs. + completed := t.Run("g", func(t *testing.T) { + for _, g := range groups { + runGroup := func(t *testing.T) { + for _, v := range g.variants { + if aborted.Load() { + t.Logf("compatibility group %s: skipping variant %q onwards; another run already failed", g.name, v.name) + break + } + ok := t.Run(v.name, func(t *testing.T) { + // Both sides start on the baseline version; pick moves the upgrade side's + // stateful syncs to the candidate. + var ref, upg *testutils.TestConfig + if !t.Run("run", func(t *testing.T) { + t.Run("ref", func(t *testing.T) { + t.Parallel() + ref = f.NewConfig(t, baseline.DriverVersion) + runSide(t, ref, g, v, referencePick, policies) + }) + t.Run("upg", func(t *testing.T) { + t.Parallel() + upg = f.NewConfig(t, baseline.DriverVersion) + runSide(t, upg, g, v, upgradePick, policies) + }) + }) { + t.Fatalf("a %s side failed; the comparison would be noise", v.name) + } + compareVariant(t, policies, ref, upg, g, v) + }) + if !ok { + aborted.Store(true) + t.Logf("compatibility group %s: stopping after variant %q", g.name, v.name) + break + } + } + } + t.Run(g.name, func(t *testing.T) { t.Parallel(); runGroup(t) }) + } + }) + require.True(t, completed, "a compatibility run failed") +} + +// getCompatibilityBaselines returns the baselines to run driver against: the +// OLAKE_COMPATIBILITY_TEST_BASELINE override alone, else every release in `state-versions.json` +// whose bump gated this driver, oldest first -- a bump that touched only other drivers changed +// nothing this driver's state file pins, so it is logged and left out. +func getCompatibilityBaselines(t *testing.T, rootPath, driver string) ([]string, error) { + t.Helper() + if spec := os.Getenv(compatibilityBaselineEnvVar); spec != "" { + return []string{spec}, nil + } + baselines, err := testutils.StateVersionBaselines(rootPath) + if err != nil { + return nil, err + } + slices.SortFunc(baselines, func(a, b testutils.StateVersionBaseline) int { return a.StateVersion - b.StateVersion }) + specs := make([]string, 0, len(baselines)) + for _, baseline := range baselines { + if !baseline.Gates(driver) { + t.Logf("compatibility: baseline %s not run for %s; state version %d gates only %s", baseline.ReleaseTag, driver, baseline.StateVersion, baseline.Drivers) + continue + } + // One release can cover several state versions (a release that jumps the manifest by + // more than one carries every version it skipped), and running it twice proves nothing. + if !slices.Contains(specs, baseline.ReleaseTag) { + specs = append(specs, baseline.ReleaseTag) + } + } + return specs, nil +} + +// compatibilityGroupSpecs is the writer-group fan-out: one group per destination writer, each +// naming the destination config its syncs run against and where its gates live in +// compatibility_rules.json (mode is empty for a destination that has none). +func compatibilityGroupSpecs() []compatibilityGroupSpec { + return []compatibilityGroupSpec{ + {name: "legacy", destination: "iceberg", mode: "legacy", destinationFile: "iceberg_destination.json"}, + {name: "arrow", destination: "iceberg", mode: "arrow", destinationFile: "iceberg_destination_arrow.json"}, + {name: "pq", destination: "parquet", destinationFile: "parquet_destination.json"}, + } +} + +// gateFrom picks this group's gate out of a destinations block: the destination's own gate, and +// its mode's gate when the group names one. +func (s compatibilityGroupSpec) gateFrom(destinations map[string]compatibilityDestination) compatibilityGate { + dest := destinations[s.destination] + if s.mode == "" { + return dest.compatibilityGate + } + return mergedGate(dest.compatibilityGate, dest.Modes[s.mode]) +} + +func compatibilityVariantGroups(driver string) []compatibilityGroup { + // Same fan-out as TestSync, and the same two skips. + cdc := !slices.Contains(constants.SkipCDCDrivers, constants.DriverType(driver)) + inc := driver != string(constants.Kafka) + + driverDestinations := compatibilityRules.Drivers[driver].Destinations + var groups []compatibilityGroup + for _, spec := range compatibilityGroupSpecs() { + var variants []compatibilityVariant + if cdc { + // The parquet CDC scenario ends on a delete-only batch, and parquet holds only its + // last case's files -- so both sides ending with none is its verified outcome. + variants = append(variants, compatibilityVariant{name: "cdc", kind: scenarioCDC, emptyFinalState: spec.destination == "parquet"}) + } + if inc { + variants = append(variants, compatibilityVariant{name: "inc", kind: scenarioIncremental}) + } + if len(variants) == 0 { + continue + } + gate := mergedGate(spec.gateFrom(compatibilityRules.Destinations.gates()), spec.gateFrom(driverDestinations)) + groups = append(groups, compatibilityGroup{compatibilityGroupSpec: spec, gate: gate, variants: variants}) + } + return groups +} + +// compareVariant asserts the upgrade run's destination for one scenario is indistinguishable from +// the reference run's. +func compareVariant(t *testing.T, policies *assertionPolicies, ref, upg *testutils.TestConfig, g compatibilityGroup, v compatibilityVariant) { + ctx := t.Context() + spark, err := testutils.SparkSession(ctx, t) + require.NoError(t, err, "failed to connect to Spark Connect server") + + refDB, upgDB := ref.DestinationDB, upg.DestinationDB + refTable, upgTable := ref.GetTableName(), upg.GetTableName() + var refRel, upgRel string + switch g.destination { + case "iceberg": + refRel = icebergRelation(ctx, t, spark, refDB, refTable) + upgRel = icebergRelation(ctx, t, spark, upgDB, upgTable) + case "parquet": + refRel = parquetRelation(ctx, t, spark, refDB, refTable, "ref") + upgRel = parquetRelation(ctx, t, spark, upgDB, upgTable, "upg") + // Absence is a comparable state, so it is asserted rather than skipped over. One side + // absent is a genuine finding: the binaries disagree about whether this case writes + // output. Both sides absent is the verified outcome for a variant that ENDS empty + // (emptyFinalState), and a shared failure to produce rows for any other -- the one shape + // of regression a row diff can never catch, because there are no rows to diff. + if refRel == "" || upgRel == "" { + require.Equalf(t, refRel == "", upgRel == "", + "only one run produced parquet files for %s (reference %q, upgrade %q): the binaries disagree about whether this case writes output", v.name, refDB, upgDB) + require.Truef(t, v.emptyFinalState, + "neither run left parquet files for %s (reference %q, upgrade %q), but its last case writes rows: both binaries produced nothing where output is expected", v.name, refDB, upgDB) + t.Logf("verified: neither run leaves parquet files for %s -- its last case is a delete-only batch, which writes none", v.name) + return + } + default: + t.Fatalf("unknown destination %q", g.destination) + } + + compareRelations(ctx, t, spark, refRel, upgRel, policies.typeOnly) +} + +// icebergRelation refreshes and returns the fully-qualified name of an Iceberg table: the shared +// Spark session caches snapshots, so a table written after it was built reads as empty without it. +func icebergRelation(ctx context.Context, t *testing.T, spark sql.SparkSession, db, table string) string { + name := fmt.Sprintf("%s.%s.%s", testutils.IcebergCatalog, db, table) + _, err := spark.Sql(ctx, "REFRESH TABLE "+name) + require.NoErrorf(t, err, "failed to refresh %s -- the run may not have produced it", name) + return name +} + +// parquetRelation stands a temp view over one side's parquet output; "" means the side wrote no +// files, which the caller treats as a comparable state (see the emptyFinalState assertion). +// Do NOT SET spark.sql.parquet.mergeSchema on this session: it breaks every later direct file query +// (UNSUPPORTED_DATASOURCE_FOR_DIRECT_QUERY), VerifyParquetSync's included. +func parquetRelation(ctx context.Context, t *testing.T, spark sql.SparkSession, db, table, side string) string { + view := fmt.Sprintf("`compatibility_%s_%s`", side, table) + path := fmt.Sprintf("s3a://%s/%s/%s", testutils.ParquetBucket, db, table) + _, err := spark.Sql(ctx, fmt.Sprintf("CREATE OR REPLACE TEMP VIEW %s AS SELECT * FROM parquet.`%s/*.parquet`", view, path)) + if err != nil { + require.Containsf(t, err.Error(), "PATH_NOT_FOUND", "failed to read parquet at %s", path) + return "" + } + t.Cleanup(func() { _, _ = spark.Sql(ctx, "DROP VIEW IF EXISTS "+view) }) + return view +} + +// compareRelations is the assertion. Order matters: a schema mismatch has to be reported before a +// row query that would fail confusingly because of it. +func compareRelations(ctx context.Context, t *testing.T, spark sql.SparkSession, refRel, upgRel string, volatile []string) { + // 1. Non-vacuity FIRST. Two empty tables satisfy every diff below, and an empty reference is a + // plausible outcome, not a far-fetched one: a stream the baseline binary could not validate + // is skipped with a Warn and the sync still exits 0 (protocol/sync.go, D3 in the doc). Without + // this guard that scenario reports a green. + refCount := scalarCount(ctx, t, spark, "SELECT COUNT(*) AS n FROM "+refRel) + require.Greaterf(t, refCount, int64(0), + "the reference run produced no rows in %s; it is the source of truth, so an empty one makes the whole comparison vacuous (a silently skipped stream looks exactly like this)", refRel) + upgCount := scalarCount(ctx, t, spark, "SELECT COUNT(*) AS n FROM "+upgRel) + require.Equalf(t, refCount, upgCount, "row count differs: reference %s has %d, upgrade %s has %d", refRel, refCount, upgRel, upgCount) + + // 2. Schema. Compared as a map, so a column order difference (schema evolution appends in + // record-arrival order) is not a failure while an added, dropped or retyped column is. This + // is the assertion that catches a type-mapping change -- I6 in the doc. + refSchema := describeRelation(ctx, t, spark, refRel) + upgSchema := describeRelation(ctx, t, spark, upgRel) + require.Equalf(t, refSchema, upgSchema, + "destination schema differs between the reference and upgrade runs.\n reference (%s): %v\n upgrade (%s): %v", refRel, refSchema, upgRel, upgSchema) + + // 3. Per-op-type counts, so a row diff reads as "5 'u' rows where the reference had 6" rather + // than an opaque set difference. + require.Equal(t, opTypeCounts(ctx, t, spark, refRel), opTypeCounts(ctx, t, spark, upgRel), + "per-_op_type row counts differ between the reference and upgrade runs") + + // 4. Values, both directions. This is the assertion that catches a changed record: every + // non-volatile column of every row must hold the same value on both sides. + cols := comparableColumns(refSchema, volatile) + require.NotEmpty(t, cols, "every column is volatile; there is nothing left to compare by value") + t.Logf("comparing values of %d rows over %d columns (%d volatile, type-checked only)", refCount, len(cols), len(volatile)) + + onlyInRef := rowsOnlyIn(ctx, t, spark, refRel, upgRel, cols) + onlyInUpg := rowsOnlyIn(ctx, t, spark, upgRel, refRel, cols) + if len(onlyInRef) == 0 && len(onlyInUpg) == 0 { + t.Logf("values identical: all %d rows match on all %d compared columns", refCount, len(cols)) + return + } + + // Name the columns that actually differ before dumping rows -- with 30-odd columns, a row dump + // alone leaves you diffing two long tuples by eye. + reportColumnDiffs(ctx, t, spark, refRel, upgRel, cols) + logSampleRows(t, "only in the reference run", refRel, onlyInRef) + logSampleRows(t, "only in the upgrade run", upgRel, onlyInUpg) + t.Fatalf("row values differ between the reference and upgrade runs: %d row(s) only in %s, %d row(s) only in %s", + len(onlyInRef), refRel, len(onlyInUpg), upgRel) +} + +// reportColumnDiffs names the columns whose values differ, with a sample from each side. Runs one +// query per column, so it is called only after a diff has already been found. +func reportColumnDiffs(ctx context.Context, t *testing.T, spark sql.SparkSession, refRel, upgRel string, cols []string) { + for _, col := range cols { + n := scalarCount(ctx, t, spark, fmt.Sprintf( + "SELECT COUNT(*) AS n FROM (SELECT %s FROM %s EXCEPT ALL SELECT %s FROM %s)", col, refRel, col, upgRel)) + if n == 0 { + continue + } + t.Logf(" column %s differs in %d row(s)", col, n) + t.Logf(" reference: %v", sampleColumn(ctx, spark, refRel, col)) + t.Logf(" upgrade: %v", sampleColumn(ctx, spark, upgRel, col)) + } +} + +// sampleColumn returns up to three values of one column, for a failure message. +func sampleColumn(ctx context.Context, spark sql.SparkSession, relation, col string) []any { + df, err := spark.Sql(ctx, fmt.Sprintf("SELECT %s AS v FROM %s LIMIT 3", col, relation)) + if err != nil { + return nil + } + rows, err := df.Collect(ctx) + if err != nil { + return nil + } + values := make([]any, 0, len(rows)) + for _, row := range rows { + values = append(values, row.Value("v")) + } + return values +} + +func logSampleRows(t *testing.T, what, relation string, rows []types.Row) { + for i, row := range rows { + if i == 5 { + t.Logf(" ... and %d more %s", len(rows)-5, what) + break + } + t.Logf(" %s (%s): %v", what, relation, row) + } +} + +// comparableColumns is the sorted, back-quoted projection compared by value. +func comparableColumns(schema map[string]string, volatile []string) []string { + var cols []string + for col := range schema { + if !slices.Contains(volatile, col) { + cols = append(cols, "`"+col+"`") + } + } + slices.Sort(cols) + return cols +} + +// rowsOnlyIn returns the rows of `left` that `right` does not hold, comparing every column in +// cols by value. +// +// EXCEPT ALL, not EXCEPT: the plain form is DISTINCT-based and would hide a duplicate-row +// regression (five identical rows reading as equal to six). The EXCEPT family is also NULL-safe, +// which a join-based diff would not be, and these tables are full of nullable columns. +func rowsOnlyIn(ctx context.Context, t *testing.T, spark sql.SparkSession, left, right string, cols []string) []types.Row { + projection := strings.Join(cols, ", ") + query := fmt.Sprintf("SELECT %s FROM %s EXCEPT ALL SELECT %s FROM %s", projection, left, projection, right) + df, err := spark.Sql(ctx, query) + require.NoErrorf(t, err, "failed to diff %s against %s", left, right) + rows, err := df.Collect(ctx) + require.NoError(t, err, "failed to collect the row diff") + return rows +} + +func scalarCount(ctx context.Context, t *testing.T, spark sql.SparkSession, query string) int64 { + df, err := spark.Sql(ctx, query) + require.NoErrorf(t, err, "failed to run %q", query) + rows, err := df.Collect(ctx) + require.NoErrorf(t, err, "failed to collect %q", query) + require.NotEmpty(t, rows, "no result for %q", query) + n, ok := rows[0].Value("n").(int64) + require.Truef(t, ok, "count is not int64: %T", rows[0].Value("n")) + return n +} + +func describeRelation(ctx context.Context, t *testing.T, spark sql.SparkSession, relation string) map[string]string { + df, err := spark.Sql(ctx, "DESCRIBE TABLE "+relation) + require.NoErrorf(t, err, "failed to describe %s", relation) + rows, err := df.Collect(ctx) + require.NoErrorf(t, err, "failed to collect the description of %s", relation) + + schema := make(map[string]string, len(rows)) + for _, row := range rows { + col, _ := row.Value("col_name").(string) + dataType, _ := row.Value("data_type").(string) + // DESCRIBE appends partition/metadata sections, all introduced by a "#" heading. + if col != "" && !strings.HasPrefix(col, "#") { + schema[col] = dataType + } + } + return schema +} + +func opTypeCounts(ctx context.Context, t *testing.T, spark sql.SparkSession, relation string) map[string]int64 { + query := fmt.Sprintf("SELECT `_op_type` AS op, COUNT(*) AS n FROM %s GROUP BY 1", relation) + df, err := spark.Sql(ctx, query) + require.NoErrorf(t, err, "failed to count op types in %s", relation) + rows, err := df.Collect(ctx) + require.NoErrorf(t, err, "failed to collect op type counts for %s", relation) + + counts := make(map[string]int64, len(rows)) + for _, row := range rows { + op, _ := row.Value("op").(string) + n, _ := row.Value("n").(int64) + counts[op] = n + } + return counts +} + +// compatibilityGlobalFloor is the oldest baseline the suite runs for any driver: the oldest entry in the +// product's state-versions.json. Derived rather than restated, so adding or retiring a baseline +// moves the floor with it. +func compatibilityGlobalFloor(rootPath string) (string, error) { + baselines, err := testutils.StateVersionBaselines(rootPath) + if err != nil { + return "", err + } + oldest := baselines[0] + for _, baseline := range baselines[1:] { + if baseline.StateVersion < oldest.StateVersion { + oldest = baseline + } + } + return oldest.ReleaseTag, nil +} diff --git a/tests/testutils/compatibility/compatibility_columns.go b/tests/testutils/compatibility/compatibility_columns.go new file mode 100644 index 000000000..ac050b7f3 --- /dev/null +++ b/tests/testutils/compatibility/compatibility_columns.go @@ -0,0 +1,173 @@ +package compatibility + +import ( + "fmt" + "maps" + "os" + "slices" + "strings" +) + +// ColumnRule is one column's backward-compatibility assertion policy, keyed on the baseline release +// under test. No rule -- the default -- means the column is asserted in full, on type and value. +type ColumnRule struct { + Column string + // ExcludeBelow drops the column from the seed data and the catalog when the baseline is older + // than this release. For hard fails only: a baseline that cannot carry the column at any price. + ExcludeBelow string + // AssertValueFrom value-compares the column only when the baseline is at or after this release; + // older baselines still assert its type through the schema comparison. + AssertValueFrom string +} + +// columnPolicies is a rule set applied to one baseline: which columns stay out of the seed data, +// which are compared by type only, and a log-ready line per decision so the run's assertion +// surface is explicit in its output. +type columnPolicies struct { + seedExcluded []string + assertDatatypeOnly []string + notes []string +} + +// resolveColumnPolicies evaluates a driver's rules against the baseline spec. A baseline that +// cannot be dated ("latest", an image ref, a commit sha) is treated as newest -- ExcludeBelow and +// AssertValueFrom never fire, only TypeOnly -- mirroring resolveInputGeneration's fallback. +// Malformed rules are an error, never a skip: a typo'd version must not silently change what a +// green run proves. +func resolveColumnPolicies(rules []ColumnRule, spec string) (*columnPolicies, error) { + version, canCompare := parseReleaseTag(spec) + policies := &columnPolicies{} + seen := make(map[string]bool, len(rules)) + for _, rule := range rules { + if rule.Column == "" { + return nil, fmt.Errorf("compatibility column rule with an empty column name: %+v", rule) + } + if seen[rule.Column] { + return nil, fmt.Errorf("duplicate compatibility column rule for %q; one rule carries every policy for a column", rule.Column) + } + seen[rule.Column] = true + if rule.ExcludeBelow == "" && rule.AssertValueFrom == "" { + return nil, fmt.Errorf("compatibility column rule for %q declares no policy", rule.Column) + } + + if rule.ExcludeBelow != "" { + boundary, ok := parseReleaseTag(rule.ExcludeBelow) + if !ok { + return nil, fmt.Errorf("compatibility column rule for %q: ExcludeBelow %q is not a release tag", rule.Column, rule.ExcludeBelow) + } + if canCompare && compareRelease(version, boundary) < 0 { + policies.seedExcluded = append(policies.seedExcluded, rule.Column) + policies.notes = append(policies.notes, fmt.Sprintf( + "column %s: excluded from the seed data, baseline %s is older than %s", rule.Column, spec, rule.ExcludeBelow)) + // Absent from both runs, so its assertion policy is moot. + continue + } + } + + switch { + case rule.AssertValueFrom != "": + boundary, ok := parseReleaseTag(rule.AssertValueFrom) + if !ok { + return nil, fmt.Errorf("compatibility column rule for %q: AssertValueFrom %q is not a release tag", rule.Column, rule.AssertValueFrom) + } + if canCompare && compareRelease(version, boundary) < 0 { + policies.assertDatatypeOnly = append(policies.assertDatatypeOnly, rule.Column) + policies.notes = append(policies.notes, fmt.Sprintf( + "column %s: type-only, baseline %s is older than %s", rule.Column, spec, rule.AssertValueFrom)) + } + } + } + if len(policies.notes) == 0 && len(rules) > 0 { + policies.notes = append(policies.notes, fmt.Sprintf( + "all %d column rules inactive against baseline %s; every column is fully asserted", len(rules), spec)) + } + return policies, nil +} + +// assertionPolicies is every rule resolved against one baseline: the one set the run applies. The +// scenarios read catalogExcluded, the fixture's seeding reads seedExcluded, and the comparison +// reads typeOnly; nothing else consults the rules again. +type assertionPolicies struct { + seedExcluded []string + catalogExcluded []string + typeOnly []string + notes []string +} + +// resolveAssertionPolicies folds the driver's and variant's rules -- type-keyed and column-keyed, +// dated and unconditional -- with the always-volatile columns into one policy set for this +// baseline. Thresholds at or below the sweep's floor are dead config and fail loudly. +func resolveAssertionPolicies(fixture *Test, spec, floorTag string, globalFloor [3]int, driverRules compatibilityDriverRules, variantRules compatibilityVariantRules) (*assertionPolicies, error) { + // The destinations' shared rules first (olake's own columns), then the driver's -- destination + // columns and source columns are separate lists in the json -- then the variant's. + typeRules := slices.Clone(compatibilityRules.Destinations.Rules) + typeRules = append(typeRules, driverRules.DestinationRules...) + typeRules = append(typeRules, driverRules.Rules...) + typeRules = append(typeRules, variantRules.Rules...) + for _, rule := range typeRules { + for _, threshold := range []string{rule.ExcludeBelow, rule.AssertValueFrom} { + if threshold == "" { + continue + } + bound, ok := parseReleaseTag(threshold) + if !ok { + return nil, fmt.Errorf("compatibility_rules.json: %q is not a release tag", threshold) + } + if compareRelease(bound, globalFloor) <= 0 { + return nil, fmt.Errorf("compatibility_rules.json: rule threshold %s is at or below the oldest reachable baseline %s, so it can never fire (%s); drop the rule or record it as a note", + threshold, floorTag, rule.Note) + } + } + } + + // The columns a data_types rule can select: the driver's declared schema, the fixture's own + // tags, and the json column_types tags. + columnTypes := map[string][]string{} + for column, declared := range fixture.DeclaredSchema { + if declared = strings.ToLower(strings.TrimSpace(declared)); declared != "" { + columnTypes[column] = append(columnTypes[column], declared) + } + } + for _, tags := range []map[string][]string{fixture.ColumnTypes, driverRules.ColumnTypes, variantRules.ColumnTypes} { + for column, columnTags := range tags { + for _, tag := range columnTags { + if !slices.Contains(columnTypes[column], tag) { + columnTypes[column] = append(columnTypes[column], tag) + } + } + } + } + + columnRules, alwaysTypeOnly, err := resolveTypeRules(typeRules, columnTypes) + if err != nil { + return nil, err + } + dated, err := resolveColumnPolicies(columnRules, spec) + if err != nil { + return nil, err + } + policies := &assertionPolicies{seedExcluded: dated.seedExcluded, notes: dated.notes} + // Seed-excluded columns leave the catalog too, so streams.json never selects a column the + // fixture left out of the table; the env sweep hook appends. + policies.catalogExcluded = slices.Clone(dated.seedExcluded) + if raw := os.Getenv(compatibilityExcludeColumnsEnvVar); raw != "" { + policies.catalogExcluded = append(policies.catalogExcluded, strings.Split(raw, ",")...) + } + + // Compared by type but never by value: the driver's CDC columns (source-log coordinates), + // the dated rules' columns, and the unconditional type_only ones -- olake's own and any + // driver's exceptions among them, all from the json. The destinations' value_compared list + // carves the deterministic columns back out of the CDC set; a driver re-adds one with a rule. + volatile := map[string]bool{} + for column := range fixture.CDCColumnsSchema { + volatile[column] = true + } + for _, column := range compatibilityRules.Destinations.ValueCompared.Columns { + delete(volatile, column) + } + for _, column := range append(dated.assertDatatypeOnly, alwaysTypeOnly...) { + volatile[column] = true + } + policies.typeOnly = slices.Sorted(maps.Keys(volatile)) + return policies, nil +} diff --git a/tests/testutils/compatibility/compatibility_rules.go b/tests/testutils/compatibility/compatibility_rules.go new file mode 100644 index 000000000..f0924e891 --- /dev/null +++ b/tests/testutils/compatibility/compatibility_rules.go @@ -0,0 +1,395 @@ +package compatibility + +import ( + "bytes" + _ "embed" + "encoding/json" + "fmt" + "maps" + "slices" + "strconv" + "strings" + + "github.com/datazip-inc/olake/tests/testutils/constants" +) + +// compatibility_rules.json is the compatibility suite's whole configuration: per-driver and per-group +// baseline gates, and the column rules that decide what each column can be asserted on. Adding a +// driver, a gate or a rule is an edit to that file alone. What stays in code is only what binds a +// name to behavior: the group list in compatibilityGroupSpecs, since each variant calls a harness func. + +// compatibilityGate bounds which baselines a scope runs against. Empty fields mean no bound. +type compatibilityGate struct { + MinBaseline string `json:"min_baseline"` + SkipBaselines []string `json:"skip_baselines"` + Note string `json:"note"` +} + +// compatibilityTypeRule selects columns by data-type tag (resolved against the declared schema plus +// the json column_types maps) or by an +// olake-owned column name, and carries the same policy fields as ColumnRule. +type compatibilityTypeRule struct { + DataTypes []string `json:"data_types"` + Column string `json:"column"` + ExcludeBelow string `json:"exclude_below"` + AssertValueFrom string `json:"assert_value_from"` + TypeOnly bool `json:"type_only"` + Note string `json:"note"` +} + +// compatibilityDestination is one destination's gates. A destination may gate itself (parquet) and/or +// carry named modes (iceberg's arrow and legacy); both shapes decode into this one type, so adding +// a destination or a mode is a config edit. +type compatibilityDestination struct { + compatibilityGate + Modes map[string]compatibilityGate +} + +var compatibilityGateFields = map[string]bool{"min_baseline": true, "skip_baselines": true, "note": true} + +func (d *compatibilityDestination) UnmarshalJSON(data []byte) error { + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + gate := map[string]json.RawMessage{} + d.Modes = map[string]compatibilityGate{} + for key, value := range raw { + if compatibilityGateFields[key] { + gate[key] = value + continue + } + var mode compatibilityGate + if err := strictUnmarshal(value, &mode); err != nil { + return fmt.Errorf("mode %q: %w", key, err) + } + d.Modes[key] = mode + } + if len(gate) == 0 { + return nil + } + encoded, err := json.Marshal(gate) + if err != nil { + return err + } + return strictUnmarshal(encoded, &d.compatibilityGate) +} + +// strictUnmarshal rejects unknown keys. A custom UnmarshalJSON does not inherit the outer +// decoder's strictness, so nested gates re-apply it here; a typo must never fail open. +func strictUnmarshal(data []byte, target any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + return dec.Decode(target) +} + +// compatibilityVariantRules gates and rules for one source data format (s3's csv/json/parquet). +type compatibilityVariantRules struct { + compatibilityGate + Rules []compatibilityTypeRule `json:"rules"` + // ColumnTypes tags fixture columns with type identifiers the declared destination schema + // cannot express (a charset, a parquet physical type); data_types rules select on both. + ColumnTypes map[string][]string `json:"column_types"` +} + +type compatibilityDriverRules struct { + compatibilityGate + Destinations map[string]compatibilityDestination `json:"destinations"` + Rules []compatibilityTypeRule `json:"rules"` + // DestinationRules covers the columns this driver makes APPEAR in the destination (its + // materialized key, olake's metadata) as opposed to the source columns its fixture seeds. + DestinationRules []compatibilityTypeRule `json:"destination_rules"` + Variants map[string]compatibilityVariantRules `json:"variants"` + ColumnTypes map[string][]string `json:"column_types"` +} + +// compatibilityDestinationsRules is the destinations block: the rules for the columns every +// destination writer emits (olake's own metadata), and each destination's gates. +type compatibilityDestinationsRules struct { + Rules []compatibilityTypeRule `json:"rules"` + ValueCompared struct { + Columns []string `json:"columns"` + Note string `json:"note"` + } `json:"value_compared"` + Iceberg compatibilityDestination `json:"iceberg"` + Parquet compatibilityDestination `json:"parquet"` +} + +// gates keys the destinations the way the writer groups look them up. +func (r compatibilityDestinationsRules) gates() map[string]compatibilityDestination { + return map[string]compatibilityDestination{"iceberg": r.Iceberg, "parquet": r.Parquet} +} + +type compatibilityRulesConfig struct { + Drivers map[string]compatibilityDriverRules `json:"drivers"` + Destinations compatibilityDestinationsRules `json:"destinations"` +} + +//go:embed compatibility_rules.json +var rawCompatibilityRules []byte + +// compatibilityRules is parsed and validated at package init, so a malformed or misspelled rules file +// fails every suite loudly instead of silently under-enforcing. +var compatibilityRules = func() compatibilityRulesConfig { + var cfg compatibilityRulesConfig + dec := json.NewDecoder(bytes.NewReader(rawCompatibilityRules)) + dec.DisallowUnknownFields() + if err := dec.Decode(&cfg); err != nil { + panic("tests/testutils/compatibility_rules.json: " + err.Error()) + } + if err := cfg.validate(); err != nil { + panic("tests/testutils/compatibility_rules.json: " + err.Error()) + } + return cfg +}() + +var knownCompatibilityDrivers = []constants.DriverType{ + constants.MongoDB, constants.Postgres, constants.MySQL, constants.Oracle, + constants.DB2, constants.S3, constants.Kafka, constants.MSSQL, +} + +// validate rejects what would otherwise fail open: a driver or group name nothing matches, and a +// baseline that is not a release tag. Either would silently drop the gate or rule it carries. +func (c compatibilityRulesConfig) validate() error { + modes := map[string][]string{} + for _, spec := range compatibilityGroupSpecs() { + modes[spec.destination] = append(modes[spec.destination], spec.mode) + } + checkDestinations := func(scope string, destinations map[string]compatibilityDestination) error { + for name, dest := range destinations { + known, ok := modes[name] + if !ok { + return fmt.Errorf("%s: unknown destination %q (known: %v)", scope, name, slices.Sorted(maps.Keys(modes))) + } + if err := dest.compatibilityGate.validate(scope + " destination " + name); err != nil { + return err + } + for mode, gate := range dest.Modes { + if !slices.Contains(known, mode) { + return fmt.Errorf("%s destination %s: unknown mode %q (known: %v)", scope, name, mode, known) + } + if err := gate.validate(fmt.Sprintf("%s destination %s mode %s", scope, name, mode)); err != nil { + return err + } + } + } + return nil + } + if err := checkDestinations("destinations", c.Destinations.gates()); err != nil { + return err + } + if err := validateRules("destinations", c.Destinations.Rules); err != nil { + return err + } + for _, column := range c.Destinations.ValueCompared.Columns { + if strings.TrimSpace(column) == "" { + return fmt.Errorf("destinations.value_compared carries an empty column name") + } + } + for name, driver := range c.Drivers { + if !slices.Contains(knownCompatibilityDrivers, constants.DriverType(name)) { + return fmt.Errorf("unknown driver %q (known: %v)", name, knownCompatibilityDrivers) + } + if err := driver.validate(name); err != nil { + return err + } + if err := checkDestinations("driver "+name, driver.Destinations); err != nil { + return err + } + for format, variant := range driver.Variants { + scope := fmt.Sprintf("driver %s variant %s", name, format) + if err := variant.validate(scope); err != nil { + return err + } + if err := validateRules(scope, variant.Rules); err != nil { + return err + } + } + if err := validateRules("driver "+name, driver.Rules); err != nil { + return err + } + if err := validateRules("driver "+name+" destination_rules", driver.DestinationRules); err != nil { + return err + } + } + return nil +} + +func (g compatibilityGate) validate(scope string) error { + for _, tag := range append([]string{g.MinBaseline}, g.SkipBaselines...) { + if tag == "" { + continue + } + if _, ok := parseReleaseTag(tag); !ok { + return fmt.Errorf("%s: %q is not a release tag", scope, tag) + } + } + return nil +} + +func validateRules(scope string, rules []compatibilityTypeRule) error { + for i, r := range rules { + if r.Column == "" && len(r.DataTypes) == 0 { + return fmt.Errorf("%s rule %d: selects nothing (needs column or data_types): %s", scope, i, r.Note) + } + if r.Column != "" && len(r.DataTypes) > 0 { + return fmt.Errorf("%s rule %d: sets both column and data_types; use one", scope, i) + } + if r.ExcludeBelow == "" && r.AssertValueFrom == "" && !r.TypeOnly { + return fmt.Errorf("%s rule %d: asserts nothing (needs exclude_below, assert_value_from or type_only): %s", scope, i, r.Note) + } + for _, tag := range []string{r.ExcludeBelow, r.AssertValueFrom} { + if tag == "" { + continue + } + if _, ok := parseReleaseTag(tag); !ok { + return fmt.Errorf("%s rule %d: %q is not a release tag", scope, i, tag) + } + } + } + return nil +} + +// skipReason says why a baseline is out of this gate's range ("" = it runs). Tags are validated at +// load, so anything unparseable here is a bug rather than bad config. +func (g compatibilityGate) skipReason(version [3]int, dated bool) string { + if !dated { + return "" + } + if g.MinBaseline != "" { + if boundary, ok := parseReleaseTag(g.MinBaseline); ok && compareRelease(version, boundary) < 0 { + return fmt.Sprintf("baseline is older than %s", g.MinBaseline) + } + } + for _, skip := range g.SkipBaselines { + if boundary, ok := parseReleaseTag(skip); ok && compareRelease(version, boundary) == 0 { + return fmt.Sprintf("baseline %s is a known bounded regression here", skip) + } + } + return "" +} + +// mergedGate overlays a driver's gate for a group on the global one: the higher floor wins and the +// skip windows union, so a driver can only ever narrow what it runs. +func mergedGate(global, driver compatibilityGate) compatibilityGate { + merged := compatibilityGate{MinBaseline: global.MinBaseline, Note: global.Note} + if driver.MinBaseline != "" && (merged.MinBaseline == "" || releaseTagLess(merged.MinBaseline, driver.MinBaseline)) { + merged.MinBaseline, merged.Note = driver.MinBaseline, driver.Note + } + merged.SkipBaselines = append(append([]string{}, global.SkipBaselines...), driver.SkipBaselines...) + return merged +} + +func releaseTagLess(a, b string) bool { + av, aok := parseReleaseTag(a) + bv, bok := parseReleaseTag(b) + return aok && bok && compareRelease(av, bv) < 0 +} + +// resolveTypeRules maps type-keyed rules onto the fixture's declared column types, merging +// multiple matches per column into one ColumnRule. A data_types rule matching no declared +// column is an error: the fixture does not carry the type, so the rule would assert nothing. +func resolveTypeRules(rules []compatibilityTypeRule, columnTypes map[string][]string) ([]ColumnRule, []string, error) { + alwaysTypeOnly := map[string]bool{} + merged := map[string]*ColumnRule{} + var order []string + apply := func(column string, r compatibilityTypeRule) error { + if r.TypeOnly { + alwaysTypeOnly[column] = true + } + // Only a dated policy becomes a ColumnRule; type_only alone is carried by alwaysTypeOnly. + if r.ExcludeBelow == "" && r.AssertValueFrom == "" { + return nil + } + rule, ok := merged[column] + if !ok { + rule = &ColumnRule{Column: column} + merged[column] = rule + order = append(order, column) + } + if r.ExcludeBelow != "" { + if rule.ExcludeBelow != "" && rule.ExcludeBelow != r.ExcludeBelow { + return fmt.Errorf("column %s: conflicting exclude_below %s and %s", column, rule.ExcludeBelow, r.ExcludeBelow) + } + rule.ExcludeBelow = r.ExcludeBelow + } + if r.AssertValueFrom != "" { + if rule.AssertValueFrom != "" && rule.AssertValueFrom != r.AssertValueFrom { + return fmt.Errorf("column %s: conflicting assert_value_from %s and %s", column, rule.AssertValueFrom, r.AssertValueFrom) + } + rule.AssertValueFrom = r.AssertValueFrom + } + return nil + } + + columns := make([]string, 0, len(columnTypes)) + for column := range columnTypes { + columns = append(columns, column) + } + slices.Sort(columns) + + for _, r := range rules { + switch { + case r.Column != "": + if err := apply(r.Column, r); err != nil { + return nil, nil, err + } + case len(r.DataTypes) > 0: + found := false + for _, column := range columns { + matches := slices.ContainsFunc(r.DataTypes, func(dt string) bool { + return slices.Contains(columnTypes[column], dt) + }) + if !matches { + continue + } + if err := apply(column, r); err != nil { + return nil, nil, err + } + found = true + } + if !found { + return nil, nil, fmt.Errorf("no declared column matches data_types %v (%s); tag the column in the fixture's ColumnTypes or drop the rule", r.DataTypes, r.Note) + } + } + } + + out := make([]ColumnRule, 0, len(order)) + for _, column := range order { + out = append(out, *merged[column]) + } + return out, slices.Sorted(maps.Keys(alwaysTypeOnly)), nil +} + +// parseReleaseTag reads "vX.Y.Z" (optionally behind a "repo:tag" prefix) into a comparable triple; +// ok is false for anything that is not a release tag (a sha, "latest", a bare image). +func parseReleaseTag(spec string) ([3]int, bool) { + var version [3]int + if i := strings.LastIndex(spec, ":"); i >= 0 { + spec = spec[i+1:] + } + parts := strings.Split(strings.TrimPrefix(strings.TrimSpace(spec), "v"), ".") + if len(parts) != 3 { + return version, false + } + for i, part := range parts { + n, err := strconv.Atoi(part) + if err != nil || n < 0 { + return version, false + } + version[i] = n + } + return version, true +} + +func compareRelease(a, b [3]int) int { + for i := range a { + switch { + case a[i] < b[i]: + return -1 + case a[i] > b[i]: + return 1 + } + } + return 0 +} diff --git a/tests/testutils/compatibility/compatibility_rules.json b/tests/testutils/compatibility/compatibility_rules.json new file mode 100644 index 000000000..f2efc5125 --- /dev/null +++ b/tests/testutils/compatibility/compatibility_rules.json @@ -0,0 +1,101 @@ +{ + "destinations": { + "rules": [ + {"column": "_olake_timestamp", "type_only": true, "note": "olake's write stamp: wall-clock, never value-comparable across two runs"} + ], + "iceberg": { + "arrow": { + "min_baseline": "v0.3.17", + "note": "P1: v0.3.6 through v0.3.16 arrow integer widths disagree with today's; below v0.3.6 no arrow writer exists" + } + }, + "parquet": { + "skip_baselines": ["v0.3.16"], + "note": "P2: v0.3.16 adds the `data` column unconditionally, fixed in v0.3.17; below that the no-CDC drivers' output lacks _cdc_timestamp" + } + }, + "drivers": { + "postgres": { + "destination_rules": [ + {"column": "_cdc_timestamp", "type_only": true, "note": "wall clock of the sync"}, + {"column": "_cdc_lsn", "type_only": true, "note": "source-log coordinate"} + ] + }, + "oracle": { + "destinations": { + "parquet": {"min_baseline": "v0.3.17", "note": "P2 for a no-CDC driver: below v0.3.17 the parquet output lacks _cdc_timestamp, and v0.3.11's writer panics feeding an int64 into an INT32 column on the incremental path"} + } + }, + "mysql": { + "destination_rules": [ + {"column": "_cdc_timestamp", "type_only": true}, + {"column": "_cdc_binlog_file_name", "type_only": true}, + {"column": "_cdc_binlog_file_pos", "type_only": true} + ], + "rules": [ + {"data_types": ["ucs2", "utf16le", "latin1"], "exclude_below": "v0.7.2", "note": "non-UTF-8 charset bytes reach the writer as invalid UTF-8; gRPC marshal fails, retry backoff looks like a hang"}, + {"data_types": ["set"], "assert_value_from": "v0.7.2", "note": "M1: SET columns emitted the numeric bitmask on the binlog path before the fix"}, + {"data_types": ["unsigned mediumint"], "assert_value_from": "v0.9.3", "note": "unsigned MEDIUMINT was sign-extended on the binlog path before v0.9.3 (-1 where the value is 16777215)"}, + {"data_types": ["unsigned bigint"], "exclude_below": "v0.9.0", "note": "BIGINT UNSIGNED at or above 2^63 reaches the incremental scan as a byte slice; before v0.9.0 (state v6, ReformatInt64) it cannot be converted: v0.3.11's Iceberg writer fatals on the mixed long/string batch and a candidate on an older state file retries into a hang"} + ], + "note": "M2 (ENUM serialization, fixed v0.3.9) and M3 (DECIMAL/NUMERIC via float32, fixed v0.3.7) need no rule: both thresholds sit below v0.3.11, the oldest baseline any sweep reaches" + }, + "mongodb": { + "destination_rules": [ + {"column": "_cdc_timestamp", "type_only": true}, + {"column": "_cdc_resume_token", "type_only": true}, + {"column": "_id", "type_only": true, "note": "server-generated ObjectID; cannot match across two independent runs"}, + {"column": "_olake_id", "type_only": true, "note": "hashes the server-generated _id, so it is as non-deterministic as its source"} + ], + "rules": [ + {"data_types": ["regex"], "assert_value_from": "v0.3.14", "note": "G1: BSON regex serialized with Go field names, not lowercase keys, until the fix landed"} + ] + }, + "mssql": { + "min_baseline": "v0.3.15", + "note": "first release carrying the driver", + "destination_rules": [ + {"column": "_cdc_timestamp", "type_only": true}, + {"column": "_cdc_start_lsn", "type_only": true}, + {"column": "_cdc_seqval", "type_only": true} + ] + }, + "kafka": { + "min_baseline": "v0.3.16", + "note": "v0.3.12 added decoder.UseNumber() but the flattener only learned json.Number in v0.3.16; pre-v0.3.12 numbers all typed double against today's catalog", + "destination_rules": [ + {"column": "_cdc_timestamp", "type_only": true}, + {"column": "_kafka_timestamp", "type_only": true, "note": "broker receive time"}, + {"column": "_kafka_offset", "type_only": true, "note": "assignment order varies across concurrent partition writes"} + ], + "variants": { + "json": {}, + "avro": {} + } + }, + "db2": { + "min_baseline": "v0.3.14", + "note": "first release carrying the driver; refine after the first sweep if early db2 images are missing from the registry", + "destinations": { + "parquet": {"min_baseline": "v0.3.17", "note": "P2 for a no-CDC driver: below v0.3.17 the parquet output lacks _cdc_timestamp"} + }, + "rules": [ + {"data_types": ["decfloat"], "assert_value_from": "v0.7.6", "note": "ReformatValue rendered a float into a String column with %d"} + ] + }, + "s3": { + "min_baseline": "v0.9.2", + "note": "first release carrying the driver", + "destination_rules": [ + {"column": "_last_modified_time", "type_only": true, "note": "the cursor IS the object's upload stamp; each run uploads its own copy"}, + {"column": "_olake_id", "type_only": true, "note": "hashes the record carrying _last_modified_time"} + ], + "variants": { + "csv": {}, + "json": {}, + "parquet": {}, + "xml": {"min_baseline": "v0.9.5", "note": "the XML source format landed on 2026-08-07, after v0.9.4; older binaries reject file_format xml at config validation"} + } + } + } +} diff --git a/tests/testutils/compatibility/scenarios.go b/tests/testutils/compatibility/scenarios.go new file mode 100644 index 000000000..efeaef2e4 --- /dev/null +++ b/tests/testutils/compatibility/scenarios.go @@ -0,0 +1,232 @@ +package compatibility + +import ( + "context" + "fmt" + "testing" + + "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/constants" + "github.com/datazip-inc/olake/tests/testutils/require" +) + +const ( + scenarioCDC = "cdc" + scenarioIncremental = "inc" +) + +// compatibilityVariant is one scenario, run once per side. Each pair gets its own destination +// namespace through its subtest-derived suite, so every scenario's output survives to be compared. +type compatibilityVariant struct { + name string + kind string + // emptyFinalState marks a variant whose LAST case writes no output; both sides ending empty is + // then the asserted outcome, for every other variant it is a failure no row diff would catch. + emptyFinalState bool +} + +// compatibilityGroup is one destination writer's scenarios. Groups are the parallelism unit, and +// the unit of the writer-level version gates from compatibility_rules.json. +type compatibilityGroup struct { + compatibilityGroupSpec + gate compatibilityGate + variants []compatibilityVariant +} + +type compatibilityGroupSpec struct { + name string + // destination and mode locate this group's gates in compatibility_rules.json; destinationFile + // is the destination config its syncs run against. + destination string + mode string + destinationFile string +} + +// syncCase is one sync of a scenario: the DML that precedes it and whether it reads state. +type syncCase struct { + operation string + useState bool +} + +// scenarioCases is the same case sequence TestSync runs, minus its verification: the comparison +// against the reference run is this suite's only assertion. +func scenarioCases(driver, kind string) []syncCase { + if kind == scenarioIncremental { + return []syncCase{{operation: "", useState: false}, {operation: "insert", useState: true}, {operation: "update", useState: true}} + } + if driver == string(constants.Kafka) { + // Kafka is strict-CDC: no stateless full load, and no deletes to replay. + return []syncCase{{operation: "", useState: false}, {operation: "update", useState: true}} + } + return []syncCase{ + {operation: "", useState: false}, + {operation: "insert", useState: true}, + {operation: "update", useState: true}, + {operation: "delete", useState: true}, + } +} + +// evolvesSchema mirrors TestSync's evolve-schema fan-out: which drivers alter the table before the +// update case, per scenario kind. +func evolvesSchema(driver, kind string) bool { + if kind == scenarioIncremental { + return driver != string(constants.MongoDB) && driver != string(constants.MSSQL) + } + return driver != string(constants.MongoDB) && driver != string(constants.MSSQL) && driver != string(constants.Kafka) +} + +// runSide seeds, syncs and tears down one side of a variant on its own config. pick routes each +// sync to a driver version: the reference side always answers the baseline, the upgrade side +// hands stateful syncs to the candidate. +func runSide( + t *testing.T, + cfg *testutils.TestConfig, + g compatibilityGroup, + v compatibilityVariant, + pick func(useState bool) string, + policies *assertionPolicies, +) { + ctx := t.Context() + table := cfg.GetTableName() + t.Logf("compatibility side %q: source table %s", cfg.Suite, table) + + // The driver's own ExecuteQuery reads this, so a column a rule excluded for this baseline + // never reaches the seed DDL or DML. + cfg.SeedExcludedColumns = policies.seedExcluded + + // Stream selection on the freshly rendered catalog. No filter and no column selection: this + // suite compares what a sync produces, and either would only narrow both sides equally. + require.NoError(t, testutils.UpdateSelectedStreams(cfg, cfg.Namespace, cfg.PartitionRegex, "", []string{table}, "", policies.catalogExcluded...), + "failed to select the compatibility stream") + // A seed-excluded column is absent from the table, so it leaves the catalog's schema too: a + // binary that predates column selection writes every column the catalog declares. + require.NoError(t, dropCatalogColumns(cfg, policies.seedExcluded), "failed to drop the seed-excluded columns from streams.json") + if v.kind == scenarioIncremental { + require.NoError(t, setIncrementalMode(cfg, table), "failed to patch streams.json for incremental") + } + + // Whatever a previous invocation of this same test name left behind, cleared up front; the + // scenarios themselves never clear, so the candidate binary meets the table the baseline made. + clearDestination(t, g, cfg.DestinationDB, table) + + // The slot lives as long as the source config that names it, and olake validates the CDC + // configuration at startup for every sync -- the incremental ones included; only postgres needs it. + if cfg.Driver == string(constants.Postgres) { + cfg.ExecuteQuery(ctx, t, cfg, "create-slot") + defer cfg.ExecuteQuery(ctx, t, cfg, "drop-slot") + } + if testutils.KeepTestData() { + t.Logf("compatibility side %q: leaving source table %s in place (%s is set); it holds the last case's data", + cfg.Suite, table, testutils.KeepTestDataEnvVar) + } else { + defer cfg.ExecuteQuery(ctx, t, cfg, "drop") + } + + // Seed the source: the same reset every TestSync scenario starts from. + cfg.ExecuteQuery(ctx, t, cfg, "drop") + cfg.ExecuteQuery(ctx, t, cfg, "create") + cfg.ExecuteQuery(ctx, t, cfg, "add") + if cfg.Driver == string(constants.DB2) { + cfg.ExecuteQuery(ctx, t, cfg, "populate-stats") + } + // The seed rows sit in the CDC log, and before #843 the mssql driver captured its initial LSN + // without waiting for the async capture agent -- wait here so every binary snapshots past the seed. + if v.kind == scenarioCDC && cfg.Driver == string(constants.MSSQL) { + cfg.ExecuteQuery(ctx, t, cfg, "wait-cdc-catchup") + } + if v.kind == scenarioIncremental { + require.NoError(t, testutils.ResetStateFile(cfg), "failed to reset state for incremental") + } + + for _, c := range scenarioCases(cfg.Driver, v.kind) { + if c.operation == "update" && evolvesSchema(cfg.Driver, v.kind) { + cfg.ExecuteQuery(ctx, t, cfg, "evolve-schema") + } + if c.useState && c.operation != "" { + cfg.ExecuteQuery(ctx, t, cfg, c.operation) + if v.kind == scenarioCDC && cfg.Driver == string(constants.MSSQL) { + cfg.ExecuteQuery(ctx, t, cfg, "wait-cdc-catchup") + } + } + // Successive syncs write the same parquet column with different types, which Spark refuses + // to read together (CANNOT_MERGE_SCHEMAS; F2 in docs/backward-compatibility.md) -- so a + // parquet variant holds, and compares, only its last case's files. + if g.destination == "parquet" { + require.NoErrorf(t, testutils.DeleteParquetFiles(t, cfg.DestinationDB, table), "failed to clear parquet files before %q", c.operation) + } + runSync(ctx, t, cfg, g.destinationFile, pick(c.useState), c.useState) + } +} + +// runSync runs one sync of the scenario on the image of the given driver version. +func runSync(ctx context.Context, t *testing.T, cfg *testutils.TestConfig, destinationFile, version string, useState bool) { + t.Helper() + flags := []string{"--destination-database-prefix", cfg.UniqueID()} + cfg.DriverVersion = version + t.Logf("running %s sync on image %s", testutils.Ternary(useState, "stateful", "stateless").(string), cfg.GetDriverImage()) + + code, out, err := testutils.RunOlake(ctx, cfg, testutils.SyncArgs(useState, destinationFile, flags...)...) + if err != nil || code != 0 { + t.Fatal(testutils.RenderOlakeFailure(code, err, out)) + } +} + +// setIncrementalMode patches the catalog's stream to incremental with the driver's cursor, the +// same edit TestSync's incremental scenarios make. +func setIncrementalMode(cfg *testutils.TestConfig, table string) error { + streamName := testutils.NormalizeStreamName(cfg.Driver, table) + return testutils.EditJSONFile(cfg.GetFilePath("streams.json"), func(doc map[string]interface{}) error { + entries, _ := doc["streams"].([]interface{}) + for _, raw := range entries { + wrapper, ok := raw.(map[string]interface{}) + if !ok { + continue + } + stream, ok := wrapper["stream"].(map[string]interface{}) + if !ok { + continue + } + if stream["name"] == streamName && stream["namespace"] == cfg.Namespace { + stream["sync_mode"] = "incremental" + if cfg.CursorField != "" { + stream["cursor_field"] = cfg.CursorField + } + return nil + } + } + return fmt.Errorf("stream %s.%s not found in streams.json", cfg.Namespace, streamName) + }) +} + +// dropCatalogColumns removes columns the seed left out of the table from the stream's type_schema. +func dropCatalogColumns(cfg *testutils.TestConfig, columns []string) error { + if len(columns) == 0 { + return nil + } + return testutils.EditJSONFile(cfg.GetFilePath("streams.json"), func(doc map[string]interface{}) error { + entries, _ := doc["streams"].([]interface{}) + for _, raw := range entries { + wrapper, _ := raw.(map[string]interface{}) + stream, _ := wrapper["stream"].(map[string]interface{}) + schema, _ := stream["type_schema"].(map[string]interface{}) + properties, _ := schema["properties"].(map[string]interface{}) + for _, column := range columns { + delete(properties, column) + } + } + return nil + }) +} + +// clearDestination drops whatever a previous invocation of this test name left at the variant's +// destination; missing tables and empty prefixes are simply nothing to clear. +func clearDestination(t *testing.T, g compatibilityGroup, db, table string) { + switch g.destination { + case "iceberg": + testutils.DropIcebergTable(t, table, db) + case "parquet": + if err := testutils.DeleteParquetFiles(t, db, table); err != nil { + t.Logf("could not clear parquet files at %s/%s (likely absent): %s", db, table, err) + } + } +} diff --git a/tests/testutils/ddl.go b/tests/testutils/ddl.go new file mode 100644 index 000000000..3a9e9920d --- /dev/null +++ b/tests/testutils/ddl.go @@ -0,0 +1,34 @@ +package testutils + +import ( + "regexp" + "strings" +) + +var ddlCharset = regexp.MustCompile(`(?i)CHARACTER SET (\w+)`) + +// DDLColumnTypes reads column type tags off a CREATE TABLE column list -- the base type, its +// unsigned form and charset where the dialect has them -- so a fixture declares nothing by hand. +func DDLColumnTypes(ddl string) map[string][]string { + types := map[string][]string{} + for _, line := range strings.Split(ddl, "\n") { + fields := strings.Fields(strings.TrimSuffix(strings.TrimSpace(line), ",")) + if len(fields) < 2 { + continue + } + switch strings.ToUpper(fields[0]) { + case "PRIMARY", "KEY", "UNIQUE", "INDEX", "CONSTRAINT": + continue + } + typ, _, _ := strings.Cut(strings.ToLower(fields[1]), "(") + tags := []string{typ} + if strings.Contains(strings.ToUpper(line), " UNSIGNED") { + tags = append(tags, "unsigned "+typ) + } + if m := ddlCharset.FindStringSubmatch(line); m != nil { + tags = append(tags, strings.ToLower(m[1])) + } + types[fields[0]] = tags + } + return types +} diff --git a/tests/testutils/docker.go b/tests/testutils/docker.go index 1d2a86f83..d365c997d 100644 --- a/tests/testutils/docker.go +++ b/tests/testutils/docker.go @@ -9,47 +9,125 @@ import ( "strings" "sync" "sync/atomic" + "testing" ) const ( + // CurrentDriverVersion refers to the version of driver image of current code + CurrentDriverVersion = "local" + // containerTestDataDir is where the driver's testdata is mounted in the container; every // olake input and output lives under it, since the CLI writes next to --config containerTestDataDir = "/testdata" - // driverImageEnvVar pins the image under test, so a caller that has already built or pulled it - // (CI does) is not made to build it again. - driverImageEnvVar = "OLAKE_DRIVER_IMAGE" - // driverVersionEnvVar is used to specify what is the version of the driver to run the test for // by default the driver version is the current code, which is built as `local` driverVersionEnvVar = "OLAKE_DRIVER_VERSION" - // currentDriverVersion refers to the version of driver image of current code - currentDriverVersion = "local" + // preBuiltImageEnvVar means the caller already built the local image and the harness must not + // rebuild it. CI sets it after its own build step; a local run leaves it unset, so the build + // still runs and picks up current code. + preBuiltImageEnvVar = "OLAKE_PRE_BUILT_IMAGE" ) -func getDriverImage(driver, version string) string { - return fmt.Sprintf("olake/source-%s:%s", driver, version) -} - var ( - ensureImageOnce sync.Once - ensureImageErr error - containerSeq atomic.Int64 + // imageResolutions runs each image's resolution once per process, keyed by image ref, so the + // configs of concurrent subtests that name the same version share one build or pull. + imageResolutions sync.Map + containerSeq atomic.Int64 ) +type imageResolution struct { + once sync.Once + err error +} + +// resolveImageOnce hands every caller naming image the result of the one resolve that ran. +func resolveImageOnce(image string, resolve func() error) error { + entry, _ := imageResolutions.LoadOrStore(image, &imageResolution{}) + resolution := entry.(*imageResolution) + resolution.once.Do(func() { resolution.err = resolve() }) + return resolution.err +} + // buildDriverImage builds the driver image if needed. -func buildDriverImage(cfg *TestConfig) error { - ensureImageOnce.Do(func() { - cmd := exec.Command("make", fmt.Sprintf("docker.%s.build", cfg.Driver), fmt.Sprintf("IMAGE_TAG=%s", currentDriverVersion)) +func buildDriverImage(t *testing.T, cfg *TestConfig) error { + t.Helper() + image := cfg.GetDriverImage() + if os.Getenv(preBuiltImageEnvVar) != "" { + t.Logf("skipping the build of %s: %s is set, so the caller already built it", image, preBuiltImageEnvVar) + return nil + } + return resolveImageOnce(image, func() error { + t.Logf("building driver image %s with `make docker.%s.build` to pick up the latest local changes", image, cfg.Driver) + defer TrackPhaseTiming(t, "driver-image", "build "+image)() + cmd := exec.Command("make", fmt.Sprintf("docker.%s.build", cfg.Driver), fmt.Sprintf("IMAGE_TAG=%s", CurrentDriverVersion)) cmd.Dir = cfg.OlakeRootPath - out, err := cmd.CombinedOutput() + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("`make docker.%s.build` failed in %s: %s\n%s", cfg.Driver, cfg.OlakeRootPath, err, out) + } + return nil + }) +} + +// buildBaselineFromCommit builds a driver image from a detached worktree at sha. This is a +// debugging affordance for bisecting a break, not the supported path -- released tags need no +// worktree, no maven and no old-toolchain build, and they ship the exact artifact users run. +// +// Two things the old tree needs that the released path does not: its OWN Iceberg writer jar (the +// Dockerfile copies the jar out of the build context, and the old Go side speaks the old jar's +// RPC), and a build entry point that exists in that tree -- `make docker..build IMAGE_TAG=...` +// is recent, so fall back to a plain `docker build`, whose DRIVER_NAME build-arg is far older. +func buildImageFromCommit(cfg *TestConfig, commitID string) error { + imageTag := cfg.GetDriverImage() + return resolveImageOnce(imageTag, func() error { + if exec.Command("docker", "image", "inspect", imageTag).Run() == nil { + return nil + } + + worktree := filepath.Join(cfg.TestWorkingDir, "olake-compatibility-"+commitID) + run := func(what string, name string, args ...string) error { + cmd := exec.Command(name, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error during %s: %v\noutput: %s", what, err, out) + } + + return nil + } + + err := run("create the worktree", "git", "-C", cfg.OlakeRootPath, "worktree", "add", "--detach", worktree, commitID) if err != nil { - ensureImageErr = fmt.Errorf("`make docker.%s.build` failed in %s: %s\n%s", cfg.Driver, cfg.OlakeRootPath, err, out) + return err } + // The working dir it lives in is removed when the test ends, which would leave the + // registration behind for every later `git worktree` call in the repo. + defer func() { + _ = run("remove the worktree", "git", "-C", cfg.OlakeRootPath, "worktree", "remove", "--force", worktree) + }() + if err := run("build the iceberg writer jar", "make", "-C", worktree, "iceberg.jar"); err != nil { + return err + } + return run("build the image", "docker", "build", "--build-arg", "DRIVER_NAME="+cfg.Driver, "-t", imageTag, worktree) + }) +} + +func ensureImagePresent(t *testing.T, image string) error { + t.Helper() + return resolveImageOnce(image, func() error { + defer TrackPhaseTiming(t, "driver-image", "ensure "+image)() + if err := exec.Command("docker", "image", "inspect", image).Run(); err == nil { + return nil + } + t.Logf("pulling driver image %s", image) + defer TrackPhaseTiming(t, "driver-image", "pull "+image)() + args := []string{"pull", image} + if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil { + return fmt.Errorf("failed to pull %s: %s\n%s", image, err, out) + } + return nil }) - return ensureImageErr } // DockerRunArgs builds the `docker run` argument list that invokes the driver image exactly @@ -58,7 +136,7 @@ func buildDriverImage(cfg *TestConfig) error { // shared with the host and the CLI writes its outputs (streams.json, state.json, ...) back // there. extraFlags carries per-invocation docker flags (host gateway, network, name); image is // explicit rather than derived so one suite can hand successive syncs to different images. -func DockerRunArgs(cfg *TestConfig, image string, extraFlags []string, olakeArgs []string) []string { +func DockerRunArgs(cfg *TestConfig, extraFlags []string, olakeArgs []string) []string { args := []string{ "run", "--rm", "-v", fmt.Sprintf("%s:%s", cfg.TestWorkingDir, containerTestDataDir), @@ -71,7 +149,7 @@ func DockerRunArgs(cfg *TestConfig, image string, extraFlags []string, olakeArgs args = append(args, "--platform", cfg.ImagePlatform) } args = append(args, extraFlags...) - args = append(args, image) + args = append(args, cfg.GetDriverImage()) return append(args, olakeArgs...) } @@ -81,10 +159,10 @@ func generateUniqueContainerName(cfg *TestConfig) string { // RunOlake runs the driver image once, exactly like a real user would: // -// docker run --rm -v :/testdata olake/source-:local +// docker run --rm -v :/testdata olakego/source-:local func RunOlake(ctx context.Context, cfg *TestConfig, olakeArgs ...string) (int, []byte, error) { name := generateUniqueContainerName(cfg) - args := DockerRunArgs(cfg, cfg.DriverImage, []string{"--add-host", "host.docker.internal:host-gateway", "--name", name}, olakeArgs) + args := DockerRunArgs(cfg, []string{"--add-host", "host.docker.internal:host-gateway", "--name", name}, olakeArgs) runCtx, cancel := context.WithTimeout(ctx, SyncTimeout) defer cancel() @@ -101,27 +179,6 @@ func RunOlake(ctx context.Context, cfg *TestConfig, olakeArgs ...string) (int, [ return DockerExitResult(out, err, olakeArgs[0]) } -// ensureImagePresent pulls image unless the local daemon already has it. Used for compatibility -// baselines, which come from a registry rather than from a build; platform is passed through so -// an amd64-only baseline can be pulled on an arm64 host. -// -// A pull failure is returned rather than fataled: a baseline tag older than the driver itself -// legitimately has no image, and the caller turns that into a skip, not a failure. -func EnsureImagePresent(image, platform string) error { - if err := exec.Command("docker", "image", "inspect", image).Run(); err == nil { - return nil - } - args := []string{"pull"} - if platform != "" { - args = append(args, "--platform", platform) - } - args = append(args, image) - if out, err := exec.Command("docker", args...).CombinedOutput(); err != nil { - return fmt.Errorf("failed to pull %s: %s\n%s", image, err, out) - } - return nil -} - // logContainerTimings re-emits the `[timing]` lines the driver wrote inside the container. A // successful `docker run`'s output is otherwise dropped on the floor, so without this the // in-container breakdown is invisible and every sync reads as one opaque span. The leading diff --git a/tests/testutils/integration/iceberg.go b/tests/testutils/iceberg.go similarity index 95% rename from tests/testutils/integration/iceberg.go rename to tests/testutils/iceberg.go index f27efff12..9db344aac 100644 --- a/tests/testutils/integration/iceberg.go +++ b/tests/testutils/iceberg.go @@ -1,4 +1,4 @@ -package integration +package testutils import ( "context" @@ -8,7 +8,6 @@ import ( "time" "github.com/apache/spark-connect-go/v35/spark/sql" - "github.com/datazip-inc/olake/tests/testutils" ) const ( @@ -32,7 +31,7 @@ func SparkSession(ctx context.Context, t *testing.T) (sql.SparkSession, error) { // The shared session outlives whichever test builds it, so its construction must not be // tied to that test's context (t.Context cancels when the test ends). ctx := context.WithoutCancel(ctx) - defer testutils.TrackPhaseTiming(t, "spark", "session build")() + defer TrackPhaseTiming(t, "spark", "session build")() for attempt := 1; ; attempt++ { sharedSpark, sharedSparkErr = sql.NewSessionBuilder().Remote(sparkConnectAddress).Build(ctx) if sharedSparkErr == nil || attempt == 3 { diff --git a/tests/testutils/integration/2pc.go b/tests/testutils/integration/2pc.go index 3bcdb7813..5aa30580c 100644 --- a/tests/testutils/integration/2pc.go +++ b/tests/testutils/integration/2pc.go @@ -84,7 +84,7 @@ func (cfg *Test) Iceberg2PCCDCRecovery( // Drop the Iceberg table and reset state before the first sync, so stale rows and the // olake_2pc table property left by a previous run can't leak into this run's recovery timeline. - DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + testutils.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) if err := testutils.ResetStateFile(cfg.TestConfig); err != nil { return fmt.Errorf("failed to reset state: %w", err) } @@ -159,7 +159,7 @@ func (cfg *Test) Iceberg2PCCDCRecovery( } t.Log("Iceberg 2PC CDC Recovery tests completed successfully") - DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + testutils.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) t.Logf("Dropped Iceberg table after 2PC CDC tests: %s", testTable) return nil } @@ -183,7 +183,7 @@ func (cfg *Test) Iceberg2PCIncrementalRecovery( // Drop the Iceberg table before the first sync, so stale rows and the olake_2pc table // property left by a previous run can't leak into this run's recovery timeline. - DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + testutils.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) // Patch streams.json: set sync_mode = incremental, cursor_field if err := updateStreamConfig(cfg.TestConfig, cfg.TestConfig.Namespace, testTable, "incremental", cfg.TestConfig.CursorField); err != nil { @@ -266,7 +266,7 @@ func (cfg *Test) Iceberg2PCIncrementalRecovery( } t.Log("Iceberg 2PC Incremental Recovery tests completed successfully") - DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + testutils.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) t.Logf("Dropped Iceberg table after 2PC Incremental tests: %s", testTable) return nil } diff --git a/tests/testutils/integration/parquet_rolling.go b/tests/testutils/integration/parquet_rolling.go index 9d11be2a1..68a9e0ac4 100644 --- a/tests/testutils/integration/parquet_rolling.go +++ b/tests/testutils/integration/parquet_rolling.go @@ -61,11 +61,11 @@ func (cfg *Test) testParquetRolling(ctx context.Context, t *testing.T, testTable // Start from an empty destination folder — the earlier parquet sub-tests leave files and // destination metadata behind, while this is an independent initial sync. - if err := deleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + if err := testutils.DeleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { return fmt.Errorf("failed to reset parquet table before rolling sync: %s", err) } defer func() { - if err := deleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + if err := testutils.DeleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { t.Logf("cleanup: failed to reset parquet table: %v", err) } }() @@ -86,10 +86,10 @@ func (cfg *Test) verifyParquetRolling(t *testing.T, table string) { t.Helper() ctx := t.Context() - client, err := newMinIOClient() + client, err := testutils.NewMinIOClient() require.NoError(t, err) - objects, err := listParquetObjects(ctx, client, cfg.TestConfig.DestinationDB, table) + objects, err := testutils.ListParquetObjects(ctx, client, cfg.TestConfig.DestinationDB, table) require.NoError(t, err, "failed to list rolled parquet files") var totalSize int64 @@ -107,7 +107,7 @@ func (cfg *Test) verifyParquetRolling(t *testing.T, table string) { // Sum footer row counts across every rolled file — no rows may be lost at a roll boundary. var totalRows int64 for _, obj := range objects { - reader, gerr := client.GetObject(ctx, parquetTestBucket, obj.Key, minio.GetObjectOptions{}) + reader, gerr := client.GetObject(ctx, testutils.ParquetBucket, obj.Key, minio.GetObjectOptions{}) require.NoErrorf(t, gerr, "failed to get object %s", obj.Key) data, rerr := io.ReadAll(reader) _ = reader.Close() diff --git a/tests/testutils/integration/sync.go b/tests/testutils/integration/sync.go index cd1ef51a2..f9a808be0 100644 --- a/tests/testutils/integration/sync.go +++ b/tests/testutils/integration/sync.go @@ -198,7 +198,7 @@ func (cfg *Test) IcebergFullLoadAndCDC( t.Logf("keeping %s source data (%s) is set", cfg.TestConfig.Driver, testutils.KeepTestDataEnvVar) } else { // Drop the Iceberg table after all tests are finished - DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + testutils.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) t.Logf("Dropped Iceberg table: %s", testTable) } @@ -216,7 +216,7 @@ func (cfg *Test) ParquetFullLoadAndCDC( if err := cfg.resetTable(ctx, t); err != nil { return fmt.Errorf("failed to reset table: %s", err) } - if err := deleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + if err := testutils.DeleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { return fmt.Errorf("failed to reset parquet table: %s", err) } @@ -297,7 +297,7 @@ func (cfg *Test) ParquetFullLoadAndCDC( // not. That is F2 in docs/backward-compatibility.md (parquet has no schema evolution; // the break surfaces in the reader). The consequence for compatibility is that a parquet // variant compares only its LAST case's output; see compareVariant. - if err := DeleteParquetFiles(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + if err := testutils.DeleteParquetFiles(t, cfg.TestConfig.DestinationDB, testTable); err != nil { t.Fatalf("Failed to delete parquet files before %s: %v", tc.name, err) } @@ -383,7 +383,7 @@ func (cfg *Test) IcebergFullLoadAndIncremental( } // drop iceberg table before sync - DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + testutils.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) t.Logf("Dropped Iceberg table: %s", testTable) if err := cfg.runSyncAndVerify( @@ -407,7 +407,7 @@ func (cfg *Test) IcebergFullLoadAndIncremental( if testutils.KeepTestData() { t.Logf("keeping %s source data (%s) is set", cfg.TestConfig.Driver, testutils.KeepTestDataEnvVar) } else { - DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) + testutils.DropIcebergTable(t, testTable, cfg.TestConfig.DestinationDB) t.Logf("Dropped Iceberg table: %s", testTable) } @@ -425,7 +425,7 @@ func (cfg *Test) ParquetFullLoadAndIncremental( if err := cfg.resetTable(ctx, t); err != nil { return fmt.Errorf("failed to reset table: %s", err) } - if err := deleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + if err := testutils.DeleteParquetTable(t, cfg.TestConfig.DestinationDB, testTable); err != nil { return fmt.Errorf("failed to reset parquet table: %s", err) } @@ -482,7 +482,7 @@ func (cfg *Test) ParquetFullLoadAndIncremental( // not. That is F2 in docs/backward-compatibility.md (parquet has no schema evolution; // the break surfaces in the reader). The consequence for compatibility is that a parquet // variant compares only its LAST case's output; see compareVariant. - if err := DeleteParquetFiles(t, cfg.TestConfig.DestinationDB, testTable); err != nil { + if err := testutils.DeleteParquetFiles(t, cfg.TestConfig.DestinationDB, testTable); err != nil { t.Fatalf("Failed to delete parquet files before %s: %v", tc.name, err) } diff --git a/tests/testutils/integration/verify.go b/tests/testutils/integration/verify.go index 4806e4437..e57f0d91f 100644 --- a/tests/testutils/integration/verify.go +++ b/tests/testutils/integration/verify.go @@ -20,10 +20,10 @@ import ( func VerifyIcebergSync(t *testing.T, tableName, icebergDB string, datatypeSchema map[string]string, defaultCDCColumnsSchema map[string]string, schema map[string]interface{}, opSymbol, partitionRegex, driver string, isCDC bool, excludedColumn string) { t.Helper() ctx := t.Context() - spark, err := SparkSession(ctx, t) + spark, err := testutils.SparkSession(ctx, t) require.NoError(t, err, "Failed to connect to Spark Connect server") - fullTableName := fmt.Sprintf("%s.%s.%s", IcebergCatalog, icebergDB, tableName) + fullTableName := fmt.Sprintf("%s.%s.%s", testutils.IcebergCatalog, icebergDB, tableName) // The shared session caches table snapshots, so refresh to see the rows the sync just committed. // Non-fatal: on a first sync the table may not exist yet, which the retry loop below handles. if _, refreshErr := spark.Sql(ctx, fmt.Sprintf("REFRESH TABLE %s", fullTableName)); refreshErr != nil { @@ -205,10 +205,10 @@ func VerifyIcebergSync(t *testing.T, tableName, icebergDB string, datatypeSchema func VerifyIcebergNoDuplicates(ctx context.Context, t *testing.T, tableName, icebergDB, opSymbol string, expectedRowCountByOpType int64) { t.Helper() - spark, err := SparkSession(ctx, t) + spark, err := testutils.SparkSession(ctx, t) require.NoError(t, err, "Failed to connect to Spark Connect server for duplicate check") - fullTableName := fmt.Sprintf("%s.%s.%s", IcebergCatalog, icebergDB, tableName) + fullTableName := fmt.Sprintf("%s.%s.%s", testutils.IcebergCatalog, icebergDB, tableName) // Refresh to get the latest committed Iceberg snapshot. refreshQuery := fmt.Sprintf("REFRESH TABLE %s", fullTableName) @@ -260,7 +260,7 @@ func VerifyParquetSync(t *testing.T, tableName, parquetDB string, datatypeSchema t.Helper() ctx := t.Context() - spark, err := SparkSession(ctx, t) + spark, err := testutils.SparkSession(ctx, t) require.NoError(t, err, "Failed to connect to Spark Connect server") parquetPath := fmt.Sprintf("s3a://warehouse/%s/%s", parquetDB, tableName) diff --git a/tests/testutils/integration/parquet.go b/tests/testutils/parquet.go similarity index 67% rename from tests/testutils/integration/parquet.go rename to tests/testutils/parquet.go index cd46bb209..f20849ea4 100644 --- a/tests/testutils/integration/parquet.go +++ b/tests/testutils/parquet.go @@ -1,4 +1,4 @@ -package integration +package testutils import ( "context" @@ -10,10 +10,10 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" ) -const parquetTestBucket = "warehouse" +const ParquetBucket = "warehouse" -// newMinIOClient returns a client for the MinIO instance backing the parquet destination in tests. -func newMinIOClient() (*minio.Client, error) { +// NewMinIOClient returns a client for the MinIO instance backing the parquet destination in tests. +func NewMinIOClient() (*minio.Client, error) { client, err := minio.New("localhost:9000", &minio.Options{ Creds: credentials.NewStaticV4("admin", "password", ""), Secure: false, @@ -24,10 +24,10 @@ func newMinIOClient() (*minio.Client, error) { return client, nil } -// listParquetObjects lists the .parquet objects lying directly in a table's folder in MinIO. -func listParquetObjects(ctx context.Context, client *minio.Client, parquetDB, tableName string) ([]minio.ObjectInfo, error) { +// ListParquetObjects lists the .parquet objects lying directly in a table's folder in MinIO. +func ListParquetObjects(ctx context.Context, client *minio.Client, parquetDB, tableName string) ([]minio.ObjectInfo, error) { objects := []minio.ObjectInfo{} - for object := range client.ListObjects(ctx, parquetTestBucket, minio.ListObjectsOptions{ + for object := range client.ListObjects(ctx, ParquetBucket, minio.ListObjectsOptions{ Prefix: parquetTablePath(parquetDB, tableName), Recursive: false, }) { @@ -51,16 +51,16 @@ func DeleteParquetFiles(t *testing.T, parquetDB, tableName string) error { t.Helper() parquetPath := parquetTablePath(parquetDB, tableName) - t.Logf("Cleaning up .parquet files in: s3a://%s/%s", parquetTestBucket, parquetPath) + t.Logf("Cleaning up .parquet files in: s3a://%s/%s", ParquetBucket, parquetPath) - minioClient, err := newMinIOClient() + minioClient, err := NewMinIOClient() if err != nil { return err } ctx := t.Context() - objects, err := listParquetObjects(ctx, minioClient, parquetDB, tableName) + objects, err := ListParquetObjects(ctx, minioClient, parquetDB, tableName) if err != nil { return err } @@ -68,7 +68,7 @@ func DeleteParquetFiles(t *testing.T, parquetDB, tableName string) error { for _, object := range objects { t.Logf("Deleting: %s", strings.TrimPrefix(object.Key, parquetPath)) - if err := minioClient.RemoveObject(ctx, parquetTestBucket, object.Key, minio.RemoveObjectOptions{}); err != nil { + if err := minioClient.RemoveObject(ctx, ParquetBucket, object.Key, minio.RemoveObjectOptions{}); err != nil { return fmt.Errorf("failed to delete %s: %s", object.Key, err) } } @@ -77,27 +77,27 @@ func DeleteParquetFiles(t *testing.T, parquetDB, tableName string) error { return nil } -// deleteParquetTable wipes a table's prefix recursively, unlike DeleteParquetFiles: it takes the +// DeleteParquetTable wipes a table's prefix recursively, unlike DeleteParquetFiles: it takes the // destination metadata with it, so the next sync starts as a genuinely initial one. -func deleteParquetTable(t *testing.T, parquetDB, tableName string) error { +func DeleteParquetTable(t *testing.T, parquetDB, tableName string) error { t.Helper() parquetPath := parquetTablePath(parquetDB, tableName) - minioClient, err := newMinIOClient() + minioClient, err := NewMinIOClient() if err != nil { return err } ctx := context.Background() deletedCount := 0 - for object := range minioClient.ListObjects(ctx, parquetTestBucket, minio.ListObjectsOptions{ + for object := range minioClient.ListObjects(ctx, ParquetBucket, minio.ListObjectsOptions{ Prefix: parquetPath, Recursive: true, }) { if object.Err != nil { return fmt.Errorf("error listing objects: %s", object.Err) } - if err := minioClient.RemoveObject(ctx, parquetTestBucket, object.Key, minio.RemoveObjectOptions{}); err != nil { + if err := minioClient.RemoveObject(ctx, ParquetBucket, object.Key, minio.RemoveObjectOptions{}); err != nil { return fmt.Errorf("failed to delete %s: %s", object.Key, err) } deletedCount++ diff --git a/tests/testutils/performance/performance.go b/tests/testutils/performance/performance.go index 21b7577c6..493d38a79 100644 --- a/tests/testutils/performance/performance.go +++ b/tests/testutils/performance/performance.go @@ -147,7 +147,7 @@ func (cfg *Test) destinationPrefix() []string { // runOlake runs the driver image with host networking so the benchmark reaches the external // instances directly, exactly as a deployed sync would. func (cfg *Test) runOlake(ctx context.Context, olakeArgs ...string) (int, []byte, error) { - args := testutils.DockerRunArgs(cfg.TestConfig, cfg.DriverImage, []string{"--network", "host"}, olakeArgs) + args := testutils.DockerRunArgs(cfg.TestConfig, []string{"--network", "host"}, olakeArgs) out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput() return testutils.DockerExitResult(out, err, olakeArgs[0]) } @@ -164,7 +164,7 @@ func (cfg *Test) timedSync(ctx context.Context, useState bool) ([]byte, error) { defer cancel() olakeArgs := testutils.SyncArgs(useState, icebergDestinationFile, cfg.destinationPrefix()...) - args := testutils.DockerRunArgs(cfg.TestConfig, cfg.DriverImage, []string{"--network", "host", "--name", name}, olakeArgs) + args := testutils.DockerRunArgs(cfg.TestConfig, []string{"--network", "host", "--name", name}, olakeArgs) out, err := exec.CommandContext(timedCtx, "docker", args...).CombinedOutput() if timedCtx.Err() == context.DeadlineExceeded { _ = exec.Command("docker", "kill", name).Run() diff --git a/tests/testutils/require/require.go b/tests/testutils/require/require.go index 24c712d44..520338afd 100644 --- a/tests/testutils/require/require.go +++ b/tests/testutils/require/require.go @@ -4,6 +4,7 @@ package require import ( "fmt" + "strings" "testing" "time" @@ -36,13 +37,23 @@ func run(t *testing.T, check func(c *failT)) { c := &failT{} check(c) for _, message := range c.messages { - t.Errorf(red+"%s"+reset, message) + t.Errorf("%s", colorize(message)) } if c.failed { t.FailNow() } } +func colorize(message string) string { + lines := strings.Split(message, "\n") + for i, line := range lines { + if line != "" { + lines[i] = red + line + reset + } + } + return strings.Join(lines, "\n") +} + func Contains(t *testing.T, s, contains any, msgAndArgs ...any) { t.Helper() run(t, func(c *failT) { trequire.Contains(c, s, contains, msgAndArgs...) }) diff --git a/tests/testutils/state_version.go b/tests/testutils/state_version.go index cdf6deedb..7923c2be5 100644 --- a/tests/testutils/state_version.go +++ b/tests/testutils/state_version.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" ) @@ -22,6 +23,17 @@ type StateVersionBaseline struct { Note string `json:"note"` } +// Gates reports whether this release's bump changed driver's semantics: the manifest names the +// drivers it touched, comma-separated, or "*" for all. +func (b StateVersionBaseline) Gates(driver string) bool { + for _, gated := range strings.Split(b.Drivers, ",") { + if gated = strings.TrimSpace(gated); gated == "*" || gated == driver { + return true + } + } + return false +} + type stateVersionManifest struct { LatestStateVersion int `json:"latest_state_version"` Baselines []StateVersionBaseline `json:"baselines"` @@ -33,9 +45,14 @@ var ( stateVersionErr error ) +// StateVersionManifestPath names the manifest file, for messages that point readers at it. +func StateVersionManifestPath(rootPath string) string { + return filepath.Join(rootPath, "constants", "state-versions.json") +} + func readStateVersionManifest(rootPath string) (stateVersionManifest, error) { stateVersionOnce.Do(func() { - path := filepath.Join(rootPath, "constants", "state-versions.json") + path := StateVersionManifestPath(rootPath) data, err := os.ReadFile(path) if err != nil { stateVersionErr = fmt.Errorf("failed to read the product state versions at %s: %w", path, err) diff --git a/tests/testutils/test_utils.go b/tests/testutils/test_utils.go index 81798e265..ad12925e6 100644 --- a/tests/testutils/test_utils.go +++ b/tests/testutils/test_utils.go @@ -39,8 +39,9 @@ type TestConfig struct { // only for amd64 and run emulated elsewhere. ImagePlatform string - // DriverImage the test is intedned to run under. Defaults to the building the image of the current codebase - DriverImage string + // DriverVersion to run the test against. This can be commit id or a released version + // Defaults to local codebase + DriverVersion string // OlakeRootPath is the repo the tests run from, the directory `make docker..build` // runs in and the committed fixtures are read from. Resolved by setupWorkingDir. @@ -51,6 +52,12 @@ type TestConfig struct { // is addressed by name through GetFilePath, so there is no path to keep a field for. TestWorkingDir string + // SeedExcludedColumns names the columns this suite must leave out of its seed data entirely -- + // columns the binary under test cannot sync at any price. The backward-compatibility runner + // fills it in per baseline from its rules; every other suite leaves it empty, so a driver's + // ExecuteQuery reads it and seeds everything by default. + SeedExcludedColumns []string `json:"-"` + // SourceBaseConfig is the working copy of source.json, parsed: the suite's own credentials, // database and prefixes, after applySuite renamed what it isolates. ExecuteQuery connects with // it, so the harness and olake always drive the same source. @@ -102,6 +109,12 @@ func NewTestConfig(t *testing.T, driver constants.DriverType, namespace, destina return cfg, nil } +func WithDriverVersion(version string) TestConfigOption { + return func(c *TestConfig) { + c.DriverVersion = version + } +} + func WithImagePlatform(platform string) TestConfigOption { return func(c *TestConfig) { c.ImagePlatform = platform @@ -140,14 +153,14 @@ func (c *TestConfig) setup(t *testing.T) error { t.Helper() c.generateSuiteName(t) + c.addTimingLogsMiddleware() + if err := c.setupWorkingDir(t); err != nil { return err } - if err := c.getOrBuildDriverImage(); err != nil { + if err := c.pullOrBuildDriverImage(t); err != nil { return err } - c.addTimingLogsMiddleware() - if err := c.applySuite(); err != nil { return err } @@ -166,25 +179,46 @@ func (c *TestConfig) String() string { return string(config) } -// getOrBuildDriverImage just sets the driver image in case builds the driver image against current codebase -func (c *TestConfig) getOrBuildDriverImage() error { - if c.DriverImage != "" { - return nil +// pullOrBuildDriverImage just sets the driver image in case builds the driver image against current codebase +func (c *TestConfig) pullOrBuildDriverImage(t *testing.T) (err error) { + if c.DriverVersion == "" { + c.DriverVersion = CurrentDriverVersion } - if image := os.Getenv(driverImageEnvVar); image != "" { - c.DriverImage = image - return nil + + if driverVersionEnv := os.Getenv(driverVersionEnvVar); driverVersionEnv != "" { + c.DriverVersion = driverVersionEnv } - driverVersion := os.Getenv(driverVersionEnvVar) - if driverVersion == "" { - if err := buildDriverImage(c); err != nil { - return fmt.Errorf("failed to build the %s driver image from the current codebase: %s", c.Driver, err) + return c.resolveImage(t) +} + +// resolveImage turns a version string into an image ref present on the local daemon: +// +// "local" -> built from current code +// "latest", "v0.6.5" -> olakego/source-:, pulled +// "9f3c1ab", "sha:9f3c1ab" -> built from a detached worktree at that commit +func (c *TestConfig) resolveImage(t *testing.T) error { + if c.DriverVersion == CurrentDriverVersion { + err := buildDriverImage(t, c) + if err != nil { + return err + } + } else { + commitID, ok := ResolveToCommit(c.OlakeRootPath, c.DriverVersion) + if ok { + c.DriverVersion = commitID + err := buildImageFromCommit(c, commitID) + if err != nil { + return err + } + } else { + err := ensureImagePresent(t, c.GetDriverImage()) + if err != nil { + return err + } } - driverVersion = currentDriverVersion } - c.DriverImage = getDriverImage(c.Driver, driverVersion) return nil } @@ -212,6 +246,10 @@ func (c *TestConfig) GetTableName() string { return Combine("test_table_olake", c.Suite) } +func (c *TestConfig) GetDriverImage() string { + return fmt.Sprintf("olakego/source-%s:%s", c.Driver, c.DriverVersion) +} + // GetFilePath addresses a file in the suite's working directory by name -- the configs, the // catalog, state and stats all live there, and the container reads them under the same names. func (c *TestConfig) GetFilePath(fileName string) string { @@ -232,7 +270,7 @@ func (c *TestConfig) GetFixturePath(fileName string, dataFormat ...string) strin func (c *TestConfig) setupWorkingDir(t *testing.T) (err error) { c.TestWorkingDir = t.TempDir() - c.OlakeRootPath, err = repoRoot() + c.OlakeRootPath, err = RepoRoot() if err != nil { return fmt.Errorf("failed to determine the repo root; the tests run from a git checkout: %s", err) } @@ -240,7 +278,7 @@ func (c *TestConfig) setupWorkingDir(t *testing.T) (err error) { commonFixturesDir := filepath.Join(c.OlakeRootPath, "tests/testdata") driverFixuresDir := filepath.Join(c.OlakeRootPath, "tests", c.Driver, "testdata", c.DataFormat) for _, fixtures := range []string{commonFixturesDir, driverFixuresDir} { - if err := copyDirFiles(fixtures, c.TestWorkingDir); err != nil { + if err := CopyDirFiles(fixtures, c.TestWorkingDir); err != nil { return fmt.Errorf("failed to copy the fixtures of %s into %s: %s", fixtures, c.TestWorkingDir, err) } } diff --git a/tests/testutils/utils.go b/tests/testutils/utils.go index 537f0cd03..da93a4638 100644 --- a/tests/testutils/utils.go +++ b/tests/testutils/utils.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "slices" "sort" "strconv" @@ -233,9 +234,9 @@ func SeedColumnsExcluded(excluded, supported []string) (map[string]bool, error) return drop, nil } -// copyDirFiles copies every file in src into dst, replacing what is already there. Files only: +// CopyDirFiles copies every file in src into dst, replacing what is already there. Files only: // a driver's data-format fixtures are a directory of their own, copied as their own source. -func copyDirFiles(src, dst string) error { +func CopyDirFiles(src, dst string) error { entries, err := os.ReadDir(src) if err != nil { return fmt.Errorf("failed to read %s: %s", src, err) @@ -251,12 +252,31 @@ func copyDirFiles(src, dst string) error { return nil } -// repoRoot is the git checkout the tests run from. Read straight from git rather than derived from +// RepoRoot is the git checkout the tests run from. Read straight from git rather than derived from // the running test's directory, which is the one thing here the repo layout does not fix. -func repoRoot() (string, error) { +func RepoRoot() (string, error) { root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() if err != nil { return "", err } return strings.TrimSpace(string(root)), nil } + +// ResolveToCommit tells if the string passed resolved to a git commit and returns it, abbreviated +// the way git itself abbreviates: the short form is what names the image and every path and +// identifier derived from it, and a full 40-char sha overruns limits those have -- Postgres caps a +// replication slot name at 63 characters and truncates the overflow silently. +func ResolveToCommit(gitRootPath, str string) (string, bool) { + str = strings.TrimPrefix(str, "sha:") + commitPattern := regexp.MustCompile(`^[0-9a-f]{7,40}$`) + if !commitPattern.MatchString(str) { + return "", false + } + // --short both verifies the commit exists and picks a length git considers unambiguous here. + short, err := exec.Command("git", "-C", gitRootPath, "rev-parse", "--short", str+"^{commit}").Output() + if err != nil { + return "", false + } + + return strings.TrimSpace(string(short)), true +} From bb9f7558f59e4f1f0260067cc55852041704034e Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Thu, 27 Aug 2026 18:50:30 +0530 Subject: [PATCH 05/20] chore: compatibility test with integration tests --- .github/actions/commit-image/action.yml | 63 +++++++ .github/actions/commit-image/commit-image.sh | 28 +++ .github/actions/detect-drivers/action.yml | 5 + .../actions/detect-drivers/detect-drivers.sh | 3 + .github/workflows/test-preflight.yml | 30 +--- .github/workflows/tests.yml | 169 ++++++++++-------- Makefile | 10 ++ .../testutils/compatibility/compatibility.go | 65 +++++-- tests/testutils/docker.go | 10 +- tests/testutils/test_utils.go | 26 +-- 10 files changed, 273 insertions(+), 136 deletions(-) create mode 100644 .github/actions/commit-image/action.yml create mode 100755 .github/actions/commit-image/commit-image.sh diff --git a/.github/actions/commit-image/action.yml b/.github/actions/commit-image/action.yml new file mode 100644 index 000000000..db0b931e1 --- /dev/null +++ b/.github/actions/commit-image/action.yml @@ -0,0 +1,63 @@ +name: Commit driver image +description: > + Makes the driver image for an arbitrary commit available locally, caching it in a container + registry so the first run that needs a commit builds it and every run after pulls. Tagged with the + abbreviated sha, which is what the test harness resolves a commit id to and the ref it looks up -- + it then finds the image present and skips its own build. + +inputs: + driver: + description: Driver to build the image for. + required: true + sha: + description: Commit to build. Empty is a no-op, for events that have no commit to compare against. + required: false + default: '' + cache-repo: + description: Registry repository the cached images live under, e.g. ghcr.io/owner/repo. + required: true + registry: + description: Registry to authenticate against. + required: false + default: ghcr.io + username: + description: Registry user. + required: true + token: + description: Registry token. A read-only one still pulls; publishing is skipped. + required: true + path: + description: > + Where to check the commit's tree out. actions/checkout refuses anything outside the workspace, + so it lands beside the caller's own checkout and must sit somewhere .dockerignore already + excludes -- otherwise it joins the context of every other image the job builds, changing that + context and evicting its layer cache. tests/ is excluded in this repo, hence the default. + required: false + default: tests/.commit-image-src + +runs: + using: composite + steps: + - name: Check out the commit + if: inputs.sha != '' + uses: actions/checkout@v7 + with: + ref: ${{ inputs.sha }} + path: ${{ inputs.path }} + + - name: Log in to the cache registry + if: inputs.sha != '' + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ inputs.token }} + + - name: Pull or build the commit's image + shell: bash + env: + DRIVER: ${{ inputs.driver }} + SHA: ${{ inputs.sha }} + SRC: ${{ inputs.path }} + CACHE_REPO: ${{ inputs.cache-repo }} + run: ${{ github.action_path }}/commit-image.sh diff --git a/.github/actions/commit-image/commit-image.sh b/.github/actions/commit-image/commit-image.sh new file mode 100755 index 000000000..ebb65cac7 --- /dev/null +++ b/.github/actions/commit-image/commit-image.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ -z "${SHA:-}" ]; then + echo "::notice::no commit to prepare an image for; skipping" + exit 0 +fi + +tag=$(git rev-parse --short "$SHA") +image="olakego/source-$DRIVER:$tag" +cache="$CACHE_REPO/source-$DRIVER:$tag" + +if docker pull -q "$cache"; then + docker tag "$cache" "$image" + echo "restored $image from $cache" + exit 0 +fi + +# That tree's own Iceberg jar: the Dockerfile copies it out of the build context, and the commit's +# Go side speaks that jar's RPC. +make -C "$SRC" iceberg.jar +docker buildx build --progress=plain --cache-from type=gha,scope=olake-base \ + --load --build-arg DRIVER_NAME="$DRIVER" -t "$image" "$SRC" + +# Non-fatal: a fork's token is read-only, and failing to publish only costs the next run the build +# this one just did. +docker tag "$image" "$cache" +docker push "$cache" || echo "::notice::could not publish $cache (read-only token?)" diff --git a/.github/actions/detect-drivers/action.yml b/.github/actions/detect-drivers/action.yml index 47d02b4dc..958c23a57 100644 --- a/.github/actions/detect-drivers/action.yml +++ b/.github/actions/detect-drivers/action.yml @@ -14,6 +14,11 @@ outputs: drivers: description: JSON array of drivers to run; empty when a change touches no tested driver. value: ${{ steps.resolve.outputs.drivers }} + driver-labels: + description: > + JSON object mapping each driver to its display name, for job titles: expressions cannot + change case, so the capitalised form is produced in the script. + value: ${{ steps.resolve.outputs.driver-labels }} runs: using: composite diff --git a/.github/actions/detect-drivers/detect-drivers.sh b/.github/actions/detect-drivers/detect-drivers.sh index af423c768..9c9999223 100755 --- a/.github/actions/detect-drivers/detect-drivers.sh +++ b/.github/actions/detect-drivers/detect-drivers.sh @@ -26,3 +26,6 @@ fi drivers=$(printf '%s\n' $selected | sort -u | jq -Rc '[., inputs] | map(select(. != ""))') echo "drivers=$drivers" >> "$GITHUB_OUTPUT" echo "Affected drivers: $drivers" + +labels=$(printf '%s' "$drivers" | jq -c 'map({key: ., value: ((.[0:1] | ascii_upcase) + .[1:])}) | from_entries') +echo "driver-labels=$labels" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/test-preflight.yml b/.github/workflows/test-preflight.yml index bda1c0c3d..31d37f5d4 100644 --- a/.github/workflows/test-preflight.yml +++ b/.github/workflows/test-preflight.yml @@ -3,15 +3,6 @@ name: Test Preflight on: workflow_call: inputs: - suite: - description: 'Suite being gated, for the approval job name.' - required: true - type: string - environment: - description: 'Approval environment. Empty runs ungated, which is what a push to a protected branch does.' - required: false - default: '' - type: string go-checks: description: 'Run lint + gosec with the Go cache job. The suite that owns the checks sets it; the others just consume the cache.' required: false @@ -21,9 +12,9 @@ on: drivers: description: 'JSON array of drivers the caller should fan its matrix out over; [] means nothing to test.' value: ${{ jobs.preflight.outputs.drivers }} - attempt: - description: 'The run attempt the approval was given on, so a caller can re-prompt on a re-run.' - value: ${{ jobs.approve.outputs.attempt }} + driver-labels: + description: 'JSON object of driver -> display name, for the caller job titles.' + value: ${{ jobs.preflight.outputs.driver-labels }} jobs: # Ungated, like the three cache jobs below it: the environment prompts once per wave of jobs that @@ -37,6 +28,7 @@ jobs: pull-requests: read outputs: drivers: ${{ steps.drivers.outputs.drivers }} + driver-labels: ${{ steps.drivers.outputs.driver-labels }} jar-cached: ${{ steps.jar-cache.outputs.cache-hit }} go-cached: ${{ steps.go-cache.outputs.cache-hit }} go-checks-changed: ${{ steps.go-checks.outputs.changed }} @@ -92,20 +84,6 @@ jobs: key: ${{ runner.os }}-aptwarm-${{ hashFiles('Dockerfile') }} lookup-only: true - # The run's single approval, last before the matrix so a pre-job failure fails or skips it too -- - # then any re-run re-executes it, and every attempt that reaches the drivers prompts exactly once. - approve: - name: Approve ${{ inputs.suite }} tests - needs: [preflight, build-jar, apt-warm, go-cache] - if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.preflight.outputs.drivers != '[]' && needs.build-jar.result != 'failure' && needs.apt-warm.result != 'failure' && needs.go-cache.result != 'failure' }} - runs-on: ubuntu-latest - timeout-minutes: 5 - environment: ${{ inputs.environment }} - outputs: - attempt: ${{ github.run_attempt }} - steps: - - run: echo "Approved -- running the ${{ inputs.suite }} matrix." - # Only when the jar is missing: preflight already looked up the cache, so an unchanged writer # skips Maven and this whole job. build-jar: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 79ab75dbf..f5340c347 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,35 +42,64 @@ on: jobs: # Detect changes, publish the caches, build the jar, take the run's single approval -- once for - # both matrices below, and shared with performance-test.yml (test-preflight.yml). + # every matrix below. # go-checks runs here: this workflow owns the tests modules' lint and gosec. preflight: - # The called workflow's jobs render as " / ", so this is what keeps - # the internal name out of the run's job list. The job id stays preflight for `needs:`. - name: Tests + # The called workflow's jobs render as " / ". + name: Pre-Validation uses: ./.github/workflows/test-preflight.yml permissions: contents: read pull-requests: read with: - suite: integration + compatibility - environment: ${{ github.event_name == 'pull_request' && 'integration_tests' || '' }} go-checks: true secrets: inherit - # One job per driver, each on its own VM with its own Docker daemon: it brings up its own source - # plus destination stack, builds its own image and runs the driver's suites against it. Every step - # up to the last is anchored here and aliased by the compatibility job, which needs the same setup. - # - # No `if:` needed: `needs` already requires preflight to have succeeded, and a matrix over an - # empty driver array is skipped on its own. - integration-tests: - name: Integration Test ${{ matrix.driver }} + # The run's single approval, its own job rather than the last one inside the called workflow: it + # gates the suites below, so it belongs beside them rather than under the pre-validation heading. + # `needs: preflight` covers that whole workflow, so this still runs only once change detection and + # every cache job has finished. + approve: + name: Approve Test Deployment needs: preflight + if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.preflight.outputs.drivers != '[]' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + environment: ${{ github.event_name == 'pull_request' && 'integration_tests' || '' }} + outputs: + attempt: ${{ github.run_attempt }} + env: + GATE_ENVIRONMENT: integration_tests + steps: + - name: Verify the approval gate is configured + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + rules=$(gh api "repos/$GITHUB_REPOSITORY/environments/$GATE_ENVIRONMENT" --jq '[.protection_rules[].type] | join(",")') + echo "protection rules on $GATE_ENVIRONMENT: ${rules:-none}" + case ",$rules," in + *,required_reviewers,*) echo "the approval gate is configured" ;; + *) echo "::error::environment $GATE_ENVIRONMENT has no required_reviewers, so nothing was reviewed; add them in Settings -> Environments" + exit 1 ;; + esac + + - run: echo "Approved -- running the end to end tests" + + e2e-tests: + name: E2E Test ${{ matrix.driver }} + needs: [preflight, approve] + # packages: write publishes the baseline image below; a fork PR's token stays read-only. + permissions: + contents: read + packages: write if: needs.preflight.outputs.drivers != '[]' runs-on: 16gb-runner - environment: ${{ github.event_name == 'pull_request' && needs.preflight.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} - timeout-minutes: 45 + environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} + # The two suites run concurrently, so this is the slower of them plus what they cost each other + # in contention -- not the sum. + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -104,6 +133,20 @@ jobs: timeout-minutes: 15 run: make -j2 --output-sync=target olake.${{ matrix.driver }}.up olake.destination.all.up + # Compatibility pulls its baseline images from a shared-egress runner pool, which is exactly + # where Docker Hub's anonymous rate limit bites. + - &docker-login + name: Log in to Docker Hub + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + if [ -n "$DOCKER_USERNAME" ]; then + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + else + echo "::notice::no Docker Hub credentials configured; pulling anonymously" + fi + - &buildx name: Set up Docker Buildx uses: ./.github/actions/buildx @@ -121,6 +164,21 @@ jobs: timeout-minutes: 20 run: make docker.${{ matrix.driver }}.build DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --cache-from type=gha,scope=olake-${{ matrix.driver }} --load" + - name: Build base branch image + id: base-image + background: true + timeout-minutes: 20 + uses: ./.github/actions/commit-image + with: + driver: ${{ matrix.driver }} + sha: ${{ github.event.pull_request.base.sha }} + cache-repo: ghcr.io/${{ github.repository }} + username: ${{ github.actor }} + token: ${{ secrets.GITHUB_TOKEN }} + # Under tests/, which .dockerignore already excludes, so it stays out of the candidate + # image's build context without an entry of its own. + path: tests/.compat-base + # setup-go's cache keys on go.sum, which this repo does not track, so it would cache nothing. - &go name: Set up Go @@ -144,7 +202,7 @@ jobs: - &wait-background name: Wait for background jobs - wait: [containers, image, go-build] + wait: [containers, image, base-image, go-build] - &wait-ready name: Wait for source + destination readiness @@ -165,53 +223,36 @@ jobs: "olakego/source-${{ matrix.driver }}:local" \ check --destination /testdata/iceberg_destination.json - # Integration + 2PC + (kafka) Rebalance in one go test, against the image built above. - # The pin is what stops the harness rebuilding that image: it rebuilds by default so a local - # run tests current code, and only a caller that has already built it says otherwise. - - name: Run tests + - name: Run E2E tests env: OLAKE_PRE_BUILT_IMAGE: olakego/source-${{ matrix.driver }}:local - run: make test.integration.${{ matrix.driver }} + COMPATIBILITY_BASELINE: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + if [ -n "$COMPATIBILITY_BASELINE" ]; then + make test.e2e.${{ matrix.driver }} COMPATIBILITY_BASELINE="$COMPATIBILITY_BASELINE" + else + make test.integration.${{ matrix.driver }} + fi - # The same drivers against a baseline image: see the header for what "baseline" means per event. - # Its own job, on its own machine, so a compatibility failure is re-run without re-running integration -- - # the setup is the integration job's, aliased rather than repeated. - compatibility-tests: - name: Backward Compatibility ${{ matrix.driver }} - needs: preflight - # Skipped on a staging push: the sweep is release-gating, and the PR into staging already ran it. - # The drivers check is for visibility, as on the integration job above. - if: (github.event_name != 'push' || github.ref == 'refs/heads/master') && needs.preflight.outputs.drivers != '[]' + + compatibility-sweep: + name: ${{ fromJSON(needs.preflight.outputs.driver-labels)[matrix.driver] }} Backward Compatibility Sweep + needs: [preflight, approve] runs-on: 16gb-runner - environment: ${{ github.event_name == 'pull_request' && needs.preflight.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} - # Longer than the integration job's 45: a sweep runs one pipeline pair per baseline, in series. + environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} + # One pipeline pair per baseline, run one after another. timeout-minutes: 90 strategy: fail-fast: false matrix: - driver: ${{ fromJSON(needs.preflight.outputs.drivers) }} - env: - # Empty is the sweep; a PR pins the baseline to its own base commit. - COMPATIBILITY_BASELINE: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.baseline }} + driver: ${{ fromJSON((github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.base_ref == 'master')) && needs.preflight.outputs.drivers || '[]') }} steps: - *checkout - *data-dirs - *driver-deps - *containers - - # The sweep pulls one image per baseline from a shared-egress runner pool, which is exactly - # where Docker Hub's anonymous rate limit bites. - - name: Log in to Docker Hub - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: | - if [ -n "$DOCKER_USERNAME" ]; then - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - else - echo "::notice::no Docker Hub credentials configured; pulling anonymously" - fi - + - *docker-login - *buildx - *jar - *driver-image @@ -225,20 +266,18 @@ jobs: - *wait-ready - *catalog - # Pinned like the integration job: the candidate side runs the image built above, and only - # the baseline image is resolved (pulled or built) by the harness. - - name: Run compatibility tests + # Empty baseline is the whole manifest; a manual run can pin one instead. + - name: Run the compatibility sweep env: OLAKE_PRE_BUILT_IMAGE: olakego/source-${{ matrix.driver }}:local - run: make test.compatibility.${{ matrix.driver }} COMPATIBILITY_BASELINE="$COMPATIBILITY_BASELINE" + run: make test.compatibility.${{ matrix.driver }} COMPATIBILITY_BASELINE="${{ inputs.baseline }}" # Benchmarks against the remote instances, so it shares this run's preflight but none of its # stack: no local containers, and the approval is its own environment (which is also where its AWS # and VPN secrets live). Staging pushes only, which is where the benchmark history is kept. performance-tests: name: Performance ${{ matrix.driver }} - needs: preflight - if: needs.preflight.outputs.drivers != '[]' && github.event_name == 'push' && github.ref == 'refs/heads/staging' + needs: [preflight, approve] environment: Performance Testing permissions: actions: read @@ -247,21 +286,9 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # TODO: add benchmark tests for mongodb and oracle (instance ids TBD). matrix: - include: - - driver: mysql - driver_upper: MYSQL - instance_id: 8 - - driver: postgres - driver_upper: POSTGRES - instance_id: 9 - # TODO: add benchmark tests for the below databases - # - driver: mongodb - # driver_upper: MONGODB - # instance_id: - # - driver: oracle - # driver_upper: ORACLE - # instance_id: + include: ${{ fromJSON((github.event_name == 'push' && github.ref == 'refs/heads/staging' && needs.preflight.outputs.drivers != '[]') && '[{"driver":"mysql","driver_upper":"MYSQL","instance_id":8},{"driver":"postgres","driver_upper":"POSTGRES","instance_id":9}]' || '[]') }} steps: - *checkout diff --git a/Makefile b/Makefile index f4d679117..3a4577f6f 100644 --- a/Makefile +++ b/Makefile @@ -340,6 +340,16 @@ test.integration.$(1): prepare.$(1) endef $(foreach d,$(SOURCE_DRIVERS),$(eval $(call DRIVER_TEST_template,$(d)))) +define E2E_TEST_template +.PHONY: test.e2e.$(1) +test.e2e.$(1): prepare.$(1) + @$$(call driver_test_setup,$(1)) + $$(GO_ENV.$(1)) cd tests && \ + OLAKE_COMPATIBILITY_TEST_BASELINE=$$(COMPATIBILITY_BASELINE) \ + go test -v ./$(1)/... -timeout 0 -count=1 -parallel 8 -skip 'Performance' +endef +$(foreach d,$(SOURCE_DRIVERS),$(eval $(call E2E_TEST_template,$(d)))) + define DRIVER_SUITE_template .PHONY: test.$(2).$(1) test.$(2).$(1): prepare.$(1) diff --git a/tests/testutils/compatibility/compatibility.go b/tests/testutils/compatibility/compatibility.go index e6eba01e0..5c5577d9f 100644 --- a/tests/testutils/compatibility/compatibility.go +++ b/tests/testutils/compatibility/compatibility.go @@ -100,44 +100,71 @@ func (f *Test) RunBackwardCompatibility(t *testing.T) { require.NoError(t, err) for _, version := range baselineVersions { + // Before NewConfig, which resolves the baseline's image: a driver younger than a release has + // no image published for it, so building the config first turns a declared skip into + // "failed to pull olakego/source-:" -- a hard failure the gate exists to + // prevent. Only the driver-level gate can be answered here; the variant gate keys on the + // config's data format and stays in runCompatibilityBaseline. + reason, err := baselineSkipReason(currentConf.OlakeRootPath, currentConf.Driver, version) + require.NoError(t, err) + if reason != "" { + t.Run(version, func(t *testing.T) { t.Skip(reason) }) + continue + } + baselineConf := f.NewConfig(t, version) - if !t.Run(baselineConf.DriverVersion, func(t *testing.T) { f.runCompatibilityBaseline(t, baselineConf, currentConf) }) { + if !t.Run(baselineConf.DriverVersion, func(t *testing.T) { + t.Parallel() + f.runCompatibilityBaseline(t, baselineConf, currentConf) + }) { t.Logf("compatibility: stopping the sweep at %s; the later baselines carry newer code and would repeat it", version) return } } } +// baselineSkipReason is why this driver does not run against this baseline at all, or "" when it +// does: the global floor from state-versions.json, then the driver's own gate in +// compatibility_rules.json. Both are answerable from the driver name alone, which is what lets the +// caller skip a baseline before paying for its image. +func baselineSkipReason(rootPath, driver, spec string) (string, error) { + version, dated := parseReleaseTag(spec) + floorTag, err := compatibilityGlobalFloor(rootPath) + if err != nil { + return "", err + } + if globalFloor, _ := parseReleaseTag(floorTag); dated && compareRelease(version, globalFloor) < 0 { + return fmt.Sprintf("baseline %s predates %s, the oldest state-version baseline; the compatibility suite does not run below it", + spec, floorTag), nil + } + gate := compatibilityRules.Drivers[driver].compatibilityGate + if reason := gate.skipReason(version, dated); reason != "" { + return fmt.Sprintf("%s cannot run baseline %s: %s (compatibility_rules.json: %s)", driver, spec, reason, gate.Note), nil + } + + return "", nil +} + // runCompatibilityBaseline runs every writer group's variants against one baseline: the reference // side on baseline's image throughout, the upgrade side handing its stateful syncs to upgrade's. func (f *Test) runCompatibilityBaseline(t *testing.T, baseline, upgrade *testutils.TestConfig) { spec := baseline.DriverVersion driver, dataFormat := baseline.Driver, baseline.DataFormat - // The driver's own floor. A skip, not a failure: the driver declares it cannot run against - // releases this old (the why lives next to the declaration in compatibility_rules.json), and - // that limitation is data, not a regression. + // The variant's own floor. A skip, not a failure: the driver declares this data format cannot + // run against releases this old (the why lives next to the declaration in + // compatibility_rules.json), and that limitation is data, not a regression. The driver-level + // gate and the global floor were already answered by the caller, before this baseline's image + // was resolved -- see baselineSkipReason. baselineVersion, baselineDated := parseReleaseTag(spec) floorTag, err := compatibilityGlobalFloor(baseline.OlakeRootPath) require.NoError(t, err) globalFloor, _ := parseReleaseTag(floorTag) - if baselineDated && compareRelease(baselineVersion, globalFloor) < 0 { - t.Skipf("baseline %s predates %s, the oldest state-version baseline; the compatibility suite does not run below it", - spec, floorTag) - } driverRules := compatibilityRules.Drivers[driver] variantRules := driverRules.Variants[dataFormat] - for _, scoped := range []struct { - scope string - gate compatibilityGate - }{ - {driver, driverRules.compatibilityGate}, - {driver + "/" + dataFormat, variantRules.compatibilityGate}, - } { - if reason := scoped.gate.skipReason(baselineVersion, baselineDated); reason != "" { - t.Skipf("%s cannot run baseline %s: %s (compatibility_rules.json: %s)", - scoped.scope, spec, reason, scoped.gate.Note) - } + if reason := variantRules.compatibilityGate.skipReason(baselineVersion, baselineDated); reason != "" { + t.Skipf("%s/%s cannot run baseline %s: %s (compatibility_rules.json: %s)", + driver, dataFormat, spec, reason, variantRules.compatibilityGate.Note) } // Both images were pulled or built when the caller constructed the two configs, serially, diff --git a/tests/testutils/docker.go b/tests/testutils/docker.go index d365c997d..fe1c40387 100644 --- a/tests/testutils/docker.go +++ b/tests/testutils/docker.go @@ -79,15 +79,23 @@ func buildDriverImage(t *testing.T, cfg *TestConfig) error { // Dockerfile copies the jar out of the build context, and the old Go side speaks the old jar's // RPC), and a build entry point that exists in that tree -- `make docker..build IMAGE_TAG=...` // is recent, so fall back to a plain `docker build`, whose DRIVER_NAME build-arg is far older. -func buildImageFromCommit(cfg *TestConfig, commitID string) error { +func buildImageFromCommit(t *testing.T, cfg *TestConfig, commitID string) error { + t.Helper() imageTag := cfg.GetDriverImage() return resolveImageOnce(imageTag, func() error { if exec.Command("docker", "image", "inspect", imageTag).Run() == nil { + t.Logf("driver image %s is already present; not rebuilding it from %s", imageTag, commitID) return nil } + t.Logf("building driver image %s from a worktree at %s", imageTag, commitID) + defer TrackPhaseTiming(t, "driver-image", "build "+imageTag)() worktree := filepath.Join(cfg.TestWorkingDir, "olake-compatibility-"+commitID) + // Each step is minutes long -- maven, then a full image build off an old tree -- so time + // them separately; without it the whole thing is one silent span. run := func(what string, name string, args ...string) error { + t.Logf(" %s (%s)", what, commitID) + defer TrackPhaseTiming(t, "driver-image", what)() cmd := exec.Command(name, args...) out, err := cmd.CombinedOutput() if err != nil { diff --git a/tests/testutils/test_utils.go b/tests/testutils/test_utils.go index ad12925e6..4e541ed62 100644 --- a/tests/testutils/test_utils.go +++ b/tests/testutils/test_utils.go @@ -199,27 +199,15 @@ func (c *TestConfig) pullOrBuildDriverImage(t *testing.T) (err error) { // "9f3c1ab", "sha:9f3c1ab" -> built from a detached worktree at that commit func (c *TestConfig) resolveImage(t *testing.T) error { if c.DriverVersion == CurrentDriverVersion { - err := buildDriverImage(t, c) - if err != nil { - return err - } - } else { - commitID, ok := ResolveToCommit(c.OlakeRootPath, c.DriverVersion) - if ok { - c.DriverVersion = commitID - err := buildImageFromCommit(c, commitID) - if err != nil { - return err - } - } else { - err := ensureImagePresent(t, c.GetDriverImage()) - if err != nil { - return err - } - } + return buildDriverImage(t, c) + } + // A commit id is abbreviated on the way in, and that short form becomes the image tag. + if commitID, ok := ResolveToCommit(c.OlakeRootPath, c.DriverVersion); ok { + c.DriverVersion = commitID + return buildImageFromCommit(t, c, commitID) } - return nil + return ensureImagePresent(t, c.GetDriverImage()) } func (c *TestConfig) addTimingLogsMiddleware() { From 3fed000dcb1b5e4133d2244e91a1c8027b695858 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Fri, 28 Aug 2026 11:48:28 +0530 Subject: [PATCH 06/20] chore: single machine for single compatibility check --- .github/actions/commit-image/action.yml | 27 +-- .../commit-image/build-images-from-commit.sh | 60 +++++++ .github/actions/commit-image/commit-image.sh | 28 ---- .github/workflows/tests.yml | 117 +++++++++---- Makefile | 17 +- .../testutils/compatibility/compatibility.go | 68 ++++---- .../compatibility/compatibility_failure.go | 157 ++++++++++++++++++ 7 files changed, 362 insertions(+), 112 deletions(-) create mode 100755 .github/actions/commit-image/build-images-from-commit.sh delete mode 100755 .github/actions/commit-image/commit-image.sh create mode 100644 tests/testutils/compatibility/compatibility_failure.go diff --git a/.github/actions/commit-image/action.yml b/.github/actions/commit-image/action.yml index db0b931e1..528f0657d 100644 --- a/.github/actions/commit-image/action.yml +++ b/.github/actions/commit-image/action.yml @@ -1,13 +1,13 @@ -name: Commit driver image +name: Commit driver images description: > - Makes the driver image for an arbitrary commit available locally, caching it in a container - registry so the first run that needs a commit builds it and every run after pulls. Tagged with the - abbreviated sha, which is what the test harness resolves a commit id to and the ref it looks up -- - it then finds the image present and skips its own build. + Makes the driver images for an arbitrary commit available locally, caching them in a container + registry so the first run that needs a commit builds them and every run after pulls. Tagged with + the abbreviated sha, which is what the test harness resolves a commit id to and the ref it looks + up -- it then finds the image present and skips its own build. inputs: - driver: - description: Driver to build the image for. + drivers: + description: Space separated drivers to build images for. required: true sha: description: Commit to build. Empty is a no-op, for events that have no commit to compare against. @@ -26,6 +26,12 @@ inputs: token: description: Registry token. A read-only one still pulls; publishing is skipped. required: true + max-parallel: + description: > + How many images to build at once. Each is a full Go compile, and the job that calls this + usually has its own builds running beside it. + required: false + default: '3' path: description: > Where to check the commit's tree out. actions/checkout refuses anything outside the workspace, @@ -53,11 +59,12 @@ runs: username: ${{ inputs.username }} password: ${{ inputs.token }} - - name: Pull or build the commit's image + - name: Pull or build the commit's images shell: bash env: - DRIVER: ${{ inputs.driver }} + DRIVERS: ${{ inputs.drivers }} SHA: ${{ inputs.sha }} SRC: ${{ inputs.path }} CACHE_REPO: ${{ inputs.cache-repo }} - run: ${{ github.action_path }}/commit-image.sh + MAX_PARALLEL: ${{ inputs.max-parallel }} + run: ${{ github.action_path }}/build-images-from-commit.sh diff --git a/.github/actions/commit-image/build-images-from-commit.sh b/.github/actions/commit-image/build-images-from-commit.sh new file mode 100755 index 000000000..7e79812ad --- /dev/null +++ b/.github/actions/commit-image/build-images-from-commit.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ -z "${SHA:-}" ]; then + echo "::notice::no commit to prepare images for; skipping" + exit 0 +fi + +TAG=$(git rev-parse --short "$SHA") + +# The pull pass first: it is what decides whether the jar below has to be built at all, and a hit +# costs seconds against the minutes a build does. +missing=() +for driver in $DRIVERS; do + if docker pull -q "$CACHE_REPO/source-$driver:$TAG"; then + docker tag "$CACHE_REPO/source-$driver:$TAG" "olakego/source-$driver:$TAG" + echo "restored olakego/source-$driver:$TAG from the cache" + else + echo "no cached image for $driver at $TAG; it will be built" + missing+=("$driver") + fi +done + +if [ ${#missing[@]} -eq 0 ]; then + echo "every image for $TAG came from $CACHE_REPO" + exit 0 +fi + +# That tree's own Iceberg writer jar, once for all of them: the Dockerfile copies it out of the +# build context, the commit's Go side speaks that jar's RPC, and parallel builds would otherwise +# race maven on one output directory. Kept off stdout unless it fails -- several hundred lines of +# maven nobody reads when it works. +echo "building the iceberg jar for $TAG..." +started=$SECONDS +if ! jar_log=$(make -C "$SRC" iceberg.jar 2>&1); then + echo "$jar_log" + echo "::error::failed to build the iceberg jar at $TAG" + exit 1 +fi +echo "built the iceberg jar in $((SECONDS - started))s" + +# TODO: we can use make command for build once this PR merges as local builds gets tagged as olake/source... instead of olakego/source... +build_one() { + docker buildx build --progress=plain --cache-from type=gha,scope=olake-base \ + --load --build-arg DRIVER_NAME="$1" -t "olakego/source-$1:$TAG" "$SRC" + + # Non-fatal: a fork's token is read-only, and failing to publish only costs the next run the + # build this one just did. + docker tag "olakego/source-$1:$TAG" "$CACHE_REPO/source-$1:$TAG" + docker push "$CACHE_REPO/source-$1:$TAG" || echo "::notice::could not publish source-$1:$TAG (read-only token?)" +} +export -f build_one +export TAG SRC CACHE_REPO + +# xargs -P fans these out with no pids to track and no marker files: if any build fails it still +# finishes the rest and exits 123, which set -e turns into this script's failure. -e on the child +# shell too, so a failed build stops before it tags and pushes what it did not produce. Bounded, +# because each one is a full Go compile and the caller has its own builds running beside it. +echo "building ${missing[*]}" +printf '%s\n' "${missing[@]}" | xargs -P "${MAX_PARALLEL:-3}" -I{} bash -euc 'build_one {}' diff --git a/.github/actions/commit-image/commit-image.sh b/.github/actions/commit-image/commit-image.sh deleted file mode 100755 index ebb65cac7..000000000 --- a/.github/actions/commit-image/commit-image.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [ -z "${SHA:-}" ]; then - echo "::notice::no commit to prepare an image for; skipping" - exit 0 -fi - -tag=$(git rev-parse --short "$SHA") -image="olakego/source-$DRIVER:$tag" -cache="$CACHE_REPO/source-$DRIVER:$tag" - -if docker pull -q "$cache"; then - docker tag "$cache" "$image" - echo "restored $image from $cache" - exit 0 -fi - -# That tree's own Iceberg jar: the Dockerfile copies it out of the build context, and the commit's -# Go side speaks that jar's RPC. -make -C "$SRC" iceberg.jar -docker buildx build --progress=plain --cache-from type=gha,scope=olake-base \ - --load --build-arg DRIVER_NAME="$DRIVER" -t "$image" "$SRC" - -# Non-fatal: a fork's token is read-only, and failing to publish only costs the next run the build -# this one just did. -docker tag "$image" "$cache" -docker push "$cache" || echo "::notice::could not publish $cache (read-only token?)" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f5340c347..87db6dcfe 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,18 +87,12 @@ jobs: - run: echo "Approved -- running the end to end tests" - e2e-tests: - name: E2E Test ${{ matrix.driver }} + integration-tests: + name: Integration Test ${{ matrix.driver }} needs: [preflight, approve] - # packages: write publishes the baseline image below; a fork PR's token stays read-only. - permissions: - contents: read - packages: write if: needs.preflight.outputs.drivers != '[]' runs-on: 16gb-runner environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} - # The two suites run concurrently, so this is the slower of them plus what they cost each other - # in contention -- not the sum. timeout-minutes: 60 strategy: fail-fast: false @@ -164,21 +158,6 @@ jobs: timeout-minutes: 20 run: make docker.${{ matrix.driver }}.build DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --cache-from type=gha,scope=olake-${{ matrix.driver }} --load" - - name: Build base branch image - id: base-image - background: true - timeout-minutes: 20 - uses: ./.github/actions/commit-image - with: - driver: ${{ matrix.driver }} - sha: ${{ github.event.pull_request.base.sha }} - cache-repo: ghcr.io/${{ github.repository }} - username: ${{ github.actor }} - token: ${{ secrets.GITHUB_TOKEN }} - # Under tests/, which .dockerignore already excludes, so it stays out of the candidate - # image's build context without an entry of its own. - path: tests/.compat-base - # setup-go's cache keys on go.sum, which this repo does not track, so it would cache nothing. - &go name: Set up Go @@ -202,7 +181,7 @@ jobs: - &wait-background name: Wait for background jobs - wait: [containers, image, base-image, go-build] + wait: [containers, image, go-build] - &wait-ready name: Wait for source + destination readiness @@ -223,18 +202,90 @@ jobs: "olakego/source-${{ matrix.driver }}:local" \ check --destination /testdata/iceberg_destination.json - - name: Run E2E tests + - name: Run integration tests env: OLAKE_PRE_BUILT_IMAGE: olakego/source-${{ matrix.driver }}:local - COMPATIBILITY_BASELINE: ${{ github.event.pull_request.base.sha }} + run: make test.integration.${{ matrix.driver }} + + compatibility-base: + name: Backwards Compatibility with ${{ github.event.pull_request.base.ref || 'the base branch' }} + needs: [preflight, approve, integration-tests] + permissions: + contents: read + packages: write + if: github.event_name == 'pull_request' && needs.preflight.outputs.drivers != '[]' + runs-on: 32gb-runner + environment: ${{ needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} + timeout-minutes: 90 + env: + DRIVERS: ${{ join(fromJSON(needs.preflight.outputs.drivers), ' ') }} + steps: + - *checkout + - *data-dirs + - *driver-deps + + - name: Start containers + id: containers + background: true + timeout-minutes: 20 + run: make -j --output-sync=target olake.all.up DRIVERS="$DRIVERS" + + - *docker-login + - *buildx + - *jar + + - name: Build driver images + id: image + background: true + timeout-minutes: 30 + run: make -j3 --output-sync=target docker.all.build DRIVERS="$DRIVERS" DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load" + + - name: Build base branch images + id: base-image + background: true + timeout-minutes: 30 + uses: ./.github/actions/commit-image + with: + drivers: ${{ join(fromJSON(needs.preflight.outputs.drivers), ' ') }} + sha: ${{ github.event.pull_request.base.sha }} + cache-repo: ghcr.io/${{ github.repository }} + username: ${{ github.actor }} + token: ${{ secrets.GITHUB_TOKEN }} + path: tests/.compat-base + + - *go + - *go-caches + + - name: Install Dependencies + id: go-build + timeout-minutes: 20 + run: make test.build.all DRIVERS="$DRIVERS" + + - name: Wait for background jobs + wait: [containers, image, base-image, go-build] + + - name: Wait for source + destination readiness + timeout-minutes: 5 + run: make -j --output-sync=target olake.all.wait DRIVERS="$DRIVERS" + + - name: Check destination (bootstrap Iceberg catalog) run: | set -euo pipefail - if [ -n "$COMPATIBILITY_BASELINE" ]; then - make test.e2e.${{ matrix.driver }} COMPATIBILITY_BASELINE="$COMPATIBILITY_BASELINE" - else - make test.integration.${{ matrix.driver }} - fi + driver=${DRIVERS%% *} + echo "Bootstrapping Iceberg catalog via '$driver'..." + docker run --rm \ + -v "$PWD/tests/testdata:/testdata" \ + --add-host host.docker.internal:host-gateway \ + -e TELEMETRY_DISABLED=true \ + "olakego/source-$driver:local" \ + check --destination /testdata/iceberg_destination.json + # All of them at once: -k so one driver failing still leaves the rest reported, and + # --output-sync so each driver's log arrives whole instead of woven into the others'. + - name: Run the compatibility tests + env: + OLAKE_PRE_BUILT_IMAGE: "true" + run: make -k -j --output-sync=target test.compatibility DRIVERS="$DRIVERS" COMPATIBILITY_BASELINE="${{ github.event.pull_request.base.sha }}" compatibility-sweep: name: ${{ fromJSON(needs.preflight.outputs.driver-labels)[matrix.driver] }} Backward Compatibility Sweep @@ -260,9 +311,7 @@ jobs: - *go - *go-caches - *test-build - - name: Wait for background jobs - wait: [containers, image, go-build] - + - *wait-background - *wait-ready - *catalog diff --git a/Makefile b/Makefile index 3a4577f6f..2160ba4c9 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,9 @@ $(addsuffix .build,$(addprefix docker.,$(DRIVERS))): docker.%.build: --build-arg DRIVER_NAME=$* \ -t olakego/source-$*:$(IMAGE_TAG) . +.PHONY: docker.all.build +docker.all.build: $(addsuffix .build,$(addprefix docker.,$(DRIVERS))) + gomod: find . -name go.mod -execdir go mod tidy \; @@ -340,16 +343,6 @@ test.integration.$(1): prepare.$(1) endef $(foreach d,$(SOURCE_DRIVERS),$(eval $(call DRIVER_TEST_template,$(d)))) -define E2E_TEST_template -.PHONY: test.e2e.$(1) -test.e2e.$(1): prepare.$(1) - @$$(call driver_test_setup,$(1)) - $$(GO_ENV.$(1)) cd tests && \ - OLAKE_COMPATIBILITY_TEST_BASELINE=$$(COMPATIBILITY_BASELINE) \ - go test -v ./$(1)/... -timeout 0 -count=1 -parallel 8 -skip 'Performance' -endef -$(foreach d,$(SOURCE_DRIVERS),$(eval $(call E2E_TEST_template,$(d)))) - define DRIVER_SUITE_template .PHONY: test.$(2).$(1) test.$(2).$(1): prepare.$(1) @@ -388,7 +381,7 @@ test.compatibility.$(1): prepare.$(1) @$$(call driver_test_setup,$(1)) $$(GO_ENV.$(1)) cd tests && \ OLAKE_COMPATIBILITY_TEST_BASELINE=$$(COMPATIBILITY_BASELINE) \ - go test -v ./$(1)/... -timeout 0 -count=1 -parallel 8 -run 'Compatibility' + go test -v ./$(1)/... -timeout 0 -count=1 -run 'Compatibility' endef $(foreach d,$(SOURCE_DRIVERS),$(eval $(call COMPATIBILITY_TEST_template,$(d)))) @@ -450,7 +443,7 @@ help: @$(foreach d,$(SOURCE_DRIVERS),printf " %-44s %s\n" "test.performance.$(d)" "benchmark suite for $(d) (remote instances, no local stack)";) @$(foreach d,$(SOURCE_DRIVERS),printf " %-44s %s\n" "test.compatibility.$(d)" "backward-compatibility for upgrading from baseline to latest $(d): COMPATIBILITY_BASELINE=, empty = sweep every baseline in state-versions.json";) @printf " %-44s %s\n" "test.discover | test.sync | test.2pc | test.unit" "aggregate runs (all drivers at once)" - @printf " %-44s %s\n" "test.compatibility" "backward-compatibility for every driver, sequentially (COMPATIBILITY_BASELINE as above)" + @printf " %-44s %s\n" "test.compatibility" "backward-compatibility for every driver (add -j to run them at once; COMPATIBILITY_BASELINE as above)" @printf " %-44s %s\n" "test.build.all" "compile every driver's test binary (CI cache warm)" @if [ -n "$(strip $(HELP_TARGETS))" ]; then \ echo ""; \ diff --git a/tests/testutils/compatibility/compatibility.go b/tests/testutils/compatibility/compatibility.go index 5c5577d9f..24b1b6d89 100644 --- a/tests/testutils/compatibility/compatibility.go +++ b/tests/testutils/compatibility/compatibility.go @@ -220,6 +220,9 @@ func (f *Test) runCompatibilityBaseline(t *testing.T, baseline, upgrade *testuti // Whichever side fails first stops every group at its next variant boundary: the comparison // is skipped either way, so the remaining syncs would be minutes of output nothing reads. aborted := &atomic.Bool{} + // What every failed variant found, so the assertion at the end of this function -- the one CI + // shows in red -- can report the findings themselves rather than the fact that there were some. + report := &failureReport{driver: driver, spec: spec, baseline: baselineImage, candidate: candidateImage} // Subtest names double as suite segments, so they stay terse: the table and (postgres) slot // names built from the suite must clear a 63-byte identifier limit on sweep runs. completed := t.Run("g", func(t *testing.T) { @@ -230,6 +233,7 @@ func (f *Test) runCompatibilityBaseline(t *testing.T, baseline, upgrade *testuti t.Logf("compatibility group %s: skipping variant %q onwards; another run already failed", g.name, v.name) break } + diag := &diagnostics{} ok := t.Run(v.name, func(t *testing.T) { // Both sides start on the baseline version; pick moves the upgrade side's // stateful syncs to the candidate. @@ -246,11 +250,12 @@ func (f *Test) runCompatibilityBaseline(t *testing.T, baseline, upgrade *testuti runSide(t, upg, g, v, upgradePick, policies) }) }) { - t.Fatalf("a %s side failed; the comparison would be noise", v.name) + diag.fatalf(t, "a %s side never finished, so the two destinations were never compared; the side's own subtest output has why", v.name) } - compareVariant(t, policies, ref, upg, g, v) + compareVariant(t, diag, policies, ref, upg, g, v) }) if !ok { + report.add(g.name, v.name, diag) aborted.Store(true) t.Logf("compatibility group %s: stopping after variant %q", g.name, v.name) break @@ -260,7 +265,7 @@ func (f *Test) runCompatibilityBaseline(t *testing.T, baseline, upgrade *testuti t.Run(g.name, func(t *testing.T) { t.Parallel(); runGroup(t) }) } }) - require.True(t, completed, "a compatibility run failed") + require.Truef(t, completed, "%s", report.render(baseline.OlakeRootPath)) } // getCompatibilityBaselines returns the baselines to run driver against: the @@ -341,7 +346,7 @@ func compatibilityVariantGroups(driver string) []compatibilityGroup { // compareVariant asserts the upgrade run's destination for one scenario is indistinguishable from // the reference run's. -func compareVariant(t *testing.T, policies *assertionPolicies, ref, upg *testutils.TestConfig, g compatibilityGroup, v compatibilityVariant) { +func compareVariant(t *testing.T, diag *diagnostics, policies *assertionPolicies, ref, upg *testutils.TestConfig, g compatibilityGroup, v compatibilityVariant) { ctx := t.Context() spark, err := testutils.SparkSession(ctx, t) require.NoError(t, err, "failed to connect to Spark Connect server") @@ -362,10 +367,12 @@ func compareVariant(t *testing.T, policies *assertionPolicies, ref, upg *testuti // (emptyFinalState), and a shared failure to produce rows for any other -- the one shape // of regression a row diff can never catch, because there are no rows to diff. if refRel == "" || upgRel == "" { - require.Equalf(t, refRel == "", upgRel == "", - "only one run produced parquet files for %s (reference %q, upgrade %q): the binaries disagree about whether this case writes output", v.name, refDB, upgDB) - require.Truef(t, v.emptyFinalState, - "neither run left parquet files for %s (reference %q, upgrade %q), but its last case writes rows: both binaries produced nothing where output is expected", v.name, refDB, upgDB) + if (refRel == "") != (upgRel == "") { + diag.fatalf(t, "only one run produced parquet files for %s (reference %q, upgrade %q): the binaries disagree about whether this case writes output", v.name, refDB, upgDB) + } + if !v.emptyFinalState { + diag.fatalf(t, "neither run left parquet files for %s (reference %q, upgrade %q), but its last case writes rows: both binaries produced nothing where output is expected", v.name, refDB, upgDB) + } t.Logf("verified: neither run leaves parquet files for %s -- its last case is a delete-only batch, which writes none", v.name) return } @@ -373,7 +380,7 @@ func compareVariant(t *testing.T, policies *assertionPolicies, ref, upg *testuti t.Fatalf("unknown destination %q", g.destination) } - compareRelations(ctx, t, spark, refRel, upgRel, policies.typeOnly) + compareRelations(ctx, t, diag, spark, refRel, upgRel, policies.typeOnly) } // icebergRelation refreshes and returns the fully-qualified name of an Iceberg table: the shared @@ -403,29 +410,35 @@ func parquetRelation(ctx context.Context, t *testing.T, spark sql.SparkSession, // compareRelations is the assertion. Order matters: a schema mismatch has to be reported before a // row query that would fail confusingly because of it. -func compareRelations(ctx context.Context, t *testing.T, spark sql.SparkSession, refRel, upgRel string, volatile []string) { +func compareRelations(ctx context.Context, t *testing.T, diag *diagnostics, spark sql.SparkSession, refRel, upgRel string, volatile []string) { // 1. Non-vacuity FIRST. Two empty tables satisfy every diff below, and an empty reference is a // plausible outcome, not a far-fetched one: a stream the baseline binary could not validate // is skipped with a Warn and the sync still exits 0 (protocol/sync.go, D3 in the doc). Without // this guard that scenario reports a green. refCount := scalarCount(ctx, t, spark, "SELECT COUNT(*) AS n FROM "+refRel) - require.Greaterf(t, refCount, int64(0), - "the reference run produced no rows in %s; it is the source of truth, so an empty one makes the whole comparison vacuous (a silently skipped stream looks exactly like this)", refRel) + if refCount == 0 { + diag.fatalf(t, "the reference run produced no rows in %s; it is the source of truth, so an empty one makes the whole comparison vacuous (a silently skipped stream looks exactly like this)", refRel) + } upgCount := scalarCount(ctx, t, spark, "SELECT COUNT(*) AS n FROM "+upgRel) - require.Equalf(t, refCount, upgCount, "row count differs: reference %s has %d, upgrade %s has %d", refRel, refCount, upgRel, upgCount) + if refCount != upgCount { + diag.fatalf(t, "row count differs: reference %s has %d, upgrade %s has %d", refRel, refCount, upgRel, upgCount) + } // 2. Schema. Compared as a map, so a column order difference (schema evolution appends in // record-arrival order) is not a failure while an added, dropped or retyped column is. This // is the assertion that catches a type-mapping change -- I6 in the doc. refSchema := describeRelation(ctx, t, spark, refRel) upgSchema := describeRelation(ctx, t, spark, upgRel) - require.Equalf(t, refSchema, upgSchema, - "destination schema differs between the reference and upgrade runs.\n reference (%s): %v\n upgrade (%s): %v", refRel, refSchema, upgRel, upgSchema) + if !maps.Equal(refSchema, upgSchema) { + diag.fatalf(t, "destination schema differs between the reference and upgrade runs.\n reference (%s): %v\n upgrade (%s): %v", refRel, refSchema, upgRel, upgSchema) + } // 3. Per-op-type counts, so a row diff reads as "5 'u' rows where the reference had 6" rather // than an opaque set difference. - require.Equal(t, opTypeCounts(ctx, t, spark, refRel), opTypeCounts(ctx, t, spark, upgRel), - "per-_op_type row counts differ between the reference and upgrade runs") + refOps, upgOps := opTypeCounts(ctx, t, spark, refRel), opTypeCounts(ctx, t, spark, upgRel) + if !maps.Equal(refOps, upgOps) { + diag.fatalf(t, "per-_op_type row counts differ between the reference and upgrade runs: reference %v, upgrade %v", refOps, upgOps) + } // 4. Values, both directions. This is the assertion that catches a changed record: every // non-volatile column of every row must hold the same value on both sides. @@ -442,25 +455,24 @@ func compareRelations(ctx context.Context, t *testing.T, spark sql.SparkSession, // Name the columns that actually differ before dumping rows -- with 30-odd columns, a row dump // alone leaves you diffing two long tuples by eye. - reportColumnDiffs(ctx, t, spark, refRel, upgRel, cols) - logSampleRows(t, "only in the reference run", refRel, onlyInRef) - logSampleRows(t, "only in the upgrade run", upgRel, onlyInUpg) - t.Fatalf("row values differ between the reference and upgrade runs: %d row(s) only in %s, %d row(s) only in %s", + reportColumnDiffs(ctx, t, diag, spark, refRel, upgRel, cols) + logSampleRows(t, diag, "only in the reference run", refRel, onlyInRef) + logSampleRows(t, diag, "only in the upgrade run", upgRel, onlyInUpg) + diag.fatalf(t, "row values differ between the reference and upgrade runs: %d row(s) only in %s, %d row(s) only in %s", len(onlyInRef), refRel, len(onlyInUpg), upgRel) } // reportColumnDiffs names the columns whose values differ, with a sample from each side. Runs one // query per column, so it is called only after a diff has already been found. -func reportColumnDiffs(ctx context.Context, t *testing.T, spark sql.SparkSession, refRel, upgRel string, cols []string) { +func reportColumnDiffs(ctx context.Context, t *testing.T, diag *diagnostics, spark sql.SparkSession, refRel, upgRel string, cols []string) { for _, col := range cols { n := scalarCount(ctx, t, spark, fmt.Sprintf( "SELECT COUNT(*) AS n FROM (SELECT %s FROM %s EXCEPT ALL SELECT %s FROM %s)", col, refRel, col, upgRel)) if n == 0 { continue } - t.Logf(" column %s differs in %d row(s)", col, n) - t.Logf(" reference: %v", sampleColumn(ctx, spark, refRel, col)) - t.Logf(" upgrade: %v", sampleColumn(ctx, spark, upgRel, col)) + diag.logf(t, "column %s differs in %d row(s)\n reference: %v\n upgrade: %v", + col, n, sampleColumn(ctx, spark, refRel, col), sampleColumn(ctx, spark, upgRel, col)) } } @@ -481,13 +493,13 @@ func sampleColumn(ctx context.Context, spark sql.SparkSession, relation, col str return values } -func logSampleRows(t *testing.T, what, relation string, rows []types.Row) { +func logSampleRows(t *testing.T, diag *diagnostics, what, relation string, rows []types.Row) { for i, row := range rows { if i == 5 { - t.Logf(" ... and %d more %s", len(rows)-5, what) + diag.logf(t, "... and %d more %s", len(rows)-5, what) break } - t.Logf(" %s (%s): %v", what, relation, row) + diag.logf(t, "%s (%s): %v", what, relation, row) } } diff --git a/tests/testutils/compatibility/compatibility_failure.go b/tests/testutils/compatibility/compatibility_failure.go new file mode 100644 index 000000000..0ed335740 --- /dev/null +++ b/tests/testutils/compatibility/compatibility_failure.go @@ -0,0 +1,157 @@ +package compatibility + +// What a failed compatibility run reports. +// +// The evidence a run produces -- the columns that differ, the rows only one side has -- is found +// deep inside the variant subtests, and Go attributes it to them. The assertion CI shows in red is +// the one at the end of runCompatibilityBaseline, which knew only that something below it had +// failed. The two types here carry the evidence back up to it: diagnostics collects a variant's +// findings as they are produced, and failureReport gathers every failed variant into the single +// message that assertion reports. + +import ( + "fmt" + "strings" + "sync" + "testing" + + "github.com/datazip-inc/olake/tests/testutils" +) + +// diagnostics is one variant's account of its own failure: the reason it stopped, and whatever +// detail was gathered before it did. +type diagnostics struct { + mu sync.Mutex + reason string + lines []string +} + +// logf logs a line of detail and keeps a copy. The subtest's own output remains the fuller story +// -- it carries the passing variants too -- but everything logged through here also survives into +// the final report. +func (d *diagnostics) logf(t *testing.T, format string, args ...any) { + t.Helper() + line := fmt.Sprintf(format, args...) + t.Log(line) + d.mu.Lock() + defer d.mu.Unlock() + d.lines = append(d.lines, line) +} + +// fatalf records why the variant stopped and then stops it. The reason leads the report, ahead of +// the detail, however late it was discovered. +func (d *diagnostics) fatalf(t *testing.T, format string, args ...any) { + t.Helper() + line := fmt.Sprintf(format, args...) + d.mu.Lock() + d.reason = line + d.mu.Unlock() + t.Fatal(line) +} + +func (d *diagnostics) collected() (string, []string) { + d.mu.Lock() + defer d.mu.Unlock() + return d.reason, d.lines +} + +// variantFailure is one broken variant of one writer group. +type variantFailure struct { + group, variant string + reason string + detail []string +} + +// failureReport accumulates the variants that failed one baseline and renders the message the +// run's final assertion reports. Written from the group goroutines, which run in parallel. +type failureReport struct { + mu sync.Mutex + driver string + spec string + baseline string // the image both sides start on + candidate string // the image the upgrade side hands off to + failures []variantFailure +} + +func (r *failureReport) add(group, variant string, d *diagnostics) { + reason, detail := d.collected() + r.mu.Lock() + defer r.mu.Unlock() + r.failures = append(r.failures, variantFailure{group: group, variant: variant, reason: reason, detail: detail}) +} + +// render is the whole of what CI shows in red. +// +// It opens with the state version, in capitals, because that is the thing a reader has to reason +// about: "v0.3.15 failed" names the release we compared against, not what about it matters, while +// the state version IS the contract under test -- it is what a state file written by that release +// tells this build to honor. The manifest's note for the bump says what that meant, then every +// failed variant with the detail it gathered, then what the two runs actually were. +func (r *failureReport) render(rootPath string) string { + r.mu.Lock() + defer r.mu.Unlock() + + headline, note := r.headline(rootPath) + var b strings.Builder + fmt.Fprintf(&b, "\n%s\n", headline) + if note != "" { + fmt.Fprintf(&b, "what that state version changed: %s\n", note) + } + + fmt.Fprintf(&b, "\n%s failed:\n", plural(len(r.failures), "scenario")) + for _, f := range r.failures { + fmt.Fprintf(&b, "\n %s/%s\n", f.group, f.variant) + if f.reason != "" { + fmt.Fprintf(&b, "%s\n", indent(f.reason, " ")) + } + for _, line := range f.detail { + fmt.Fprintf(&b, "%s\n", indent(line, " ")) + } + if f.reason == "" && len(f.detail) == 0 { + fmt.Fprintf(&b, " failed before the two destinations could be compared; its own subtest output has why\n") + } + } + + fmt.Fprintf(&b, "\nreference run: every sync on %s\n", r.baseline) + fmt.Fprintf(&b, "upgrade run: the stateless load on %s, every sync after it on %s\n", r.baseline, r.candidate) + return b.String() +} + +// headline names the state version the baseline introduced, in capitals, and returns the +// manifest's note for it. A baseline that is not a release in the manifest -- a commit, an image +// ref, the base branch a pull request merges into -- has no state version to name, so it says what +// it does have. +func (r *failureReport) headline(rootPath string) (string, string) { + if baselines, err := testutils.StateVersionBaselines(rootPath); err == nil { + for _, b := range baselines { + if b.ReleaseTag == r.spec { + return fmt.Sprintf("STATE VERSION %d FAILED for %s -- baseline %s is the release that introduced it", + b.StateVersion, r.driver, b.ReleaseTag), b.Note + } + } + } + // No state version to name, so name the one this build reads it with: that is still the half + // of the contract the reader can act on. + if current, err := testutils.ProductStateVersion(rootPath); err == nil { + return fmt.Sprintf("BASELINE %s FAILED for %s -- state written by that build is not read the same way by this one, which is at state version %d", + r.spec, r.driver, current), "" + } + return fmt.Sprintf("BASELINE %s FAILED for %s", r.spec, r.driver), "" +} + +// indent prefixes every line, not just the first: a message that carries its own detail (a schema +// diff, a row dump) is several lines long and reads as one block only if all of them move. +func indent(s, prefix string) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + for i, line := range lines { + lines[i] = prefix + line + } + return strings.Join(lines, "\n") +} + +func plural(n int, noun string) string { + if n == 1 { + return fmt.Sprintf("1 %s", noun) + } + return fmt.Sprintf("%d %ss", n, noun) +} From d62e69b193322509708d3e31331ed82b4aabf13b Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Fri, 28 Aug 2026 12:19:28 +0530 Subject: [PATCH 07/20] chore: temp - skip the ghcr image restore to measure the worst case --- .../commit-image/build-images-from-commit.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/actions/commit-image/build-images-from-commit.sh b/.github/actions/commit-image/build-images-from-commit.sh index 7e79812ad..510c1948a 100755 --- a/.github/actions/commit-image/build-images-from-commit.sh +++ b/.github/actions/commit-image/build-images-from-commit.sh @@ -12,13 +12,15 @@ TAG=$(git rev-parse --short "$SHA") # costs seconds against the minutes a build does. missing=() for driver in $DRIVERS; do - if docker pull -q "$CACHE_REPO/source-$driver:$TAG"; then - docker tag "$CACHE_REPO/source-$driver:$TAG" "olakego/source-$driver:$TAG" - echo "restored olakego/source-$driver:$TAG from the cache" - else - echo "no cached image for $driver at $TAG; it will be built" - missing+=("$driver") - fi + # TEMPORARY: the restore is commented out to measure the worst case, every image built from + # scratch. Restore before merging. + # if docker pull -q "$CACHE_REPO/source-$driver:$TAG"; then + # docker tag "$CACHE_REPO/source-$driver:$TAG" "olakego/source-$driver:$TAG" + # echo "restored olakego/source-$driver:$TAG from the cache" + # continue + # fi + echo "no cached image for $driver at $TAG; it will be built" + missing+=("$driver") done if [ ${#missing[@]} -eq 0 ]; then From cfc9d9344633a648c01b062926becb3f5115b57b Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Fri, 28 Aug 2026 12:56:20 +0530 Subject: [PATCH 08/20] chore: single machine for single compatibility check --- .../commit-image/build-images-from-commit.sh | 4 +- .github/workflows/test-preflight.yml | 72 +++++++++++++++++++ .github/workflows/tests.yml | 16 +++-- 3 files changed, 83 insertions(+), 9 deletions(-) diff --git a/.github/actions/commit-image/build-images-from-commit.sh b/.github/actions/commit-image/build-images-from-commit.sh index 510c1948a..719013e94 100755 --- a/.github/actions/commit-image/build-images-from-commit.sh +++ b/.github/actions/commit-image/build-images-from-commit.sh @@ -12,8 +12,8 @@ TAG=$(git rev-parse --short "$SHA") # costs seconds against the minutes a build does. missing=() for driver in $DRIVERS; do - # TEMPORARY: the restore is commented out to measure the worst case, every image built from - # scratch. Restore before merging. + # TEMPORARY: the restore is commented out so every image is built from scratch, to time the worst + # case. Restore before merging, together with the lookup in test-preflight.yml. # if docker pull -q "$CACHE_REPO/source-$driver:$TAG"; then # docker tag "$CACHE_REPO/source-$driver:$TAG" "olakego/source-$driver:$TAG" # echo "restored olakego/source-$driver:$TAG from the cache" diff --git a/.github/workflows/test-preflight.yml b/.github/workflows/test-preflight.yml index 31d37f5d4..73ee4189a 100644 --- a/.github/workflows/test-preflight.yml +++ b/.github/workflows/test-preflight.yml @@ -26,6 +26,7 @@ jobs: permissions: contents: read pull-requests: read + packages: read outputs: drivers: ${{ steps.drivers.outputs.drivers }} driver-labels: ${{ steps.drivers.outputs.driver-labels }} @@ -33,9 +34,12 @@ jobs: go-cached: ${{ steps.go-cache.outputs.cache-hit }} go-checks-changed: ${{ steps.go-checks.outputs.changed }} apt-warmed: ${{ steps.apt-cache.outputs.cache-hit }} + base-image-misses: ${{ steps.base-images.outputs.misses }} steps: - name: Checkout code uses: actions/checkout@v7 + with: + fetch-depth: 0 # Shared with any workflow that wants to skip untouched drivers; the caller's matrix fans out # over whatever it returns, and an empty array skips the driver jobs entirely. @@ -62,6 +66,29 @@ jobs: echo "changed=$changed" >> "$GITHUB_OUTPUT" echo "Go-check inputs changed: $changed" + - name: Look up base branch images + id: base-images + if: github.event_name == 'pull_request' + env: + DOCKER_CLI_EXPERIMENTAL: enabled + BASE_SHA: ${{ github.event.pull_request.base.sha }} + DRIVERS: ${{ steps.drivers.outputs.drivers }} + run: | + set -euo pipefail + echo "${{ github.token }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + tag=$(git rev-parse --short "$BASE_SHA") + # TEMPORARY: the cache check is commented out so every driver counts as a miss and the + # build job below runs on the full set, to time the worst case. Restore before merging. + misses=$(for d in $(jq -r '.[]' <<<"$DRIVERS"); do + # if docker manifest inspect "ghcr.io/$GITHUB_REPOSITORY/source-$d:$tag" >/dev/null 2>&1; then + # echo "cached: source-$d:$tag" >&2 + # else + echo "$d" + # fi + done | jq -Rsc 'split("\n") | map(select(length > 0))') + echo "misses=$misses" >> "$GITHUB_OUTPUT" + echo "base branch images to build at $tag: $misses" + - name: Look up Iceberg writer jar cache id: jar-cache uses: ./.github/actions/iceberg-jar @@ -84,6 +111,51 @@ jobs: key: ${{ runner.os }}-aptwarm-${{ hashFiles('Dockerfile') }} lookup-only: true + base-images: + name: Build base branch image + needs: preflight + # Guarded rather than left to an empty matrix: inside a called workflow an empty matrix never + # resolves, and the calling job hangs on it instead of completing -- which skips everything + # gated on that job. A skipped job, which is what the three below do, is resolved and fine. + if: needs.preflight.outputs.base-image-misses != '' && needs.preflight.outputs.base-image-misses != '[]' + permissions: + contents: read + packages: write + # One machine rather than a driver each: the images share one Iceberg jar, which a job per + # driver would rebuild per driver, and the script fans the builds out beneath it. Hosted, since + # this overlaps the approval wait and the self-hosted pool is about to be wanted by the matrix. + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout code + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Log in to Docker Hub + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + if [ -n "$DOCKER_USERNAME" ]; then + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + else + echo "::notice::no Docker Hub credentials configured; pulling anonymously" + fi + + - name: Set up Docker Buildx + uses: ./.github/actions/buildx + + - name: Build base branch image + uses: ./.github/actions/commit-image + with: + drivers: ${{ join(fromJSON(needs.preflight.outputs.base-image-misses), ' ') }} + sha: ${{ github.event.pull_request.base.sha }} + cache-repo: ghcr.io/${{ github.repository }} + username: ${{ github.actor }} + token: ${{ secrets.GITHUB_TOKEN }} + path: tests/.compat-base + # Only when the jar is missing: preflight already looked up the cache, so an unchanged writer # skips Maven and this whole job. build-jar: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 87db6dcfe..7548f537b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,18 +51,20 @@ jobs: permissions: contents: read pull-requests: read + packages: write with: go-checks: true secrets: inherit # The run's single approval, its own job rather than the last one inside the called workflow: it # gates the suites below, so it belongs beside them rather than under the pre-validation heading. - # `needs: preflight` covers that whole workflow, so this still runs only once change detection and - # every cache job has finished. + # + # Deliberately depends on nothing. The prompt is a human wait and everything preflight does is + # ungated cache warming that runs regardless, so the two belong side by side: the reviewer is + # asked the moment the run starts and the caches warm while they decide. Every job below gates on + # `drivers != '[]'` for itself, so this needs nothing from preflight to know whether to ask. approve: name: Approve Test Deployment - needs: preflight - if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.preflight.outputs.drivers != '[]' }} runs-on: ubuntu-latest timeout-minutes: 5 environment: ${{ github.event_name == 'pull_request' && 'integration_tests' || '' }} @@ -209,7 +211,7 @@ jobs: compatibility-base: name: Backwards Compatibility with ${{ github.event.pull_request.base.ref || 'the base branch' }} - needs: [preflight, approve, integration-tests] + needs: [preflight, approve] permissions: contents: read packages: write @@ -238,9 +240,9 @@ jobs: id: image background: true timeout-minutes: 30 - run: make -j3 --output-sync=target docker.all.build DRIVERS="$DRIVERS" DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load" + run: make -j --output-sync=target docker.all.build DRIVERS="$DRIVERS" DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load" - - name: Build base branch images + - name: Restore base branch images id: base-image background: true timeout-minutes: 30 From 2c42541f8dde525f9f9a7eee69b49083f31ad4e8 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Fri, 28 Aug 2026 17:28:49 +0530 Subject: [PATCH 09/20] chore: cache driver docker images --- .../commit-image/build-images-from-commit.sh | 24 +++++- .github/workflows/tests.yml | 83 ++++++++++++++----- 2 files changed, 81 insertions(+), 26 deletions(-) diff --git a/.github/actions/commit-image/build-images-from-commit.sh b/.github/actions/commit-image/build-images-from-commit.sh index 719013e94..38e8f9fff 100755 --- a/.github/actions/commit-image/build-images-from-commit.sh +++ b/.github/actions/commit-image/build-images-from-commit.sh @@ -42,17 +42,33 @@ fi echo "built the iceberg jar in $((SECONDS - started))s" # TODO: we can use make command for build once this PR merges as local builds gets tagged as olake/source... instead of olakego/source... +# +# Output is collected and printed whole, in a group per driver: several of these run at once, and +# interleaved --progress=plain streams leave you unable to tell how many images actually built. build_one() { - docker buildx build --progress=plain --cache-from type=gha,scope=olake-base \ - --load --build-arg DRIVER_NAME="$1" -t "olakego/source-$1:$TAG" "$SRC" + local log="$WORK/$1.log" + if ! docker buildx build --progress=plain --cache-from type=gha,scope=olake-base \ + --load --build-arg DRIVER_NAME="$1" -t "olakego/source-$1:$TAG" "$SRC" > "$log" 2>&1; then + echo "::group::build $1 -- FAILED" + cat "$log" + echo "::endgroup::" + return 1 + fi # Non-fatal: a fork's token is read-only, and failing to publish only costs the next run the # build this one just did. docker tag "olakego/source-$1:$TAG" "$CACHE_REPO/source-$1:$TAG" - docker push "$CACHE_REPO/source-$1:$TAG" || echo "::notice::could not publish source-$1:$TAG (read-only token?)" + docker push "$CACHE_REPO/source-$1:$TAG" >> "$log" 2>&1 \ + || echo "::notice::could not publish source-$1:$TAG (read-only token?)" + + echo "::group::build $1 -- built and published" + cat "$log" + echo "::endgroup::" } export -f build_one -export TAG SRC CACHE_REPO +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +export TAG SRC CACHE_REPO WORK # xargs -P fans these out with no pids to track and no marker files: if any build fails it still # finishes the rest and exits 123, which set -e turns into this script's failure. -e on the child diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7548f537b..8a24b6a9e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,18 +58,21 @@ jobs: # The run's single approval, its own job rather than the last one inside the called workflow: it # gates the suites below, so it belongs beside them rather than under the pre-validation heading. - # - # Deliberately depends on nothing. The prompt is a human wait and everything preflight does is - # ungated cache warming that runs regardless, so the two belong side by side: the reviewer is - # asked the moment the run starts and the caches warm while they decide. Every job below gates on - # `drivers != '[]'` for itself, so this needs nothing from preflight to know whether to ask. + # `needs: preflight` covers that whole workflow, so this runs once change detection and every + # cache job has finished -- which is why nothing below needs preflight as well. It forwards what + # they read from it, so the chain is preflight -> approve -> suites and each job names only the + # thing it actually waits on. approve: name: Approve Test Deployment + needs: preflight + if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.preflight.outputs.drivers != '[]' }} runs-on: ubuntu-latest timeout-minutes: 5 environment: ${{ github.event_name == 'pull_request' && 'integration_tests' || '' }} outputs: attempt: ${{ github.run_attempt }} + drivers: ${{ needs.preflight.outputs.drivers }} + driver-labels: ${{ needs.preflight.outputs.driver-labels }} env: GATE_ENVIRONMENT: integration_tests steps: @@ -91,15 +94,18 @@ jobs: integration-tests: name: Integration Test ${{ matrix.driver }} - needs: [preflight, approve] - if: needs.preflight.outputs.drivers != '[]' + needs: approve + permissions: + contents: read + packages: write + if: needs.approve.outputs.drivers != '[]' runs-on: 16gb-runner environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} timeout-minutes: 60 strategy: fail-fast: false matrix: - driver: ${{ fromJSON(needs.preflight.outputs.drivers) }} + driver: ${{ fromJSON(needs.approve.outputs.drivers) }} steps: # Full history so the compatibility job can build the PR's base commit; the extra objects cost this # job a few seconds and keep one checkout definition for both. @@ -129,19 +135,22 @@ jobs: timeout-minutes: 15 run: make -j2 --output-sync=target olake.${{ matrix.driver }}.up olake.destination.all.up - # Compatibility pulls its baseline images from a shared-egress runner pool, which is exactly - # where Docker Hub's anonymous rate limit bites. + # Docker Hub: compatibility pulls its baseline images from a shared-egress runner pool, which + # is exactly where the anonymous rate limit bites. ghcr: where this run's driver images are + # published for the compatibility job to reuse. - &docker-login - name: Log in to Docker Hub + name: Log in to the image registries env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if [ -n "$DOCKER_USERNAME" ]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin else echo "::notice::no Docker Hub credentials configured; pulling anonymously" fi + echo "$GHCR_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - &buildx name: Set up Docker Buildx @@ -209,18 +218,28 @@ jobs: OLAKE_PRE_BUILT_IMAGE: olakego/source-${{ matrix.driver }}:local run: make test.integration.${{ matrix.driver }} + - name: Publish the driver image for the compatibility job + env: + CANDIDATE: ghcr.io/${{ github.repository }}/source-${{ matrix.driver }}:${{ github.sha }} + run: | + set -euo pipefail + docker tag "olakego/source-${{ matrix.driver }}:local" "$CANDIDATE" + # Non-fatal: a fork's token is read-only, and the compatibility job builds its own copy + # of whatever it cannot pull. + docker push "$CANDIDATE" || echo "::notice::could not publish $CANDIDATE (read-only token?)" + compatibility-base: name: Backwards Compatibility with ${{ github.event.pull_request.base.ref || 'the base branch' }} - needs: [preflight, approve] + needs: [approve, integration-tests] permissions: contents: read packages: write - if: github.event_name == 'pull_request' && needs.preflight.outputs.drivers != '[]' + if: github.event_name == 'pull_request' && needs.approve.outputs.drivers != '[]' runs-on: 32gb-runner environment: ${{ needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} timeout-minutes: 90 env: - DRIVERS: ${{ join(fromJSON(needs.preflight.outputs.drivers), ' ') }} + DRIVERS: ${{ join(fromJSON(needs.approve.outputs.drivers), ' ') }} steps: - *checkout - *data-dirs @@ -236,11 +255,31 @@ jobs: - *buildx - *jar - - name: Build driver images + - name: Restore driver images id: image background: true timeout-minutes: 30 - run: make -j --output-sync=target docker.all.build DRIVERS="$DRIVERS" DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load" + env: + CANDIDATE_REPO: ghcr.io/${{ github.repository }} + run: | + set -euo pipefail + missing=() + for d in $DRIVERS; do + if docker pull -q "$CANDIDATE_REPO/source-$d:$GITHUB_SHA"; then + docker tag "$CANDIDATE_REPO/source-$d:$GITHUB_SHA" "olakego/source-$d:local" + echo "restored olakego/source-$d:local from this run's integration build" + else + missing+=("$d") + fi + done + # An if, not `[ ... ] && exit 0`: on the false branch that list returns non-zero and + # set -e would end the step right here, silently skipping the build below. + if [ ${#missing[@]} -eq 0 ]; then + echo "every driver image came from this run's integration build" + exit 0 + fi + echo "::notice::nothing was published for ${missing[*]}; building them here" + make -j --output-sync=target docker.all.build DRIVERS="${missing[*]}" DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load" - name: Restore base branch images id: base-image @@ -248,7 +287,7 @@ jobs: timeout-minutes: 30 uses: ./.github/actions/commit-image with: - drivers: ${{ join(fromJSON(needs.preflight.outputs.drivers), ' ') }} + drivers: ${{ join(fromJSON(needs.approve.outputs.drivers), ' ') }} sha: ${{ github.event.pull_request.base.sha }} cache-repo: ghcr.io/${{ github.repository }} username: ${{ github.actor }} @@ -290,8 +329,8 @@ jobs: run: make -k -j --output-sync=target test.compatibility DRIVERS="$DRIVERS" COMPATIBILITY_BASELINE="${{ github.event.pull_request.base.sha }}" compatibility-sweep: - name: ${{ fromJSON(needs.preflight.outputs.driver-labels)[matrix.driver] }} Backward Compatibility Sweep - needs: [preflight, approve] + name: ${{ fromJSON(needs.approve.outputs.driver-labels)[matrix.driver] }} Backward Compatibility Sweep + needs: approve runs-on: 16gb-runner environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} # One pipeline pair per baseline, run one after another. @@ -299,7 +338,7 @@ jobs: strategy: fail-fast: false matrix: - driver: ${{ fromJSON((github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.base_ref == 'master')) && needs.preflight.outputs.drivers || '[]') }} + driver: ${{ fromJSON((github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.base_ref == 'master')) && needs.approve.outputs.drivers || '[]') }} steps: - *checkout - *data-dirs @@ -328,7 +367,7 @@ jobs: # and VPN secrets live). Staging pushes only, which is where the benchmark history is kept. performance-tests: name: Performance ${{ matrix.driver }} - needs: [preflight, approve] + needs: approve environment: Performance Testing permissions: actions: read @@ -339,7 +378,7 @@ jobs: fail-fast: false # TODO: add benchmark tests for mongodb and oracle (instance ids TBD). matrix: - include: ${{ fromJSON((github.event_name == 'push' && github.ref == 'refs/heads/staging' && needs.preflight.outputs.drivers != '[]') && '[{"driver":"mysql","driver_upper":"MYSQL","instance_id":8},{"driver":"postgres","driver_upper":"POSTGRES","instance_id":9}]' || '[]') }} + include: ${{ fromJSON((github.event_name == 'push' && github.ref == 'refs/heads/staging' && needs.approve.outputs.drivers != '[]') && '[{"driver":"mysql","driver_upper":"MYSQL","instance_id":8},{"driver":"postgres","driver_upper":"POSTGRES","instance_id":9}]' || '[]') }} steps: - *checkout From bee8d8d24ca3527aae70aec23230332ccc8e7ff0 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Fri, 28 Aug 2026 17:57:33 +0530 Subject: [PATCH 10/20] chore: artifacts for candidate docker image --- .../commit-image/build-images-from-commit.sh | 29 +++++++---- .github/workflows/test-preflight.yml | 12 ++--- .github/workflows/tests.yml | 48 ++++++++++--------- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/.github/actions/commit-image/build-images-from-commit.sh b/.github/actions/commit-image/build-images-from-commit.sh index 38e8f9fff..2daeb2a36 100755 --- a/.github/actions/commit-image/build-images-from-commit.sh +++ b/.github/actions/commit-image/build-images-from-commit.sh @@ -9,18 +9,27 @@ fi TAG=$(git rev-parse --short "$SHA") # The pull pass first: it is what decides whether the jar below has to be built at all, and a hit -# costs seconds against the minutes a build does. +# costs seconds against the minutes a build does. All at once -- these are network bound, and eight +# of them one after another is minutes spent waiting on nothing. No -e on the child shell: a pull +# that misses is the expected case here, not a failure. +pull_one() { + if docker pull -q "$CACHE_REPO/source-$1:$TAG" >/dev/null 2>&1; then + docker tag "$CACHE_REPO/source-$1:$TAG" "olakego/source-$1:$TAG" + echo "restored olakego/source-$1:$TAG from the cache" + fi +} +export -f pull_one +export TAG CACHE_REPO +printf '%s\n' $DRIVERS | xargs -P 0 -I{} bash -c 'pull_one {}' + +# Presence decides what is missing, rather than each pull reporting back: the pulls ran in their own +# shells, and an image either landed locally or it did not. missing=() for driver in $DRIVERS; do - # TEMPORARY: the restore is commented out so every image is built from scratch, to time the worst - # case. Restore before merging, together with the lookup in test-preflight.yml. - # if docker pull -q "$CACHE_REPO/source-$driver:$TAG"; then - # docker tag "$CACHE_REPO/source-$driver:$TAG" "olakego/source-$driver:$TAG" - # echo "restored olakego/source-$driver:$TAG from the cache" - # continue - # fi - echo "no cached image for $driver at $TAG; it will be built" - missing+=("$driver") + if ! docker image inspect "olakego/source-$driver:$TAG" >/dev/null 2>&1; then + echo "no cached image for $driver at $TAG; it will be built" + missing+=("$driver") + fi done if [ ${#missing[@]} -eq 0 ]; then diff --git a/.github/workflows/test-preflight.yml b/.github/workflows/test-preflight.yml index 73ee4189a..ac143618c 100644 --- a/.github/workflows/test-preflight.yml +++ b/.github/workflows/test-preflight.yml @@ -77,14 +77,12 @@ jobs: set -euo pipefail echo "${{ github.token }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin tag=$(git rev-parse --short "$BASE_SHA") - # TEMPORARY: the cache check is commented out so every driver counts as a miss and the - # build job below runs on the full set, to time the worst case. Restore before merging. misses=$(for d in $(jq -r '.[]' <<<"$DRIVERS"); do - # if docker manifest inspect "ghcr.io/$GITHUB_REPOSITORY/source-$d:$tag" >/dev/null 2>&1; then - # echo "cached: source-$d:$tag" >&2 - # else - echo "$d" - # fi + if docker manifest inspect "ghcr.io/$GITHUB_REPOSITORY/source-$d:$tag" >/dev/null 2>&1; then + echo "cached: source-$d:$tag" >&2 + else + echo "$d" + fi done | jq -Rsc 'split("\n") | map(select(length > 0))') echo "misses=$misses" >> "$GITHUB_OUTPUT" echo "base branch images to build at $tag: $misses" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8a24b6a9e..e91f3a462 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -95,9 +95,6 @@ jobs: integration-tests: name: Integration Test ${{ matrix.driver }} needs: approve - permissions: - contents: read - packages: write if: needs.approve.outputs.drivers != '[]' runs-on: 16gb-runner environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} @@ -135,22 +132,19 @@ jobs: timeout-minutes: 15 run: make -j2 --output-sync=target olake.${{ matrix.driver }}.up olake.destination.all.up - # Docker Hub: compatibility pulls its baseline images from a shared-egress runner pool, which - # is exactly where the anonymous rate limit bites. ghcr: where this run's driver images are - # published for the compatibility job to reuse. + # Compatibility pulls its baseline images from a shared-egress runner pool, which is exactly + # where Docker Hub's anonymous rate limit bites. - &docker-login - name: Log in to the image registries + name: Log in to Docker Hub env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if [ -n "$DOCKER_USERNAME" ]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin else echo "::notice::no Docker Hub credentials configured; pulling anonymously" fi - echo "$GHCR_TOKEN" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - &buildx name: Set up Docker Buildx @@ -194,6 +188,19 @@ jobs: name: Wait for background jobs wait: [containers, image, go-build] + - name: Save the driver image for the compatibility job + run: docker save -o "$RUNNER_TEMP/source-${{ matrix.driver }}.tar" "olakego/source-${{ matrix.driver }}:local" + + - name: Upload the driver image + id: publish + background: true + timeout-minutes: 20 + uses: actions/upload-artifact@v7 + with: + name: driver-image-${{ matrix.driver }} + path: ${{ runner.temp }}/source-${{ matrix.driver }}.tar + retention-days: 1 + - &wait-ready name: Wait for source + destination readiness timeout-minutes: 3 @@ -218,15 +225,8 @@ jobs: OLAKE_PRE_BUILT_IMAGE: olakego/source-${{ matrix.driver }}:local run: make test.integration.${{ matrix.driver }} - - name: Publish the driver image for the compatibility job - env: - CANDIDATE: ghcr.io/${{ github.repository }}/source-${{ matrix.driver }}:${{ github.sha }} - run: | - set -euo pipefail - docker tag "olakego/source-${{ matrix.driver }}:local" "$CANDIDATE" - # Non-fatal: a fork's token is read-only, and the compatibility job builds its own copy - # of whatever it cannot pull. - docker push "$CANDIDATE" || echo "::notice::could not publish $CANDIDATE (read-only token?)" + - name: Wait for the image upload + wait: [publish] compatibility-base: name: Backwards Compatibility with ${{ github.event.pull_request.base.ref || 'the base branch' }} @@ -234,6 +234,7 @@ jobs: permissions: contents: read packages: write + actions: read if: github.event_name == 'pull_request' && needs.approve.outputs.drivers != '[]' runs-on: 32gb-runner environment: ${{ needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} @@ -260,14 +261,15 @@ jobs: background: true timeout-minutes: 30 env: - CANDIDATE_REPO: ghcr.io/${{ github.repository }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail + gh run download "$GITHUB_RUN_ID" --pattern 'driver-image-*' --dir "$RUNNER_TEMP/images" || true missing=() for d in $DRIVERS; do - if docker pull -q "$CANDIDATE_REPO/source-$d:$GITHUB_SHA"; then - docker tag "$CANDIDATE_REPO/source-$d:$GITHUB_SHA" "olakego/source-$d:local" - echo "restored olakego/source-$d:local from this run's integration build" + tar="$RUNNER_TEMP/images/driver-image-$d/source-$d.tar" + if [ -f "$tar" ]; then + docker load -i "$tar" else missing+=("$d") fi @@ -278,7 +280,7 @@ jobs: echo "every driver image came from this run's integration build" exit 0 fi - echo "::notice::nothing was published for ${missing[*]}; building them here" + echo "::notice::no artifact for ${missing[*]}; building them here" make -j --output-sync=target docker.all.build DRIVERS="${missing[*]}" DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load" - name: Restore base branch images From 3f858c42dfac8cc23e520051666d14a29d2a5921 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Fri, 28 Aug 2026 18:00:16 +0530 Subject: [PATCH 11/20] chore: temp --- .github/workflows/tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e91f3a462..33c467f02 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -323,12 +323,10 @@ jobs: "olakego/source-$driver:local" \ check --destination /testdata/iceberg_destination.json - # All of them at once: -k so one driver failing still leaves the rest reported, and - # --output-sync so each driver's log arrives whole instead of woven into the others'. - name: Run the compatibility tests env: OLAKE_PRE_BUILT_IMAGE: "true" - run: make -k -j --output-sync=target test.compatibility DRIVERS="$DRIVERS" COMPATIBILITY_BASELINE="${{ github.event.pull_request.base.sha }}" + run: make -k -j4 --output-sync=target test.compatibility DRIVERS="$DRIVERS" COMPATIBILITY_BASELINE="${{ github.event.pull_request.base.sha }}" compatibility-sweep: name: ${{ fromJSON(needs.approve.outputs.driver-labels)[matrix.driver] }} Backward Compatibility Sweep From c4035ec90f79226f794c34109217ee355cee3de1 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Fri, 28 Aug 2026 19:55:43 +0530 Subject: [PATCH 12/20] chore: single docker image export job --- .github/actions/driver-image/action.yml | 28 +++++++++++++++++++++++++ .github/workflows/tests.yml | 11 +++------- 2 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 .github/actions/driver-image/action.yml diff --git a/.github/actions/driver-image/action.yml b/.github/actions/driver-image/action.yml new file mode 100644 index 000000000..5e646ebfd --- /dev/null +++ b/.github/actions/driver-image/action.yml @@ -0,0 +1,28 @@ +name: Driver image artifact +description: > + Exports the driver image this run built and uploads it as a run artifact, so a later job can load + it instead of building its own copy. Run-scoped on purpose: the image is read within the run and + never after it, so retention expires it without anything having to clean up. + +inputs: + driver: + description: Driver whose local image to export. + required: true + retention-days: + description: How long the artifact lives. A day is already far longer than the run that reads it. + required: false + default: '1' + +runs: + using: composite + steps: + - name: Export the image + shell: bash + run: docker save -o "$RUNNER_TEMP/source-${{ inputs.driver }}.tar" "olakego/source-${{ inputs.driver }}:local" + + - name: Upload the image + uses: actions/upload-artifact@v7 + with: + name: driver-image-${{ inputs.driver }} + path: ${{ runner.temp }}/source-${{ inputs.driver }}.tar + retention-days: ${{ inputs.retention-days }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 33c467f02..0c5f0c1fc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -188,18 +188,13 @@ jobs: name: Wait for background jobs wait: [containers, image, go-build] - - name: Save the driver image for the compatibility job - run: docker save -o "$RUNNER_TEMP/source-${{ matrix.driver }}.tar" "olakego/source-${{ matrix.driver }}:local" - - - name: Upload the driver image + - name: Publish the driver image id: publish background: true timeout-minutes: 20 - uses: actions/upload-artifact@v7 + uses: ./.github/actions/driver-image with: - name: driver-image-${{ matrix.driver }} - path: ${{ runner.temp }}/source-${{ matrix.driver }}.tar - retention-days: 1 + driver: ${{ matrix.driver }} - &wait-ready name: Wait for source + destination readiness From f2185150b2469a063ab6e534e6a5ee186c8cce16 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Fri, 28 Aug 2026 20:21:24 +0530 Subject: [PATCH 13/20] chore: parallel image restore --- .github/actions/driver-image/action.yml | 1 + .github/scripts/restore-driver-images.sh | 32 ++++++++++++++++++++++++ .github/workflows/tests.yml | 23 ++--------------- 3 files changed, 35 insertions(+), 21 deletions(-) create mode 100755 .github/scripts/restore-driver-images.sh diff --git a/.github/actions/driver-image/action.yml b/.github/actions/driver-image/action.yml index 5e646ebfd..8f45ab06d 100644 --- a/.github/actions/driver-image/action.yml +++ b/.github/actions/driver-image/action.yml @@ -26,3 +26,4 @@ runs: name: driver-image-${{ inputs.driver }} path: ${{ runner.temp }}/source-${{ inputs.driver }}.tar retention-days: ${{ inputs.retention-days }} + overwrite: true diff --git a/.github/scripts/restore-driver-images.sh b/.github/scripts/restore-driver-images.sh new file mode 100755 index 000000000..f91354fe2 --- /dev/null +++ b/.github/scripts/restore-driver-images.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Loads this run's driver images, which the integration matrix exported as one artifact per driver, +# and builds whatever did not arrive -- a fork PR publishes nothing, and the job still has to stand +# on its own. Reads DRIVERS, plus GH_TOKEN for the download. +set -euo pipefail + +restore_single_image() { + local dir="$RUNNER_TEMP/images/$1" + gh run download "$GITHUB_RUN_ID" -n "driver-image-$1" -D "$dir" >/dev/null 2>&1 || return 0 + docker load -i "$dir/source-$1.tar" +} +export -f restore_single_image +export GH_TOKEN GITHUB_RUN_ID RUNNER_TEMP +printf '%s\n' $DRIVERS | xargs -P "${MAX_PARALLEL:-4}" -I{} bash -c 'restore_single_image {}' + +# Presence decides what is missing, the way the base branch images do it: the restores ran in their +# own shells, and an image either landed in the daemon or it did not. +missing=() +for driver in $DRIVERS; do + docker image inspect "olakego/source-$driver:local" >/dev/null 2>&1 || missing+=("$driver") +done + +# An if, not `[ ... ] && exit 0`: on the false branch that list returns non-zero and set -e would +# end the script right here, silently skipping the build below. +if [ ${#missing[@]} -eq 0 ]; then + echo "every driver image came from this run's integration build" + exit 0 +fi + +echo "::notice::no artifact for ${missing[*]}; building them here" +make -j --output-sync=target docker.all.build DRIVERS="${missing[*]}" \ + DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0c5f0c1fc..c4f383817 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -223,7 +223,7 @@ jobs: - name: Wait for the image upload wait: [publish] - compatibility-base: + test-backward-compatibility: name: Backwards Compatibility with ${{ github.event.pull_request.base.ref || 'the base branch' }} needs: [approve, integration-tests] permissions: @@ -257,26 +257,7 @@ jobs: timeout-minutes: 30 env: GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - gh run download "$GITHUB_RUN_ID" --pattern 'driver-image-*' --dir "$RUNNER_TEMP/images" || true - missing=() - for d in $DRIVERS; do - tar="$RUNNER_TEMP/images/driver-image-$d/source-$d.tar" - if [ -f "$tar" ]; then - docker load -i "$tar" - else - missing+=("$d") - fi - done - # An if, not `[ ... ] && exit 0`: on the false branch that list returns non-zero and - # set -e would end the step right here, silently skipping the build below. - if [ ${#missing[@]} -eq 0 ]; then - echo "every driver image came from this run's integration build" - exit 0 - fi - echo "::notice::no artifact for ${missing[*]}; building them here" - make -j --output-sync=target docker.all.build DRIVERS="${missing[*]}" DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load" + run: .github/scripts/restore-driver-images.sh - name: Restore base branch images id: base-image From e6447b134b7c4683a0bae2f07f26190dfdd495fd Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Mon, 31 Aug 2026 10:43:48 +0530 Subject: [PATCH 14/20] chore: minor changes --- .github/CODEOWNERS | 7 +- .github/actions/commit-image/action.yml | 5 - .../commit-image/build-images-from-commit.sh | 27 +- .github/actions/detect-drivers/action.yml | 3 +- .../actions/detect-drivers/detect-drivers.sh | 6 +- .github/actions/driver-image/action.yml | 27 +- .github/rulesets/state-version-approval.json | 2 +- .github/scripts/restore-driver-images.sh | 18 +- .github/workflows/test-preflight.yml | 36 +- .github/workflows/tests.yml | 33 +- constants/state_version_test.go | 99 ++++ tests/db2/db2_util_test.go | 270 ++++++----- tests/kafka/kafka_util_test.go | 10 +- tests/mongodb/mongodb_test.go | 25 +- tests/mongodb/mongodb_util_test.go | 81 ++-- tests/mssql/mssql_util_test.go | 21 +- tests/mysql/mysql_test.go | 8 +- tests/mysql/mysql_util_test.go | 421 ++++++------------ tests/postgres/postgres_test.go | 23 +- tests/postgres/postgres_util_test.go | 30 -- tests/s3/s3_test.go | 1 - tests/s3/s3_util_test.go | 131 ++---- .../testutils/compatibility/compatibility.go | 69 +-- .../compatibility/compatibility_rules.json | 66 +-- tests/testutils/compatibility/scenarios.go | 17 - tests/testutils/ddl.go | 36 +- tests/testutils/docker.go | 23 +- tests/testutils/integration/integration.go | 16 - tests/testutils/integration/sync.go | 16 - tests/testutils/state_version.go | 7 - tests/testutils/test_utils.go | 12 - 31 files changed, 593 insertions(+), 953 deletions(-) create mode 100644 constants/state_version_test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 88c19d811..6f4f3e4aa 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,9 +2,14 @@ ## Ownership and review routing only -- the 2-approval requirement for these paths is a repository ruleset (required reviewer rule on the same teams), since CODEOWNERS with branch protection is satisfied by any one owner. A team needs write access to the repo for CODEOWNERS to resolve it +## GitHub reads this file from the branch a pull request targets, so it governs merges into master and staging once it is on them, and never the branch proposing it. An owner it cannot resolve is dropped rather than enforced, silently -- the ruleset in .github/rulesets carries the enforcement for these paths, naming the teams directly + ## The state configuration and the compat rules pin backward-compatibility semantics forever, so changes need 2 sign-offs from the people who own those semantics. CODEOWNERS takes one pattern per line, so the team repeats; the ruleset groups both paths under a single rule +## TODO(settings): create @datazip-inc/olake-admins and @datazip-inc/state-version-owners with WRITE access -- read-only resolves to nobody and the entry is dropped +## TODO(settings): the counts these paths need live in .github/rulesets/state-version-approval.json, which is not applied yet -- see that directory's README + /.github/CODEOWNERS @datazip-inc/olake-admins /constants/state-versions.json @datazip-inc/state-version-owners -/tests/testutils/compatibility_rules.json @datazip-inc/state-version-owners +/tests/testutils/compatibility/compatibility_rules.json @datazip-inc/state-version-owners diff --git a/.github/actions/commit-image/action.yml b/.github/actions/commit-image/action.yml index 528f0657d..42efcbffb 100644 --- a/.github/actions/commit-image/action.yml +++ b/.github/actions/commit-image/action.yml @@ -33,11 +33,6 @@ inputs: required: false default: '3' path: - description: > - Where to check the commit's tree out. actions/checkout refuses anything outside the workspace, - so it lands beside the caller's own checkout and must sit somewhere .dockerignore already - excludes -- otherwise it joins the context of every other image the job builds, changing that - context and evicting its layer cache. tests/ is excluded in this repo, hence the default. required: false default: tests/.commit-image-src diff --git a/.github/actions/commit-image/build-images-from-commit.sh b/.github/actions/commit-image/build-images-from-commit.sh index 2daeb2a36..37d91ff4a 100755 --- a/.github/actions/commit-image/build-images-from-commit.sh +++ b/.github/actions/commit-image/build-images-from-commit.sh @@ -8,19 +8,15 @@ fi TAG=$(git rev-parse --short "$SHA") -# The pull pass first: it is what decides whether the jar below has to be built at all, and a hit -# costs seconds against the minutes a build does. All at once -- these are network bound, and eight -# of them one after another is minutes spent waiting on nothing. No -e on the child shell: a pull -# that misses is the expected case here, not a failure. -pull_one() { +pull_driver_image() { if docker pull -q "$CACHE_REPO/source-$1:$TAG" >/dev/null 2>&1; then docker tag "$CACHE_REPO/source-$1:$TAG" "olakego/source-$1:$TAG" echo "restored olakego/source-$1:$TAG from the cache" fi } -export -f pull_one +export -f pull_driver_image export TAG CACHE_REPO -printf '%s\n' $DRIVERS | xargs -P 0 -I{} bash -c 'pull_one {}' +printf '%s\n' $DRIVERS | xargs -P 0 -I{} bash -c 'pull_driver_image {}' # Presence decides what is missing, rather than each pull reporting back: the pulls ran in their own # shells, and an image either landed locally or it did not. @@ -37,10 +33,6 @@ if [ ${#missing[@]} -eq 0 ]; then exit 0 fi -# That tree's own Iceberg writer jar, once for all of them: the Dockerfile copies it out of the -# build context, the commit's Go side speaks that jar's RPC, and parallel builds would otherwise -# race maven on one output directory. Kept off stdout unless it fails -- several hundred lines of -# maven nobody reads when it works. echo "building the iceberg jar for $TAG..." started=$SECONDS if ! jar_log=$(make -C "$SRC" iceberg.jar 2>&1); then @@ -51,10 +43,7 @@ fi echo "built the iceberg jar in $((SECONDS - started))s" # TODO: we can use make command for build once this PR merges as local builds gets tagged as olake/source... instead of olakego/source... -# -# Output is collected and printed whole, in a group per driver: several of these run at once, and -# interleaved --progress=plain streams leave you unable to tell how many images actually built. -build_one() { +build_driver_image() { local log="$WORK/$1.log" if ! docker buildx build --progress=plain --cache-from type=gha,scope=olake-base \ --load --build-arg DRIVER_NAME="$1" -t "olakego/source-$1:$TAG" "$SRC" > "$log" 2>&1; then @@ -74,14 +63,10 @@ build_one() { cat "$log" echo "::endgroup::" } -export -f build_one +export -f build_driver_image WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT export TAG SRC CACHE_REPO WORK -# xargs -P fans these out with no pids to track and no marker files: if any build fails it still -# finishes the rest and exits 123, which set -e turns into this script's failure. -e on the child -# shell too, so a failed build stops before it tags and pushes what it did not produce. Bounded, -# because each one is a full Go compile and the caller has its own builds running beside it. echo "building ${missing[*]}" -printf '%s\n' "${missing[@]}" | xargs -P "${MAX_PARALLEL:-3}" -I{} bash -euc 'build_one {}' +printf '%s\n' "${missing[@]}" | xargs -P "${MAX_PARALLEL:-3}" -I{} bash -euc 'build_driver_image {}' diff --git a/.github/actions/detect-drivers/action.yml b/.github/actions/detect-drivers/action.yml index 958c23a57..bb2cdc406 100644 --- a/.github/actions/detect-drivers/action.yml +++ b/.github/actions/detect-drivers/action.yml @@ -16,8 +16,7 @@ outputs: value: ${{ steps.resolve.outputs.drivers }} driver-labels: description: > - JSON object mapping each driver to its display name, for job titles: expressions cannot - change case, so the capitalised form is produced in the script. + JSON object mapping each driver to its display name value: ${{ steps.resolve.outputs.driver-labels }} runs: diff --git a/.github/actions/detect-drivers/detect-drivers.sh b/.github/actions/detect-drivers/detect-drivers.sh index 9c9999223..bdc58b9d1 100755 --- a/.github/actions/detect-drivers/detect-drivers.sh +++ b/.github/actions/detect-drivers/detect-drivers.sh @@ -27,5 +27,9 @@ drivers=$(printf '%s\n' $selected | sort -u | jq -Rc '[., inputs] | map(select(. echo "drivers=$drivers" >> "$GITHUB_OUTPUT" echo "Affected drivers: $drivers" -labels=$(printf '%s' "$drivers" | jq -c 'map({key: ., value: ((.[0:1] | ascii_upcase) + .[1:])}) | from_entries') +title_case() { + jq -c 'map({key: ., value: ((.[0:1] | ascii_upcase) + .[1:])}) | from_entries' +} + +labels=$(printf '%s' "$drivers" | title_case) echo "driver-labels=$labels" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/driver-image/action.yml b/.github/actions/driver-image/action.yml index 8f45ab06d..ab8d5b133 100644 --- a/.github/actions/driver-image/action.yml +++ b/.github/actions/driver-image/action.yml @@ -1,13 +1,25 @@ name: Driver image artifact description: > - Exports the driver image this run built and uploads it as a run artifact, so a later job can load - it instead of building its own copy. Run-scoped on purpose: the image is read within the run and - never after it, so retention expires it without anything having to clean up. + Exports the driver image this run built and uploads it as a run artifact inputs: driver: - description: Driver whose local image to export. + description: Driver whose image to export. required: true + tag: + description: > + Tag to export. The suites build and run the current code as `local`; a released baseline or a + commit-tagged build can be exported by naming its tag instead. + required: false + default: local + artifact-name: + description: > + What to publish under. Defaults to driver-image-, which is the name the compatibility + job looks for. Exporting more than one tag of the same driver in a run needs distinct names + here: an artifact name can only be uploaded once, and this action overwrites, so a second + export under the same name would replace the first rather than fail. + required: false + default: '' retention-days: description: How long the artifact lives. A day is already far longer than the run that reads it. required: false @@ -18,12 +30,15 @@ runs: steps: - name: Export the image shell: bash - run: docker save -o "$RUNNER_TEMP/source-${{ inputs.driver }}.tar" "olakego/source-${{ inputs.driver }}:local" + env: + IMAGE: olakego/source-${{ inputs.driver }}:${{ inputs.tag }} + TAR: ${{ runner.temp }}/source-${{ inputs.driver }}.tar + run: docker save -o "$TAR" "$IMAGE" - name: Upload the image uses: actions/upload-artifact@v7 with: - name: driver-image-${{ inputs.driver }} + name: ${{ inputs.artifact-name || format('driver-image-{0}', inputs.driver) }} path: ${{ runner.temp }}/source-${{ inputs.driver }}.tar retention-days: ${{ inputs.retention-days }} overwrite: true diff --git a/.github/rulesets/state-version-approval.json b/.github/rulesets/state-version-approval.json index e67eff3c3..7c6771f18 100644 --- a/.github/rulesets/state-version-approval.json +++ b/.github/rulesets/state-version-approval.json @@ -24,7 +24,7 @@ { "file_patterns": [ "constants/state-versions.json", - "tests/testutils/compatibility_rules.json" + "tests/testutils/compatibility/compatibility_rules.json" ], "minimum_approvals": 2, "reviewer": { diff --git a/.github/scripts/restore-driver-images.sh b/.github/scripts/restore-driver-images.sh index f91354fe2..29fae76d3 100755 --- a/.github/scripts/restore-driver-images.sh +++ b/.github/scripts/restore-driver-images.sh @@ -1,32 +1,28 @@ #!/usr/bin/env bash -# Loads this run's driver images, which the integration matrix exported as one artifact per driver, -# and builds whatever did not arrive -- a fork PR publishes nothing, and the job still has to stand -# on its own. Reads DRIVERS, plus GH_TOKEN for the download. set -euo pipefail +TAG=${TAG:-local} +ARTIFACT_PREFIX=${ARTIFACT_PREFIX:-driver-image} + restore_single_image() { local dir="$RUNNER_TEMP/images/$1" - gh run download "$GITHUB_RUN_ID" -n "driver-image-$1" -D "$dir" >/dev/null 2>&1 || return 0 + gh run download "$GITHUB_RUN_ID" -n "$ARTIFACT_PREFIX-$1" -D "$dir" >/dev/null 2>&1 || return 0 docker load -i "$dir/source-$1.tar" } export -f restore_single_image -export GH_TOKEN GITHUB_RUN_ID RUNNER_TEMP +export GH_TOKEN GITHUB_RUN_ID RUNNER_TEMP ARTIFACT_PREFIX printf '%s\n' $DRIVERS | xargs -P "${MAX_PARALLEL:-4}" -I{} bash -c 'restore_single_image {}' -# Presence decides what is missing, the way the base branch images do it: the restores ran in their -# own shells, and an image either landed in the daemon or it did not. missing=() for driver in $DRIVERS; do - docker image inspect "olakego/source-$driver:local" >/dev/null 2>&1 || missing+=("$driver") + docker image inspect "olakego/source-$driver:$TAG" >/dev/null 2>&1 || missing+=("$driver") done -# An if, not `[ ... ] && exit 0`: on the false branch that list returns non-zero and set -e would -# end the script right here, silently skipping the build below. if [ ${#missing[@]} -eq 0 ]; then echo "every driver image came from this run's integration build" exit 0 fi echo "::notice::no artifact for ${missing[*]}; building them here" -make -j --output-sync=target docker.all.build DRIVERS="${missing[*]}" \ +make -j --output-sync=target docker.all.build DRIVERS="${missing[*]}" IMAGE_TAG="$TAG" \ DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --load" diff --git a/.github/workflows/test-preflight.yml b/.github/workflows/test-preflight.yml index ac143618c..6f589bd9f 100644 --- a/.github/workflows/test-preflight.yml +++ b/.github/workflows/test-preflight.yml @@ -17,8 +17,6 @@ on: value: ${{ jobs.preflight.outputs.driver-labels }} jobs: - # Ungated, like the three cache jobs below it: the environment prompts once per wave of jobs that - # reach it, so gating these too would cost three approvals a run. Cheap: a file list and 3 lookups. preflight: name: Detect changes runs-on: ubuntu-latest @@ -41,8 +39,6 @@ jobs: with: fetch-depth: 0 - # Shared with any workflow that wants to skip untouched drivers; the caller's matrix fans out - # over whatever it returns, and an empty array skips the driver jobs entirely. - name: Resolve driver matrix id: drivers uses: ./.github/actions/detect-drivers @@ -99,8 +95,6 @@ jobs: with: mode: lookup - # A marker, since buildx's own gha cache keys are internal and cannot be looked up: it records - # that the runtime-base stage was warmed. Eviction can outdate it, and a miss is only a slow build. - name: Look up base docker layer marker id: apt-cache uses: actions/cache/restore@v4 @@ -112,16 +106,10 @@ jobs: base-images: name: Build base branch image needs: preflight - # Guarded rather than left to an empty matrix: inside a called workflow an empty matrix never - # resolves, and the calling job hangs on it instead of completing -- which skips everything - # gated on that job. A skipped job, which is what the three below do, is resolved and fine. if: needs.preflight.outputs.base-image-misses != '' && needs.preflight.outputs.base-image-misses != '[]' permissions: contents: read packages: write - # One machine rather than a driver each: the images share one Iceberg jar, which a job per - # driver would rebuild per driver, and the script fans the builds out beneath it. Hosted, since - # this overlaps the approval wait and the self-hosted pool is about to be wanted by the matrix. runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -154,8 +142,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} path: tests/.compat-base - # Only when the jar is missing: preflight already looked up the cache, so an unchanged writer - # skips Maven and this whole job. build-jar: name: Build Iceberg writer jar needs: preflight @@ -181,8 +167,6 @@ jobs: with: mode: save - # Skipped while the Dockerfile is unchanged and already warmed. Failure here is never fatal: the - # driver build just re-runs the layers it would have hit from cache. apt-warm: name: Warm base docker layers needs: preflight @@ -196,8 +180,6 @@ jobs: - name: Set up Docker Buildx uses: ./.github/actions/buildx - # Exits 0 either way: a failed warm must not block the matrix, it only leaves those layers to - # the driver build. The marker is what tells the next run to try again. - name: Warm base docker layers run: | docker buildx build --target runtime-base \ @@ -205,7 +187,6 @@ jobs: && echo "$GITHUB_SHA" > .apt-warmed \ || echo "::warning::base layer warm failed (non-fatal)" - # Only on success, so a failed warm leaves the marker missing and the next run retries. - name: Save base layer marker if: hashFiles('.apt-warmed') != '' uses: actions/cache/save@v4 @@ -213,8 +194,6 @@ jobs: path: .apt-warmed key: ${{ runner.os }}-aptwarm-${{ hashFiles('Dockerfile') }} - # The modules and compiled shared deps every driver job restores, plus the tests modules' lint - # and gosec; skipped when the cache is published and no go-check input changed in the PR. go-cache: name: Go checks + cache needs: preflight @@ -225,7 +204,6 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - # setup-go's cache keys on go.sum, which this repo does not track, so it would cache nothing. - name: Set up Go uses: actions/setup-go@v4 with: @@ -242,18 +220,13 @@ jobs: with: save: "true" - # Every tests module, not just testutils, so the matrix jobs download nothing. - - name: Download Go modules - run: cd tests && for d in $(go list -m -f '{{.Dir}}'); do go -C "$d" mod download; done - - # The same targets the driver jobs run, so their compile and link both hit this cache. - name: Compile the driver test binaries run: make test.build.all - # Installs golangci-lint as a side effect (the Makefile guard skips it when the restored - # cache already carries the binary), so it rides into the post-job save with gosec below. - name: golangci-lint tests modules - if: inputs.go-checks + id: lint + background: true + timeout-minutes: 15 run: make test.lint - name: install gosec @@ -266,3 +239,6 @@ jobs: if: inputs.go-checks working-directory: tests run: $(go env GOPATH)/bin/gosec -exclude=G115 -tests -severity=high -confidence=medium ./... + + - name: Wait for the lint + wait: [lint] diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c4f383817..86b6f5d27 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,13 +8,11 @@ on: required: false default: '' type: string - # A push runs the suite post-merge and publishes the caches every pull request restores from. push: branches: - "master" - "staging" paths: &paths - # olake itself: change any of it and the binary the suites run changes. - '**/*.go' - '**/go.mod' - '**/go.work' @@ -26,7 +24,6 @@ on: - 'drivers/*/driver.mk' - 'drivers/**.conf' - 'constants/state-versions.json' - # what the suites are run with: the harness, the stacks they need, and the CI itself. - 'tests/**' - 'drivers/*/docker-compose.yml' - 'destination/iceberg/local-test/**' @@ -41,11 +38,7 @@ on: paths: *paths jobs: - # Detect changes, publish the caches, build the jar, take the run's single approval -- once for - # every matrix below. - # go-checks runs here: this workflow owns the tests modules' lint and gosec. preflight: - # The called workflow's jobs render as " / ". name: Pre-Validation uses: ./.github/workflows/test-preflight.yml permissions: @@ -56,16 +49,10 @@ jobs: go-checks: true secrets: inherit - # The run's single approval, its own job rather than the last one inside the called workflow: it - # gates the suites below, so it belongs beside them rather than under the pre-validation heading. - # `needs: preflight` covers that whole workflow, so this runs once change detection and every - # cache job has finished -- which is why nothing below needs preflight as well. It forwards what - # they read from it, so the chain is preflight -> approve -> suites and each job names only the - # thing it actually waits on. approve: name: Approve Test Deployment needs: preflight - if: ${{ !cancelled() && needs.preflight.result == 'success' && needs.preflight.outputs.drivers != '[]' }} + if: needs.preflight.outputs.drivers != '[]' runs-on: ubuntu-latest timeout-minutes: 5 environment: ${{ github.event_name == 'pull_request' && 'integration_tests' || '' }} @@ -95,7 +82,6 @@ jobs: integration-tests: name: Integration Test ${{ matrix.driver }} needs: approve - if: needs.approve.outputs.drivers != '[]' runs-on: 16gb-runner environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} timeout-minutes: 60 @@ -104,8 +90,6 @@ jobs: matrix: driver: ${{ fromJSON(needs.approve.outputs.drivers) }} steps: - # Full history so the compatibility job can build the PR's base commit; the extra objects cost this - # job a few seconds and keep one checkout definition for both. - &checkout name: Checkout code uses: actions/checkout@v7 @@ -132,8 +116,6 @@ jobs: timeout-minutes: 15 run: make -j2 --output-sync=target olake.${{ matrix.driver }}.up olake.destination.all.up - # Compatibility pulls its baseline images from a shared-egress runner pool, which is exactly - # where Docker Hub's anonymous rate limit bites. - &docker-login name: Log in to Docker Hub env: @@ -154,8 +136,6 @@ jobs: name: Restore Iceberg writer jar uses: ./.github/actions/iceberg-jar - # Background: the ~600MB Go cache restore below overlaps it instead of delaying it. No - # --cache-to -- exporting this build measured ~119s against the ~85s build it would save. - &driver-image name: Build driver image id: image @@ -163,7 +143,6 @@ jobs: timeout-minutes: 20 run: make docker.${{ matrix.driver }}.build DOCKER_BUILD="docker buildx build --progress=plain --cache-from type=gha,scope=olake-base --cache-from type=gha,scope=olake-${{ matrix.driver }} --load" - # setup-go's cache keys on go.sum, which this repo does not track, so it would cache nothing. - &go name: Set up Go uses: actions/setup-go@v4 @@ -171,13 +150,10 @@ jobs: go-version-file: "go.mod" cache: false - # Restore-only: the modules and compiled shared deps are identical for every driver, and seven - # jobs saving a ~350MB copy each would evict the jar from the cache. - &go-caches name: Restore shared Go caches uses: ./.github/actions/go-caches - # Compiles the test binary while the container pull and the image build are still in flight. - &test-build name: Install Dependencies id: go-build @@ -230,7 +206,7 @@ jobs: contents: read packages: write actions: read - if: github.event_name == 'pull_request' && needs.approve.outputs.drivers != '[]' + if: github.event_name == 'pull_request' runs-on: 32gb-runner environment: ${{ needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} timeout-minutes: 90 @@ -309,7 +285,6 @@ jobs: needs: approve runs-on: 16gb-runner environment: ${{ github.event_name == 'pull_request' && needs.approve.outputs.attempt != github.run_attempt && 'integration_tests' || '' }} - # One pipeline pair per baseline, run one after another. timeout-minutes: 90 strategy: fail-fast: false @@ -324,7 +299,6 @@ jobs: - *buildx - *jar - *driver-image - - *go - *go-caches - *test-build @@ -332,7 +306,6 @@ jobs: - *wait-ready - *catalog - # Empty baseline is the whole manifest; a manual run can pin one instead. - name: Run the compatibility sweep env: OLAKE_PRE_BUILT_IMAGE: olakego/source-${{ matrix.driver }}:local @@ -354,7 +327,7 @@ jobs: fail-fast: false # TODO: add benchmark tests for mongodb and oracle (instance ids TBD). matrix: - include: ${{ fromJSON((github.event_name == 'push' && github.ref == 'refs/heads/staging' && needs.approve.outputs.drivers != '[]') && '[{"driver":"mysql","driver_upper":"MYSQL","instance_id":8},{"driver":"postgres","driver_upper":"POSTGRES","instance_id":9}]' || '[]') }} + include: ${{ fromJSON((github.event_name == 'push' && github.ref == 'refs/heads/staging') && '[{"driver":"mysql","driver_upper":"MYSQL","instance_id":8},{"driver":"postgres","driver_upper":"POSTGRES","instance_id":9}]' || '[]') }} steps: - *checkout diff --git a/constants/state_version_test.go b/constants/state_version_test.go new file mode 100644 index 000000000..c5c2a5c77 --- /dev/null +++ b/constants/state_version_test.go @@ -0,0 +1,99 @@ +package constants + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type stateVersionEntry struct { + StateVersion int `json:"state_version"` + ReleaseTag string `json:"release_tag"` + Drivers string `json:"drivers"` + Note string `json:"note"` +} + +// A published release: vMAJOR.MINOR.PATCH, which is what the driver images are tagged with. +var releaseTagPattern = regexp.MustCompile(`^v\d+\.\d+\.\d+$`) + +func loadStateVersions(t *testing.T) (int, []stateVersionEntry) { + t.Helper() + var doc struct { + LatestStateVersion int `json:"latest_state_version"` + Baselines []stateVersionEntry `json:"baselines"` + } + require.NoError(t, json.Unmarshal(rawStateVersions, &doc), "state-versions.json is not valid JSON") + require.NotEmpty(t, doc.Baselines, "state-versions.json lists no baselines") + return doc.LatestStateVersion, doc.Baselines +} + +func TestStateVersionsAreContiguous(t *testing.T) { + latest, baselines := loadStateVersions(t) + + seen := make(map[int]string, len(baselines)) + for _, baseline := range baselines { + previous, duplicate := seen[baseline.StateVersion] + assert.Falsef(t, duplicate, "state version %d is listed twice, by %s and %s", + baseline.StateVersion, previous, baseline.ReleaseTag) + seen[baseline.StateVersion] = baseline.ReleaseTag + } + + for version := 0; version <= latest; version++ { + assert.Containsf(t, seen, version, + "no entry for state version %d; every version from 0 to latest_state_version (%d) needs one", version, latest) + } + + highest := 0 + for version := range seen { + if version > highest { + highest = version + } + } + assert.Equalf(t, latest, highest, + "latest_state_version is %d but the newest entry is %d; bumping one without the other leaves the build writing a version it cannot describe", + latest, highest) +} + +func TestStateVersionReleaseTagsAreValid(t *testing.T) { + _, baselines := loadStateVersions(t) + + for _, baseline := range baselines { + t.Run(fmt.Sprintf("v%d", baseline.StateVersion), func(t *testing.T) { + assert.Regexpf(t, releaseTagPattern, baseline.ReleaseTag, + "release_tag %q is not a vMAJOR.MINOR.PATCH release", baseline.ReleaseTag) + }) + } +} + +// Which drivers a bump changed semantics for: "*" for all of them, otherwise a comma separated list +// of driver names. The suite skips a baseline whose bump touched no driver it is testing, so a name +// that matches nothing silently drops that baseline from the sweep. +func TestStateVersionDriversAreKnown(t *testing.T) { + _, baselines := loadStateVersions(t) + + known := map[string]bool{} + for _, driver := range []DriverType{MongoDB, Postgres, MySQL, Oracle, DB2, S3, Kafka, MSSQL} { + known[string(driver)] = true + } + + for _, baseline := range baselines { + t.Run(fmt.Sprintf("v%d", baseline.StateVersion), func(t *testing.T) { + require.NotEmptyf(t, baseline.Drivers, + "state version %d names no drivers; use \"*\" when a bump changes every driver", baseline.StateVersion) + if baseline.Drivers == "*" { + return + } + for driver := range strings.SplitSeq(baseline.Drivers, ",") { + trimmed := strings.TrimSpace(driver) + assert.NotEmptyf(t, trimmed, "drivers %q has an empty entry", baseline.Drivers) + assert.Containsf(t, known, trimmed, + "drivers names %q, which is not a driver; expected \"*\" or a comma separated list of known drivers", trimmed) + } + }) + } +} diff --git a/tests/db2/db2_util_test.go b/tests/db2/db2_util_test.go index 465a7e020..24930945e 100644 --- a/tests/db2/db2_util_test.go +++ b/tests/db2/db2_util_test.go @@ -3,6 +3,7 @@ package db2 import ( "context" "fmt" + "strconv" "strings" "sync" "testing" @@ -15,36 +16,106 @@ import ( "github.com/jmoiron/sqlx" ) -// seedTableDDL is the seed table's column list, the one place the fixture's schema lives: create -// renders it and seedColumnTypes reads it. -const seedTableDDL = ` - id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, - col_cursor BIGINT, - col_bigint BIGINT, - col_char CHAR(1), - col_character CHAR(10), - col_varchar VARCHAR(50), - col_date DATE, - col_decimal DECIMAL(10, 2), - col_decfloat DECFLOAT, - col_double DOUBLE, - col_real REAL, - col_int INTEGER, - col_smallint SMALLINT, - col_bool BOOLEAN, - col_clob CLOB(1M), - col_blob BLOB(1M), - col_timestamp TIMESTAMP, - col_time TIME, - col_graphic GRAPHIC(11), - col_vargraphic VARGRAPHIC(14), - excludedColumn INT NULL - ` - -// seedColumnTypes derives every seed column's type tags from the DDL, so a data_types rule in -// compatibility_rules.json follows a seed edit with nothing to declare. +type seedColumn struct { + name string + datatype string + value string + filtered string + updated string +} + +func (c seedColumn) definition() string { return c.name + " " + c.datatype } + +var seedColumns = []seedColumn{ + {name: "id", datatype: "BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY", value: "", filtered: "", updated: ""}, + {name: "col_cursor", datatype: "BIGINT", value: "", filtered: "-1", updated: "NULL"}, + {name: "col_bigint", datatype: "BIGINT", value: "12345678901234", filtered: "111111111111111", updated: ""}, + {name: "col_char", datatype: "CHAR(1)", value: "'c'", filtered: "'x'", updated: ""}, + {name: "col_character", datatype: "CHAR(10)", value: "'char_val'", filtered: "'filtered'", updated: ""}, + {name: "col_varchar", datatype: "VARCHAR(50)", value: "'varchar_val'", filtered: "'filtered_val'", updated: ""}, + {name: "col_date", datatype: "DATE", value: "DATE('2023-01-01')", filtered: "DATE('2022-06-15')", updated: ""}, + {name: "col_decimal", datatype: "DECIMAL(10, 2)", value: "123.45", filtered: "50.123", updated: ""}, + {name: "col_decfloat", datatype: "DECFLOAT", value: "123.45", filtered: "50.123", updated: ""}, + {name: "col_double", datatype: "DOUBLE", value: "123.456789", filtered: "50.123", updated: ""}, + {name: "col_real", datatype: "REAL", value: "123.5", filtered: "50.0", updated: ""}, + {name: "col_int", datatype: "INTEGER", value: "123", filtered: "0", updated: ""}, + {name: "col_smallint", datatype: "SMALLINT", value: "123", filtered: "0", updated: "321"}, + {name: "col_bool", datatype: "BOOLEAN", value: "TRUE", filtered: "FALSE", updated: ""}, + {name: "col_clob", datatype: "CLOB(1M)", value: "CLOB('sample text')", filtered: "CLOB('filtered text')", updated: ""}, + {name: "col_blob", datatype: "BLOB(1M)", value: "BLOB(X'424C4F422044415441204F4E45')", filtered: "BLOB(X'00')", updated: ""}, + {name: "col_timestamp", datatype: "TIMESTAMP", value: "TIMESTAMP('2023-01-01-12.00.00.000000')", filtered: "TIMESTAMP('2022-06-15-10.00.00.000000')", updated: "TIMESTAMP('2024-01-01-12.00.00.000000')"}, + {name: "col_time", datatype: "TIME", value: "TIME('12.00.00')", filtered: "TIME('10.00.00')", updated: ""}, + {name: "col_graphic", datatype: "GRAPHIC(11)", value: "GRAPHIC('graphic_val')", filtered: "GRAPHIC('filtered')", updated: ""}, + {name: "col_vargraphic", datatype: "VARGRAPHIC(14)", value: "VARGRAPHIC('vargraphic_val')", filtered: "VARGRAPHIC('filtered')", updated: ""}, + {name: "excludedColumn", datatype: "INT NULL", value: "", filtered: "", updated: "102"}, +} + +// seedColumnTypes derives every seed column's type tags, excluded ones included, so a data_types +// rule in compatibility_rules.json follows a seed edit with nothing to declare. func seedColumnTypes() map[string][]string { - return testutils.DDLColumnTypes(seedTableDDL) + types := make(map[string][]string, len(seedColumns)) + for _, col := range seedColumns { + types[col.name] = testutils.DataTypeTags(col.datatype) + } + return types +} + +func filterSeedColumns(t *testing.T, excluded []string) []seedColumn { + t.Helper() + names := make([]string, 0, len(seedColumns)) + for _, col := range seedColumns { + names = append(names, col.name) + } + drop, err := testutils.SeedColumnsExcluded(excluded, names) + require.NoError(t, err, "db2 seed exclusion") + + kept := make([]seedColumn, 0, len(seedColumns)) + for _, col := range seedColumns { + if !drop[col.name] { + kept = append(kept, col) + } + } + return kept +} + +func createTableQuery(table string, cols []seedColumn) string { + defs := make([]string, 0, len(cols)) + for _, col := range cols { + defs = append(defs, col.definition()) + } + return fmt.Sprintf("CREATE TABLE %s (\n\t%s\n)", table, strings.Join(defs, ",\n\t")) +} + +func insertRowQuery(table string, cols []seedColumn, filtered bool, overrides map[string]string) string { + names := make([]string, 0, len(cols)) + values := make([]string, 0, len(cols)) + for _, col := range cols { + value := col.value + if filtered { + value = col.filtered + } + if override, ok := overrides[col.name]; ok { + value = override + } + if value == "" { + continue + } + names = append(names, col.name) + values = append(values, value) + } + return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", table, strings.Join(names, ", "), strings.Join(values, ", ")) +} + +func updateRowQuery(table string, cols []seedColumn) string { + sets := make([]string, 0, len(cols)+1) + for _, col := range cols { + if col.updated == "" { + continue + } + sets = append(sets, col.name+" = "+col.updated) + } + sets = append(sets, "includedColumn = 202") + return fmt.Sprintf("UPDATE %s SET %s WHERE id = 1", table, strings.Join(sets, ", ")) } var ( @@ -118,6 +189,8 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, var err error integrationTestTable := conf.GetTableName() + excludedColumns := conf.SeedExcludedColumns + seedCols := filterSeedColumns(t, excludedColumns) var query string switch operation { @@ -130,7 +203,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, END` case "create": - query = fmt.Sprintf("CREATE TABLE %s (%s)", integrationTestTable, seedTableDDL) + query = createTableQuery(integrationTestTable, seedCols) // DB2 has no CREATE TABLE IF NOT EXISTS; tolerate an existing table (SQL0601N, // SQLSTATE 42710) to match the other drivers' create semantics. if cerr := exec(ctx, db, query); cerr != nil && !strings.Contains(cerr.Error(), "SQL0601N") { @@ -150,84 +223,27 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, query = fmt.Sprintf("DELETE FROM %s", integrationTestTable) case "add": - insertTestData(ctx, t, db, integrationTestTable) + insertTestData(ctx, t, db, integrationTestTable, excludedColumns) + require.NoError(t, exec(ctx, db, fmt.Sprintf(`CALL SYSPROC.ADMIN_CMD('RUNSTATS ON TABLE DB2INST1.%s AND INDEXES ALL')`, integrationTestTable)), + "Failed to populate stats after seeding") return case "insert": - query = fmt.Sprintf(` - INSERT INTO %s ( - col_cursor, col_bigint, col_char, col_character, - col_varchar, col_date, col_decimal, col_decfloat, - col_double, col_real, col_int, col_smallint, - col_clob, col_blob, col_timestamp, col_time, - col_graphic, col_vargraphic, col_bool, excludedColumn - ) VALUES ( - 6, 12345678901234, 'c', 'char_val', - 'varchar_val', DATE('2023-01-01'), 123.45, 123.45, - 123.456789, 123.5, 123, 123, - CLOB('sample text'), BLOB(X'424C4F422044415441204F4E45'), - TIMESTAMP('2023-01-01-12.00.00.000000'), - TIME('12.00.00'), - GRAPHIC('graphic_val'), - VARGRAPHIC('vargraphic_val'), - TRUE, - 101 - )`, integrationTestTable) - err = exec(ctx, db, query) + err = exec(ctx, db, insertRowQuery(integrationTestTable, seedCols, false, + map[string]string{"col_cursor": "6", "excludedColumn": "101"})) require.NoError(t, err, "Failed to execute %s operation", operation) // insert a filtered row — timestamp is before the filter threshold, so it won't be synced - filteredQuery := fmt.Sprintf(` - INSERT INTO %s ( - col_cursor, col_bigint, col_char, col_character, - col_varchar, col_date, col_decimal, col_decfloat, - col_double, col_real, col_int, col_smallint, - col_clob, col_blob, col_timestamp, col_time, - col_graphic, col_vargraphic, col_bool, excludedColumn - ) VALUES ( - -1, 111111111111111, 'x', 'filtered', - 'filtered_val', DATE('2022-06-15'), 50.123, 50.123, - 50.123, 50.0, 0, 0, - CLOB('filtered text'), BLOB(X'00'), - TIMESTAMP('2022-06-15-10.00.00.000000'), - TIME('10.00.00'), - GRAPHIC('filtered'), - VARGRAPHIC('filtered'), - FALSE, - 200 - )`, integrationTestTable) - err = exec(ctx, db, filteredQuery) + err = exec(ctx, db, insertRowQuery(integrationTestTable, seedCols, true, + map[string]string{"excludedColumn": "200"})) require.NoError(t, err, "Failed to insert filtered test data row") return case "insert_2pc": - query = fmt.Sprintf(` - INSERT INTO %s ( - col_cursor, col_bigint, col_char, col_character, - col_varchar, col_date, col_decimal, col_decfloat, - col_double, col_real, col_int, col_smallint, - col_clob, col_blob, col_timestamp, col_time, - col_graphic, col_vargraphic, col_bool - ) VALUES ( - 7, 12345678901234, 'c', 'char_val', - 'varchar_val', DATE('2023-01-01'), 123.45, 123.45, - 123.456789, 123.5, 123, 123, - CLOB('sample text'), BLOB(X'424C4F422044415441204F4E45'), - TIMESTAMP('2023-01-01-12.00.00.000000'), - TIME('12.00.00'), - GRAPHIC('graphic_val'), - VARGRAPHIC('vargraphic_val'), - TRUE - )`, integrationTestTable) + query = insertRowQuery(integrationTestTable, seedCols, false, + map[string]string{"col_cursor": "7"}) case "update": - query = fmt.Sprintf(` - UPDATE %s SET - col_cursor = NULL, - col_smallint = 321, - col_timestamp = TIMESTAMP('2024-01-01-12.00.00.000000'), - excludedColumn = 102, - includedColumn = 202 - WHERE id = 1`, integrationTestTable) + query = updateRowQuery(integrationTestTable, seedCols) case "delete": query = fmt.Sprintf("DELETE FROM %s WHERE id = 1", integrationTestTable) @@ -244,10 +260,6 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, // to clear REORG pending state of DB2 after schema evolution query = fmt.Sprintf(`CALL SYSPROC.ADMIN_CMD('REORG TABLE DB2INST1.%s')`, integrationTestTable) - case "populate-stats": - // if table exists, run stats for DB2 to populate stats - query = fmt.Sprintf(`CALL SYSPROC.ADMIN_CMD('RUNSTATS ON TABLE DB2INST1.%s AND INDEXES ALL')`, integrationTestTable) - default: t.Fatalf("Unsupported operation: %s", operation) } @@ -269,53 +281,25 @@ func requireEmptyTable(ctx context.Context, t *testing.T, db *sqlx.DB, table, op require.Zerof(t, rows, "%s left %d rows in %s", operation, rows, table) } -func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName string) { +func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName string, excludedColumns []string) { t.Helper() + seedCols := filterSeedColumns(t, excludedColumns) for i := 1; i <= 5; i++ { - query := fmt.Sprintf(` - INSERT INTO %s ( - col_cursor, col_bigint, col_char, col_character, - col_varchar, col_date, col_decimal, col_decfloat, - col_double, col_real, col_int, col_smallint, - col_clob, col_blob, col_timestamp, col_time, - col_graphic, col_vargraphic, col_bool, excludedColumn - ) VALUES ( - %d, 12345678901234, 'c', 'char_val', - 'varchar_val', DATE('2023-01-01'), 123.45, 123.45, - 123.456789, 123.5, 123, 123, - CLOB('sample text'), BLOB(X'424C4F422044415441204F4E45'), - TIMESTAMP('2023-01-01-12.00.00.000000'), - TIME('12.00.00'), - GRAPHIC('graphic_val'), - VARGRAPHIC('vargraphic_val'), - TRUE, - 100 - )`, tableName, i) - - require.NoError(t, exec(ctx, db, query), "Failed to insert test data") + row := map[string]string{"col_cursor": strconv.Itoa(i), "excludedColumn": "100"} + require.NoError(t, exec(ctx, db, insertRowQuery(tableName, seedCols, false, row)), "Failed to insert test data") } // insert a filtered row — timestamp is before the filter threshold, so it won't be synced - filteredQuery := fmt.Sprintf(` - INSERT INTO %s ( - col_cursor, col_bigint, col_char, col_character, - col_varchar, col_date, col_decimal, col_decfloat, - col_double, col_real, col_int, col_smallint, - col_clob, col_blob, col_timestamp, col_time, - col_graphic, col_vargraphic, col_bool, excludedColumn - ) VALUES ( - -1, 111111111111111, 'x', 'filtered', - 'filtered_val', DATE('2021-06-15'), 500234.123, 500234.123, - 500234.123, 500234.0, 0, 0, - CLOB('filtered text'), BLOB(X'00'), - TIMESTAMP('2021-06-15-10.00.00.000000'), - TIME('10.00.00'), - GRAPHIC('filtered'), - VARGRAPHIC('filtered'), - FALSE, - 200 - )`, tableName) - require.NoError(t, exec(ctx, db, filteredQuery), "Failed to insert filtered test data row") + filteredRow := map[string]string{ + "excludedColumn": "200", + "col_date": "DATE('2021-06-15')", + "col_decimal": "500234.123", + "col_decfloat": "500234.123", + "col_double": "500234.123", + "col_real": "500234.0", + "col_timestamp": "TIMESTAMP('2021-06-15-10.00.00.000000')", + } + require.NoError(t, exec(ctx, db, insertRowQuery(tableName, seedCols, true, filteredRow)), "Failed to insert filtered test data row") } var ExpectedDB2Data = map[string]interface{}{ diff --git a/tests/kafka/kafka_util_test.go b/tests/kafka/kafka_util_test.go index 0393be897..dc3455009 100644 --- a/tests/kafka/kafka_util_test.go +++ b/tests/kafka/kafka_util_test.go @@ -23,12 +23,10 @@ import ( ) const ( - partitionCount = 5 - rebalanceBulkMessageCount = 100_000 - rebalanceBulkPartition = int32(0) - rebalanceBulkBatchSize = 500 - // The broker advertises a listener per network: source.json names the one the driver container - // reaches (host.docker.internal:39092), and dialing it from the host fails on the advertised name. + partitionCount = 5 + rebalanceBulkMessageCount = 100_000 + rebalanceBulkPartition = int32(0) + rebalanceBulkBatchSize = 500 kafkaJSONIntegrationBroker = "127.0.0.1:29092" kafkaAvroIntegrationBroker = "127.0.0.1:29192" avroSchemaRegistryURL = "http://127.0.0.1:8081" diff --git a/tests/mongodb/mongodb_test.go b/tests/mongodb/mongodb_test.go index 95407d808..42b40a869 100644 --- a/tests/mongodb/mongodb_test.go +++ b/tests/mongodb/mongodb_test.go @@ -7,6 +7,7 @@ import ( "github.com/datazip-inc/olake/tests/testutils/compatibility" "github.com/datazip-inc/olake/tests/testutils/constants" "github.com/datazip-inc/olake/tests/testutils/integration" + "github.com/datazip-inc/olake/tests/testutils/performance" "github.com/datazip-inc/olake/tests/testutils/require" ) @@ -58,26 +59,22 @@ func TestMongodb2PC(t *testing.T) { mongodbBaseConfig(t).Test2PCIntegration(t) } -// func TestMongodbPerformance(t *testing.T) { -// cfg, err := testutils.NewTestConfig(constants.MongoDB, "twitter_data", "", ExecuteQuery, "") -// require.NoError(t, err, "failed to build the test config") +func TestMongodbPerformance(t *testing.T) { + cfg, err := testutils.NewTestConfig(t, constants.MongoDB, "twitter_data", "", ExecuteQuery) + require.NoError(t, err, "failed to build the test config") -// perf := &performance.Test{ -// TestConfig: cfg, -// BackfillStreams: performance.GetBackfillStreamsFromCDC(performanceCDCStreams), -// CDCStreams: performanceCDCStreams, -// } + perf := &performance.Test{ + TestConfig: cfg, + BackfillStreams: performance.GetBackfillStreamsFromCDC(performanceCDCStreams), + CDCStreams: performanceCDCStreams, + } -// perf.TestPerformance(t) -// } + perf.TestPerformance(t) +} // TestMongodbCompatibility pins the backward-compatibility contract for the driver owning the v5 gate // (BSON DateTime decoded as UTC time.Time at any depth, constants/state_version.go). v0.6.1 is // the newest release still on state version 4, so it is the one that exercises it. -// -// _id and _olake_id are volatile here, unlike every other driver: the seed inserts documents -// without an _id, so the server generates a fresh ObjectID per run and _olake_id, which hashes the -// primary key, follows it. Both are still compared by TYPE -- only their values are exempt. func TestMongodbCompatibility(t *testing.T) { t.Parallel() fixture := &compatibility.Test{ diff --git a/tests/mongodb/mongodb_util_test.go b/tests/mongodb/mongodb_util_test.go index 80277b1df..7f686f961 100644 --- a/tests/mongodb/mongodb_util_test.go +++ b/tests/mongodb/mongodb_util_test.go @@ -9,6 +9,7 @@ import ( "github.com/apache/arrow-go/v18/arrow" "github.com/datazip-inc/olake/tests/testutils" + "github.com/datazip-inc/olake/tests/testutils/performance" "github.com/datazip-inc/olake/tests/testutils/require" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -18,14 +19,8 @@ import ( var ( nestedDoc = bson.M{ - "nested_string": "nested_value", - "nested_int": 42, - // A BSON DateTime below the top level, which is what the state-version-5 gate governs - // (drivers/mongodb/internal/mon.go: at v>=5 a custom registry decodes it to a UTC - // time.Time, at v<=4 the stock decoder yields a primitive.DateTime). Both marshal to the - // same string for an in-range year -- primitive.DateTime.MarshalJSON already normalizes to - // UTC -- so this pins that the decoder swap did NOT change in-range values. The versions - // only diverge outside [0,9999], where v<=4 fails json.Marshal outright. + "nested_string": "nested_value", + "nested_int": 42, "nested_timestamp": time.Date(2023, 1, 1, 12, 0, 0, 0, time.UTC), } ) @@ -173,44 +168,44 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, } return - // case "bulk_cdc_data_insert": - // backfillStreams := performance.GetBackfillStreamsFromCDC(performanceCDCStreams) - // totalRows := 15000000 + case "bulk_cdc_data_insert": + backfillStreams := performance.GetBackfillStreamsFromCDC(performanceCDCStreams) + totalRows := 15000000 - // // TODO: insert data in batch - // // insert the data into the cdc tables concurrently - // err := testutils.Concurrent(ctx, performanceCDCStreams, len(performanceCDCStreams), func(ctx context.Context, cdcStream string, executionNumber int) error { - // srcColl := client.Database(config.String("database")).Collection(backfillStreams[executionNumber]) - // destColl := client.Database(config.String("database")).Collection(cdcStream) + // TODO: insert data in batch + // insert the data into the cdc tables concurrently + err := testutils.Concurrent(ctx, performanceCDCStreams, len(performanceCDCStreams), func(ctx context.Context, cdcStream string, executionNumber int) error { + srcColl := client.Database(config.String("database")).Collection(backfillStreams[executionNumber]) + destColl := client.Database(config.String("database")).Collection(cdcStream) - // cursor, err := srcColl.Find(ctx, bson.D{}, options.Find().SetLimit(int64(totalRows))) - // if err != nil { - // return fmt.Errorf("stream: %s, error: %s", cdcStream, err) - // } - // defer cursor.Close(ctx) + cursor, err := srcColl.Find(ctx, bson.D{}, options.Find().SetLimit(int64(totalRows))) + if err != nil { + return fmt.Errorf("stream: %s, error: %s", cdcStream, err) + } + defer cursor.Close(ctx) - // var docs []interface{} - // for cursor.Next(ctx) { - // var doc bson.M - // if err := cursor.Decode(&doc); err != nil { - // return err - // } - // docs = append(docs, doc) - // } - // if err := cursor.Err(); err != nil { - // return err - // } - // if len(docs) == 0 { - // return nil - // } - // _, err = destColl.InsertMany(ctx, docs) - // if err != nil { - // return fmt.Errorf("stream: %s, error: %s", cdcStream, err) - // } - // return nil - // }) - // require.NoError(t, err, fmt.Sprintf("failed to execute %s operation", operation), err) - // return + var docs []interface{} + for cursor.Next(ctx) { + var doc bson.M + if err := cursor.Decode(&doc); err != nil { + return err + } + docs = append(docs, doc) + } + if err := cursor.Err(); err != nil { + return err + } + if len(docs) == 0 { + return nil + } + _, err = destColl.InsertMany(ctx, docs) + if err != nil { + return fmt.Errorf("stream: %s, error: %s", cdcStream, err) + } + return nil + }) + require.NoError(t, err, fmt.Sprintf("failed to execute %s operation", operation), err) + return } } diff --git a/tests/mssql/mssql_util_test.go b/tests/mssql/mssql_util_test.go index bc483f144..2ad331e3b 100644 --- a/tests/mssql/mssql_util_test.go +++ b/tests/mssql/mssql_util_test.go @@ -220,7 +220,6 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, case "add": insertTestData(ctx, t, db, integrationTestTable) - return case "insert": insertOne := fmt.Sprintf(` @@ -359,14 +358,14 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, _, err := db.ExecContext(ctx, stmt) require.NoError(t, err, "failed to evolve schema") - case "wait-cdc-catchup": - // The caller just committed DML it expects the next CDC sync to pick up; wait for the - // asynchronous capture job to scan it. - waitForCDCCapture(ctx, t, db) - default: t.Fatalf("Unsupported operation: %s", operation) } + + switch operation { + case "add", "insert", "insert_2pc", "update", "delete": + waitForCDCCapture(ctx, t, db) + } } // suiteDatabasesEnsured tracks the databases this process has provisioned, so ensureSuiteDatabase @@ -376,10 +375,7 @@ var ( suiteDatabasesEnsuredMu sync.Mutex ) -// ensureSuiteDatabase creates the suite's CDC-enabled database when the volume lacks it. Lazy and -// harness-owned rather than 01-init.sql: an init script runs only on a fresh volume, and every new -// suite needed a hand-edit there plus a refresh -- this way any volume converges on first touch. -// Runs against master, since the suite connection names a database that cannot exist before it does. +// ensureSuiteDatabase creates the suite's CDC-enabled database when the volume lacks it. func ensureSuiteDatabase(ctx context.Context, t *testing.T, config testutils.SourceConfig, dbName string) { t.Helper() suiteDatabasesEnsuredMu.Lock() @@ -458,10 +454,7 @@ func startCDCCapture(ctx context.Context, t *testing.T, db *sqlx.DB) { } // ensureFastCDCPolling drops this database's CDC capture job to the minimum 1s polling interval -// (default 5s) -- the cycle every create / wait-cdc-catchup, and the driver's own catch-up, waits -// out. Only the interval is written: it takes effect on the next start, which is startCDCCapture's -// job, and starting it here too leaves that one racing the agent's own pending request. Best-effort -// -- a job still on 5s is slower, not wrong -- and a no-op once it reports 1s. +// (default 5s) -- the cycle every create / wait-cdc-catchup, and the driver's own catch-up, waits out. func ensureFastCDCPolling(ctx context.Context, t *testing.T, db *sqlx.DB) { t.Helper() diff --git a/tests/mysql/mysql_test.go b/tests/mysql/mysql_test.go index 997a3da0f..882fba7c3 100644 --- a/tests/mysql/mysql_test.go +++ b/tests/mysql/mysql_test.go @@ -75,13 +75,7 @@ func TestMySQLPerformance(t *testing.T) { // TestMySQLCompatibility pins the backward-compatibility contract for the driver that owns three of the // six version gates -- the binlog timestamp location (v2), the timezone offset (v3) and the -// UNSIGNED widening (v4), see constants/state_version.go. Note that a passing run is the -// contract HOLDING: the candidate reading a state file at version N reproduces version N's types, -// so it agrees with the baseline. A diff here means a gate stopped firing. -// -// Baseline defaults to the newest release; OLAKE_COMPATIBILITY_BASELINE picks another tag, image or -// commit. v0.4.0 is the newest release still on state version 3, so it is the one that exercises -// the UNSIGNED gate. +// UNSIGNED widening (v4), see constants/state_version.go. func TestMySQLCompatibility(t *testing.T) { t.Parallel() fixture := &compatibility.Test{ diff --git a/tests/mysql/mysql_util_test.go b/tests/mysql/mysql_util_test.go index 02a9a39a4..b5b31a688 100644 --- a/tests/mysql/mysql_util_test.go +++ b/tests/mysql/mysql_util_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math" + "strconv" "strings" "testing" "time" @@ -20,105 +21,134 @@ import ( // PerformanceTest config and the perf operations below. var performanceCDCStreams = []string{"trips_cdc", "fhv_trips_cdc"} -// versionedSeedColumns are the columns a suite can leave out of the seed data through -// TestConfig.SeedExcludedColumns -- the backward-compatibility suite drops the ones an old -// baseline cannot sync; every other suite leaves the list empty and seeds all of them. -var versionedSeedColumns = []struct { - name, ddl, value, filteredValue, updateExpr string -}{ - {"name_ucs2", "name_ucs2 VARCHAR(100) CHARACTER SET ucs2", "'ucs2_val'", "'filtered ucs2'", "name_ucs2 = 'updated ucs2'"}, - {"name_utf16le", "name_utf16le VARCHAR(100) CHARACTER SET utf16le", "'utf16le_val'", "'filtered utf16le'", "name_utf16le = 'updated utf16le'"}, - {"grade", "grade ENUM('naïve','café','résumé') CHARACTER SET latin1", "'naïve'", "'naïve'", "grade = 'café'"}, - {"name_latin1", "name_latin1 VARCHAR(100) CHARACTER SET latin1", "'latin1_val'", "'filtered latin1'", "name_latin1 = 'updated latin1'"}, - {"permissions", "permissions SET('read','write','execute') CHARACTER SET latin1 DEFAULT NULL", "'read,write'", "'execute'", "permissions = 'read,write,execute'"}, - {"id_bigint_unsigned", "id_bigint_unsigned BIGINT UNSIGNED", "5003", "0", "id_bigint_unsigned = 6003"}, - {"id_bigint_unsigned_signbit", "id_bigint_unsigned_signbit BIGINT UNSIGNED", "9223372036854775808", "0", "id_bigint_unsigned_signbit = 9223372036854775809"}, - {"id_bigint_unsigned_max", "id_bigint_unsigned_max BIGINT UNSIGNED", "18446744073709551615", "0", "id_bigint_unsigned_max = 18446744073709551614"}, +type seedColumn struct { + name string + datatype string + value string + filtered string + updated string } -// seedTableDDL is the seed table's column list, the one place the fixture's schema lives: create -// renders it and seedColumnTypes reads it; the versioned columns splice in at %s. -const seedTableDDL = ` - id INT UNSIGNED NOT NULL AUTO_INCREMENT, - id_bigint BIGINT, - id_int INT, - id_cursor INT, - id_int_unsigned INT UNSIGNED, - id_integer INT, - id_integer_unsigned INT UNSIGNED, - id_mediumint MEDIUMINT, - id_mediumint_unsigned MEDIUMINT UNSIGNED, - id_smallint SMALLINT, - id_smallint_unsigned SMALLINT UNSIGNED, - id_tinyint TINYINT, - id_tinyint_unsigned TINYINT UNSIGNED, - id_tinyint_unsigned_max TINYINT UNSIGNED, - id_smallint_unsigned_max SMALLINT UNSIGNED, - id_mediumint_unsigned_max MEDIUMINT UNSIGNED, - id_mediumint_unsigned_signbit MEDIUMINT UNSIGNED, - id_int_unsigned_max INT UNSIGNED, - price_decimal DECIMAL(10,2), - amount_decimal_9_2 DECIMAL(9,2), - price_double DOUBLE, - price_double_precision DOUBLE, - price_float FLOAT, - price_numeric DECIMAL(10,2), - price_real DOUBLE, - name_char CHAR(50), - name_varchar VARCHAR(100), - name_text TEXT, - name_tinytext TINYTEXT, - name_mediumtext MEDIUMTEXT, - name_longtext LONGTEXT, - created_date DATETIME, - created_timestamp TIMESTAMP NULL, - is_active TINYINT(1), - long_varchar MEDIUMTEXT, - name_bool TINYINT(1) DEFAULT '1', - status ENUM('active','inactive','pending') DEFAULT NULL, - priority ENUM('low','medium','high') DEFAULT 'low',%s - tags SET('sports','music','gaming','reading') DEFAULT NULL, - PRIMARY KEY (id), - excludedColumn INT - ` - -// seedColumnTypes derives every seed column's type tags from the DDL, versioned columns included, -// so a data_types rule in compatibility_rules.json follows a seed edit with nothing to declare. +func (c seedColumn) definition() string { return c.name + " " + c.datatype } + +var seedColumns = []seedColumn{ + {name: "id", datatype: "INT UNSIGNED NOT NULL AUTO_INCREMENT", value: "", filtered: "", updated: ""}, + {name: "id_bigint", datatype: "BIGINT", value: "123456789012345", filtered: "111111111111111", updated: "987654321098765"}, + {name: "id_int", datatype: "INT", value: "100", filtered: "0", updated: "200"}, + {name: "id_cursor", datatype: "INT", value: "", filtered: "-1", updated: "NULL"}, + {name: "id_int_unsigned", datatype: "INT UNSIGNED", value: "4294967295", filtered: "0", updated: "4294967293"}, + {name: "id_integer", datatype: "INT", value: "102", filtered: "0", updated: "202"}, + {name: "id_integer_unsigned", datatype: "INT UNSIGNED", value: "4294967294", filtered: "0", updated: "4294967292"}, + {name: "id_mediumint", datatype: "MEDIUMINT", value: "5001", filtered: "0", updated: "6001"}, + {name: "id_mediumint_unsigned", datatype: "MEDIUMINT UNSIGNED", value: "5002", filtered: "0", updated: "6002"}, + {name: "id_smallint", datatype: "SMALLINT", value: "101", filtered: "0", updated: "201"}, + {name: "id_smallint_unsigned", datatype: "SMALLINT UNSIGNED", value: "102", filtered: "0", updated: "202"}, + {name: "id_tinyint", datatype: "TINYINT", value: "50", filtered: "0", updated: "60"}, + {name: "id_tinyint_unsigned", datatype: "TINYINT UNSIGNED", value: "51", filtered: "0", updated: "61"}, + {name: "id_tinyint_unsigned_max", datatype: "TINYINT UNSIGNED", value: "255", filtered: "0", updated: "254"}, + {name: "id_smallint_unsigned_max", datatype: "SMALLINT UNSIGNED", value: "65535", filtered: "0", updated: "65534"}, + {name: "id_mediumint_unsigned_max", datatype: "MEDIUMINT UNSIGNED", value: "16777215", filtered: "0", updated: "16777214"}, + {name: "id_mediumint_unsigned_signbit", datatype: "MEDIUMINT UNSIGNED", value: "8388608", filtered: "0", updated: "8388609"}, + {name: "id_int_unsigned_max", datatype: "INT UNSIGNED", value: "4294967295", filtered: "0", updated: "4294967294"}, + {name: "price_decimal", datatype: "DECIMAL(10,2)", value: "123.45", filtered: "50.123", updated: "543.21"}, + {name: "amount_decimal_9_2", datatype: "DECIMAL(9,2)", value: "5330197.27", filtered: "50.12", updated: "1234567.89"}, + {name: "price_double", datatype: "DOUBLE", value: "123.456", filtered: "50.123", updated: "654.321"}, + {name: "price_double_precision", datatype: "DOUBLE", value: "123.456", filtered: "50.123", updated: "654.321"}, + {name: "price_float", datatype: "FLOAT", value: "123.45", filtered: "50.0", updated: "543.21"}, + {name: "price_numeric", datatype: "DECIMAL(10,2)", value: "123.45", filtered: "50.123", updated: "543.21"}, + {name: "price_real", datatype: "DOUBLE", value: "123.456", filtered: "50.123", updated: "654.321"}, + {name: "name_char", datatype: "CHAR(50)", value: "'c'", filtered: "'x'", updated: "'X'"}, + {name: "name_varchar", datatype: "VARCHAR(100)", value: "'varchar_val'", filtered: "'filtered_val'", updated: "'updated varchar'"}, + {name: "name_text", datatype: "TEXT", value: "'text_val'", filtered: "'filtered text'", updated: "'updated text'"}, + {name: "name_tinytext", datatype: "TINYTEXT", value: "'tinytext_val'", filtered: "'filtered tiny'", updated: "'upd tiny'"}, + {name: "name_mediumtext", datatype: "MEDIUMTEXT", value: "'mediumtext_val'", filtered: "'filtered medium'", updated: "'upd medium'"}, + {name: "name_longtext", datatype: "LONGTEXT", value: "'longtext_val'", filtered: "'filtered long'", updated: "'upd long'"}, + {name: "created_date", datatype: "DATETIME", value: "'2023-01-01 12:00:00'", filtered: "'2022-06-15 10:00:00'", updated: "'2024-07-01 15:30:00'"}, + {name: "created_timestamp", datatype: "TIMESTAMP NULL", value: "'2023-01-01 12:00:00'", filtered: "'2021-06-15 10:00:00'", updated: "'2024-07-01 15:30:00'"}, + {name: "is_active", datatype: "TINYINT(1)", value: "1", filtered: "0", updated: "0"}, + {name: "long_varchar", datatype: "MEDIUMTEXT", value: "'long_varchar_val'", filtered: "'filtered long varchar'", updated: "'updated long...'"}, + {name: "name_bool", datatype: "TINYINT(1) DEFAULT '1'", value: "1", filtered: "0", updated: "0"}, + {name: "status", datatype: "ENUM('active','inactive','pending') DEFAULT NULL", value: "'active'", filtered: "'inactive'", updated: "'pending'"}, + {name: "priority", datatype: "ENUM('low','medium','high') DEFAULT 'low'", value: "'high'", filtered: "'low'", updated: "'low'"}, + {name: "name_ucs2", datatype: "VARCHAR(100) CHARACTER SET ucs2", value: "'ucs2_val'", filtered: "'filtered ucs2'", updated: "'updated ucs2'"}, + {name: "name_utf16le", datatype: "VARCHAR(100) CHARACTER SET utf16le", value: "'utf16le_val'", filtered: "'filtered utf16le'", updated: "'updated utf16le'"}, + {name: "grade", datatype: "ENUM('naïve','café','résumé') CHARACTER SET latin1", value: "'naïve'", filtered: "'naïve'", updated: "'café'"}, + {name: "name_latin1", datatype: "VARCHAR(100) CHARACTER SET latin1", value: "'latin1_val'", filtered: "'filtered latin1'", updated: "'updated latin1'"}, + {name: "permissions", datatype: "SET('read','write','execute') CHARACTER SET latin1 DEFAULT NULL", value: "'read,write'", filtered: "'execute'", updated: "'read,write,execute'"}, + {name: "id_bigint_unsigned", datatype: "BIGINT UNSIGNED", value: "5003", filtered: "0", updated: "6003"}, + {name: "id_bigint_unsigned_signbit", datatype: "BIGINT UNSIGNED", value: "9223372036854775808", filtered: "0", updated: "9223372036854775809"}, + {name: "id_bigint_unsigned_max", datatype: "BIGINT UNSIGNED", value: "18446744073709551615", filtered: "0", updated: "18446744073709551614"}, + {name: "tags", datatype: "SET('sports','music','gaming','reading') DEFAULT NULL", value: "'sports,reading'", filtered: "'music'", updated: "'gaming,reading'"}, + {name: "excludedColumn", datatype: "INT", value: "", filtered: "", updated: "102"}, +} + +// seedColumnTypes derives every seed column's type tags, excluded ones included, so a data_types +// rule in compatibility_rules.json follows a seed edit with nothing to declare. func seedColumnTypes() map[string][]string { - ddl := fmt.Sprintf(seedTableDDL, "") - for _, col := range versionedSeedColumns { - ddl += "\n" + col.ddl + types := make(map[string][]string, len(seedColumns)) + for _, col := range seedColumns { + types[col.name] = testutils.DataTypeTags(col.datatype) } - return testutils.DDLColumnTypes(ddl) + return types } -// seedColumnFragments renders the versioned columns NOT being excluded as the fragments each seed -// statement splices in after name_latin1; excluding nothing reproduces the full fixture. -func seedColumnFragments(t *testing.T, excluded []string) (ddl, cols, vals, filteredVals, updates string) { +func filterSeedColumns(t *testing.T, excluded []string) []seedColumn { t.Helper() - supported := make([]string, 0, len(versionedSeedColumns)) - for _, col := range versionedSeedColumns { - supported = append(supported, col.name) + names := make([]string, 0, len(seedColumns)) + for _, col := range seedColumns { + names = append(names, col.name) } - drop, err := testutils.SeedColumnsExcluded(excluded, supported) + drop, err := testutils.SeedColumnsExcluded(excluded, names) require.NoError(t, err, "mysql seed exclusion") - var names, values, filtered, sets []string - for _, col := range versionedSeedColumns { - if drop[col.name] { + kept := make([]seedColumn, 0, len(seedColumns)) + for _, col := range seedColumns { + if !drop[col.name] { + kept = append(kept, col) + } + } + return kept +} + +func createTableQuery(table string, cols []seedColumn) string { + defs := make([]string, 0, len(cols)+1) + for _, col := range cols { + defs = append(defs, col.definition()) + } + defs = append(defs, "PRIMARY KEY (id)") + return fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n\t%s\n)", table, strings.Join(defs, ",\n\t")) +} + +func insertRowQuery(table string, cols []seedColumn, filtered bool, overrides map[string]string) string { + names := make([]string, 0, len(cols)) + values := make([]string, 0, len(cols)) + for _, col := range cols { + value := col.value + if filtered { + value = col.filtered + } + if override, ok := overrides[col.name]; ok { + value = override + } + if value == "" { continue } - ddl += "\n\t\t" + col.ddl + "," names = append(names, col.name) - values = append(values, col.value) - filtered = append(filtered, col.filteredValue) - sets = append(sets, col.updateExpr) + values = append(values, value) } - if len(names) == 0 { - return "", "", "", "", "" + return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", table, strings.Join(names, ", "), strings.Join(values, ", ")) +} + +func updateRowQuery(table string, cols []seedColumn) string { + sets := make([]string, 0, len(cols)+1) + for _, col := range cols { + if col.updated == "" { + continue + } + sets = append(sets, col.name+" = "+col.updated) } - join := func(parts []string) string { return " " + strings.Join(parts, ", ") + "," } - return ddl, join(names), join(values), join(filtered), join(sets) + sets = append(sets, "includedColumn = 202") + return fmt.Sprintf("UPDATE %s SET %s WHERE id = 1", table, strings.Join(sets, ", ")) } // ExecuteQuery executes MySQL queries for testing based on the operation type. Columns named in @@ -127,7 +157,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, t.Helper() excludedColumns := conf.SeedExcludedColumns - seedDDL, seedCols, seedVals, seedFilteredVals, seedUpdates := seedColumnFragments(t, excludedColumns) + seedCols := filterSeedColumns(t, excludedColumns) var connStr, database string config := conf.SourceBaseConfig @@ -151,7 +181,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, switch operation { case "create": - query = fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (%s)", integrationTestTable, fmt.Sprintf(seedTableDDL, seedDDL)) + query = createTableQuery(integrationTestTable, seedCols) case "drop": query = fmt.Sprintf("DROP TABLE IF EXISTS %s", integrationTestTable) @@ -169,143 +199,21 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, return // Early return since we handle all inserts in the helper function case "insert": - query = fmt.Sprintf(` - INSERT INTO %s ( - id_cursor, id, id_bigint, - id_int, id_int_unsigned, id_integer, id_integer_unsigned, - id_mediumint, id_mediumint_unsigned, id_smallint, id_smallint_unsigned, - id_tinyint, id_tinyint_unsigned, - id_tinyint_unsigned_max, id_smallint_unsigned_max, - id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - price_decimal, amount_decimal_9_2, price_double, - price_double_precision, price_float, price_numeric, price_real, - name_char, name_varchar, name_text, name_tinytext, - name_mediumtext, name_longtext, created_date, - created_timestamp, is_active, - long_varchar, name_bool, status, priority, - %s - tags, - excludedColumn - ) VALUES ( - 6, 6, 123456789012345, - 100, 4294967295, 102, 4294967294, - 5001, 5002, 101, 102, - 50, 51, - 255, 65535, - 16777215, 8388608, 4294967295, - 123.45, 5330197.27, 123.456, - 123.456, 123.45, 123.45, 123.456, - 'c', 'varchar_val', 'text_val', 'tinytext_val', - 'mediumtext_val', 'longtext_val', '2023-01-01 12:00:00', - '2023-01-01 12:00:00', 1, - 'long_varchar_val', 1, 'active', 'high', - %s - 'sports,reading', - 101 - )`, integrationTestTable, seedCols, seedVals) - _, err = db.ExecContext(ctx, query) + _, err = db.ExecContext(ctx, insertRowQuery(integrationTestTable, seedCols, false, + map[string]string{"id": "6", "id_cursor": "6", "excludedColumn": "101"})) require.NoError(t, err, "Failed to execute %s operation", operation) // insert a filtered doc, it would be filtered out by the filter, won't be synced into the destination - filteredQuery := fmt.Sprintf(` - INSERT INTO %s ( - id_cursor, id, id_bigint, - id_int, id_int_unsigned, id_integer, id_integer_unsigned, - id_mediumint, id_mediumint_unsigned, id_smallint, id_smallint_unsigned, - id_tinyint, id_tinyint_unsigned, - id_tinyint_unsigned_max, id_smallint_unsigned_max, - id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - price_decimal, amount_decimal_9_2, price_double, - price_double_precision, price_float, price_numeric, price_real, - name_char, name_varchar, name_text, name_tinytext, - name_mediumtext, name_longtext, created_date, - created_timestamp, is_active, - long_varchar, name_bool, status, priority, - %s - tags, - excludedColumn - ) VALUES ( - -1, 999, 111111111111111, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, - 0, 0, - 0, 0, 0, - 50.123, 50.12, 50.123, - 50.123, 50.0, 50.123, 50.123, - 'x', 'filtered_val', 'filtered text', 'filtered tiny', - 'filtered medium', 'filtered long', '2022-06-15 10:00:00', - '2021-06-15 10:00:00', 0, - 'filtered long varchar', 0, 'inactive', 'low', - %s - 'music', - 200 - )`, integrationTestTable, seedCols, seedFilteredVals) - _, err = db.ExecContext(ctx, filteredQuery) + _, err = db.ExecContext(ctx, insertRowQuery(integrationTestTable, seedCols, true, + map[string]string{"id": "999", "excludedColumn": "200"})) require.NoError(t, err, "Failed to insert filtered test data row") return case "insert_2pc": - query = fmt.Sprintf(` - INSERT INTO %s ( - id_cursor, id, id_bigint, - id_int, id_int_unsigned, id_integer, id_integer_unsigned, - id_mediumint, id_mediumint_unsigned, id_smallint, id_smallint_unsigned, - id_tinyint, id_tinyint_unsigned, - id_tinyint_unsigned_max, id_smallint_unsigned_max, - id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - price_decimal, amount_decimal_9_2, price_double, - price_double_precision, price_float, price_numeric, price_real, - name_char, name_varchar, name_text, name_tinytext, - name_mediumtext, name_longtext, created_date, - created_timestamp, is_active, - long_varchar, name_bool, status, priority, - %s - tags - ) VALUES ( - 7, 7, 123456789012345, - 100, 4294967295, 102, 4294967294, - 5001, 5002, 101, 102, - 50, 51, - 255, 65535, - 16777215, 8388608, 4294967295, - 123.45, 5330197.27, 123.456, - 123.456, 123.45, 123.45, 123.456, - 'c', 'varchar_val', 'text_val', 'tinytext_val', - 'mediumtext_val', 'longtext_val', '2023-01-01 12:00:00', - '2023-01-01 12:00:00', 1, - 'long_varchar_val', 1, 'active', 'high', - %s - 'sports,reading' - )`, integrationTestTable, seedCols, seedVals) + query = insertRowQuery(integrationTestTable, seedCols, false, + map[string]string{"id": "7", "id_cursor": "7"}) case "update": - query = fmt.Sprintf(` - UPDATE %s SET - id_cursor = NULL, - id_bigint = 987654321098765, - id_int = 200, id_int_unsigned = 4294967293, - id_integer = 202, id_integer_unsigned = 4294967292, - id_mediumint = 6001, id_mediumint_unsigned = 6002, - id_smallint = 201, id_smallint_unsigned = 202, - id_tinyint = 60, id_tinyint_unsigned = 61, - id_tinyint_unsigned_max = 254, id_smallint_unsigned_max = 65534, - id_mediumint_unsigned_max = 16777214, id_mediumint_unsigned_signbit = 8388609, - id_int_unsigned_max = 4294967294, - price_decimal = 543.21, amount_decimal_9_2 = 1234567.89, price_double = 654.321, - price_double_precision = 654.321, price_float = 543.21, - price_numeric = 543.21, price_real = 654.321, - name_char = 'X', name_varchar = 'updated varchar', - name_text = 'updated text', name_tinytext = 'upd tiny', - name_mediumtext = 'upd medium', name_longtext = 'upd long', - created_date = '2024-07-01 15:30:00', - created_timestamp = '2024-07-01 15:30:00', is_active = 0, - long_varchar = 'updated long...', name_bool = 0, - status = 'pending', priority = 'low', - %s - tags = 'gaming,reading', - excludedColumn = 102, - includedColumn = 202 - WHERE id = 1`, integrationTestTable, seedUpdates) + query = updateRowQuery(integrationTestTable, seedCols) case "delete": query = fmt.Sprintf("DELETE FROM %s WHERE id = 1", integrationTestTable) @@ -359,78 +267,25 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, func insertTestData(ctx context.Context, t *testing.T, db *sqlx.DB, tableName string, excludedColumns []string) { t.Helper() - _, seedCols, seedVals, seedFilteredVals, _ := seedColumnFragments(t, excludedColumns) + seedCols := filterSeedColumns(t, excludedColumns) for i := 1; i <= 5; i++ { - query := fmt.Sprintf(` - INSERT INTO %s ( - id_cursor, id, id_bigint, - id_int, id_int_unsigned, id_integer, id_integer_unsigned, - id_mediumint, id_mediumint_unsigned, id_smallint, id_smallint_unsigned, - id_tinyint, id_tinyint_unsigned, - id_tinyint_unsigned_max, id_smallint_unsigned_max, - id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - price_decimal, amount_decimal_9_2, price_double, - price_double_precision, price_float, price_numeric, price_real, - name_char, name_varchar, name_text, name_tinytext, - name_mediumtext, name_longtext, created_date, - created_timestamp, is_active, long_varchar, name_bool, status, priority, - %s - tags, - excludedColumn - ) VALUES ( - %d, %d, 123456789012345, - 100, 4294967295, 102, 4294967294, - 5001, 5002, 101, 102, - 50, 51, - 255, 65535, - 16777215, 8388608, 4294967295, - 123.45, 5330197.27, 123.456, - 123.456, 123.45, 123.45, 123.456, - 'c', 'varchar_val', 'text_val', 'tinytext_val', - 'mediumtext_val', 'longtext_val', '2023-01-01 12:00:00', - '2023-01-01 12:00:00', 1, 'long_varchar_val', 1, 'active', 'high', - %s - 'sports,reading', - 100 - )`, tableName, seedCols, i, i, seedVals) - - _, err := db.ExecContext(ctx, query) + _, err := db.ExecContext(ctx, insertRowQuery(tableName, seedCols, false, + map[string]string{"id": strconv.Itoa(i), "id_cursor": strconv.Itoa(i), "excludedColumn": "100"})) require.NoError(t, err, "Failed to insert test data row %d", i) } // insert a filtered doc, it would be filtered out by the filter, won't be synced into the destination - filteredQuery := fmt.Sprintf(` - INSERT INTO %s ( - id_cursor, id, id_bigint, - id_int, id_int_unsigned, id_integer, id_integer_unsigned, - id_mediumint, id_mediumint_unsigned, id_smallint, id_smallint_unsigned, - id_tinyint, id_tinyint_unsigned, - id_tinyint_unsigned_max, id_smallint_unsigned_max, - id_mediumint_unsigned_max, id_mediumint_unsigned_signbit, id_int_unsigned_max, - price_decimal, amount_decimal_9_2, price_double, - price_double_precision, price_float, price_numeric, price_real, - name_char, name_varchar, name_text, name_tinytext, - name_mediumtext, name_longtext, created_date, - created_timestamp, is_active, long_varchar, name_bool, status, priority, - %s - tags, - excludedColumn - ) VALUES ( - -1, 998, 111111111111111, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, - 0, 0, - 0, 0, 0, - 500234.123, 500234.12, 500234.123, - 500234.123, 500234.0, 500234.123, 500234.123, - 'x', 'filtered_val', 'filtered text', 'filtered tiny', - 'filtered medium', 'filtered long', '2021-06-15 10:00:00', - '2021-06-15 10:00:00', 0, 'filtered long varchar', 0, 'inactive', 'low', - %s - 'music', - 200 - )`, tableName, seedCols, seedFilteredVals) - _, err := db.ExecContext(ctx, filteredQuery) + _, err := db.ExecContext(ctx, insertRowQuery(tableName, seedCols, true, map[string]string{ + "id": "998", + "excludedColumn": "200", + "price_decimal": "500234.123", + "amount_decimal_9_2": "500234.12", + "price_double": "500234.123", + "price_double_precision": "500234.123", + "price_float": "500234.0", + "price_numeric": "500234.123", + "price_real": "500234.123", + "created_date": "'2021-06-15 10:00:00'", + })) require.NoError(t, err, "Failed to insert filtered test data row") } diff --git a/tests/postgres/postgres_test.go b/tests/postgres/postgres_test.go index a7518aea3..0feec2156 100644 --- a/tests/postgres/postgres_test.go +++ b/tests/postgres/postgres_test.go @@ -1,6 +1,7 @@ package postgres import ( + "context" "testing" "github.com/datazip-inc/olake/tests/testutils" @@ -44,8 +45,19 @@ func postgresBaseConfig(t *testing.T, opts ...testutils.TestConfigOption) *integ } } +func createReplicationSlot(t *testing.T, cfg *testutils.TestConfig) { + t.Helper() + ctx := t.Context() + ExecuteQuery(ctx, t, cfg, "create-slot") + t.Cleanup(func() { + ExecuteQuery(context.WithoutCancel(ctx), t, cfg, "drop-slot") + }) +} + func TestPostgresDiscover(t *testing.T) { - postgresBaseConfig(t).TestDiscover(t) + cfg := postgresBaseConfig(t) + createReplicationSlot(t, cfg.TestConfig) + cfg.TestDiscover(t) } func TestPostgresSync(t *testing.T) { @@ -53,12 +65,15 @@ func TestPostgresSync(t *testing.T) { cfg := postgresBaseConfig(t) cfg.ExpectedUpdatedData = ExpectedUpdatedData cfg.UpdatedDestinationDataTypeSchema = UpdatedPostgresToDestinationSchema + createReplicationSlot(t, cfg.TestConfig) cfg.TestSync(t) } func TestPostgres2PC(t *testing.T) { t.Parallel() - postgresBaseConfig(t).Test2PCIntegration(t) + cfg := postgresBaseConfig(t) + createReplicationSlot(t, cfg.TestConfig) + cfg.Test2PCIntegration(t) } func TestPostgresPerformance(t *testing.T) { @@ -83,7 +98,9 @@ func TestPostgresCompatibility(t *testing.T) { // No column rules: postgres compares clean on every reachable baseline (COMPAT_RESULTS_v2.md). fixture := &compatibility.Test{ NewConfig: func(t *testing.T, version string) *testutils.TestConfig { - return postgresBaseConfig(t, testutils.WithDriverVersion(version)).TestConfig + cfg := postgresBaseConfig(t, testutils.WithDriverVersion(version)).TestConfig + createReplicationSlot(t, cfg) + return cfg }, DeclaredSchema: PostgresToDestinationSchema, CDCColumnsSchema: ExpectedPostgresDefaultCDCColumnsSchema, diff --git a/tests/postgres/postgres_util_test.go b/tests/postgres/postgres_util_test.go index 5f585d47c..03f3a7e90 100644 --- a/tests/postgres/postgres_util_test.go +++ b/tests/postgres/postgres_util_test.go @@ -3,7 +3,6 @@ package postgres import ( "context" "fmt" - "sync" "testing" "time" @@ -55,8 +54,6 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, switch operation { case "create": - ensureReplicationSlot(ctx, t, conf, replicationSlot) - query = fmt.Sprintf(` CREATE TABLE IF NOT EXISTS %s ( col_bigint BIGINT, @@ -510,30 +507,3 @@ var ExpectedPostgresDefaultCDCColumnsSchema = map[string]string{ "_cdc_timestamp": "timestamp", "_cdc_lsn": "string", } - -// ensureReplicationSlot creates the slot this suite's source config names, once. olake validates -// the CDC config on every command it runs, so the slot has to outlive the whole suite -- hence the -// once: "create" is called again by every subtest that resets the table, and a t.Cleanup registered -// there would drop the slot while the suite is still running. -func ensureReplicationSlot(ctx context.Context, t *testing.T, conf *testutils.TestConfig, slot string) { - t.Helper() - slotsEnsuredMu.Lock() - defer slotsEnsuredMu.Unlock() - if slotsEnsured[slot] { - return - } - slotsEnsured[slot] = true - - ExecuteQuery(ctx, t, conf, "create-slot") - t.Cleanup(func() { - slotsEnsuredMu.Lock() - delete(slotsEnsured, slot) - slotsEnsuredMu.Unlock() - ExecuteQuery(context.WithoutCancel(ctx), t, conf, "drop-slot") - }) -} - -var ( - slotsEnsuredMu sync.Mutex - slotsEnsured = map[string]bool{} -) diff --git a/tests/s3/s3_test.go b/tests/s3/s3_test.go index 18401d532..8e5629816 100644 --- a/tests/s3/s3_test.go +++ b/tests/s3/s3_test.go @@ -64,7 +64,6 @@ func TestS3Compatibility(t *testing.T) { t.Parallel() fixture := &compatibility.Test{ DeclaredSchema: variant.DestinationSchema, - ColumnTypes: variant.ColumnTypes(), } fixture.NewConfig = func(t *testing.T, version string) *testutils.TestConfig { return s3BaseConfig(t, variant, testutils.WithDriverVersion(version)).TestConfig diff --git a/tests/s3/s3_util_test.go b/tests/s3/s3_util_test.go index 0b4321dd9..5ffe5cc3b 100644 --- a/tests/s3/s3_util_test.go +++ b/tests/s3/s3_util_test.go @@ -12,6 +12,7 @@ import ( "math/big" "net/url" "os" + "slices" "strings" "testing" "time" @@ -781,17 +782,17 @@ func ExecuteQueryFactory(variant S3TestVariant, cfg *integration.Test) func(ctx case "add": // One plain file and, where the variant allows it, one gzipped: a single stream // mixing both proves compression is detected per file rather than per stream. - variant.putFile(ctx, t, src, prefix, "seed_1", variant.BuildFile, 1, seedValues, false, excluded) - variant.putFile(ctx, t, src, prefix, "seed_2", variant.BuildFile, 4, seedValues, variant.Gzipped, excluded) + variant.putFile(ctx, t, src, prefix, "seed_1", variant.BuildFile, 1, seedValues, false, false, excluded) + variant.putFile(ctx, t, src, prefix, "seed_2", variant.BuildFile, 4, seedValues, variant.Gzipped, false, excluded) case "insert": // A file the incremental cursor has not seen: it is stamped after the previous // sync, so only its rows are re-read. - variant.putFilePastCursor(ctx, t, src, prefix, "insert_1", variant.BuildFile, 7, seedValues, false, excluded) + variant.putFile(ctx, t, src, prefix, "insert_1", variant.BuildFile, 7, seedValues, false, true, excluded) case "update": // Object stores have no in-place update: changed data arrives as another file. - variant.putFilePastCursor(ctx, t, src, prefix, "update_1", variant.BuildFile, 10, updatedValues, false, excluded) + variant.putFile(ctx, t, src, prefix, "update_1", variant.BuildFile, 10, updatedValues, false, true, excluded) case "evolve-schema": // An object store's ALTER TABLE: a file whose rows carry a column discover has @@ -801,7 +802,7 @@ func ExecuteQueryFactory(variant S3TestVariant, cfg *integration.Test) func(ctx // ExpectedUpdatedData; evolvedColumn itself is asserted through the schema, not // per row, since the "update" file's rows sync a null there. if variant.BuildEvolvedFile != nil { - variant.putFilePastCursor(ctx, t, src, prefix, "evolve_1", variant.BuildEvolvedFile, 13, updatedValues, false, excluded) + variant.putFile(ctx, t, src, prefix, "evolve_1", variant.BuildEvolvedFile, 13, updatedValues, false, true, excluded) } default: @@ -813,7 +814,7 @@ func ExecuteQueryFactory(variant S3TestVariant, cfg *integration.Test) func(ctx // putFile renders one file with build and uploads it under prefix. Gzipped files get a // ".gz" suffix, which is what both the driver's file matcher and its reader use to detect // compression. -func (v S3TestVariant) putFile(ctx context.Context, t *testing.T, src s3Source, prefix, name string, build buildFileFn, startID int64, vals rowValues, gzipped bool, excluded []string) string { +func (v S3TestVariant) putFile(ctx context.Context, t *testing.T, src s3Source, prefix, name string, build buildFileFn, startID int64, vals rowValues, gzipped, wait bool, excluded []string) { t.Helper() data := build(t, startID, vals, excluded) @@ -824,38 +825,17 @@ func (v S3TestVariant) putFile(ctx context.Context, t *testing.T, src s3Source, } key := prefix + name + ext + + // TODO: the driver should handle same-second arrivals itself (`>=` plus tracking the file keys + // already synced at the cursor's second); this guard papers over real, silent data loss. + if wait { + t.Logf("waiting before putting file to avoid s3 driver skipping the new file...") + time.Sleep(1 * time.Second) + } + _, err := src.client.PutObject(ctx, src.bucket, key, bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}) require.NoError(t, err, "failed to upload %s", key) t.Logf("Uploaded s3://%s/%s (%d bytes)", src.bucket, key, len(data)) - return key -} - -// putFilePastCursor is putFile for a file the next incremental sync must pick up: the driver's -// cursor keeps LastModified at whole seconds with a strict >, so a file landing in the same second -// as the previous sync's newest object is silently skipped. Re-upload until the object's second is -// past every object already under the prefix. -// TODO: the driver should handle same-second arrivals itself (`>=` plus tracking the file keys -// already synced at the cursor's second); this guard papers over real, silent data loss. -func (v S3TestVariant) putFilePastCursor(ctx context.Context, t *testing.T, src s3Source, prefix, name string, build buildFileFn, startID int64, vals rowValues, gzipped bool, excluded []string) { - t.Helper() - - var prevMax time.Time - for obj := range src.client.ListObjects(ctx, src.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { - require.NoError(t, obj.Err, "failed to list objects under %s", prefix) - if obj.LastModified.After(prevMax) { - prevMax = obj.LastModified - } - } - prevSecond := prevMax.UTC().Truncate(time.Second) - for { - key := v.putFile(ctx, t, src, prefix, name, build, startID, vals, gzipped, excluded) - info, err := src.client.StatObject(ctx, src.bucket, key, minio.StatObjectOptions{}) - require.NoError(t, err, "failed to stat %s", key) - if info.LastModified.UTC().Truncate(time.Second).After(prevSecond) { - return - } - time.Sleep(200 * time.Millisecond) - } } // Sub-second timestamp layouts for the text variants. Fixed-width fractions (not the @@ -1138,18 +1118,22 @@ func parquetTestGroup() pq.Group { } } -// parquetExcludableColumns are the seed columns the fixture knows how to leave out, all of them -// #1020 (v0.9.1) fixes: the group-typed three panic a older parser on Kind(), and int96 is typed -// timestamptz by discover while the reader hands back a string, which the iceberg flush rejects. -var parquetExcludableColumns = []string{"map_col", "struct_col", "list_col", "int96_col"} - // parquetGroupExcluding is the seed's column group minus the columns this baseline cannot read. +// Any column the group declares can be left out, so a rule in compatibility_rules.json naming a new +// one needs nothing declared here. func parquetGroupExcluding(t *testing.T, excluded []string) pq.Group { t.Helper() - drop, err := testutils.SeedColumnsExcluded(excluded, parquetExcludableColumns) + group := parquetTestGroup() + + names := make([]string, 0, len(group)+1) + for column := range group { + names = append(names, column) + } + names = append(names, evolvedColumn) + + drop, err := testutils.SeedColumnsExcluded(excluded, names) require.NoError(t, err, "s3 parquet seed exclusion") - group := parquetTestGroup() for column := range drop { delete(group, column) } @@ -1262,7 +1246,9 @@ func buildEvolvedParquetFile(t *testing.T, startID int64, vals rowValues, exclud } group := parquetGroupExcluding(t, excluded) - group[evolvedColumn] = pq.String() + if !slices.Contains(excluded, evolvedColumn) { + group[evolvedColumn] = pq.String() + } return writeParquetRows(t, pq.NewSchema("s3_parquet_row", group), rows) } @@ -1297,62 +1283,3 @@ func gzipBytes(t *testing.T, data []byte) []byte { require.NoError(t, writer.Close(), "failed to close gzip writer") return buf.Bytes() } - -// ColumnTypes derives type tags for the variant's seed columns, which a data_types rule in -// compatibility_rules.json resolves against. Parquet is the one format with a declared schema, -// parquetTestGroup, so its tags are read off that; a text file's only source type is what the -// driver infers, which DeclaredSchema already carries. -func (v S3TestVariant) ColumnTypes() map[string][]string { - if v.DataFormat != "parquet" { - return nil - } - types := map[string][]string{} - for column, node := range parquetTestGroup() { - types[column] = parquetNodeTags(node) - } - return types -} - -// parquetNodeTags names a leaf by its physical kind and logical type, with the width, sign or time -// unit that tells one apart (uint32, timestamp(nanos)); a group by its shape. -func parquetNodeTags(node pq.Node) []string { - logical := node.Type().LogicalType() - if !node.Leaf() { - switch { - case logical != nil && logical.Map != nil: - return []string{"map"} - case logical != nil && logical.List != nil: - return []string{"list"} - } - return []string{"struct"} - } - tags := []string{strings.ToLower(node.Type().Kind().String())} - if logical == nil { - return tags - } - switch { - case logical.UTF8 != nil: - tags = append(tags, "string") - case logical.Enum != nil: - tags = append(tags, "enum") - case logical.Decimal != nil: - tags = append(tags, "decimal") - case logical.Date != nil: - tags = append(tags, "date") - case logical.Time != nil: - tags = append(tags, "time", "time("+strings.ToLower(logical.Time.Unit.String())+")") - case logical.Timestamp != nil: - tags = append(tags, "timestamp", "timestamp("+strings.ToLower(logical.Timestamp.Unit.String())+")") - case logical.Integer != nil: - sign := "" - if !logical.Integer.IsSigned { - sign = "u" - } - tags = append(tags, fmt.Sprintf("%sint%d", sign, logical.Integer.BitWidth)) - case logical.Json != nil: - tags = append(tags, "json") - case logical.UUID != nil: - tags = append(tags, "uuid") - } - return tags -} diff --git a/tests/testutils/compatibility/compatibility.go b/tests/testutils/compatibility/compatibility.go index 24b1b6d89..064d3d965 100644 --- a/tests/testutils/compatibility/compatibility.go +++ b/tests/testutils/compatibility/compatibility.go @@ -2,9 +2,8 @@ package compatibility // Backward-compatibility suite. // -// The contract being tested is docs/backward-compatibility.md: upgrading the OLake binary must not -// change the records or the column types an existing pipeline produces. The state file's `version` -// pins that, so a candidate binary reading a state file an older binary wrote must keep the older +// Upgrading the OLake binary must not change the records or the column types an existing pipeline produces. +// The state file's `version` pins that, so a candidate binary reading a state file an older binary wrote must keep the older // binary's semantics. // // Rather than encode per-version expectations -- which rot, and which nobody remembers to add when @@ -17,11 +16,6 @@ package compatibility // and then asserts the two destinations are indistinguishable. The reference run IS the // expectation. A gate that stopped firing, a type map that shifted, a state key that got renamed: // each shows up as a diff between two tables, with no expectation file to maintain. -// -// What this does NOT cover, deliberately: discover output (both runs are seeded from the same -// frozen test_streams.json, so they differ only in the binary -- and discover is ungated by design, -// see A4 in the doc); the reverse direction (a new state file fed to an old image is not a -// supported operation); and any gate older than the baseline being tested. import ( "context" @@ -46,32 +40,14 @@ const ( // per-driver overrides use the suffixed form, OLAKE_COMPATIBILITY_BASELINE_POSTGRES. compatibilityBaselineEnvVar = "OLAKE_COMPATIBILITY_TEST_BASELINE" - // compatibilityExcludeColumnsEnvVar appends catalog-level column exclusions to every compatibility run, a - // sweep affordance for probing a baseline without editing the driver's rules. + // compatibilityExcludeColumnsEnvVar appends catalog-level column exclusions to every compatibility run compatibilityExcludeColumnsEnvVar = "OLAKE_COMPATIBILITY_EXCLUDE_COLUMNS" ) -// Test is the basic compatibility check: one scenario run on two images, whose destinations must -// be indistinguishable. A driver declares one -- NewConfig plus its two-image vocabulary -- and -// the runner fills Reference and Upgrade per variant: the reference config runs the baseline end -// to end, the upgrade config writes its stateless load on the baseline and every stateful sync -// after it on the candidate. type Test struct { - // NewConfig builds one side's TestConfig from the subtest it runs in; the suite derived from - // t.Name() is what isolates the sides ((baseline x group x variant x side)). - NewConfig func(t *testing.T, DriverVersion string) *testutils.TestConfig - - // DeclaredSchema is the driver's column -> destination-type map, what a data_types rule in - // compatibility_rules.json resolves against. The sync suite asserts the same map, so it cannot - // drift from the fixture. - DeclaredSchema map[string]string - - // ColumnTypes tags columns with what DeclaredSchema cannot express (a charset, a modifier), - // derived by the fixture from its own seed DDL; a data_types rule selects on both. - ColumnTypes map[string][]string - - // CDCColumnsSchema names the driver's CDC metadata columns, which carry source-log coordinates - // and so are compared by type but never by value (see volatileColumns). + NewConfig func(t *testing.T, DriverVersion string) *testutils.TestConfig + DeclaredSchema map[string]string + ColumnTypes map[string][]string CDCColumnsSchema map[string]string } @@ -86,25 +62,12 @@ func (f *Test) Validate(t *testing.T) { // image and an upgrade run that hands off to the candidate after the initial load -- then asserts // the two destinations match. Both sides of all three writer groups (iceberg legacy, iceberg // arrow, parquet) run in parallel, six isolated pipelines at once. -// -// newConfig MUST build a fresh Test from the t it is handed: the suite -- and so every path and -// name the side owns -- derives from that subtest's name. -// RunBackwardCompatibility runs the compatibility scenarios against every baseline the manifest lists, -// oldest first, stopping at the first that fails -- later baselines are newer code and would only -// repeat it. A single explicit baseline runs on its own, without the extra subtest level. func (f *Test) RunBackwardCompatibility(t *testing.T) { - f.Validate(t) - currentConf := f.NewConfig(t, testutils.CurrentDriverVersion) baselineVersions, err := getCompatibilityBaselines(t, currentConf.OlakeRootPath, currentConf.Driver) require.NoError(t, err) for _, version := range baselineVersions { - // Before NewConfig, which resolves the baseline's image: a driver younger than a release has - // no image published for it, so building the config first turns a declared skip into - // "failed to pull olakego/source-:" -- a hard failure the gate exists to - // prevent. Only the driver-level gate can be answered here; the variant gate keys on the - // config's data format and stays in runCompatibilityBaseline. reason, err := baselineSkipReason(currentConf.OlakeRootPath, currentConf.Driver, version) require.NoError(t, err) if reason != "" { @@ -151,11 +114,6 @@ func (f *Test) runCompatibilityBaseline(t *testing.T, baseline, upgrade *testuti spec := baseline.DriverVersion driver, dataFormat := baseline.Driver, baseline.DataFormat - // The variant's own floor. A skip, not a failure: the driver declares this data format cannot - // run against releases this old (the why lives next to the declaration in - // compatibility_rules.json), and that limitation is data, not a regression. The driver-level - // gate and the global floor were already answered by the caller, before this baseline's image - // was resolved -- see baselineSkipReason. baselineVersion, baselineDated := parseReleaseTag(spec) floorTag, err := compatibilityGlobalFloor(baseline.OlakeRootPath) require.NoError(t, err) @@ -209,22 +167,20 @@ func (f *Test) runCompatibilityBaseline(t *testing.T, baseline, upgrade *testuti // Each variant runs as its own pair of parallel subtests -- reference entirely on the // baseline, upgrade handing off to the candidate after the stateless load -- and is compared - // as soon as both sides finish. Every side builds its config inside its own subtest, so the - // suite t.Name() derives is what isolates the pipelines: (baseline x group x variant x side). + // as soon as both sides finish referencePick := func(bool) string { return baseline.DriverVersion } upgradePick := func(useState bool) string { - // useState is the upgrade boundary: the stateless initial load writes the state file on - // the baseline binary, and every sync after it reads that file on the candidate. return testutils.Ternary(useState, upgrade.DriverVersion, baseline.DriverVersion).(string) } // Whichever side fails first stops every group at its next variant boundary: the comparison // is skipped either way, so the remaining syncs would be minutes of output nothing reads. + aborted := &atomic.Bool{} + // What every failed variant found, so the assertion at the end of this function -- the one CI // shows in red -- can report the findings themselves rather than the fact that there were some. report := &failureReport{driver: driver, spec: spec, baseline: baselineImage, candidate: candidateImage} - // Subtest names double as suite segments, so they stay terse: the table and (postgres) slot - // names built from the suite must clear a 63-byte identifier limit on sweep runs. + completed := t.Run("g", func(t *testing.T) { for _, g := range groups { runGroup := func(t *testing.T) { @@ -361,11 +317,6 @@ func compareVariant(t *testing.T, diag *diagnostics, policies *assertionPolicies case "parquet": refRel = parquetRelation(ctx, t, spark, refDB, refTable, "ref") upgRel = parquetRelation(ctx, t, spark, upgDB, upgTable, "upg") - // Absence is a comparable state, so it is asserted rather than skipped over. One side - // absent is a genuine finding: the binaries disagree about whether this case writes - // output. Both sides absent is the verified outcome for a variant that ENDS empty - // (emptyFinalState), and a shared failure to produce rows for any other -- the one shape - // of regression a row diff can never catch, because there are no rows to diff. if refRel == "" || upgRel == "" { if (refRel == "") != (upgRel == "") { diag.fatalf(t, "only one run produced parquet files for %s (reference %q, upgrade %q): the binaries disagree about whether this case writes output", v.name, refDB, upgDB) diff --git a/tests/testutils/compatibility/compatibility_rules.json b/tests/testutils/compatibility/compatibility_rules.json index f2efc5125..688988904 100644 --- a/tests/testutils/compatibility/compatibility_rules.json +++ b/tests/testutils/compatibility/compatibility_rules.json @@ -1,19 +1,4 @@ { - "destinations": { - "rules": [ - {"column": "_olake_timestamp", "type_only": true, "note": "olake's write stamp: wall-clock, never value-comparable across two runs"} - ], - "iceberg": { - "arrow": { - "min_baseline": "v0.3.17", - "note": "P1: v0.3.6 through v0.3.16 arrow integer widths disagree with today's; below v0.3.6 no arrow writer exists" - } - }, - "parquet": { - "skip_baselines": ["v0.3.16"], - "note": "P2: v0.3.16 adds the `data` column unconditionally, fixed in v0.3.17; below that the no-CDC drivers' output lacks _cdc_timestamp" - } - }, "drivers": { "postgres": { "destination_rules": [ @@ -27,28 +12,28 @@ } }, "mysql": { - "destination_rules": [ - {"column": "_cdc_timestamp", "type_only": true}, - {"column": "_cdc_binlog_file_name", "type_only": true}, - {"column": "_cdc_binlog_file_pos", "type_only": true} - ], "rules": [ {"data_types": ["ucs2", "utf16le", "latin1"], "exclude_below": "v0.7.2", "note": "non-UTF-8 charset bytes reach the writer as invalid UTF-8; gRPC marshal fails, retry backoff looks like a hang"}, {"data_types": ["set"], "assert_value_from": "v0.7.2", "note": "M1: SET columns emitted the numeric bitmask on the binlog path before the fix"}, {"data_types": ["unsigned mediumint"], "assert_value_from": "v0.9.3", "note": "unsigned MEDIUMINT was sign-extended on the binlog path before v0.9.3 (-1 where the value is 16777215)"}, {"data_types": ["unsigned bigint"], "exclude_below": "v0.9.0", "note": "BIGINT UNSIGNED at or above 2^63 reaches the incremental scan as a byte slice; before v0.9.0 (state v6, ReformatInt64) it cannot be converted: v0.3.11's Iceberg writer fatals on the mixed long/string batch and a candidate on an older state file retries into a hang"} ], + "destination_rules": [ + {"column": "_cdc_timestamp", "type_only": true}, + {"column": "_cdc_binlog_file_name", "type_only": true}, + {"column": "_cdc_binlog_file_pos", "type_only": true} + ], "note": "M2 (ENUM serialization, fixed v0.3.9) and M3 (DECIMAL/NUMERIC via float32, fixed v0.3.7) need no rule: both thresholds sit below v0.3.11, the oldest baseline any sweep reaches" }, "mongodb": { + "rules": [ + {"data_types": ["regex"], "assert_value_from": "v0.3.14", "note": "G1: BSON regex serialized with Go field names, not lowercase keys, until the fix landed"} + ], "destination_rules": [ {"column": "_cdc_timestamp", "type_only": true}, {"column": "_cdc_resume_token", "type_only": true}, {"column": "_id", "type_only": true, "note": "server-generated ObjectID; cannot match across two independent runs"}, {"column": "_olake_id", "type_only": true, "note": "hashes the server-generated _id, so it is as non-deterministic as its source"} - ], - "rules": [ - {"data_types": ["regex"], "assert_value_from": "v0.3.14", "note": "G1: BSON regex serialized with Go field names, not lowercase keys, until the fix landed"} ] }, "mssql": { @@ -76,12 +61,13 @@ "db2": { "min_baseline": "v0.3.14", "note": "first release carrying the driver; refine after the first sweep if early db2 images are missing from the registry", - "destinations": { - "parquet": {"min_baseline": "v0.3.17", "note": "P2 for a no-CDC driver: below v0.3.17 the parquet output lacks _cdc_timestamp"} - }, "rules": [ {"data_types": ["decfloat"], "assert_value_from": "v0.7.6", "note": "ReformatValue rendered a float into a String column with %d"} - ] + ], + "destinations": { + "parquet": {"min_baseline": "v0.3.17", "note": "P2 for a no-CDC driver: below v0.3.17 the parquet output lacks _cdc_timestamp"} + } + }, "s3": { "min_baseline": "v0.9.2", @@ -93,9 +79,31 @@ "variants": { "csv": {}, "json": {}, - "parquet": {}, - "xml": {"min_baseline": "v0.9.5", "note": "the XML source format landed on 2026-08-07, after v0.9.4; older binaries reject file_format xml at config validation"} + "parquet": { + "rules": [ + {"column": "map_col", "exclude_below": "v0.9.1", "note": "a group-typed column panics the parser on Kind() before v0.9.1"}, + {"column": "struct_col", "exclude_below": "v0.9.1", "note": "as map_col"}, + {"column": "list_col", "exclude_below": "v0.9.1", "note": "as map_col"}, + {"column": "int96_col", "exclude_below": "v0.9.1", "note": "discover types INT96 timestamptz while the reader hands back a string, which the iceberg flush rejects"} + ] + }, + "xml": {"min_baseline": "v0.9.3", "note": "the XML source format landed in v0.9.3"} } } + }, + "destinations": { + "rules": [ + {"column": "_olake_timestamp", "type_only": true, "note": "olake's write stamp: wall-clock, never value-comparable across two runs"} + ], + "iceberg": { + "arrow": { + "min_baseline": "v0.3.17", + "note": "P1: v0.3.6 through v0.3.16 arrow integer widths disagree with today's; below v0.3.6 no arrow writer exists" + } + }, + "parquet": { + "skip_baselines": ["v0.3.16"], + "note": "P2: v0.3.16 adds the `data` column unconditionally, fixed in v0.3.17; below that the no-CDC drivers' output lacks _cdc_timestamp" + } } } diff --git a/tests/testutils/compatibility/scenarios.go b/tests/testutils/compatibility/scenarios.go index efeaef2e4..1a9c7ab0e 100644 --- a/tests/testutils/compatibility/scenarios.go +++ b/tests/testutils/compatibility/scenarios.go @@ -109,12 +109,6 @@ func runSide( // scenarios themselves never clear, so the candidate binary meets the table the baseline made. clearDestination(t, g, cfg.DestinationDB, table) - // The slot lives as long as the source config that names it, and olake validates the CDC - // configuration at startup for every sync -- the incremental ones included; only postgres needs it. - if cfg.Driver == string(constants.Postgres) { - cfg.ExecuteQuery(ctx, t, cfg, "create-slot") - defer cfg.ExecuteQuery(ctx, t, cfg, "drop-slot") - } if testutils.KeepTestData() { t.Logf("compatibility side %q: leaving source table %s in place (%s is set); it holds the last case's data", cfg.Suite, table, testutils.KeepTestDataEnvVar) @@ -126,14 +120,6 @@ func runSide( cfg.ExecuteQuery(ctx, t, cfg, "drop") cfg.ExecuteQuery(ctx, t, cfg, "create") cfg.ExecuteQuery(ctx, t, cfg, "add") - if cfg.Driver == string(constants.DB2) { - cfg.ExecuteQuery(ctx, t, cfg, "populate-stats") - } - // The seed rows sit in the CDC log, and before #843 the mssql driver captured its initial LSN - // without waiting for the async capture agent -- wait here so every binary snapshots past the seed. - if v.kind == scenarioCDC && cfg.Driver == string(constants.MSSQL) { - cfg.ExecuteQuery(ctx, t, cfg, "wait-cdc-catchup") - } if v.kind == scenarioIncremental { require.NoError(t, testutils.ResetStateFile(cfg), "failed to reset state for incremental") } @@ -144,9 +130,6 @@ func runSide( } if c.useState && c.operation != "" { cfg.ExecuteQuery(ctx, t, cfg, c.operation) - if v.kind == scenarioCDC && cfg.Driver == string(constants.MSSQL) { - cfg.ExecuteQuery(ctx, t, cfg, "wait-cdc-catchup") - } } // Successive syncs write the same parquet column with different types, which Spark refuses // to read together (CANNOT_MERGE_SCHEMAS; F2 in docs/backward-compatibility.md) -- so a diff --git a/tests/testutils/ddl.go b/tests/testutils/ddl.go index 3a9e9920d..6a6717f64 100644 --- a/tests/testutils/ddl.go +++ b/tests/testutils/ddl.go @@ -7,28 +7,18 @@ import ( var ddlCharset = regexp.MustCompile(`(?i)CHARACTER SET (\w+)`) -// DDLColumnTypes reads column type tags off a CREATE TABLE column list -- the base type, its -// unsigned form and charset where the dialect has them -- so a fixture declares nothing by hand. -func DDLColumnTypes(ddl string) map[string][]string { - types := map[string][]string{} - for _, line := range strings.Split(ddl, "\n") { - fields := strings.Fields(strings.TrimSuffix(strings.TrimSpace(line), ",")) - if len(fields) < 2 { - continue - } - switch strings.ToUpper(fields[0]) { - case "PRIMARY", "KEY", "UNIQUE", "INDEX", "CONSTRAINT": - continue - } - typ, _, _ := strings.Cut(strings.ToLower(fields[1]), "(") - tags := []string{typ} - if strings.Contains(strings.ToUpper(line), " UNSIGNED") { - tags = append(tags, "unsigned "+typ) - } - if m := ddlCharset.FindStringSubmatch(line); m != nil { - tags = append(tags, strings.ToLower(m[1])) - } - types[fields[0]] = tags +func DataTypeTags(datatype string) []string { + fields := strings.Fields(datatype) + if len(fields) == 0 { + return nil } - return types + typ, _, _ := strings.Cut(strings.ToLower(fields[0]), "(") + tags := []string{typ} + if strings.Contains(strings.ToUpper(datatype), " UNSIGNED") { + tags = append(tags, "unsigned "+typ) + } + if m := ddlCharset.FindStringSubmatch(datatype); m != nil { + tags = append(tags, strings.ToLower(m[1])) + } + return tags } diff --git a/tests/testutils/docker.go b/tests/testutils/docker.go index fe1c40387..d43f8577d 100644 --- a/tests/testutils/docker.go +++ b/tests/testutils/docker.go @@ -71,14 +71,7 @@ func buildDriverImage(t *testing.T, cfg *TestConfig) error { }) } -// buildBaselineFromCommit builds a driver image from a detached worktree at sha. This is a -// debugging affordance for bisecting a break, not the supported path -- released tags need no -// worktree, no maven and no old-toolchain build, and they ship the exact artifact users run. -// -// Two things the old tree needs that the released path does not: its OWN Iceberg writer jar (the -// Dockerfile copies the jar out of the build context, and the old Go side speaks the old jar's -// RPC), and a build entry point that exists in that tree -- `make docker..build IMAGE_TAG=...` -// is recent, so fall back to a plain `docker build`, whose DRIVER_NAME build-arg is far older. +// buildBaselineFromCommit builds a driver image from a detached worktree at sha. func buildImageFromCommit(t *testing.T, cfg *TestConfig, commitID string) error { t.Helper() imageTag := cfg.GetDriverImage() @@ -139,11 +132,7 @@ func ensureImagePresent(t *testing.T, image string) error { } // DockerRunArgs builds the `docker run` argument list that invokes the driver image exactly -// as a user would: the image's ENTRYPOINT (./olake) runs with olakeArgs appended. The -// driver's testdata directory is mounted at /testdata so the config/catalog/state files are -// shared with the host and the CLI writes its outputs (streams.json, state.json, ...) back -// there. extraFlags carries per-invocation docker flags (host gateway, network, name); image is -// explicit rather than derived so one suite can hand successive syncs to different images. +// as a user would: the image's ENTRYPOINT (./olake) runs with olakeArgs appended. func DockerRunArgs(cfg *TestConfig, extraFlags []string, olakeArgs []string) []string { args := []string{ "run", "--rm", @@ -165,9 +154,6 @@ func generateUniqueContainerName(cfg *TestConfig) string { return fmt.Sprintf("olake-it-%s-%s-%d-%d", cfg.Driver, cfg.Suite, os.Getpid(), containerSeq.Add(1)) } -// RunOlake runs the driver image once, exactly like a real user would: -// -// docker run --rm -v :/testdata olakego/source-:local func RunOlake(ctx context.Context, cfg *TestConfig, olakeArgs ...string) (int, []byte, error) { name := generateUniqueContainerName(cfg) args := DockerRunArgs(cfg, []string{"--add-host", "host.docker.internal:host-gateway", "--name", name}, olakeArgs) @@ -187,10 +173,7 @@ func RunOlake(ctx context.Context, cfg *TestConfig, olakeArgs ...string) (int, [ return DockerExitResult(out, err, olakeArgs[0]) } -// logContainerTimings re-emits the `[timing]` lines the driver wrote inside the container. A -// successful `docker run`'s output is otherwise dropped on the floor, so without this the -// in-container breakdown is invisible and every sync reads as one opaque span. The leading -// log prefix is trimmed so the forwarded lines line up with the harness's own. +// logContainerTimings re-emits the `[timing]` lines the driver wrote inside the container. func ContainerTimings(out []byte) []string { var timings []string for _, line := range strings.Split(string(out), "\n") { diff --git a/tests/testutils/integration/integration.go b/tests/testutils/integration/integration.go index 45eb0bf23..6ebe9086a 100644 --- a/tests/testutils/integration/integration.go +++ b/tests/testutils/integration/integration.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/datazip-inc/olake/tests/testutils" - "github.com/datazip-inc/olake/tests/testutils/constants" ) const ( @@ -25,10 +24,6 @@ type Test struct { DestinationDataTypeSchema map[string]string UpdatedDestinationDataTypeSchema map[string]string DefaultCDCColumnsSchema map[string]string - - // The fields below exist for the backward-compatibility suite (compatibility.go) and are zero for - // every other suite, which keeps their behavior identical to before they existed. - } // reset table and add back data to the table @@ -36,10 +31,6 @@ func (cfg *Test) resetTable(ctx context.Context, t *testing.T) error { cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "drop") cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "create") cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "add") - if cfg.TestConfig.Driver == string(constants.DB2) { - // to populate stats for DB2 - cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "populate-stats") - } return nil } @@ -60,13 +51,6 @@ func (cfg *Test) runSyncAndVerify( // Execute operation before sync if needed if useState && operation != "" { cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, operation) - // SQL Server CDC is asynchronous: the capture job only picks up the DML above on its next - // transaction-log scan, and the sync's change window ends at the job's processed max LSN - // (sys.fn_cdc_get_max_lsn), so syncing too early would see no changes. Wait for the capture - // job to advance past the DML. Incremental runs read the table directly and need no wait. - if isCDC && cfg.TestConfig.Driver == "mssql" { - cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "wait-cdc-catchup") - } } // Run sync against the driver image diff --git a/tests/testutils/integration/sync.go b/tests/testutils/integration/sync.go index f9a808be0..7970c5c30 100644 --- a/tests/testutils/integration/sync.go +++ b/tests/testutils/integration/sync.go @@ -103,14 +103,6 @@ func (cfg *Test) IcebergFullLoadAndCDC( return fmt.Errorf("failed to reset table: %w", err) } - // The seed rows sit in the CDC log, and before #843 (v0.5.1) the mssql driver captured its - // initial LSN without waiting for the async capture agent -- a lagging agent puts that LSN - // before the seed, and the first stateful sync replays the seed rows as CDC inserts - // (relabeling r to c through the upsert). Wait here so every binary snapshots past the seed. - if cfg.TestConfig.Driver == "mssql" { - cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "wait-cdc-catchup") - } - dbTestCases := []syncTestCase{ { name: "Full-Refresh", @@ -220,14 +212,6 @@ func (cfg *Test) ParquetFullLoadAndCDC( return fmt.Errorf("failed to reset parquet table: %s", err) } - // The seed rows sit in the CDC log, and before #843 (v0.5.1) the mssql driver captured its - // initial LSN without waiting for the async capture agent -- a lagging agent puts that LSN - // before the seed, and the first stateful sync replays the seed rows as CDC inserts - // (relabeling r to c through the upsert). Wait here so every binary snapshots past the seed. - if cfg.TestConfig.Driver == "mssql" { - cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "wait-cdc-catchup") - } - dbTestCases := []syncTestCase{ { name: "Full-Refresh", diff --git a/tests/testutils/state_version.go b/tests/testutils/state_version.go index 7923c2be5..c6bff41ca 100644 --- a/tests/testutils/state_version.go +++ b/tests/testutils/state_version.go @@ -9,13 +9,6 @@ import ( "sync" ) -// The product's constants/state-versions.json is the single source of truth for state-file -// semantics: the version the build writes today, and the release history behind every bump. The -// harness holds no copy of its own -- go:embed cannot reach the file from another module, so it is -// read at runtime, once per process. - -// StateVersionBaseline is one entry of the manifest's release history: the release that introduced -// a state version, which drivers it gated, and why. The compatibility suite sweeps these tags. type StateVersionBaseline struct { StateVersion int `json:"state_version"` ReleaseTag string `json:"release_tag"` diff --git a/tests/testutils/test_utils.go b/tests/testutils/test_utils.go index 4e541ed62..47408d04b 100644 --- a/tests/testutils/test_utils.go +++ b/tests/testutils/test_utils.go @@ -491,14 +491,6 @@ func UpdateSelectedStreams(config *TestConfig, namespace, partitionRegex, filter // ResetStateFile clears state.json so incremental can perform its initial load // (equivalent to a full load on first run), irrespective of any previous CDC run. -// -// Every call site must keep this BEFORE a stateless (useState=false) sync, which is where they -// all sit today. The version written here is the product's current one (ProductStateVersion), and -// the stateless load that follows overwrites the file with whatever version the binary that ran -// it stamps (protocol/root.go writes state next to --config even with no --state flag). -// The compatibility suite depends on that overwrite: it is how a baseline image's own state version ends -// up pinning the candidate's syncs. Call this after a compatibility run's initial load instead and the -// pipeline is silently promoted to latest semantics -- the suite would pass while testing nothing. func ResetStateFile(config *TestConfig) error { version, err := ProductStateVersion(config.OlakeRootPath) if err != nil { @@ -543,7 +535,3 @@ func RenderOlakeFailure(code int, err error, out []byte) error { func KeepTestData() bool { return strings.EqualFold(os.Getenv(KeepTestDataEnvVar), "true") } - -// TestDiscover seeds the source with this driver's test table, runs discover against the driver -// image and asserts the catalog it writes matches the one rendered from streams.template.json exactly. -// From cdf3f0f5e16dec2b11400665f5da397f490ba677 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Tue, 1 Sep 2026 12:57:20 +0530 Subject: [PATCH 15/20] chore: tests UTs --- .github/workflows/test-preflight.yml | 11 +++- tests/postgres/postgres_util_test.go | 40 +++++++++++++- tests/testutils/utils.go | 66 +++++++++++++++++------- tests/testutils/utils_normalized_test.go | 33 ++++++++++++ 4 files changed, 127 insertions(+), 23 deletions(-) create mode 100644 tests/testutils/utils_normalized_test.go diff --git a/.github/workflows/test-preflight.yml b/.github/workflows/test-preflight.yml index 82c744525..90e12f56d 100644 --- a/.github/workflows/test-preflight.yml +++ b/.github/workflows/test-preflight.yml @@ -229,6 +229,13 @@ jobs: timeout-minutes: 15 run: make test.lint + - name: Unit tests for the test harness + id: harness-unit + background: true + timeout-minutes: 10 + working-directory: tests + run: go test -v -count=1 ./testutils/... + - name: install gosec if: inputs.go-checks env: @@ -240,5 +247,5 @@ jobs: working-directory: tests run: $(go env GOPATH)/bin/gosec -exclude=G115 -tests -severity=high -confidence=medium ./... - - name: Wait for the lint - wait: [lint] + - name: Wait for the lint and harness unit tests + wait: [lint, harness-unit] diff --git a/tests/postgres/postgres_util_test.go b/tests/postgres/postgres_util_test.go index 03f3a7e90..9cace3110 100644 --- a/tests/postgres/postgres_util_test.go +++ b/tests/postgres/postgres_util_test.go @@ -3,6 +3,7 @@ package postgres import ( "context" "fmt" + "strings" "testing" "time" @@ -314,7 +315,44 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, } _, err := db.ExecContext(ctx, query) - require.NoError(t, err, "Failed to execute %s operation", operation) + // TEMPORARY: catalogContext is debug scaffolding -- see its doc comment for how to revert. + require.NoError(t, err, "Failed to execute %s operation on %s%s", operation, integrationTestTable, catalogContext(ctx, db, integrationTestTable)) +} + +// TEMPORARY -- REMOVE ONCE THE CONCURRENT-CREATE FAILURE IS DIAGNOSED. +// +// This exists only to identify the object behind an intermittent compatibility-suite failure +// (`duplicate key value violates unique constraint "pg_class_relname_nsp_index"` on create, seen +// on CI runners and locally). It adds a catalog round-trip to every failing query in this driver +// and has no place in the suite once the cause is known. Delete this function and restore the +// call site to: +// +// require.NoError(t, err, "Failed to execute %s operation", operation) +// +// catalogContext describes what the catalog already holds under this suite's table prefix. +// +// Postgres reports a concurrent-DDL conflict as `duplicate key value violates unique constraint +// "pg_class_relname_nsp_index"` and names no relation, which leaves the two candidate causes +// indistinguishable: two suites deriving the same name, or two suites racing on names that only +// collide after the server truncates them to 63 bytes. Both are visible by comparing the name +// this suite wanted against the ones already present, so report exactly that. Returns "" when +// the query fails, so a diagnostic can never mask the error it is describing. +func catalogContext(ctx context.Context, db *sqlx.DB, table string) string { + var existing []string + if err := db.SelectContext(ctx, &existing, + `SELECT relname FROM pg_class WHERE relname LIKE 'test_table_olake%' ORDER BY relname`); err != nil { + return "" + } + + note := fmt.Sprintf("\n wanted: %s (%d bytes; postgres truncates relation names at 63)", table, len(table)) + if len(table) > 63 { + note += fmt.Sprintf("\n TRUNCATED: %s <- every suite whose name shares this prefix collides here", table[:63]) + } + if len(existing) == 0 { + return note + "\n catalog: no test_table_olake* relations present" + } + return note + fmt.Sprintf("\n catalog: %d test_table_olake* relation(s) present:\n %s", + len(existing), strings.Join(existing, "\n ")) } // insertTestData inserts test data into the specified table diff --git a/tests/testutils/utils.go b/tests/testutils/utils.go index da93a4638..06ff9ae63 100644 --- a/tests/testutils/utils.go +++ b/tests/testutils/utils.go @@ -69,38 +69,64 @@ func FileLoggerWithPath(content any, path string) error { return nil } -// NormalizedEqual compares two JSON documents ignoring whitespace and ordering. +// NormalizedEqual reports whether two JSON documents are structurally equal, ignoring +// whitespace, object key order and array order. func NormalizedEqual(strune1, strune2 string) bool { - normalize := func(s string) (string, error) { + decode := func(s string) (interface{}, bool) { + var doc interface{} + if json.Unmarshal([]byte(s), &doc) == nil { + return canonicalJSON(doc), true + } start := strings.IndexRune(s, '{') end := strings.LastIndex(s, "}") if start < 0 || end < 0 || start > end { - return "", fmt.Errorf("no valid JSON object found") + return nil, false + } + if json.Unmarshal([]byte(s[start:end+1]), &doc) != nil { + return nil, false } - core := s[start : end+1] - core = strings.ReplaceAll(core, " ", "") - core = strings.ReplaceAll(core, "\n", "") - core = strings.ReplaceAll(core, "\t", "") - return core, nil + return canonicalJSON(doc), true } - c1, err := normalize(strune1) - if err != nil { + d1, ok1 := decode(strune1) + d2, ok2 := decode(strune2) + if !ok1 || !ok2 { return false } - c2, err := normalize(strune2) - if err != nil { - return false + return jsonCanonicalString(d1) == jsonCanonicalString(d2) +} + +// canonicalJSON rewrites a decoded document so that array order carries no meaning: every array +// is sorted by its own serialization. Object key order is already canonical because encoding/json +// marshals map keys sorted. +func canonicalJSON(v interface{}) interface{} { + switch v := v.(type) { + case map[string]interface{}: + out := make(map[string]interface{}, len(v)) + for key, val := range v { + out[key] = canonicalJSON(val) + } + return out + case []interface{}: + out := make([]interface{}, 0, len(v)) + for _, elem := range v { + out = append(out, canonicalJSON(elem)) + } + sort.Slice(out, func(i, j int) bool { + return jsonCanonicalString(out[i]) < jsonCanonicalString(out[j]) + }) + return out + default: + return v } +} - rune1 := []rune(c1) - rune2 := []rune(c2) - if len(rune1) != len(rune2) { - return false +func jsonCanonicalString(v interface{}) string { + encoded, err := json.Marshal(v) + if err != nil { + return fmt.Sprint(v) } - sort.Slice(rune1, func(i, j int) bool { return rune1[i] < rune1[j] }) - sort.Slice(rune2, func(i, j int) bool { return rune2[i] < rune2[j] }) - return string(rune1) == string(rune2) + return string(encoded) } // Reformat lowercases key and replaces every non-alphanumeric symbol with '_', matching how diff --git a/tests/testutils/utils_normalized_test.go b/tests/testutils/utils_normalized_test.go new file mode 100644 index 000000000..8689975b3 --- /dev/null +++ b/tests/testutils/utils_normalized_test.go @@ -0,0 +1,33 @@ +package testutils + +import "testing" + +// The cases that must still compare equal are the ones the helper exists for: the catalog is +// marshaled from maps and sync.Map ranges, so key order and array order are both arbitrary. +// The cases that must differ are the ones a rune-sorting comparison used to accept. +func TestNormalizedEqual(t *testing.T) { + const catalog = `{"stream":{"name":"users","namespace":"public","type_schema":{"properties":{"col_int":{"type":["integer","null"]},"col_text":{"type":["string","null"]}}}}}` + + for _, tc := range []struct { + name string + other string + equal bool + }{ + {"identical", catalog, true}, + {"whitespace and indentation", "{\n \"stream\": {\n \"name\": \"users\",\n \"namespace\": \"public\",\n \"type_schema\": {\"properties\": {\"col_int\": {\"type\": [\"integer\", \"null\"]}, \"col_text\": {\"type\": [\"string\", \"null\"]}}}\n }\n}", true}, + {"object key order", `{"stream":{"namespace":"public","type_schema":{"properties":{"col_text":{"type":["string","null"]},"col_int":{"type":["integer","null"]}}},"name":"users"}}`, true}, + {"array order", `{"stream":{"name":"users","namespace":"public","type_schema":{"properties":{"col_int":{"type":["null","integer"]},"col_text":{"type":["null","string"]}}}}}`, true}, + + {"column names reversed", `{"stream":{"name":"users","namespace":"public","type_schema":{"properties":{"tni_loc":{"type":["integer","null"]},"txet_loc":{"type":["string","null"]}}}}}`, false}, + {"two columns swap types", `{"stream":{"name":"users","namespace":"public","type_schema":{"properties":{"col_int":{"type":["string","null"]},"col_text":{"type":["integer","null"]}}}}}`, false}, + {"name and namespace swapped", `{"stream":{"name":"public","namespace":"users","type_schema":{"properties":{"col_int":{"type":["integer","null"]},"col_text":{"type":["string","null"]}}}}}`, false}, + {"a column dropped", `{"stream":{"name":"users","namespace":"public","type_schema":{"properties":{"col_int":{"type":["integer","null"]}}}}}`, false}, + {"not json", `col_int`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := NormalizedEqual(catalog, tc.other); got != tc.equal { + t.Fatalf("NormalizedEqual = %v, want %v", got, tc.equal) + } + }) + } +} From d3843efaf6d30ee61fc33048916de92d8d1a4d64 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Tue, 1 Sep 2026 14:24:48 +0530 Subject: [PATCH 16/20] chore: minor changes --- .github/CODEOWNERS | 10 ++-------- .github/actions/driver-image/action.yml | 10 +--------- .github/workflows/tests.yml | 3 --- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6f4f3e4aa..95d66b8e9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,13 +1,7 @@ # CODEOWNERS -## Ownership and review routing only -- the 2-approval requirement for these paths is a repository ruleset (required reviewer rule on the same teams), since CODEOWNERS with branch protection is satisfied by any one owner. A team needs write access to the repo for CODEOWNERS to resolve it - -## GitHub reads this file from the branch a pull request targets, so it governs merges into master and staging once it is on them, and never the branch proposing it. An owner it cannot resolve is dropped rather than enforced, silently -- the ruleset in .github/rulesets carries the enforcement for these paths, naming the teams directly - -## The state configuration and the compat rules pin backward-compatibility semantics forever, so changes need 2 sign-offs from the people who own those semantics. CODEOWNERS takes one pattern per line, so the team repeats; the ruleset groups both paths under a single rule - -## TODO(settings): create @datazip-inc/olake-admins and @datazip-inc/state-version-owners with WRITE access -- read-only resolves to nobody and the entry is dropped -## TODO(settings): the counts these paths need live in .github/rulesets/state-version-approval.json, which is not applied yet -- see that directory's README +## TODO(settings): create @datazip-inc/olake-admins and @datazip-inc/state-version-owners with WRITE access +## TODO(settings): the counts these paths need live in .github/rulesets/state-version-approval.json /.github/CODEOWNERS @datazip-inc/olake-admins diff --git a/.github/actions/driver-image/action.yml b/.github/actions/driver-image/action.yml index ab8d5b133..d1dc709f1 100644 --- a/.github/actions/driver-image/action.yml +++ b/.github/actions/driver-image/action.yml @@ -12,14 +12,6 @@ inputs: commit-tagged build can be exported by naming its tag instead. required: false default: local - artifact-name: - description: > - What to publish under. Defaults to driver-image-, which is the name the compatibility - job looks for. Exporting more than one tag of the same driver in a run needs distinct names - here: an artifact name can only be uploaded once, and this action overwrites, so a second - export under the same name would replace the first rather than fail. - required: false - default: '' retention-days: description: How long the artifact lives. A day is already far longer than the run that reads it. required: false @@ -38,7 +30,7 @@ runs: - name: Upload the image uses: actions/upload-artifact@v7 with: - name: ${{ inputs.artifact-name || format('driver-image-{0}', inputs.driver) }} + name: driver-image-${{ inputs.driver }} path: ${{ runner.temp }}/source-${{ inputs.driver }}.tar retention-days: ${{ inputs.retention-days }} overwrite: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c0d62f55e..30dd83736 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -382,8 +382,6 @@ jobs: - *jar - # The image the suite runs against. Built here rather than left to the harness, which would - # otherwise build it inside `go test` where it is invisible in the step timings. - name: Build Driver Image run: make docker.${{ matrix.driver }}.build @@ -416,7 +414,6 @@ jobs: run-id: ${{ steps.last_run.outputs.id }} github-token: ${{ github.token }} - # Runs against the image built above: the harness resolves "local" to it and skips the build. - name: Run Performance Tests run: make test.performance.${{ matrix.driver }} From 24b8b12c5429acbcb7410a7839f559335d4d8d50 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Tue, 1 Sep 2026 14:39:43 +0530 Subject: [PATCH 17/20] fix: postgres issue --- tests/testutils/docker.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/testutils/docker.go b/tests/testutils/docker.go index b05ad2a52..a9a8ecf31 100644 --- a/tests/testutils/docker.go +++ b/tests/testutils/docker.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/exec" + "os/user" "path/filepath" "strings" "sync" @@ -146,6 +147,10 @@ func DockerRunArgs(cfg *TestConfig, extraFlags []string, olakeArgs []string) []s "-e", fmt.Sprintf("OLAKE_INDEX_DB_DIR=%s", containerTableIndexDir), } + if u, err := user.Current(); err == nil { + args = append(args, "--user", u.Uid+":"+u.Gid) + } + if cfg.ImagePlatform != "" { args = append(args, "--platform", cfg.ImagePlatform) } From 44be097aa9c21d4e3c0128369a298757eabdea3d Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Tue, 1 Sep 2026 14:52:25 +0530 Subject: [PATCH 18/20] chore: rename func --- tests/testutils/integration/iceberg_index.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/testutils/integration/iceberg_index.go b/tests/testutils/integration/iceberg_index.go index 5a7e61788..b7826de4d 100644 --- a/tests/testutils/integration/iceberg_index.go +++ b/tests/testutils/integration/iceberg_index.go @@ -26,8 +26,8 @@ func hasIcebergTableIndexTest(driver string) bool { return slices.Contains(icebergTableIndexTestDrivers, constants.DriverType(driver)) } -// updateUpdateType sets update_type in selected_streams for the stream identified by namespace+streamName. -func updateUpdateType(config *testutils.TestConfig, namespace, streamName, updateType string) error { +// setUpdateType sets update_type in selected_streams for the stream identified by namespace+streamName. +func setUpdateType(config *testutils.TestConfig, namespace, streamName, updateType string) error { streamName = testutils.NormalizeStreamName(config.Driver, streamName) return testutils.EditJSONFile(config.GetFilePath("streams.json"), func(doc map[string]interface{}) error { selected, _ := doc["selected_streams"].(map[string]interface{}) @@ -162,7 +162,7 @@ func (cfg *Test) testIcebergEqToPosConversion(ctx context.Context, t *testing.T, } // Step 1: full load + CDC update with equality deletes - if err := updateUpdateType(cfg.TestConfig, cfg.Namespace, testTable, "eq"); err != nil { + if err := setUpdateType(cfg.TestConfig, cfg.Namespace, testTable, "eq"); err != nil { return fmt.Errorf("failed setting delete type: %w", err) } if err := cfg.runIcebergSync(ctx, "initial full load sync"); err != nil { @@ -184,7 +184,7 @@ func (cfg *Test) testIcebergEqToPosConversion(ctx context.Context, t *testing.T, // Step 2: CDC insert with positional deletes (triggers eq -> pos conversion) cfg.TestConfig.ExecuteQuery(ctx, t, cfg.TestConfig, "insert") - if err := updateUpdateType(cfg.TestConfig, cfg.Namespace, testTable, "pos"); err != nil { + if err := setUpdateType(cfg.TestConfig, cfg.Namespace, testTable, "pos"); err != nil { return fmt.Errorf("failed setting delete type: %w", err) } if err := cfg.runIcebergSync(ctx, "cdc pos sync"); err != nil { @@ -212,7 +212,7 @@ func (cfg *Test) testIcebergCleanTablePositionalWithPebbleIndex(ctx context.Cont return err } - if err := updateUpdateType(cfg.TestConfig, cfg.Namespace, testTable, "pos"); err != nil { + if err := setUpdateType(cfg.TestConfig, cfg.Namespace, testTable, "pos"); err != nil { return fmt.Errorf("failed setting delete type: %w", err) } if err := cfg.runIcebergSync(ctx, "initial full load"); err != nil { @@ -249,7 +249,7 @@ func (cfg *Test) testIcebergRebuildIndexFromScratch(ctx context.Context, t *test return err } - if err := updateUpdateType(cfg.TestConfig, cfg.Namespace, testTable, "pos"); err != nil { + if err := setUpdateType(cfg.TestConfig, cfg.Namespace, testTable, "pos"); err != nil { return fmt.Errorf("failed setting delete type: %w", err) } if err := cfg.runIcebergSync(ctx, "initial full load"); err != nil { From c7b4571029c621c2c5471a7ef42bd3e2970fa0fd Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Tue, 1 Sep 2026 15:23:05 +0530 Subject: [PATCH 19/20] chore(tests): run driver containers as the invoking user in a fixed non-UTC zone --user: TestWorkingDir is a t.TempDir() bind-mounted into the container, and the image declares no USER. On a Linux bind mount everything the driver writes there belongs to root. A root-owned file is still removable, but a root-owned directory is not, and pkg/indexdb creates one (olake-table-index). Go's own t.TempDir cleanup then fails with permission denied and, reporting through t.Errorf, fails a test whose sync had already passed. TZ: several state-version gates only change behavior when the machine timezone is not UTC. In a UTC container the old and new branches produce identical output, so the compatibility suite could not tell a working gate from a removed one. Also drops the temporary pg_class diagnostic added while chasing the concurrent CREATE TABLE failure. --- tests/postgres/postgres_util_test.go | 40 +--------------------------- tests/testutils/docker.go | 9 +++++++ 2 files changed, 10 insertions(+), 39 deletions(-) diff --git a/tests/postgres/postgres_util_test.go b/tests/postgres/postgres_util_test.go index 9cace3110..03f3a7e90 100644 --- a/tests/postgres/postgres_util_test.go +++ b/tests/postgres/postgres_util_test.go @@ -3,7 +3,6 @@ package postgres import ( "context" "fmt" - "strings" "testing" "time" @@ -315,44 +314,7 @@ func ExecuteQuery(ctx context.Context, t *testing.T, conf *testutils.TestConfig, } _, err := db.ExecContext(ctx, query) - // TEMPORARY: catalogContext is debug scaffolding -- see its doc comment for how to revert. - require.NoError(t, err, "Failed to execute %s operation on %s%s", operation, integrationTestTable, catalogContext(ctx, db, integrationTestTable)) -} - -// TEMPORARY -- REMOVE ONCE THE CONCURRENT-CREATE FAILURE IS DIAGNOSED. -// -// This exists only to identify the object behind an intermittent compatibility-suite failure -// (`duplicate key value violates unique constraint "pg_class_relname_nsp_index"` on create, seen -// on CI runners and locally). It adds a catalog round-trip to every failing query in this driver -// and has no place in the suite once the cause is known. Delete this function and restore the -// call site to: -// -// require.NoError(t, err, "Failed to execute %s operation", operation) -// -// catalogContext describes what the catalog already holds under this suite's table prefix. -// -// Postgres reports a concurrent-DDL conflict as `duplicate key value violates unique constraint -// "pg_class_relname_nsp_index"` and names no relation, which leaves the two candidate causes -// indistinguishable: two suites deriving the same name, or two suites racing on names that only -// collide after the server truncates them to 63 bytes. Both are visible by comparing the name -// this suite wanted against the ones already present, so report exactly that. Returns "" when -// the query fails, so a diagnostic can never mask the error it is describing. -func catalogContext(ctx context.Context, db *sqlx.DB, table string) string { - var existing []string - if err := db.SelectContext(ctx, &existing, - `SELECT relname FROM pg_class WHERE relname LIKE 'test_table_olake%' ORDER BY relname`); err != nil { - return "" - } - - note := fmt.Sprintf("\n wanted: %s (%d bytes; postgres truncates relation names at 63)", table, len(table)) - if len(table) > 63 { - note += fmt.Sprintf("\n TRUNCATED: %s <- every suite whose name shares this prefix collides here", table[:63]) - } - if len(existing) == 0 { - return note + "\n catalog: no test_table_olake* relations present" - } - return note + fmt.Sprintf("\n catalog: %d test_table_olake* relation(s) present:\n %s", - len(existing), strings.Join(existing, "\n ")) + require.NoError(t, err, "Failed to execute %s operation", operation) } // insertTestData inserts test data into the specified table diff --git a/tests/testutils/docker.go b/tests/testutils/docker.go index a9a8ecf31..d108fe2ff 100644 --- a/tests/testutils/docker.go +++ b/tests/testutils/docker.go @@ -21,6 +21,9 @@ const ( // olake input and output lives under it, since the CLI writes next to --config containerTestDataDir = "/testdata" + // containerTimezone is what the driver container runs in. See the TZ note in DockerRunArgs. + containerTimezone = "Asia/Kolkata" + // containerTableIndexDir is where the table index database is mounted in the container containerTableIndexDir = containerTestDataDir + "/olake-table-index" @@ -145,6 +148,12 @@ func DockerRunArgs(cfg *TestConfig, extraFlags []string, olakeArgs []string) []s "-e", "TELEMETRY_DISABLED=true", "-e", "OLAKE_TIMING=1", "-e", fmt.Sprintf("OLAKE_INDEX_DB_DIR=%s", containerTableIndexDir), + // Deliberately not UTC. Several state-version gates only change behavior when the machine + // timezone differs from UTC -- mongodb's v5 DateTime decoding, mysql's v3 offset-format + // zones, the v2 binlog TimestampStringLocation. In a UTC container the old and new branches + // of each produce identical output, so the suite cannot tell a working gate from a removed + // one. A fixed offset zone with a half-hour offset also catches code that assumes whole hours. + "-e", "TZ=" + containerTimezone, } if u, err := user.Current(); err == nil { From f34d07f635a7d0b3740f80a6317e4c59b3760f17 Mon Sep 17 00:00:00 2001 From: mihir-datazip Date: Tue, 1 Sep 2026 17:00:25 +0530 Subject: [PATCH 20/20] revert(tests): drop the non-UTC container timezone --- tests/testutils/docker.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/testutils/docker.go b/tests/testutils/docker.go index d108fe2ff..a9a8ecf31 100644 --- a/tests/testutils/docker.go +++ b/tests/testutils/docker.go @@ -21,9 +21,6 @@ const ( // olake input and output lives under it, since the CLI writes next to --config containerTestDataDir = "/testdata" - // containerTimezone is what the driver container runs in. See the TZ note in DockerRunArgs. - containerTimezone = "Asia/Kolkata" - // containerTableIndexDir is where the table index database is mounted in the container containerTableIndexDir = containerTestDataDir + "/olake-table-index" @@ -148,12 +145,6 @@ func DockerRunArgs(cfg *TestConfig, extraFlags []string, olakeArgs []string) []s "-e", "TELEMETRY_DISABLED=true", "-e", "OLAKE_TIMING=1", "-e", fmt.Sprintf("OLAKE_INDEX_DB_DIR=%s", containerTableIndexDir), - // Deliberately not UTC. Several state-version gates only change behavior when the machine - // timezone differs from UTC -- mongodb's v5 DateTime decoding, mysql's v3 offset-format - // zones, the v2 binlog TimestampStringLocation. In a UTC container the old and new branches - // of each produce identical output, so the suite cannot tell a working gate from a removed - // one. A fixed offset zone with a half-hour offset also catches code that assumes whole hours. - "-e", "TZ=" + containerTimezone, } if u, err := user.Current(); err == nil {