diff --git a/CHANGELOG.md b/CHANGELOG.md index 512a5fc..7233887 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Clarified `retryAttempts` as the ordered history preceding the final attempt, aligned `retries` counting semantics, and updated the schema and examples accordingly ([#62](https://github.com/ctrf-io/ctrf/pull/62)). - Added an optional identity model for CTRF documents, logical runs, test cases, executions, attempts, attachments, and shards ([#57](https://github.com/ctrf-io/ctrf/pull/57)). - Clarified namespace guidance for `extra` extension keys and examples ([#56](https://github.com/ctrf-io/ctrf/pull/56)). - Clarified immutability guidance for emitted CTRF report artifacts ([#55](https://github.com/ctrf-io/ctrf/pull/55)). diff --git a/examples/comprehensive.json b/examples/comprehensive.json index 53772b6..d9c41d2 100644 --- a/examples/comprehensive.json +++ b/examples/comprehensive.json @@ -79,8 +79,8 @@ "name": "user can checkout", "status": "passed", "duration": 5200, - "start": 1609459203500, - "stop": 1609459208700, + "start": 1609459208700, + "stop": 1609459213900, "suite": ["Checkout"], "type": "e2e", "tags": ["smoke"], @@ -113,14 +113,6 @@ "path": "/artifacts/retry-1.png" } ] - }, - { - "attempt": 2, - "attemptId": "48c3f90b-95a9-4c13-a42f-0888bb0b1002", - "status": "passed", - "duration": 5200, - "start": 1609459208700, - "stop": 1609459213900 } ], "insights": { diff --git a/examples/with-retries.json b/examples/with-retries.json index b237970..34a205d 100644 --- a/examples/with-retries.json +++ b/examples/with-retries.json @@ -26,7 +26,7 @@ { "name": "flaky test", "status": "passed", - "duration": 300, + "duration": 140, "flaky": true, "retries": 2, "retryAttempts": [ @@ -41,11 +41,6 @@ "status": "failed", "duration": 130, "message": "Connection timeout" - }, - { - "attempt": 3, - "status": "passed", - "duration": 140 } ] } diff --git a/schema/ctrf.schema.json b/schema/ctrf.schema.json index 2cee012..85bd935 100644 --- a/schema/ctrf.schema.json +++ b/schema/ctrf.schema.json @@ -256,24 +256,26 @@ "type": "string" }, "retries": { - "description": "Number of retry attempts performed", - "type": "integer" + "description": "Number of times the test was re-executed after its initial attempt", + "type": "integer", + "minimum": 0 }, "retryAttempts": { - "description": "List of retry attempts for this test", + "description": "Ordered history of attempts completed before the final attempt represented by the test object", "type": "array", + "minItems": 1, "items": { - "description": "Single retry attempt result", + "description": "Attempt completed before the final attempt", "type": "object", "required": [ "attempt", "status" ], "properties": { "attempt": { - "description": "Attempt number (1 = first execution)", + "description": "Original sequence number for this attempt (1 = initial execution)", "type": "integer", "minimum": 1 }, "attemptId": { - "description": "Unique identifier for this individual retry attempt. UUID recommended", + "description": "Unique identifier for this individual attempt. UUID recommended", "type": "string", "minLength": 1 }, @@ -289,7 +291,8 @@ }, "duration": { "description": "Attempt execution time (milliseconds)", - "type": "integer" + "type": "integer", + "minimum": 0 }, "message": { "description": "Error or failure message for this attempt", diff --git a/spec/ctrf.md b/spec/ctrf.md index 850818d..f5a84e3 100644 --- a/spec/ctrf.md +++ b/spec/ctrf.md @@ -194,7 +194,7 @@ CTRF favors explicit fields and documented semantics over inferred or implicit b For example: -- test retries are represented explicitly as retry attempts +- test retry histories are represented explicitly as attempt history entries - derived metrics are represented as insights rather than inferred - hierarchy is expressed as metadata rather than structure @@ -287,9 +287,21 @@ The final outcome of a test case. The `status` field MUST be one of: `passed`, ` **Attempt**: A single execution of a test case. The first execution is attempt number 1. Subsequent executions of the same test case, performed because the test was retried, are also attempts with incrementing attempt numbers. +**Initial Attempt**: +The first execution of a test case. The initial attempt has attempt number 1 and is not a retry. + **Retry** / **Retry Attempt**: Any attempt with an attempt number greater than 1. Retries occur when a test is re-executed after a failure or based on a retry policy. +**Previous Attempt**: +Any attempt completed before the final attempt. A previous attempt may be the initial attempt or an earlier retry. + +**Final Attempt**: +The last execution of a test case. Its outcome and execution data are represented by the test object. When a test is not retried, the initial attempt is also the final attempt. + +**Retry History**: +The ordered previous attempts recorded in `retryAttempts`. Despite the field name, this history includes the initial attempt when a retry occurred and excludes the final attempt represented by the test object. + **Flaky Test**: A test is considered flaky only if its final status is `passed` and it experienced one or more failed attempts before passing. @@ -312,7 +324,7 @@ A quantitative value describing some property of a set of tests or runs (for exa A structure describing a metric’s `current` value, the corresponding value from a `baseline` run, and the computed `change` between them. **Attachment**: -An external file or resource referenced by a test case or retry attempt, such as a screenshot, log file, or other artifact. +An external file or resource referenced by a test case or attempt history entry, such as a screenshot, log file, or other artifact. --- @@ -421,8 +433,8 @@ Each layer addresses a distinct identification concern: 1. **Document**: `reportId` identifies the emitted CTRF document as a serialized artifact. When `reportId` is used, each CTRF document SHOULD have exactly one `reportId`. 2. **Run**: `runId` identifies the logical test run. Multiple CTRF documents MAY share the same `runId` when they represent shards or partitions of a single coordinated execution. 3. **Test case**: `testId` identifies the logical test case within the producer's chosen scope. It SHOULD be deterministic and stable across runs within that scope, enabling cross-run analysis, trending, and flake detection. -4. **Execution**: `executionId` identifies a concrete execution of a test case within a run. An execution MAY include multiple retry attempts represented in `retryAttempts`. -5. **Attempt**: `attemptId` identifies an individual retry attempt within an execution. +4. **Execution**: `executionId` identifies the complete execution lifecycle of a test case within a run. An execution MAY include previous attempts represented in `retryAttempts` followed by the final attempt represented by the test object. +5. **Attempt**: `attemptId` identifies an individual attempt history entry within an execution. 6. **Attachment**: `attachmentId` identifies a specific attachment reference instance. 7. **Shard**: `shardId` labels the partition or shard that produced this document within a logical run. @@ -440,7 +452,7 @@ Each identity field has an intended uniqueness scope: | `runId` | Unique per logical run | | `testId` | Stable within the producer's chosen scope for the logical test case | | `executionId` | Unique for one execution of a logical test case within a run | -| `attemptId` | Unique for one retry attempt of an execution | +| `attemptId` | Unique for one recorded attempt within an execution | | `attachmentId` | Unique within the containing parent object | | `shardId` | Unique within a logical run | @@ -457,7 +469,7 @@ Identity fields differ in whether their values persist across runs or are unique | `runId` | Stable across all documents belonging to the same logical run | | `testId` | Stable across runs for the same logical test case within the producer's chosen scope | | `executionId` | Unique per execution; not stable across runs | -| `attemptId` | Unique per attempt; not stable across runs | +| `attemptId` | Unique per recorded attempt; not stable across runs | | `attachmentId` | Unique per attachment reference instance | | `shardId` | Stable across retransmissions of the same shard | @@ -894,7 +906,9 @@ Consumers MUST NOT assume that `testId` values generated by different producers **Description:** A unique identifier for the specific reported execution of a logical test case within a run. -An execution represents a single reported outcome for a test case in this document. An execution MAY include multiple retry attempts represented in `retryAttempts`. +An execution represents a single reported outcome for a test case in this document. An execution MAY include previous attempts represented in `retryAttempts`. + +The test object represents the final attempt in that execution. When retries occur, attempts completed before the final attempt are represented in `retryAttempts`. **Requirements:** `executionId` is OPTIONAL. @@ -1058,22 +1072,30 @@ If present, it MUST be a string. ### 9.20. `retries` **Description:** -The number of retries performed for this test case. +The number of times the test case was re-executed after its initial attempt. **Requirements:** `retries` is OPTIONAL. If present, it MUST be a non-negative integer. -It SHOULD equal the count of entries in `retryAttempts`. +If `retryAttempts` is present, `retries` MUST be present and MUST equal the count of entries in `retryAttempts`. +If `retries` is greater than 0, `retryAttempts` MUST be present. +If `retries` is 0, `retryAttempts` MUST NOT be present. +The final attempt number MUST equal `retries + 1`. ### 9.21. `retryAttempts` **Description:** -A list of retry attempts performed for this test case. +An ordered history of every attempt completed before the final attempt represented by the test object. + +Despite the field name, `retryAttempts` includes the initial attempt when a retry occurred and every earlier retry when more than one retry occurred. It MUST NOT include the final attempt. **Requirements:** `retryAttempts` is OPTIONAL. -If present, it MUST be an array. -Each entry MUST follow the structure defined in Section 11 (Retry Attempt Object). +If present, it MUST be a non-empty array. +It MUST contain exactly `retries` entries. +Entries MUST be ordered by ascending attempt number. +Attempt numbers MUST be unique and contiguous, beginning with attempt number 1. +Each entry MUST follow the structure defined in Section 11 (Attempt History Entry Object). ### 9.22. `flaky` @@ -1380,44 +1402,48 @@ It is the ONLY permitted extension point directly under the environment object. --- -## 11. Retry Attempt Object +## 11. Attempt History Entry Object + +When retries occur, the `retryAttempts` array within a test object contains one or more attempt history entry objects. +Each entry represents an attempt completed before the final attempt represented by the test object. -The `retryAttempts` array within a test object contains zero or more retry attempt objects. -Each retry attempt represents a single re-execution of a test after an initial failure or according to a retry policy. +The array includes the initial attempt and every earlier retry. It MUST NOT include the final attempt. -A retry attempt object provides detailed information about that specific execution, including its outcome, duration, diagnostics, and attachments. +An attempt history entry object provides detailed information about that specific execution, including its outcome, duration, diagnostics, and attachments. -Retry attempts SHOULD be ordered by ascending attempt number. +Entries MUST be ordered by ascending attempt number. Attempt numbers MUST be unique and contiguous, beginning with attempt number 1. -Unknown fields MUST NOT appear in a retry attempt object except within an `extra` object. +Unknown fields MUST NOT appear in an attempt history entry object except within an `extra` object. -Each field within a retry attempt object is described in the subsections below. +Each field within an attempt history entry object is described in the subsections below. ### 11.1. `attempt` **Description:** -The attempt number for this execution of the test case. +The original sequence number for this attempt. **Requirements:** `attempt` MUST be present. It MUST be an integer greater than or equal to 1. -Attempt number 1 represents the first execution; higher numbers represent retries. +Attempt number 1 represents the initial execution; higher numbers represent earlier retries. +Values within `retryAttempts` MUST form a contiguous sequence beginning with attempt number 1. +The final attempt is not included in `retryAttempts`; its attempt number is `retries + 1`. ### 11.2. `attemptId` **Description:** -A unique identifier for this individual retry attempt. +A unique identifier for this individual attempt. **Requirements:** `attemptId` is OPTIONAL. If present, it MUST be a non-empty string. UUID is RECOMMENDED. -`attemptId` SHOULD be unique across attempts. +`attemptId` SHOULD be unique within the enclosing execution. ### 11.3. `status` **Description:** -The outcome of this retry attempt. +The outcome of this attempt. **Requirements:** `status` MUST be present. @@ -1426,11 +1452,11 @@ It MUST be one of the following values: `passed`, `failed`, `skipped`, `pending` ### 11.4. `duration` **Description:** -The execution time of this retry attempt, in milliseconds. +The execution time of this attempt, in milliseconds. **Requirements:** `duration` is OPTIONAL. -If present, it MUST be an integer. +If present, it MUST be a non-negative integer. ### 11.5. `message` @@ -1457,7 +1483,7 @@ The line number in the source file where the failure occurred, if applicable. **Requirements:** `line` is OPTIONAL. -If present, it MUST be a number. +If present, it MUST be an integer. ### 11.8. `snippet` @@ -1508,7 +1534,7 @@ If both `start` and `stop` are present, `stop` SHOULD be greater than or equal t ### 11.13. `attachments` **Description:** -An array of attachment objects associated with this retry attempt. +An array of attachment objects associated with this attempt. **Requirements:** `attachments` is OPTIONAL. @@ -1523,13 +1549,13 @@ An extensibility object containing arbitrary metadata. **Requirements:** `extra` is OPTIONAL. It MAY contain arbitrary fields. -It is the ONLY permitted extension point directly under a retry attempt object. +It is the ONLY permitted extension point directly under an attempt history entry object. --- ## 12. Attachment Object -The `attachments` array provides references to external artifacts associated with a test case or retry attempt. Attachments may include screenshots, logs, videos, performance traces, or any other files needed for diagnostic or reporting purposes. +The `attachments` array provides references to external artifacts associated with a test case or attempt history entry. Attachments may include screenshots, logs, videos, performance traces, or any other files needed for diagnostic or reporting purposes. Each attachment object describes a single artifact. @@ -1959,7 +1985,7 @@ Producers: - MUST output UTF-8 JSON - MUST set `reportFormat = "CTRF"` - MUST include all required fields -- MUST follow rules for retry attempts +- MUST follow rules for retry histories - MUST NOT introduce fields outside `extra` objects - MUST NOT emit invalid enum values (status, etc.) - MAY include `insights` if historical or aggregate data is available @@ -2286,24 +2312,26 @@ to this specification. "type": "string" }, "retries": { - "description": "Number of retry attempts performed", - "type": "integer" + "description": "Number of times the test was re-executed after its initial attempt", + "type": "integer", + "minimum": 0 }, "retryAttempts": { - "description": "List of retry attempts for this test", + "description": "Ordered history of attempts completed before the final attempt represented by the test object", "type": "array", + "minItems": 1, "items": { - "description": "Single retry attempt result", + "description": "Attempt completed before the final attempt", "type": "object", "required": [ "attempt", "status" ], "properties": { "attempt": { - "description": "Attempt number (1 = first execution)", + "description": "Original sequence number for this attempt (1 = initial execution)", "type": "integer", "minimum": 1 }, "attemptId": { - "description": "Unique identifier for this individual retry attempt. UUID recommended", + "description": "Unique identifier for this individual attempt. UUID recommended", "type": "string", "minLength": 1 }, @@ -2319,7 +2347,8 @@ to this specification. }, "duration": { "description": "Attempt execution time (milliseconds)", - "type": "integer" + "type": "integer", + "minimum": 0 }, "message": { "description": "Error or failure message for this attempt", @@ -2966,12 +2995,12 @@ It includes: ### D.2. CTRF Document with Retries -This example demonstrates how CTRF represents retry attempts and flaky tests. +This example demonstrates how CTRF represents previous attempts and flaky tests. It includes: - multiple test cases -- retry attempts recorded in `retryAttempts` +- previous attempts recorded in `retryAttempts` - a test that fails initially and passes on retry - a flaky test indicated by `flaky: true` - a final test status of `passed` after retries @@ -3000,7 +3029,7 @@ It includes: { "name": "retries then passes", "status": "passed", - "duration": 1800, + "duration": 900, "retries": 1, "flaky": true, "retryAttempts": [ @@ -3009,11 +3038,6 @@ It includes: "status": "failed", "duration": 900, "message": "Assertion failed" - }, - { - "attempt": 2, - "status": "passed", - "duration": 900 } ] }, @@ -3262,7 +3286,7 @@ It includes: "category": ["smoke", "staging"] }, "status": "passed", - "duration": 3000, + "duration": 1500, "retries": 1, "flaky": true, "retryAttempts": [ @@ -3274,12 +3298,6 @@ It includes: "message": "Timeout waiting for response", "stdout": ["Submitting login form"], "stderr": ["Request timed out"] - }, - { - "attempt": 2, - "attemptId": "f1e2d3c4-2222-4000-b000-000000000002", - "status": "passed", - "duration": 1500 } ], "stdout": ["Test completed successfully"],