diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 497191b50..d5fc06c15 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -60,6 +60,7 @@ You can also define your own custom checks in Python (see [Creating custom check | `is_older_than_col2_for_n_days` | Checks whether the values in one input column are at least N days older than the values in another column. | `column1`: first column to check (can be a string column name or a column expression); `column2`: second column to check (can be a string column name or a column expression); `days`: number of days; `negate`: if the condition should be negated | | `regex_match` | Checks whether the values in the input column match a given regex. | `column`: column to check (can be a string column name or a column expression); regex: regex to check; `negate`: if the condition should be negated (true) or not | | `is_valid_email` | Checks whether the values in the input column have valid email address format. | `column`: column to check (can be a string column name or a column expression) | +| `is_valid_url` | Checks whether the values in the input column are valid URLs per the RFC 3986 absolute-URI grammar. A scheme is required, so relative references such as `/path` or `example.com` are rejected. Any syntactically valid scheme is accepted, so `s3://`, `ftp://`, `mailto:` and `urn:` are valid alongside `http://` and `https://`. This validates URL syntax only, not safety: script-bearing schemes such as `javascript:alert(1)` and inline `data:` payloads are syntactically valid and pass, so do not use this check to sanitize untrusted input.| `column`: column to check (can be a string column name or a column expression) | | `has_valid_string_case` | Checks whether string values match the requested letter case: `upper` requires all alphabetic characters to be uppercase; `lower` requires all alphabetic characters to be lowercase; `title` requires the first character of each space-delimited word to be uppercase; `sentence` requires each period-delimited segment's first non-whitespace character to be uppercase. | `column`: column to check (can be a string column name or a column expression); `case`: one of `upper`, `lower`, `title`, or `sentence` | | `is_valid_national_id` | Checks whether the values in the input column are valid national identification numbers (e.g., US Social Security Numbers) for the given country. | `column`: column to check (can be a string column name or a column expression); `country`: ISO 3166 alpha-2 country code (optional, default: `US`) | | `is_valid_uuid` | Checks whether the values in the input column are valid UUIDs (RFC 9562, canonical 8-4-4-4-12 hyphenated hex form). By default validates the shape only; set `strict` to also enforce the version nibble (1-8) and variant bits per RFC 9562. | `column`: column to check (can be a string column name or a column expression); `strict`: if True, also validate the version nibble (1-8) and variant bits (8/9/a/b) per RFC 9562 (default: False) | @@ -545,6 +546,13 @@ For brevity, the `name` field in the examples is omitted and it will be auto-gen arguments: column: col1 +# is_valid_url check +- criticality: error + check: + function: is_valid_url + arguments: + column: col1 + # is_valid_uuid check - criticality: error check: @@ -1345,6 +1353,13 @@ checks = [ column="col1" ), + # is_valid_url check + DQRowRule( + criticality="error", + check_func=check_funcs.is_valid_url, + column="col1" + ), + # is_valid_uuid check DQRowRule( criticality="error", diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index ad8d9d39d..8a575cb8c 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -49,6 +49,21 @@ _EMAIL_QTEXT = r"[\x21\x23-\x5B\x5D-\x7E]" # printable ASCII except '"' (0x22) and '\' (0x5C) _EMAIL_QPAIR = r"\\[\x09\x20-\x7E]" # quoted-pair: '\' + VCHAR or WSP; valid only inside a quoted part +# URL helpers (RFC 3986 absolute-URI grammar). Every repetition below is over alternatives whose first +# characters are disjoint ('%' starts only pct-encoded, '/' only a new path segment), so matching is +# deterministic and there is no catastrophic backtracking; ReDoS-safe. +_URL_SCHEME = r"[A-Za-z][A-Za-z0-9+.\-]*" # RFC 3986 §3.1 +_URL_PCT_ENCODED = r"%[0-9A-Fa-f]{2}" # RFC 3986 §2.1 +_URL_UNRESERVED_SUB_DELIMS = r"[A-Za-z0-9\-._~!$&'()*+,;=]" # unreserved (§2.3) + sub-delims (§2.2) +_URL_USERINFO = rf"(?:{_URL_UNRESERVED_SUB_DELIMS}|{_URL_PCT_ENCODED}|:)*" # §3.2.1 +# reg-name (§3.2.2) also covers IPv4address, since digits and '.' are unreserved. +_URL_HOST = rf"(?:\[[A-Fa-f0-9:.]+\]|(?:{_URL_UNRESERVED_SUB_DELIMS}|{_URL_PCT_ENCODED})*)" +_URL_AUTHORITY = rf"(?:{_URL_USERINFO}@)?{_URL_HOST}(?::\d*)?" # §3.2 +_URL_PCHAR = rf"(?:{_URL_UNRESERVED_SUB_DELIMS}|{_URL_PCT_ENCODED}|[:@])" # §3.3 +_URL_PATH_ABEMPTY = rf"(?:/{_URL_PCHAR}*)*" # §3.3; each iteration consumes at least the '/' +_URL_PATH_NO_AUTHORITY = rf"(?:/?{_URL_PCHAR}+{_URL_PATH_ABEMPTY})?" # path-absolute / rootless / empty +_URL_QUERY_OR_FRAGMENT = rf"(?:{_URL_PCHAR}|[/?])*" # §3.4, §3.5 + # Curated aggregate functions for data quality checks # These are univariate (single-column) aggregate functions suitable for DQ monitoring # Maps function names to human-readable display names for error messages @@ -118,6 +133,20 @@ class DQPattern(Enum): # 000/666/9xx (9xx covers ITINs), group 00, serial 0000. Anchored, fixed-width; ReDoS-safe. SSN_US = r"\A(?!000|666|9\d{2})\d{3}([- ]?)(?!00)\d{2}\1(?!0000)\d{4}\z" + # RFC 3986 §4.3 absolute-URI: scheme ":" hier-part [ "?" query ] [ "#" fragment ]. A scheme is + # required, so relative references ("/path", "example.com") are rejected. Any syntactically valid + # scheme is accepted, which includes non-network schemes such as "javascript:" and "data:" - this + # validates URL *syntax*, not safety. Note that RFC 3986 permits an empty host ("file:///path"), + # so host presence is not enforced here. + # \A...\z anchors (not ^...$) so a trailing newline is rejected under Java regex - see IPV4_ADDRESS. + URL = ( + rf"\A{_URL_SCHEME}:" + rf"(?://{_URL_AUTHORITY}{_URL_PATH_ABEMPTY}|{_URL_PATH_NO_AUTHORITY})" + rf"(?:\?{_URL_QUERY_OR_FRAGMENT})?" + rf"(?:#{_URL_QUERY_OR_FRAGMENT})?" + rf"\z" + ) + # Canonical UUID form per RFC 9562: 8-4-4-4-12 hex groups. UUID validates the shape # only, so RFC-defined Nil/Max sentinels and legacy variant GUIDs pass; UUID_STRICT # also pins the version nibble to 1-8 and variant bits to 8/9/a/b. Anchored, fixed-width; ReDoS-safe. @@ -1165,6 +1194,38 @@ def is_valid_email(column: str | Column) -> Column: return _matches_pattern(column, DQPattern.EMAIL_ADDRESS) +@register_rule("row") +def is_valid_url(column: str | Column) -> Column: + """Checks whether the values in the input column are valid URLs. + + Validates against the RFC 3986 §4.3 *absolute-URI* grammar: *scheme ":" hier-part* with an + optional *"?" query* and *"#" fragment*. A scheme is required, so relative references such as + */path* or *example.com* are rejected, and reserved characters must be percent-encoded to be + accepted inside a path, query, or fragment. + + Any syntactically valid scheme is accepted, which keeps non-network URLs such as *s3://*, + *ftp://*, *mailto:* and *urn:* valid alongside *http://* and *https://*. Two consequences are + worth noting: + + * This validates URL *syntax*, not safety or reachability. Script-bearing schemes + (*javascript:alert(1)*) and inline payloads (*data:text/plain,hello*) are syntactically valid + URLs and pass. Do not rely on this check to sanitize untrusted input before rendering or + fetching it; gate the scheme explicitly for that, for example with *is_in_list* on an extracted + scheme column or a *sql_expression* check. + * RFC 3986 permits an empty host, so *file:///path* passes. Host presence is not enforced. + + Validation is purely syntactic: it does not verify that the host resolves or that the resource + exists. Null values will pass the check with no violation reported. + + Args: + column: column to check; can be a string column name or a column expression + + Returns: + Column object for condition + """ + return _matches_pattern(column, DQPattern.URL) + + @register_rule("row") def is_valid_national_id(column: str | Column, country: str = "US") -> Column: """Checks whether the values in the input column are valid national identification diff --git a/tests/integration/test_apply_checks.py b/tests/integration/test_apply_checks.py index 3626f6d5b..008f08257 100755 --- a/tests/integration/test_apply_checks.py +++ b/tests/integration/test_apply_checks.py @@ -6002,7 +6002,7 @@ def test_apply_checks_all_row_checks_as_yaml_with_streaming(ws, make_schema, mak "col7: map, col8: struct, col10: int, col11: string, " "col_ipv4: string, col_ipv6: string, col_json_str: string, col_json_str2: string, " "col_email: string, col_uuid: string, col_ssn: string, col_country: string, col_currency: string, " - "col_subdivision: string, col_language: string" + "col_subdivision: string, col_language: string, col_url: string" ) test_df = spark.createDataFrame( [ @@ -6028,6 +6028,7 @@ def test_apply_checks_all_row_checks_as_yaml_with_streaming(ws, make_schema, mak "USD", "US-CA", "en", + "https://example.com/a", ], [ "val2", @@ -6051,6 +6052,7 @@ def test_apply_checks_all_row_checks_as_yaml_with_streaming(ws, make_schema, mak "EUR", "GB-ENG", "en", + "https://sub.example.org/p?q=1", ], [ "val3", @@ -6074,6 +6076,7 @@ def test_apply_checks_all_row_checks_as_yaml_with_streaming(ws, make_schema, mak "GBP", "DE-BY", "de", + "ftp://files.example.org/f.txt", ], ], schema, @@ -6121,6 +6124,7 @@ def test_apply_checks_all_row_checks_as_yaml_with_streaming(ws, make_schema, mak "USD", "US-CA", "en", + "https://example.com/a", None, None, ], @@ -6146,6 +6150,7 @@ def test_apply_checks_all_row_checks_as_yaml_with_streaming(ws, make_schema, mak "EUR", "GB-ENG", "en", + "https://sub.example.org/p?q=1", None, None, ], @@ -6171,6 +6176,7 @@ def test_apply_checks_all_row_checks_as_yaml_with_streaming(ws, make_schema, mak "GBP", "DE-BY", "de", + "ftp://files.example.org/f.txt", None, None, ], @@ -6326,7 +6332,7 @@ def test_apply_checks_all_checks_as_yaml(ws, spark): "col7: map, col8: struct, col10: int, col11: string, " "col_ipv4: string, col_ipv6: string, col_json_str: string, col_json_str2: string, " "col_email: string, col_uuid: string, col_ssn: string, col_country: string, col_currency: string, " - "col_subdivision: string, col_language: string" + "col_subdivision: string, col_language: string, col_url: string" ) test_df = spark.createDataFrame( [ @@ -6352,6 +6358,7 @@ def test_apply_checks_all_checks_as_yaml(ws, spark): "USD", "US-CA", "en", + "https://example.com/a", ], [ "val2", @@ -6375,6 +6382,7 @@ def test_apply_checks_all_checks_as_yaml(ws, spark): "EUR", "GB-ENG", "en", + "https://sub.example.org/p?q=1", ], [ "val3", @@ -6398,6 +6406,7 @@ def test_apply_checks_all_checks_as_yaml(ws, spark): "GBP", "DE-BY", "de", + "ftp://files.example.org/f.txt", ], ], schema, @@ -6433,6 +6442,7 @@ def test_apply_checks_all_checks_as_yaml(ws, spark): "USD", "US-CA", "en", + "https://example.com/a", None, None, ], @@ -6458,6 +6468,7 @@ def test_apply_checks_all_checks_as_yaml(ws, spark): "EUR", "GB-ENG", "en", + "https://sub.example.org/p?q=1", None, None, ], @@ -6483,6 +6494,7 @@ def test_apply_checks_all_checks_as_yaml(ws, spark): "GBP", "DE-BY", "de", + "ftp://files.example.org/f.txt", None, None, ], @@ -7274,6 +7286,12 @@ def test_apply_checks_all_checks_using_classes(ws, spark): column="col_json_str2", check_func_kwargs={"schema": "STRUCT"}, ), + # is_valid_url check + DQRowRule( + criticality="error", + check_func=check_funcs.is_valid_url, + column="col_url", + ), # is_valid_national_id check DQRowRule( criticality="error", @@ -7315,7 +7333,8 @@ def test_apply_checks_all_checks_using_classes(ws, spark): "col1: string, col2: int, col3: int, col4 array, col5: date, col6: timestamp, " "col7: map, col8: struct, col10: int, col11: string, " "col_ipv4: string, col_ipv6: string, col_json_str: string, col_json_str2: string, col_ssn: string, " - "col_country: string, col_currency: string, col_subdivision: string, col_language: string" + "col_country: string, col_currency: string, col_subdivision: string, col_language: string, " + "col_url: string" ) test_df = spark.createDataFrame( [ @@ -7339,6 +7358,7 @@ def test_apply_checks_all_checks_using_classes(ws, spark): "USD", "US-CA", "en", + "https://example.com/a", ], [ "val2", @@ -7360,6 +7380,7 @@ def test_apply_checks_all_checks_using_classes(ws, spark): "EUR", "GB-ENG", "en", + "https://sub.example.org/p?q=1", ], [ "val3", @@ -7381,6 +7402,7 @@ def test_apply_checks_all_checks_using_classes(ws, spark): "GBP", "DE-BY", "de", + "ftp://files.example.org/f.txt", ], ], schema, @@ -7414,6 +7436,7 @@ def test_apply_checks_all_checks_using_classes(ws, spark): "USD", "US-CA", "en", + "https://example.com/a", None, None, ], @@ -7437,6 +7460,7 @@ def test_apply_checks_all_checks_using_classes(ws, spark): "EUR", "GB-ENG", "en", + "https://sub.example.org/p?q=1", None, None, ], @@ -7460,6 +7484,7 @@ def test_apply_checks_all_checks_using_classes(ws, spark): "GBP", "DE-BY", "de", + "ftp://files.example.org/f.txt", None, None, ], diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index df4013a12..2fbaea192 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -32,6 +32,7 @@ is_valid_timestamp, is_valid_ipv4_address, is_valid_email, + is_valid_url, is_valid_uuid, is_valid_national_id, is_valid_country_code, @@ -2122,6 +2123,129 @@ def violation(value: str) -> str: assertDataFrameEqual(actual, expected) +def test_col_is_valid_url(spark): + schema_url = "a: string" + test_df = spark.createDataFrame( + [ + # Valid - common web forms + ["https://example.com"], + ["http://example.com/"], + ["https://example.com/path/to/page"], + ["https://example.com/path?query=1&other=2"], + ["https://example.com/path?query=1#fragment"], + ["https://example.com#fragment"], + ["https://sub.domain.example.co.uk/a/b/c"], + ["HTTPS://EXAMPLE.COM"], # scheme and host are case-insensitive + # Valid - authority components + ["https://user@example.com/p"], + ["https://user:pw@example.com:8080/p"], + ["https://example.com:8080"], + ["https://example.com:/p"], # RFC 3986 permits an empty port + ["https://192.0.2.1/p"], # IPv4 host + ["https://[2001:db8::1]:443/p"], # IPv6 literal host + # Valid - non-network schemes (any syntactically valid scheme is accepted) + ["ftp://files.example.org/pub/file.txt"], + ["s3://bucket/key/part-00001.parquet"], + ["mailto:user@example.com"], + ["urn:isbn:0451450523"], + ["file:///var/log/app.log"], # empty host is permitted by RFC 3986 + ["custom-scheme+v2://host/p"], # scheme may contain '+', '-', '.' + # Valid - percent-encoding and sub-delims + ["https://example.com/a%20b"], + ["https://example.com/a,b;c=d"], + ["https://example.com/p?a=1+2"], + # Valid but NOT safe - syntax-only validation, see the docstring caveat + ["javascript:alert(1)"], + ["data:text/plain,hello"], + [None], # Null - passes (no violation reported) + # Invalid - missing or malformed scheme + ["example.com"], # no scheme + ["example.com/path"], + ["/relative/path"], + ["//example.com/p"], # network-path reference, not an absolute URI + ["://example.com"], # empty scheme + ["1https://example.com"], # scheme must start with a letter + ["ht tp://example.com"], # space in scheme + [""], # empty string + # Invalid - whitespace and control characters + ["https://exa mple.com"], + ["https://example.com/a b"], + ["http://example.com\n"], # trailing newline must be rejected (see issue #1440) + ["https://example.com\t/p"], + # Invalid - characters that must be percent-encoded + ["https://example.com/a str: + return f"Value '{value}' in Column 'a' does not match pattern 'URL'" + + checked_schema = "a_does_not_match_pattern_url: string" + checked_data = [ + # Valid (no violation reported) + [None], # https://example.com + [None], # http://example.com/ + [None], # https://example.com/path/to/page + [None], # https://example.com/path?query=1&other=2 + [None], # https://example.com/path?query=1#fragment + [None], # https://example.com#fragment + [None], # https://sub.domain.example.co.uk/a/b/c + [None], # HTTPS://EXAMPLE.COM + [None], # https://user@example.com/p + [None], # https://user:pw@example.com:8080/p + [None], # https://example.com:8080 + [None], # https://example.com:/p + [None], # https://192.0.2.1/p + [None], # https://[2001:db8::1]:443/p + [None], # ftp://files.example.org/pub/file.txt + [None], # s3://bucket/key/part-00001.parquet + [None], # mailto:user@example.com + [None], # urn:isbn:0451450523 + [None], # file:///var/log/app.log + [None], # custom-scheme+v2://host/p + [None], # https://example.com/a%20b + [None], # https://example.com/a,b;c=d + [None], # https://example.com/p?a=1+2 + [None], # javascript:alert(1) - syntactically valid + [None], # data:text/plain,hello - syntactically valid + [None], # Null + # Invalid - missing or malformed scheme + [violation("example.com")], + [violation("example.com/path")], + [violation("/relative/path")], + [violation("//example.com/p")], + [violation("://example.com")], + [violation("1https://example.com")], + [violation("ht tp://example.com")], + [violation("")], + # Invalid - whitespace and control characters + [violation("https://exa mple.com")], + [violation("https://example.com/a b")], + [violation("http://example.com\n")], + [violation("https://example.com\t/p")], + # Invalid - characters that must be percent-encoded + [violation("https://example.com/a