diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 497191b50..720ca77c1 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -61,7 +61,7 @@ You can also define your own custom checks in Python (see [Creating custom check | `regex_match` | Checks whether the values in the input column match a given regex. | `column`: column to check (can be a string column name or a column expression); regex: regex to check; `negate`: if the condition should be negated (true) or not | | `is_valid_email` | Checks whether the values in the input column have valid email address format. | `column`: column to check (can be a string column name or a column expression) | | `has_valid_string_case` | Checks whether string values match the requested letter case: `upper` requires all alphabetic characters to be uppercase; `lower` requires all alphabetic characters to be lowercase; `title` requires the first character of each space-delimited word to be uppercase; `sentence` requires each period-delimited segment's first non-whitespace character to be uppercase. | `column`: column to check (can be a string column name or a column expression); `case`: one of `upper`, `lower`, `title`, or `sentence` | -| `is_valid_national_id` | Checks whether the values in the input column are valid national identification numbers (e.g., US Social Security Numbers) for the given country. | `column`: column to check (can be a string column name or a column expression); `country`: ISO 3166 alpha-2 country code (optional, default: `US`) | +| `is_valid_national_id` | Checks whether the values in the input column match a supported national identification number format. Supports US Social Security Numbers (`US`), UK National Insurance numbers (`GB`), and Indian Permanent Account Numbers (`IN`). These are format checks only; they do not verify whether an identifier was issued. | `column`: column to check (can be a string column name or a column expression); `country`: ISO 3166 alpha-2 country code (optional, default: `US`) | | `is_valid_uuid` | Checks whether the values in the input column are valid UUIDs (RFC 9562, canonical 8-4-4-4-12 hyphenated hex form). By default validates the shape only; set `strict` to also enforce the version nibble (1-8) and variant bits per RFC 9562. | `column`: column to check (can be a string column name or a column expression); `strict`: if True, also validate the version nibble (1-8) and variant bits (8/9/a/b) per RFC 9562 (default: False) | | `is_valid_country_code` | Checks whether the values in the input column are valid ISO 3166-1 country codes (alpha-2, e.g. US, alpha-3, e.g. USA, or numeric, e.g. 840). Source: https://www.iso.org/iso-3166-country-codes.html | `column`: column to check (can be a string column name or a column expression); `code_format`: ISO 3166-1 representation, `alpha-2` (default), `alpha-3`, or `numeric`; `case_sensitive`: optional boolean flag for case-sensitive comparison (default: True) | | `is_valid_currency_code` | Checks whether the values in the input column are valid ISO 4217 currency codes (alphabetic, e.g. USD, or numeric, e.g. 840). Source: https://www.iso.org/iso-4217-currency-codes.html | `column`: column to check (can be a string column name or a column expression); `code_format`: ISO 4217 representation, `alphabetic` (default) or `numeric`; `case_sensitive`: optional boolean flag for case-sensitive comparison (default: True) | diff --git a/src/databricks/labs/dqx/check_funcs.py b/src/databricks/labs/dqx/check_funcs.py index ad8d9d39d..52441ff60 100644 --- a/src/databricks/labs/dqx/check_funcs.py +++ b/src/databricks/labs/dqx/check_funcs.py @@ -117,6 +117,12 @@ class DQPattern(Enum): # none) must be consistent via backreference \1. Excludes invalid ranges - area # 000/666/9xx (9xx covers ITINs), group 00, serial 0000. Anchored, fixed-width; ReDoS-safe. SSN_US = r"\A(?!000|666|9\d{2})\d{3}([- ]?)(?!00)\d{2}\1(?!0000)\d{4}\z" + # UK National Insurance Number: two-letter prefix, six digits, and an A-D + # suffix. Exclude prefixes that HMRC does not allocate. + NINO_GB = r"\A(?!(?:BG|GB|KN|NK|NT|TN|ZZ))(?!(?:[DFIQUV]))[A-Z](?![DFIOQUV])[A-Z] ?\d{2} ?\d{2} ?\d{2} ?[ABCD]\z" + # Indian Permanent Account Number (PAN): three letters, a holder-type letter, + # another letter, four digits, and a final letter. + PAN_IN = r"\A[A-Z]{3}[ABCFGHJLPT][A-Z]\d{4}[A-Z]\z" # Canonical UUID form per RFC 9562: 8-4-4-4-12 hex groups. UUID validates the shape # only, so RFC-defined Nil/Max sentinels and legacy variant GUIDs pass; UUID_STRICT @@ -130,6 +136,8 @@ class DQPattern(Enum): # alpha-2 code here. _NATIONAL_ID_PATTERNS_BY_COUNTRY: dict[str, DQPattern] = { "US": DQPattern.SSN_US, + "GB": DQPattern.NINO_GB, + "IN": DQPattern.PAN_IN, } @@ -1173,12 +1181,17 @@ def is_valid_national_id(column: str | Column, country: str = "US") -> Column: Validation is limited to *format* and *number ranges*; it does not verify that a number was actually issued. - Supported countries are keyed by ISO 3166 alpha-2 code. Currently only *US* is - supported: the *AAA-GG-SSSS* form is required, where the separators may be all - hyphens, all single spaces, or omitted entirely (e.g. *123-45-6789*, *123 45 6789* - or *123456789*), but must be used consistently. Structurally invalid ranges are - rejected (area *000*, *666* and *900-999* - the latter covering ITINs; group *00*; - serial *0000*). + Supported countries are keyed by ISO 3166 alpha-2 code. For *US*, the + *AAA-GG-SSSS* form is required, where the separators may be all hyphens, all + single spaces, or omitted entirely (e.g. *123-45-6789*, *123 45 6789* or + *123456789*), but must be used consistently. Structurally invalid ranges are + rejected (area *000*, *666* and *900-999* - the latter covering ITINs; group + *00*; serial *0000*). For *GB*, a National Insurance number consists of two + letters, six digits, and a final *A*, *B*, *C*, or *D*; unallocated prefixes + are rejected. For *IN*, a PAN consists of three letters, a holder-type letter + (*A*, *B*, *C*, *F*, *G*, *H*, *J*, *L*, *P*, or *T*), another letter, four + digits, and a final letter. These checks validate format only, not whether an + identifier was issued. Null values will pass the check with no violation reported. diff --git a/tests/integration/test_row_checks.py b/tests/integration/test_row_checks.py index df4013a12..c8fe2bd7c 100644 --- a/tests/integration/test_row_checks.py +++ b/tests/integration/test_row_checks.py @@ -2237,64 +2237,84 @@ def violation(value: str) -> str: assertDataFrameEqual(actual, expected) -def test_col_is_valid_national_id(spark): - schema_ssn = "a: string" - test_df = spark.createDataFrame( - [ - # Valid - separators must be consistent (all '-', all ' ', or none) - ["123-45-6789"], - ["123456789"], - ["123 45 6789"], - ["899-45-6789"], # area boundary just below 900 - ["667-45-6789"], # area just above 666 - ["001-01-0001"], # minimal valid area / group / serial - # Invalid - excluded number ranges - ["000-45-6789"], # area 000 - ["666-45-6789"], # area 666 - ["900-45-6789"], # area 9xx (ITIN range, rejected) - ["123-00-6789"], # group 00 - ["123-45-0000"], # serial 0000 - # Invalid - separator / structure - ["123-45 6789"], # mixed separators - ["12-45-6789"], # area too short - ["1234-45-6789"], # area too long - ["abc-de-fghi"], # non-numeric - [""], # empty string - [None], # Null - passes (no violation reported) - ], - schema_ssn, - ) - - actual = test_df.select(is_valid_national_id("a", country="US")) +@pytest.mark.parametrize( + "country, pattern_name, cases", + [ + pytest.param( + "US", + "SSN_US", + [ + # Valid - separators must be consistent (all '-', all ' ', or none) + ("123-45-6789", False), + ("123456789", False), + ("123 45 6789", False), + ("899-45-6789", False), # area boundary just below 900 + ("667-45-6789", False), # area just above 666 + ("001-01-0001", False), # minimal valid area / group / serial + # Invalid - excluded number ranges + ("000-45-6789", True), # area 000 + ("666-45-6789", True), # area 666 + ("900-45-6789", True), # area 9xx (ITIN range, rejected) + ("123-00-6789", True), # group 00 + ("123-45-0000", True), # serial 0000 + # Invalid - separator / structure + ("123-45 6789", True), # mixed separators + ("12-45-6789", True), # area too short + ("1234-45-6789", True), # area too long + ("abc-de-fghi", True), # non-numeric + ("", True), + (None, False), + ], + id="us-ssn", + ), + pytest.param( + "GB", + "NINO_GB", + [ + ("AB123456A", False), + ("AB 12 34 56 A", False), + ("BX586745C", False), + ("DF123456A", True), # invalid first letter + ("BG123456A", True), # unallocated prefix + ("AB123456E", True), # invalid suffix + ("", True), + ("AB123456A\n", True), + (" AB123456A", True), + ("AB123456A ", True), + (None, False), + ], + id="gb-nino", + ), + pytest.param( + "IN", + "PAN_IN", + [ + ("ABCPD1234F", False), + ("AACTA1234A", False), + ("ABCZD1234F", True), # Z is not a valid holder type + ("AB12E1234F", True), # letters and digits in the wrong positions + ("ABCPD12345", True), # final character must be a letter + ("", True), + ("ABCPD1234F\n", True), + (" ABCPD1234F", True), + ("ABCPD1234F ", True), + (None, False), + ], + id="in-pan", + ), + ], +) +def test_col_is_valid_national_id(spark, country, pattern_name, cases): + test_df = spark.createDataFrame([[value] for value, _ in cases], "a: string") + actual = test_df.select(is_valid_national_id("a", country=country)) def violation(value: str) -> str: - return f"Value '{value}' in Column 'a' does not match pattern 'SSN_US'" + return f"Value '{value}' in Column 'a' does not match pattern '{pattern_name}'" - checked_schema = "a_does_not_match_pattern_ssn_us: string" - checked_data = [ - # Valid (no violation reported) - [None], - [None], - [None], - [None], - [None], - [None], - # Invalid - excluded number ranges - [violation("000-45-6789")], - [violation("666-45-6789")], - [violation("900-45-6789")], - [violation("123-00-6789")], - [violation("123-45-0000")], - # Invalid - separator / structure - [violation("123-45 6789")], - [violation("12-45-6789")], - [violation("1234-45-6789")], - [violation("abc-de-fghi")], - [violation("")], - # Null passes - [None], - ] - expected = spark.createDataFrame(checked_data, checked_schema) + expected = spark.createDataFrame( + [[violation(value) if is_invalid else None] for value, is_invalid in cases], + f"a_does_not_match_pattern_{pattern_name.lower()}: string", + ) assertDataFrameEqual(actual, expected)