diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d805c8..38c0d81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- +## [Unreleased] + +### Added +- **`BulkDataObjectMixin.jsonl_download_uri`**: New accessor for the gzip-compressed JSONL download URI, replacing the removed `download_uri` field. +- **`BulkDataObjectMixin.compressed_size`**: New accessor for the compressed file size in bytes, replacing the removed `size` field. + +### Changed +- **`BulkDataObjectMixin.download()`**: Now targets `jsonl_download_uri` and parses the response as newline-delimited JSON (JSONL) rather than a single JSON array. +- **`ScryfallBulkDataData`**: TypedDict updated to match the current Scryfall bulk-data shape — `download_uri`, `size`, `content_type`, and `content_encoding` removed; `jsonl_download_uri` and `compressed_size` added. + +### Removed +- **`BulkDataObjectMixin.download_uri`**: Removed; Scryfall no longer returns this field. Use `jsonl_download_uri` instead. +- **`BulkDataObjectMixin.size`**: Removed; Scryfall no longer returns this field. Use `compressed_size` instead. +- **`BulkDataObjectMixin.content_type`**: Removed; Scryfall no longer returns this field. +- **`BulkDataObjectMixin.content_encoding`**: Removed; Scryfall no longer returns this field. + +--- + ## [2.2.0] - 2026-06-09 ### Added diff --git a/scrython/bulk_data/bulk_data_mixins.py b/scrython/bulk_data/bulk_data_mixins.py index 5f9f4a9..2ef4a05 100644 --- a/scrython/bulk_data/bulk_data_mixins.py +++ b/scrython/bulk_data/bulk_data_mixins.py @@ -65,54 +65,33 @@ def description(self) -> str: return self._scryfall_data["description"] @property - def download_uri(self) -> str: + def jsonl_download_uri(self) -> str: """ - The URI that hosts this bulk file for fetching. + The URI to download this bulk file in gzip-compressed JSONL format. Type: URI (Required) - - Note: Files may be compressed with gzip depending on CDN/proxy configuration. - The download() method automatically detects encoding from HTTP headers. - """ - return self._scryfall_data["download_uri"] - - @property - def updated_at(self) -> str: - """ - The time when this file was last updated. - - Type: Timestamp (Required) - - Note: Bulk data files are updated approximately every 12 hours. """ - return self._scryfall_data["updated_at"] + return self._scryfall_data["jsonl_download_uri"] @property - def size(self) -> int: + def compressed_size(self) -> int: """ - The size of this file in integer bytes. + The size of the compressed file in integer bytes. Type: Integer (Required) """ - return self._scryfall_data["size"] + return self._scryfall_data["compressed_size"] @property - def content_type(self) -> str: - """ - The MIME type of this file. - - Type: String (Required) + def updated_at(self) -> str: """ - return self._scryfall_data["content_type"] + The time when this file was last updated. - @property - def content_encoding(self) -> str: - """ - The Content-Encoding encoding that will be used to transmit this file when you download it. + Type: Timestamp (Required) - Type: String (Required) + Note: Bulk data files are updated approximately every 12 hours. """ - return self._scryfall_data["content_encoding"] + return self._scryfall_data["updated_at"] def download( self, @@ -122,17 +101,18 @@ def download( progress: bool = False, ) -> list[dict[str, Any]] | None: """ - Download and parse bulk data file from Scryfall. + Download and parse the bulk JSONL file from Scryfall. - The bulk data file is downloaded from Scryfall's CDN. The method automatically - detects if the response is gzip-compressed by checking HTTP Content-Encoding - headers and handles decompression accordingly. The JSON data is then parsed - and optionally saved to a file. + The bulk data file is a gzip-compressed JSONL (newline-delimited JSON) file + downloaded from Scryfall's CDN. The method automatically detects if the + response is gzip-compressed by checking HTTP Content-Encoding headers and + handles decompression accordingly. Each line is parsed as a separate JSON + object and returned as a list. Args: - filepath: Optional path to save the decompressed JSON file. + filepath: Optional path to save the parsed data as a JSON file. If None, file is not saved to disk. - return_data: If True, return parsed JSON data. If False and + return_data: If True, return parsed data. If False and filepath is provided, only saves file without returning data. Default: True. chunk_size: Download chunk size in bytes. Default: 8192. @@ -162,7 +142,7 @@ def download( Bulk data files can be very large (100+ MB compressed, 500+ MB uncompressed). Be mindful of memory usage when loading entire files into memory. """ - download_url = self.download_uri + download_url = self.jsonl_download_uri request = Request(download_url) request.add_header("User-Agent", ScrythonRequestHandler._user_agent) @@ -205,7 +185,7 @@ def download( ): data = gzip.decompress(downloaded_data) else: - # Already decompressed or plain JSON + # Already decompressed or plain JSONL data = downloaded_data else: # Download without progress bar @@ -218,11 +198,13 @@ def download( with gzip.GzipFile(fileobj=response) as gz_file: data = gz_file.read() else: - # Read plain JSON + # Read plain JSONL data = response.read() - # Parse JSON. The annotation narrows json.loads's Any return; no cast needed. - parsed_data: list[dict[str, Any]] = json.loads(data.decode("utf-8")) + # Parse JSONL: each non-empty line is a separate JSON object. + parsed_data: list[dict[str, Any]] = [ + json.loads(line) for line in data.decode("utf-8").splitlines() if line.strip() + ] # Save to file if requested if filepath: diff --git a/scrython/types.py b/scrython/types.py index ba25b68..69832e3 100644 --- a/scrython/types.py +++ b/scrython/types.py @@ -285,10 +285,8 @@ class ScryfallBulkDataData(TypedDict): uri: URI name: str description: str - download_uri: URI - size: int - content_type: str - content_encoding: str + jsonl_download_uri: URI + compressed_size: int class ScryfallListData(TypedDict): diff --git a/tests/conftest.py b/tests/conftest.py index 181a5a4..994c960 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -223,9 +223,9 @@ def sample_bulk_data(): "type": "oracle_cards", "name": "Oracle Cards", "description": "All cards, each uniquely identified by Oracle ID", - "download_uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e/download", - "updated_at": "2025-01-01T12:00:00.000Z", - "size": 123456789, + "jsonl_download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz", + "updated_at": "2026-08-11T09:01:55.863+00:00", + "compressed_size": 24502785, } diff --git a/tests/fixtures/bulk_data/all.json b/tests/fixtures/bulk_data/all.json index a6c7513..80ed64e 100644 --- a/tests/fixtures/bulk_data/all.json +++ b/tests/fixtures/bulk_data/all.json @@ -8,11 +8,10 @@ "type": "oracle_cards", "name": "Oracle Cards", "description": "A JSON file containing one Scryfall card object for each Oracle ID on Scryfall.", - "download_uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e/download", - "updated_at": "2025-01-01T12:00:00.000Z", - "size": 123456789, - "content_type": "application/json", - "content_encoding": "gzip" + "updated_at": "2026-08-11T09:01:55.863+00:00", + "uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e", + "jsonl_download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz", + "compressed_size": 24502785 }, { "object": "bulk_data", @@ -20,11 +19,10 @@ "type": "unique_artwork", "name": "Unique Artwork", "description": "A JSON file of Scryfall card objects that together contain all unique artworks.", - "download_uri": "https://api.scryfall.com/bulk-data/922288cb-4bef-45e1-bb30-0c2bd3d3534e/download", - "updated_at": "2025-01-01T12:00:00.000Z", - "size": 234567890, - "content_type": "application/json", - "content_encoding": "gzip" + "updated_at": "2026-08-11T09:01:55.863+00:00", + "uri": "https://api.scryfall.com/bulk-data/922288cb-4bef-45e1-bb30-0c2bd3d3534e", + "jsonl_download_uri": "https://data.scryfall.io/unique-artwork/unique-artwork-20260811090155.jsonl.gz", + "compressed_size": 38912400 } ] } diff --git a/tests/fixtures/bulk_data/by_id.json b/tests/fixtures/bulk_data/by_id.json index 62885d7..2ddda17 100644 --- a/tests/fixtures/bulk_data/by_id.json +++ b/tests/fixtures/bulk_data/by_id.json @@ -5,9 +5,7 @@ "type": "oracle_cards", "name": "Oracle Cards", "description": "A JSON file containing one Scryfall card object for each Oracle ID on Scryfall.", - "download_uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e/download", - "updated_at": "2025-01-01T12:00:00.000Z", - "size": 123456789, - "content_type": "application/json", - "content_encoding": "gzip" + "updated_at": "2026-08-11T09:01:55.863+00:00", + "jsonl_download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz", + "compressed_size": 24502785 } diff --git a/tests/test_bulk_data.py b/tests/test_bulk_data.py index d05cd14..e5b8618 100644 --- a/tests/test_bulk_data.py +++ b/tests/test_bulk_data.py @@ -96,11 +96,11 @@ def test_bulk_data_object_mixin_properties(self, mock_urlopen): assert bulk.type == "oracle_cards" assert bulk.name == "Oracle Cards" assert ( - bulk.download_uri - == "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e/download" + bulk.jsonl_download_uri + == "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz" ) - assert bulk.updated_at == "2025-01-01T12:00:00.000Z" - assert bulk.size == 123456789 + assert bulk.updated_at == "2026-08-11T09:01:55.863+00:00" + assert bulk.compressed_size == 24502785 def test_bulk_data_object_from_list(self, mock_urlopen): """Test that BulkDataObject wrapper works correctly.""" @@ -122,9 +122,10 @@ def test_download_returns_parsed_data(self, mock_urlopen): mock_urlopen.set_response("bulk_data/by_id.json") bulk = ByType(type="oracle_cards") - # Mock the download URL response + # Mock the download URL response with JSONL format test_data = [{"id": "card1", "name": "Test Card"}] - compressed_data = gzip.compress(json.dumps(test_data).encode("utf-8")) + jsonl_data = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8") + compressed_data = gzip.compress(jsonl_data) with patch("scrython.bulk_data.bulk_data_mixins.urlopen") as mock_download: # Wrap compressed data in BytesIO for proper file-like behavior @@ -147,7 +148,8 @@ def test_download_saves_to_file(self, mock_urlopen): bulk = ByType(type="oracle_cards") test_data = [{"id": "card1", "name": "Test Card"}] - compressed_data = gzip.compress(json.dumps(test_data).encode("utf-8")) + jsonl_data = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8") + compressed_data = gzip.compress(jsonl_data) with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json") as tmp: tmp_path = tmp.name @@ -182,7 +184,8 @@ def test_download_without_return_data(self, mock_urlopen): bulk = ByType(type="oracle_cards") test_data = [{"id": "card1", "name": "Test Card"}] - compressed_data = gzip.compress(json.dumps(test_data).encode("utf-8")) + jsonl_data = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8") + compressed_data = gzip.compress(jsonl_data) with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json") as tmp: tmp_path = tmp.name @@ -259,18 +262,18 @@ def test_download_progress_without_tqdm_raises_import_error(self, mock_urlopen): bulk.download(progress=True) def test_download_uncompressed_no_progress(self, mock_urlopen): - """Test download handles uncompressed JSON without progress bar.""" + """Test download handles uncompressed JSONL without progress bar.""" mock_urlopen.set_response("bulk_data/by_id.json") bulk = ByType(type="oracle_cards") - # Test with plain JSON (not gzip compressed) + # Test with plain JSONL (not gzip compressed) test_data = [{"id": "card1", "name": "Test Card"}] - plain_json = json.dumps(test_data).encode("utf-8") + plain_jsonl = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8") with patch("scrython.bulk_data.bulk_data_mixins.urlopen") as mock_download: # Create mock with NO Content-Encoding header (empty string) mock_response = MagicMock() - mock_response.read.return_value = plain_json + mock_response.read.return_value = plain_jsonl mock_response.info.return_value.get.return_value = "" # No encoding header mock_response.__enter__.return_value = mock_response mock_response.__exit__.return_value = None @@ -283,25 +286,25 @@ def test_download_uncompressed_no_progress(self, mock_urlopen): assert result[0]["name"] == "Test Card" def test_download_uncompressed_with_progress(self, mock_urlopen): - """Test download handles uncompressed JSON with progress bar.""" + """Test download handles uncompressed JSONL with progress bar.""" # Skip if tqdm is not installed pytest.importorskip("tqdm") mock_urlopen.set_response("bulk_data/by_id.json") bulk = ByType(type="oracle_cards") - # Test with plain JSON (not gzip compressed) + # Test with plain JSONL (not gzip compressed) test_data = [{"id": "card1", "name": "Test Card"}] - plain_json = json.dumps(test_data).encode("utf-8") + plain_jsonl = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8") with patch("scrython.bulk_data.bulk_data_mixins.urlopen") as mock_download: # Create mock with NO Content-Encoding header mock_response = MagicMock() mock_response.read.side_effect = [ - plain_json, + plain_jsonl, b"", ] # Return data then empty to signal EOF - mock_response.headers.get.return_value = str(len(plain_json)) + mock_response.headers.get.return_value = str(len(plain_jsonl)) mock_response.info.return_value.get.return_value = "" # No encoding header mock_response.__enter__.return_value = mock_response mock_response.__exit__.return_value = None @@ -325,7 +328,7 @@ def test_download_sets_headers(self, mock_urlopen): with patch("scrython.bulk_data.bulk_data_mixins.urlopen") as mock_download: # Set up mock to allow inspection of the Request object mock_response = MagicMock() - mock_response.read.return_value = b"[]" + mock_response.read.return_value = b"" mock_response.info.return_value.get.return_value = "" mock_response.__enter__.return_value = mock_response mock_response.__exit__.return_value = None diff --git a/tests/test_property_types.py b/tests/test_property_types.py index 1ff7dfc..b1204c2 100644 --- a/tests/test_property_types.py +++ b/tests/test_property_types.py @@ -226,11 +226,9 @@ def test_sets_field_type(self, mock_urlopen, prop, expected_type, nullable, desc ("type", str, False, "Bulk data type"), ("name", str, False, "Bulk data name"), ("description", str, False, "Description"), - ("download_uri", str, False, "Download URI"), + ("jsonl_download_uri", str, False, "JSONL download URI"), ("updated_at", str, False, "Last updated timestamp"), - ("size", int, False, "File size in bytes"), - ("content_type", str, False, "MIME type"), - ("content_encoding", str, False, "Content encoding"), + ("compressed_size", int, False, "Compressed file size in bytes"), ] diff --git a/tests/usage/fixtures/bulk_data_by_id__oracle_cards.json b/tests/usage/fixtures/bulk_data_by_id__oracle_cards.json index 165c4bb..d93d940 100644 --- a/tests/usage/fixtures/bulk_data_by_id__oracle_cards.json +++ b/tests/usage/fixtures/bulk_data_by_id__oracle_cards.json @@ -1,6 +1,6 @@ { "_provenance": { - "captured_at": "2026-05-30T23:30:47.712824+00:00", + "captured_at": "2026-08-11T09:01:55.863+00:00", "endpoint": "bulk-data/id", "source_url": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e", "scryfall_id": "27bf3214-1271-490b-bdfe-c0be6c23d02e" @@ -9,13 +9,11 @@ "object": "bulk_data", "id": "27bf3214-1271-490b-bdfe-c0be6c23d02e", "type": "oracle_cards", - "updated_at": "2026-05-30T21:07:31.013+00:00", + "updated_at": "2026-08-11T09:01:55.863+00:00", "uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e", "name": "Oracle Cards", "description": "A JSON file containing one Scryfall card object for each Oracle ID on Scryfall. The chosen sets for the cards are an attempt to return the most up-to-date recognizable version of the card.", - "size": 173115837, - "download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260530210731.json", - "content_type": "application/json", - "content_encoding": "gzip" + "jsonl_download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz", + "compressed_size": 24502785 } }