bulk_data accessors and download() are broken by Scryfall's bulk-data shape change

CHANGELOG.md

@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ---
 
+## [Unreleased]
+
+### Added
+- **`BulkDataObjectMixin.jsonl_download_uri`**: New accessor for the gzip-compressed JSONL download URI, replacing the removed `download_uri` field.
+- **`BulkDataObjectMixin.compressed_size`**: New accessor for the compressed file size in bytes, replacing the removed `size` field.
+
+### Changed
+- **`BulkDataObjectMixin.download()`**: Now targets `jsonl_download_uri` and parses the response as newline-delimited JSON (JSONL) rather than a single JSON array.
+- **`ScryfallBulkDataData`**: TypedDict updated to match the current Scryfall bulk-data shape — `download_uri`, `size`, `content_type`, and `content_encoding` removed; `jsonl_download_uri` and `compressed_size` added.
+
+### Removed
+- **`BulkDataObjectMixin.download_uri`**: Removed; Scryfall no longer returns this field. Use `jsonl_download_uri` instead.
+- **`BulkDataObjectMixin.size`**: Removed; Scryfall no longer returns this field. Use `compressed_size` instead.
+- **`BulkDataObjectMixin.content_type`**: Removed; Scryfall no longer returns this field.
+- **`BulkDataObjectMixin.content_encoding`**: Removed; Scryfall no longer returns this field.
+
+---
+
 ## [2.2.0] - 2026-06-09
 
 ### Added

scrython/bulk_data/bulk_data_mixins.py

@@ -65,54 +65,33 @@ def description(self) -> str:
         return self._scryfall_data["description"]
 
     @property
-    def download_uri(self) -> str:
+    def jsonl_download_uri(self) -> str:
         """
-        The URI that hosts this bulk file for fetching.
+        The URI to download this bulk file in gzip-compressed JSONL format.
 
         Type: URI (Required)
-
-        Note: Files may be compressed with gzip depending on CDN/proxy configuration.
-        The download() method automatically detects encoding from HTTP headers.
-        """
-        return self._scryfall_data["download_uri"]
-
-    @property
-    def updated_at(self) -> str:
-        """
-        The time when this file was last updated.
-
-        Type: Timestamp (Required)
-
-        Note: Bulk data files are updated approximately every 12 hours.
         """
-        return self._scryfall_data["updated_at"]
+        return self._scryfall_data["jsonl_download_uri"]
 
     @property
-    def size(self) -> int:
+    def compressed_size(self) -> int:
         """
-        The size of this file in integer bytes.
+        The size of the compressed file in integer bytes.
 
         Type: Integer (Required)
         """
-        return self._scryfall_data["size"]
+        return self._scryfall_data["compressed_size"]
 
     @property
-    def content_type(self) -> str:
-        """
-        The MIME type of this file.
-
-        Type: String (Required)
+    def updated_at(self) -> str:
         """
-        return self._scryfall_data["content_type"]
+        The time when this file was last updated.
 
-    @property
-    def content_encoding(self) -> str:
-        """
-        The Content-Encoding encoding that will be used to transmit this file when you download it.
+        Type: Timestamp (Required)
 
-        Type: String (Required)
+        Note: Bulk data files are updated approximately every 12 hours.
         """
-        return self._scryfall_data["content_encoding"]
+        return self._scryfall_data["updated_at"]
 
     def download(
         self,
@@ -122,17 +101,18 @@ def download(
         progress: bool = False,
     ) -> list[dict[str, Any]] | None:
         """
-        Download and parse bulk data file from Scryfall.
+        Download and parse the bulk JSONL file from Scryfall.
 
-        The bulk data file is downloaded from Scryfall's CDN. The method automatically
-        detects if the response is gzip-compressed by checking HTTP Content-Encoding
-        headers and handles decompression accordingly. The JSON data is then parsed
-        and optionally saved to a file.
+        The bulk data file is a gzip-compressed JSONL (newline-delimited JSON) file
+        downloaded from Scryfall's CDN. The method automatically detects if the
+        response is gzip-compressed by checking HTTP Content-Encoding headers and
+        handles decompression accordingly. Each line is parsed as a separate JSON
+        object and returned as a list.
 
         Args:
-            filepath: Optional path to save the decompressed JSON file.
+            filepath: Optional path to save the parsed data as a JSON file.
                      If None, file is not saved to disk.
-            return_data: If True, return parsed JSON data. If False and
+            return_data: If True, return parsed data. If False and
                         filepath is provided, only saves file without returning data.
                         Default: True.
             chunk_size: Download chunk size in bytes. Default: 8192.
@@ -162,7 +142,7 @@ def download(
             Bulk data files can be very large (100+ MB compressed, 500+ MB uncompressed).
             Be mindful of memory usage when loading entire files into memory.
         """
-        download_url = self.download_uri
+        download_url = self.jsonl_download_uri
 
         request = Request(download_url)
         request.add_header("User-Agent", ScrythonRequestHandler._user_agent)
@@ -205,7 +185,7 @@ def download(
                 ):
                     data = gzip.decompress(downloaded_data)
             else:
-                # Already decompressed or plain JSON
+                # Already decompressed or plain JSONL
                 data = downloaded_data
         else:
             # Download without progress bar
@@ -218,11 +198,13 @@ def download(
                     with gzip.GzipFile(fileobj=response) as gz_file:
                         data = gz_file.read()
                 else:
-                    # Read plain JSON
+                    # Read plain JSONL
                     data = response.read()
 
-        # Parse JSON. The annotation narrows json.loads's Any return; no cast needed.
-        parsed_data: list[dict[str, Any]] = json.loads(data.decode("utf-8"))
+        # Parse JSONL: each non-empty line is a separate JSON object.
+        parsed_data: list[dict[str, Any]] = [
+            json.loads(line) for line in data.decode("utf-8").splitlines() if line.strip()
+        ]
 
         # Save to file if requested
         if filepath:

scrython/types.py

@@ -285,10 +285,8 @@ class ScryfallBulkDataData(TypedDict):
     uri: URI
     name: str
     description: str
-    download_uri: URI
-    size: int
-    content_type: str
-    content_encoding: str
+    jsonl_download_uri: URI
+    compressed_size: int
 
 
 class ScryfallListData(TypedDict):

tests/conftest.py

@@ -223,9 +223,9 @@ def sample_bulk_data():
         "type": "oracle_cards",
         "name": "Oracle Cards",
         "description": "All cards, each uniquely identified by Oracle ID",
-        "download_uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e/download",
-        "updated_at": "2025-01-01T12:00:00.000Z",
-        "size": 123456789,
+        "jsonl_download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz",
+        "updated_at": "2026-08-11T09:01:55.863+00:00",
+        "compressed_size": 24502785,
     }
 
 

tests/fixtures/bulk_data/all.json

@@ -8,23 +8,21 @@
       "type": "oracle_cards",
       "name": "Oracle Cards",
       "description": "A JSON file containing one Scryfall card object for each Oracle ID on Scryfall.",
-      "download_uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e/download",
-      "updated_at": "2025-01-01T12:00:00.000Z",
-      "size": 123456789,
-      "content_type": "application/json",
-      "content_encoding": "gzip"
+      "updated_at": "2026-08-11T09:01:55.863+00:00",
+      "uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e",
+      "jsonl_download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz",
+      "compressed_size": 24502785
     },
     {
       "object": "bulk_data",
       "id": "922288cb-4bef-45e1-bb30-0c2bd3d3534e",
       "type": "unique_artwork",
       "name": "Unique Artwork",
       "description": "A JSON file of Scryfall card objects that together contain all unique artworks.",
-      "download_uri": "https://api.scryfall.com/bulk-data/922288cb-4bef-45e1-bb30-0c2bd3d3534e/download",
-      "updated_at": "2025-01-01T12:00:00.000Z",
-      "size": 234567890,
-      "content_type": "application/json",
-      "content_encoding": "gzip"
+      "updated_at": "2026-08-11T09:01:55.863+00:00",
+      "uri": "https://api.scryfall.com/bulk-data/922288cb-4bef-45e1-bb30-0c2bd3d3534e",
+      "jsonl_download_uri": "https://data.scryfall.io/unique-artwork/unique-artwork-20260811090155.jsonl.gz",
+      "compressed_size": 38912400
     }
   ]
 }

tests/fixtures/bulk_data/by_id.json

@@ -5,9 +5,7 @@
   "type": "oracle_cards",
   "name": "Oracle Cards",
   "description": "A JSON file containing one Scryfall card object for each Oracle ID on Scryfall.",
-  "download_uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e/download",
-  "updated_at": "2025-01-01T12:00:00.000Z",
-  "size": 123456789,
-  "content_type": "application/json",
-  "content_encoding": "gzip"
+  "updated_at": "2026-08-11T09:01:55.863+00:00",
+  "jsonl_download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz",
+  "compressed_size": 24502785
 }

tests/test_bulk_data.py

@@ -96,11 +96,11 @@ def test_bulk_data_object_mixin_properties(self, mock_urlopen):
         assert bulk.type == "oracle_cards"
         assert bulk.name == "Oracle Cards"
         assert (
-            bulk.download_uri
-            == "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e/download"
+            bulk.jsonl_download_uri
+            == "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz"
         )
-        assert bulk.updated_at == "2025-01-01T12:00:00.000Z"
-        assert bulk.size == 123456789
+        assert bulk.updated_at == "2026-08-11T09:01:55.863+00:00"
+        assert bulk.compressed_size == 24502785
 
     def test_bulk_data_object_from_list(self, mock_urlopen):
         """Test that BulkDataObject wrapper works correctly."""
@@ -122,9 +122,10 @@ def test_download_returns_parsed_data(self, mock_urlopen):
         mock_urlopen.set_response("bulk_data/by_id.json")
         bulk = ByType(type="oracle_cards")
 
-        # Mock the download URL response
+        # Mock the download URL response with JSONL format
         test_data = [{"id": "card1", "name": "Test Card"}]
-        compressed_data = gzip.compress(json.dumps(test_data).encode("utf-8"))
+        jsonl_data = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8")
+        compressed_data = gzip.compress(jsonl_data)
 
         with patch("scrython.bulk_data.bulk_data_mixins.urlopen") as mock_download:
             # Wrap compressed data in BytesIO for proper file-like behavior
@@ -147,7 +148,8 @@ def test_download_saves_to_file(self, mock_urlopen):
         bulk = ByType(type="oracle_cards")
 
         test_data = [{"id": "card1", "name": "Test Card"}]
-        compressed_data = gzip.compress(json.dumps(test_data).encode("utf-8"))
+        jsonl_data = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8")
+        compressed_data = gzip.compress(jsonl_data)
 
         with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json") as tmp:
             tmp_path = tmp.name
@@ -182,7 +184,8 @@ def test_download_without_return_data(self, mock_urlopen):
         bulk = ByType(type="oracle_cards")
 
         test_data = [{"id": "card1", "name": "Test Card"}]
-        compressed_data = gzip.compress(json.dumps(test_data).encode("utf-8"))
+        jsonl_data = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8")
+        compressed_data = gzip.compress(jsonl_data)
 
         with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".json") as tmp:
             tmp_path = tmp.name
@@ -259,18 +262,18 @@ def test_download_progress_without_tqdm_raises_import_error(self, mock_urlopen):
             bulk.download(progress=True)
 
     def test_download_uncompressed_no_progress(self, mock_urlopen):
-        """Test download handles uncompressed JSON without progress bar."""
+        """Test download handles uncompressed JSONL without progress bar."""
         mock_urlopen.set_response("bulk_data/by_id.json")
         bulk = ByType(type="oracle_cards")
 
-        # Test with plain JSON (not gzip compressed)
+        # Test with plain JSONL (not gzip compressed)
         test_data = [{"id": "card1", "name": "Test Card"}]
-        plain_json = json.dumps(test_data).encode("utf-8")
+        plain_jsonl = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8")
 
         with patch("scrython.bulk_data.bulk_data_mixins.urlopen") as mock_download:
             # Create mock with NO Content-Encoding header (empty string)
             mock_response = MagicMock()
-            mock_response.read.return_value = plain_json
+            mock_response.read.return_value = plain_jsonl
             mock_response.info.return_value.get.return_value = ""  # No encoding header
             mock_response.__enter__.return_value = mock_response
             mock_response.__exit__.return_value = None
@@ -283,25 +286,25 @@ def test_download_uncompressed_no_progress(self, mock_urlopen):
             assert result[0]["name"] == "Test Card"
 
     def test_download_uncompressed_with_progress(self, mock_urlopen):
-        """Test download handles uncompressed JSON with progress bar."""
+        """Test download handles uncompressed JSONL with progress bar."""
         # Skip if tqdm is not installed
         pytest.importorskip("tqdm")
 
         mock_urlopen.set_response("bulk_data/by_id.json")
         bulk = ByType(type="oracle_cards")
 
-        # Test with plain JSON (not gzip compressed)
+        # Test with plain JSONL (not gzip compressed)
         test_data = [{"id": "card1", "name": "Test Card"}]
-        plain_json = json.dumps(test_data).encode("utf-8")
+        plain_jsonl = "\n".join(json.dumps(obj) for obj in test_data).encode("utf-8")
 
         with patch("scrython.bulk_data.bulk_data_mixins.urlopen") as mock_download:
             # Create mock with NO Content-Encoding header
             mock_response = MagicMock()
             mock_response.read.side_effect = [
-                plain_json,
+                plain_jsonl,
                 b"",
             ]  # Return data then empty to signal EOF
-            mock_response.headers.get.return_value = str(len(plain_json))
+            mock_response.headers.get.return_value = str(len(plain_jsonl))
             mock_response.info.return_value.get.return_value = ""  # No encoding header
             mock_response.__enter__.return_value = mock_response
             mock_response.__exit__.return_value = None
@@ -325,7 +328,7 @@ def test_download_sets_headers(self, mock_urlopen):
         with patch("scrython.bulk_data.bulk_data_mixins.urlopen") as mock_download:
             # Set up mock to allow inspection of the Request object
             mock_response = MagicMock()
-            mock_response.read.return_value = b"[]"
+            mock_response.read.return_value = b""
             mock_response.info.return_value.get.return_value = ""
             mock_response.__enter__.return_value = mock_response
             mock_response.__exit__.return_value = None

tests/test_property_types.py

@@ -226,11 +226,9 @@ def test_sets_field_type(self, mock_urlopen, prop, expected_type, nullable, desc
     ("type", str, False, "Bulk data type"),
     ("name", str, False, "Bulk data name"),
     ("description", str, False, "Description"),
-    ("download_uri", str, False, "Download URI"),
+    ("jsonl_download_uri", str, False, "JSONL download URI"),
     ("updated_at", str, False, "Last updated timestamp"),
-    ("size", int, False, "File size in bytes"),
-    ("content_type", str, False, "MIME type"),
-    ("content_encoding", str, False, "Content encoding"),
+    ("compressed_size", int, False, "Compressed file size in bytes"),
 ]
 
 

tests/usage/fixtures/bulk_data_by_id__oracle_cards.json

@@ -1,6 +1,6 @@
 {
   "_provenance": {
-    "captured_at": "2026-05-30T23:30:47.712824+00:00",
+    "captured_at": "2026-08-11T09:01:55.863+00:00",
     "endpoint": "bulk-data/id",
     "source_url": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e",
     "scryfall_id": "27bf3214-1271-490b-bdfe-c0be6c23d02e"
@@ -9,13 +9,11 @@
     "object": "bulk_data",
     "id": "27bf3214-1271-490b-bdfe-c0be6c23d02e",
     "type": "oracle_cards",
-    "updated_at": "2026-05-30T21:07:31.013+00:00",
+    "updated_at": "2026-08-11T09:01:55.863+00:00",
     "uri": "https://api.scryfall.com/bulk-data/27bf3214-1271-490b-bdfe-c0be6c23d02e",
     "name": "Oracle Cards",
     "description": "A JSON file containing one Scryfall card object for each Oracle ID on Scryfall. The chosen sets for the cards are an attempt to return the most up-to-date recognizable version of the card.",
-    "size": 173115837,
-    "download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260530210731.json",
-    "content_type": "application/json",
-    "content_encoding": "gzip"
+    "jsonl_download_uri": "https://data.scryfall.io/oracle-cards/oracle-cards-20260811090155.jsonl.gz",
+    "compressed_size": 24502785
   }
 }