diff --git a/iiify/resolver.py b/iiify/resolver.py index 959b9bb7..066702f8 100644 --- a/iiify/resolver.py +++ b/iiify/resolver.py @@ -3,6 +3,7 @@ import os import requests from .configs import options, cors, approot, cache_root, media_root, apiurl, LINKS +from .utils.http_config import timeout_session from iiif_prezi3 import config, Collection, Manifest, Canvas, Annotation, AnnotationPage, CollectionRef, ManifestRef, CanvasRef, AnnotationPageRef, AnnotationPageRefExtended, AnnotationBody, ServiceV3, Choice, TextualBody, AccompanyingCanvas, Range from urllib.parse import urlparse, parse_qs, quote import json @@ -176,8 +177,8 @@ def create_collection3(identifier, domain, page=1, rows=MAX_API_LIMIT): # Get item metadata metadata = requests.get('%s/metadata/%s' % (ARCHIVE, identifier)).json() - # Used to build up URIs for the manifest - uri = f"{domain}{identifier}/collection.json" + # Write Collection uri with conditional page segment + uri = f"{domain}{identifier}{f'/{page}' if page != 1 else ''}/collection.json" config.configs['helpers.auto_fields.AutoLang'].auto_lang = "none" collection = Collection(id=uri, label=metadata["metadata"]["title"]) @@ -595,18 +596,32 @@ def addThumbnails(manifest, identifier, files): elif ia_thumb_files: files_to_process = ia_thumb_files - for file in files_to_process: - name = file.get("name", "") - encoded_name = quote(name.replace('/', '%2f')) - # Forward solidus before thumbnail uri must always be %2f - iiif_url = f"{IMG_SRV}/2/{identifier.strip()}%2f{encoded_name}" - try: - manifest.create_thumbnail_from_iiif(iiif_url) - except requests.HTTPError: - print(f"Failed to generate thumbnail from Cantaloupe: {iiif_url}") + MAX_IIIF_THUMB = 15 + # if we use cantaloupe for more than MAX_IIIF_THUMB + # its likely to be too slow and will fail to generate + if len(files_to_process) > MAX_IIIF_THUMB: + for file in files_to_process: + # print (f"Getting static thumbnail for {file['name']}") + name = file.get("name", "") mimetype = "image/png" if name.endswith(".png") else "image/jpeg" static_url = f"{ARCHIVE}/download/{quote(identifier)}/{quote(name)}" manifest.add_thumbnail(static_url, format=mimetype) + + else: + for file in files_to_process: + name = file.get("name", "") + encoded_name = quote(name.replace('/', '%2f')) + # Forward solidus before thumbnail uri must always be %2f + iiif_url = f"{IMG_SRV}/2/{identifier.strip()}%2f{encoded_name}" + try: + session = timeout_session(timeout=1,retry=1) + # print(f'Getting image {iiif_url}') + manifest.create_thumbnail_from_iiif(iiif_url, iiif_session=session) + except requests.exceptions.RequestException as e: + print(f"Failed to generate thumbnail from Cantaloupe: {iiif_url} due to {e}") + mimetype = "image/png" if name.endswith(".png") else "image/jpeg" + static_url = f"{ARCHIVE}/download/{quote(identifier)}/{quote(name)}" + manifest.add_thumbnail(static_url, format=mimetype) return def addThumbnailNav(manifest, identifier, files): @@ -755,10 +770,10 @@ def create_manifest3(identifier, domain=None, page=None): manifest = Manifest(id=f"{uri}/manifest.json", label=metadata["metadata"]["title"]) if 'reviews' in metadata: - reviews_as_annotations = AnnotationPageRef(__root__=AnnotationPageRefExtended( + reviews_as_annotations = AnnotationPageRefExtended( id=f"{domain.replace('iiif/', 'iiif/3/annotations/')}{identifier}/comments.json", type="AnnotationPage", - )) + ) manifest.annotations=[reviews_as_annotations] addMetadata(manifest, identifier, metadata['metadata']) addSeeAlso(manifest, identifier, metadata['files']) @@ -1106,6 +1121,7 @@ def create_manifest3(identifier, domain=None, page=None): if imgs: pageCount = video_count for file in imgs: + # print (f"Making canvas for {file['name']}") imgId = f"{identifier}/{file['name']}".replace('/','%2f') imgURL = f"{IMG_SRV}/3/{imgId}" pageCount += 1 @@ -1116,7 +1132,9 @@ def create_manifest3(identifier, domain=None, page=None): id=f"{URI_PRIFIX}/{identifier}${pageCount}/canvas", label=f"{file['name']}", anno_page_id=f"{URI_PRIFIX}/{identifier}/{slugged_id}/page", - anno_id=f"{URI_PRIFIX}/{identifier}/{slugged_id}/annotation" + anno_id=f"{URI_PRIFIX}/{identifier}/{slugged_id}/annotation", + iiif_session=timeout_session(retry=2, timeout=3 + ) ) elif mediatype == "collection": raise IsCollection diff --git a/iiify/utils/http_config.py b/iiify/utils/http_config.py new file mode 100644 index 00000000..be5a602f --- /dev/null +++ b/iiify/utils/http_config.py @@ -0,0 +1,28 @@ + +import requests +from urllib3.util import Retry +from requests.adapters import HTTPAdapter + +class TimeoutHTTPAdapter(HTTPAdapter): + def __init__(self, *args, timeout=5, **kwargs): + self.timeout = timeout + super().__init__(*args, **kwargs) + + def send(self, request, **kwargs): + if kwargs.get('timeout') is None: + kwargs['timeout'] = self.timeout + return super().send(request, **kwargs) + +def timeout_session(timeout=1, retry=1): + session = requests.Session() + retries = Retry( + total=retry, + backoff_factor=0.1, + status_forcelist=[502, 503, 504], + ) + adapter = TimeoutHTTPAdapter(max_retries=retries,timeout=timeout) + + session.mount('http://', adapter) + session.mount('https://', adapter) + + return session \ No newline at end of file diff --git a/tests/test_annotations.py b/tests/test_annotations.py index f6a300a6..1aaebf71 100644 --- a/tests/test_annotations.py +++ b/tests/test_annotations.py @@ -65,7 +65,7 @@ def test_review_annotations(self): self.assertEqual(resp.status_code, 200) self.assertEqual(annotations['@context'], "http://iiif.io/api/presentation/3/context.json","Unexpected context") self.assertEqual(annotations['type'], "AnnotationPage", "Unexpected type, expected AnnotationPage") - self.assertEqual(len(annotations['items']), 37, "Unexpected number of annotations") + self.assertEqual(len(annotations['items']), 38, "Unexpected number of annotations") ids = [] for anno in annotations['items']: self.assertEqual(anno['type'], "Annotation", "Expected type of Annotation") diff --git a/tests/test_collections.py b/tests/test_collections.py index 3d418bcb..2cca3481 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -46,6 +46,11 @@ def test_sanitized_summary(self): f"Expected summary to be sanitized but got: {collection['summary']['none'][0]}" ) + def test_unique_id_sub_page(self): + resp = self.test_app.get("/iiif/usgs_tx/2/collection.json?recache=True") + self.assertEqual(resp.status_code, 200) + collection = resp.json + self.assertEqual(collection['id'], "https://localhost/iiif/usgs_tx/2/collection.json", f"Expected unique ID for the second page of the results but got: {collection['id']}") if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/tests/test_manifests.py b/tests/test_manifests.py index f3500604..5968f150 100644 --- a/tests/test_manifests.py +++ b/tests/test_manifests.py @@ -72,7 +72,7 @@ def test_v3_128Kbps_MP3(self): self.assertEqual(resp.status_code, 200) manifest = resp.json self.assertEqual(len(manifest['items']),12,f"Expected 12 canvases but got: {len(manifest['items'])}") - self.assertEqual("128kbps mp3".lower() in resp.text.lower(), True, f"Expected the string '128kbps mp3'") + self.assertEqual("64kbps mp3".lower() in resp.text.lower(), True, f"Expected the string '64kbps mp3'") def test_v3_aiff(self): resp = self.test_app.get("/iiif/3/julian-damian-marley-with-the-uprising-band-shoreline-81697/manifest.json") diff --git a/tests/test_video.py b/tests/test_video.py index 120be6c0..473e85aa 100644 --- a/tests/test_video.py +++ b/tests/test_video.py @@ -16,13 +16,13 @@ def test_v3_single_video_manifest(self): self.assertEqual(resp.status_code, 200) manifest = resp.json - self.assertEqual(len(manifest['items']),1,f"Expected 1 canvas but got: {len(manifest['items'])}") + self.assertEqual(len(manifest['items']),2,f"Expected 2 canvas but got: {len(manifest['items'])}") def test_v3_h264_MPEG4_OGG_Theora(self): resp = self.test_app.get("/iiif/3/taboca_201002_03/manifest.json") self.assertEqual(resp.status_code, 200) manifest = resp.json - self.assertEqual(len(manifest['items']),251,f"Expected 251 canvases but got: {len(manifest['items'])}") + self.assertEqual(len(manifest['items']),503,f"Expected 503 canvases but got: {len(manifest['items'])}") self.assertEqual("h.264 MPEG4".lower() in resp.text.lower(), True, f"Expected the string 'h.264 MPEG4'") self.assertEqual("OGG Theora".lower() in resp.text.lower(), True, f"Expected the string 'OGG Theora'") @@ -31,7 +31,7 @@ def test_vtt_autogenerated(self): self.assertEqual(resp.status_code, 200) manifest = resp.json - self.assertEqual(len(manifest['items']),1,f"Expected 1 canvas but got: {len(manifest['items'])}") + self.assertEqual(len(manifest['items']),2,f"Expected 2 canvas but got: {len(manifest['items'])}") self.assertTrue('annotations' in manifest['items'][0], "Expected annotations in manifest") self.assertTrue(isinstance(manifest['items'][0]['annotations'], list), "Expected annotations to be a list") self.assertEqual(len(manifest['items'][0]['annotations']), 1, "Expected 1 item in annotations")