✨ Cache the extracted core.sqlite_zip database files

docs/source/howto/archive_profile.md

@@ -47,6 +47,38 @@ verdi -p https://example.com/process.aiida shell
 
 This creates a temporary profile that mounts the archive for the duration of the command only: nothing is added to the AiiDA configuration file and any temporary files are cleaned up when the command finishes.
 
+## Caching the database
+
+By default, the SQLite database contained in the archive is extracted (and for remote archives, downloaded) again for every Python session.
+For large or remote archives this can be avoided by caching the extracted database locally, in the `cache/sqlite_zip` subdirectory of the AiiDA configuration folder.
+Cache entries are named after the checksum and size of the database recorded in the archive, so validating the cache only requires reading the table of contents of the archive (for remote archives, a small range request), never the database itself.
+
+An existing valid cache entry is always used automatically.
+The cache is only *written* when explicitly requested, either for a single invocation with the top-level `--use-cache` option:
+
+```{code-block} console
+verdi --use-cache -p https://example.com/process.aiida process list -a
+```
+
+or when creating a profile, in which case the cached database is also recorded for the profile in the configuration file:
+
+```{code-block} console
+verdi profile setup core.sqlite_zip -n --profile-name archive --filepath process.aiida --use-cache
+```
+
+For such profiles, a deleted cache entry is transparently recreated on the next load.
+If instead the *archive itself* changed since the cache was created, loading the profile stops with an error, and the recorded cache can be updated explicitly with:
+
+```{code-block} console
+verdi profile cache-refresh archive
+```
+
+When the archive is unreachable (e.g. working offline with a remote archive), a profile with a recorded cached database can still be used by passing the top-level `--force-cache` option.
+Note that in this case the data is not validated against the archive, and repository files, which are always read directly from the archive, may not be accessible.
+
+The cache can be deleted at any time with `verdi profile cache-clear`: entries are recreated on the next load of a profile configured for caching.
+Note however that recreating an entry requires access to the archive: if you rely on `--force-cache` to work with an unreachable archive, clearing the cache deletes your only usable copy of the data.
+
 You can now inspect the contents of the `process.aiida` archive by using the `archive` profile in the same way you would a standard AiiDA profile.
 For example, you can start an interactive shell using `verdi -p archive shell` or if you are already in a notebook simply load the profile:
 

docs/source/reference/command_line.rst

@@ -412,6 +412,8 @@ Below is a list with all available subcommands.
       --help  Show this message and exit.
 
     Commands:
+      cache-clear         Delete all locally cached database files of `core.sqlite_zip`...
+      cache-refresh       Refresh the cached database for a profile using the...
       configure-rabbitmq  Configure RabbitMQ for a profile.
       delete              Delete one or more profiles.
       dump                Dump all data in an AiiDA profile's storage to disk.

src/aiida/cmdline/commands/cmd_profile.py

@@ -284,6 +284,81 @@ def _profile_set_default(profile):
     echo.echo_success(f'{profile.name} set as default profile')
 
 
+@verdi_profile.command('cache-refresh')
+@arguments.PROFILE(required=True, default=None)
+def profile_cache_refresh(profile):
+    """Refresh the cached database for a profile using the `core.sqlite_zip` storage backend.
+
+    The database file of the archive is extracted (or for remote archives, downloaded) and stored in the local
+    cache, and it is recorded for the profile in the configuration file, replacing any previously recorded version.
+    """
+    from aiida.storage.sqlite_zip.backend import SqliteZipBackend
+    from aiida.storage.sqlite_zip.cache import get_cache_dirpath
+
+    if profile.storage_backend != 'core.sqlite_zip':
+        echo.echo_critical(f'profile `{profile.name}` does not use the `core.sqlite_zip` storage backend.')
+
+    # Re-fetch the pristine profile from the configuration: the ``profile`` argument may be a detached copy carrying
+    # transient keys injected by the ``verdi --use-cache/--force-cache`` flags, which must never be persisted.
+    config = get_config()
+    profile = config.get_profile(profile.name)
+    filename_old = profile.storage_config.get('cached_database')
+
+    try:
+        filename = SqliteZipBackend.refresh_cache(profile)
+    except exceptions.AiidaException as exception:
+        echo.echo_critical(f'could not refresh the cache: {exception}')
+
+    config.update_profile(profile)
+    config.store()
+
+    if filename_old is not None and filename_old != filename:
+        # The previously recorded entry is deleted, unless another profile still records it.
+        if any(
+            other.storage_config.get('cached_database') == filename_old
+            for other in config.profiles
+            if other.name != profile.name
+        ):
+            echo.echo_report(f'The previously cached database `{filename_old}` is recorded by other profiles: kept.')
+        else:
+            (get_cache_dirpath() / filename_old).unlink(missing_ok=True)
+            echo.echo_report(f'Deleted the previously cached database `{filename_old}`.')
+
+    echo.echo_success(f'Cached database `{filename}` recorded for profile `{profile.name}`.')
+
+
+@verdi_profile.command('cache-clear')
+@options.FORCE(help='Skip any prompts for confirmation.')
+def profile_cache_clear(force):
+    """Delete all locally cached database files of `core.sqlite_zip` storages.
+
+    Cached databases are recreated on the first use of a profile configured for caching, and profiles that are not,
+    are unaffected. Note however that recreating an entry requires access to the archive: if you rely on `verdi
+    --force-cache` to work with an unreachable archive, clearing the cache deletes your only usable copy of the data.
+    """
+    from aiida.storage.sqlite_zip.cache import get_cache_dirpath
+
+    dirpath = get_cache_dirpath()
+    filepaths = sorted(filepath for filepath in dirpath.iterdir() if filepath.is_file()) if dirpath.is_dir() else []
+
+    if not filepaths:
+        echo.echo_report(f'The cache at `{dirpath}` is empty: nothing to delete.')
+        return
+
+    size_mb = sum(filepath.stat().st_size for filepath in filepaths) / 1e6
+
+    if not force:
+        echo.echo_warning(
+            f'This will delete {len(filepaths)} cached database file(s) ({size_mb:.1f} MB) at `{dirpath}`.'
+        )
+        click.confirm('Do you want to continue?', abort=True)
+
+    for filepath in filepaths:
+        filepath.unlink()
+
+    echo.echo_success(f'Deleted {len(filepaths)} cached database file(s) ({size_mb:.1f} MB).')
+
+
 @verdi_profile.command('delete')
 @options.FORCE(help='Skip any prompts for confirmation.')
 @click.option(

src/aiida/cmdline/commands/cmd_verdi.py

@@ -18,6 +18,8 @@
 
 # Pass the version explicitly to ``version_option`` otherwise editable installs can show the wrong version number
 @click.group(cls=VerdiCommandGroup, context_settings={'help_option_names': ['--help', '-h']})
+@options.USE_CACHE()
+@options.FORCE_CACHE()
 @options.PROFILE(type=types.ProfileParamType(load_profile=True, accept_archive_location=True), expose_value=False)
 @options.VERBOSITY()
 @click.version_option(__version__, package_name='aiida_core', message='AiiDA version %(version)s')

src/aiida/cmdline/params/options/__init__.py

@@ -62,6 +62,7 @@
     'FILTER_BY_LAST_DUMP_TIME',
     'FLAT',
     'FORCE',
+    'FORCE_CACHE',
     'FORMULA_MODE',
     'FREQUENCY',
     'GROUP',
@@ -118,6 +119,7 @@
     'USER_FIRST_NAME',
     'USER_INSTITUTION',
     'USER_LAST_NAME',
+    'USE_CACHE',
     'VERBOSITY',
     'VISUALIZATION_FORMAT',
     'WITH_ELEMENTS',

src/aiida/cmdline/params/options/main.py

@@ -73,6 +73,7 @@
     'FILTER_BY_LAST_DUMP_TIME',
     'FLAT',
     'FORCE',
+    'FORCE_CACHE',
     'FORMULA_MODE',
     'FREQUENCY',
     'GROUP',
@@ -129,6 +130,7 @@
     'USER_FIRST_NAME',
     'USER_INSTITUTION',
     'USER_LAST_NAME',
+    'USE_CACHE',
     'VERBOSITY',
     'VISUALIZATION_FORMAT',
     'WITH_ELEMENTS',
@@ -261,6 +263,44 @@ def set_log_level(ctx: click.Context, _param: click.Parameter, value: t.Any) ->
     help='Execute the command for this profile instead of the default profile.',
 )
 
+
+def set_use_cache(ctx: click.Context, _param: click.Parameter, value: bool) -> bool:
+    """Set the ``use_cache`` flag on the context user object, to be picked up during profile loading."""
+    if value:
+        ctx.obj.use_cache = True
+    return value
+
+
+def set_force_cache(ctx: click.Context, _param: click.Parameter, value: bool) -> bool:
+    """Set the ``force_cache`` flag on the context user object, to be picked up during profile loading."""
+    if value:
+        ctx.obj.force_cache = True
+    return value
+
+
+USE_CACHE = OverridableOption(
+    '--use-cache',
+    is_flag=True,
+    default=False,
+    is_eager=True,
+    expose_value=False,
+    callback=set_use_cache,
+    help='For profiles using the `core.sqlite_zip` storage: store the database file extracted from the archive in '
+    'the local cache, such that subsequent loads do not need to extract (or download) it again.',
+)
+
+FORCE_CACHE = OverridableOption(
+    '--force-cache',
+    is_flag=True,
+    default=False,
+    is_eager=True,
+    expose_value=False,
+    callback=set_force_cache,
+    help='For profiles using the `core.sqlite_zip` storage with a recorded cached database: use the cached database '
+    'directly, without validating it against the archive. Allows working when the archive is unreachable, but the '
+    'data may be outdated and repository files may not be accessible.',
+)
+
 CALCULATION = OverridableOption(
     '-C',
     '--calculation',

src/aiida/cmdline/params/types/profile.py

@@ -85,13 +85,59 @@ def convert(self, value: t.Any, param: click.Parameter | None, ctx: click.Contex
                 if self._cannot_exist:
                     self.fail(str(f'the profile `{value}` already exists'))
 
+        profile = self._apply_cache_flags(profile, ctx)
+
         if self._load_profile:
             load_profile(profile)
 
         ctx.obj.profile = profile  # type: ignore[union-attr]
 
         return profile
 
+    @staticmethod
+    def _apply_cache_flags(profile: Profile, ctx: click.Context | None) -> Profile:
+        """Record the ``--use-cache``/``--force-cache`` flags from the context in the profile, if they were set.
+
+        For profiles using the ``core.sqlite_zip`` storage backend, the flags are recorded in the storage
+        configuration of a detached copy of the profile, such that they are never persisted to the configuration
+        file. For other storage backends the flags do not apply and are ignored with a warning.
+        """
+        import copy
+
+        from aiida.manage.configuration import Profile
+
+        use_cache = getattr(ctx.obj, 'use_cache', False) if ctx else False
+        force_cache = getattr(ctx.obj, 'force_cache', False) if ctx else False
+
+        if not (use_cache or force_cache):
+            return profile
+
+        try:
+            storage_backend = profile.storage_backend
+        except KeyError:
+            return profile
+
+        if storage_backend != 'core.sqlite_zip':
+            from aiida.cmdline.utils import echo
+
+            names = [name for name, passed in (('--use-cache', use_cache), ('--force-cache', force_cache)) if passed]
+            flags = ' and '.join(f'`{name}`' for name in names)
+            echo.echo_warning(
+                f'The {flags} option{"s" if len(names) > 1 else ""} only appl{"y" if len(names) > 1 else "ies"} to '
+                f'profiles using the `core.sqlite_zip` storage backend, but profile `{profile.name}` uses '
+                f'`{storage_backend}`: ignoring.'
+            )
+            return profile
+
+        profile = Profile(profile.name, copy.deepcopy(profile.dictionary))
+
+        if use_cache:
+            profile.storage_config['use_cache'] = True
+        if force_cache:
+            profile.storage_config['force_cache'] = True
+
+        return profile
+
     def _convert_archive_location(
         self, value: str, param: click.Parameter | None, ctx: click.Context | None
     ) -> Profile:

src/aiida/manage/configuration/config.py

@@ -654,6 +654,14 @@ def update_profile(self, profile):
         :param profile: the profile instance to update
         :return: self
         """
+        # The ``force_cache`` key of the ``core.sqlite_zip`` storage configuration is transient: it is injected into
+        # a detached profile copy by the ``verdi --force-cache`` flag and must never be persisted, since a persisted
+        # ``force_cache`` would permanently and silently skip the validation of the archive on every load.
+        try:
+            profile.storage_config.pop('force_cache', None)
+        except KeyError:
+            # The profile does not define a storage configuration at all, e.g. a new empty profile.
+            pass
         self._profiles[profile.name] = profile
         return self