Support fetching intermediates

upki-cli/src/bin/upki.rs

@@ -12,9 +12,11 @@ use tracing::level_filters::LevelFilter;
 use tracing_subscriber::EnvFilter;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
-use upki::revocation::{Index, RevocationCheckInput};
 #[cfg(feature = "__fetch")]
-use upki::revocation::{Manifest, fetch};
+use upki::intermediates;
+#[cfg(feature = "__fetch")]
+use upki::revocation;
+use upki::revocation::{Index, RevocationCheckInput};
 use upki::{Config, ConfigPath};
 
 #[tokio::main(flavor = "current_thread")]
@@ -47,9 +49,19 @@ async fn main() -> Result<ExitCode, Report> {
 
     Ok(match args.command {
         #[cfg(feature = "__fetch")]
-        Command::Fetch { dry_run } => fetch(dry_run, &config).await?,
+        Command::Fetch { dry_run } => {
+            revocation::fetch(dry_run, &config).await?;
+            intermediates::fetch(dry_run, &config).await?;
+            ExitCode::SUCCESS
+        }
         #[cfg(feature = "__fetch")]
-        Command::Verify => Manifest::from_config(&config)?.verify(&config)?,
+        Command::Verify => {
+            revocation::Manifest::from_config(&config)?.verify(&config)?;
+            if config.intermediates.enabled {
+                intermediates::Manifest::from_config(&config)?.verify(&config)?;
+            }
+            ExitCode::SUCCESS
+        }
         Command::ShowConfigPath => unreachable!(),
         Command::ShowConfig => {
             print!(

upki-cli/tests/data/verify_non_existent_dir/config.toml

@@ -2,3 +2,7 @@ cache-dir = "not-exist/"
 
 [revocation]
 fetch-url = ""
+
+[intermediates]
+enabled = false
+fetch-url = ""

upki-cli/tests/data/verify_of_empty_intermediates_manifest/config.toml

@@ -0,0 +1,8 @@
+cache-dir = "tests/data/verify_of_empty_intermediates_manifest/"
+
+[revocation]
+fetch-url = ""
+
+[intermediates]
+enabled = true
+fetch-url = ""

upki-cli/tests/data/verify_of_empty_intermediates_manifest/intermediates/manifest.json

@@ -0,0 +1,5 @@
+{
+  "generated_at": 1765445031,
+  "comment": "empty manifest",
+  "files": []
+}

upki-cli/tests/data/verify_of_empty_intermediates_manifest/revocation/manifest.json

@@ -0,0 +1,5 @@
+{
+  "generated_at": 1765445031,
+  "comment": "empty manifest",
+  "files": []
+}

upki-cli/tests/integration.rs

@@ -49,7 +49,7 @@ fn config_unknown_fields() {
           |
         1 | cache_dir = "tests/data/config_unknown_fields/"
           | ^^^^^^^^^
-        unknown field `cache_dir`, expected `cache-dir` or `revocation`
+        unknown field `cache_dir`, expected one of `cache-dir`, `revocation`, `intermediates`
 
 
     Location:
@@ -92,6 +92,10 @@ fn show_config_fixpoint() {
     [revocation]
     fetch-url = ""
 
+    [intermediates]
+    enabled = false
+    fetch-url = ""
+
     ----- stderr -----
     "#);
 }
@@ -137,11 +141,28 @@ fn verify_of_empty_manifest() {
     ");
 }
 
+#[test]
+fn verify_of_empty_intermediates_manifest() {
+    let _filters = apply_common_filters();
+    assert_cmd_snapshot!(
+        upki()
+            .arg("--config-file")
+            .arg("tests/data/verify_of_empty_intermediates_manifest/config.toml")
+            .arg("verify"),
+        @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+
+    ----- stderr -----
+    ");
+}
+
 #[test]
 fn fetch_of_empty_manifest() {
     let _filters = apply_common_filters();
     let (server, _filters) = http_server("tests/data/verify_of_empty_manifest/");
-    let (temp, config_file, _filters) = temp_dir_and_config(server.url());
+    let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config);
 
     assert_cmd_snapshot!(
         upki()
@@ -157,7 +178,7 @@ fn fetch_of_empty_manifest() {
     ");
     assert_snapshot!(
         server.into_log(),
-        @"GET /manifest.json  ->  200 OK (79 bytes)"
+        @"GET /revocation/manifest.json  ->  200 OK (79 bytes)"
     );
     assert_eq!(
         list_dir(&temp.path().join("revocation")),
@@ -169,7 +190,7 @@ fn fetch_of_empty_manifest() {
 fn full_fetch() {
     let _filters = apply_common_filters();
     let (server, _filters) = http_server("tests/data/typical/");
-    let (temp, config_file, _filters) = temp_dir_and_config(server.url());
+    let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config);
 
     assert_cmd_snapshot!(
         upki()
@@ -186,10 +207,10 @@ fn full_fetch() {
     assert_snapshot!(
         server.into_log(),
         @r"
-    GET /manifest.json  ->  200 OK (530 bytes)
-    GET /filter1.filter  ->  200 OK (11 bytes)
-    GET /filter2.delta  ->  200 OK (14 bytes)
-    GET /filter3.delta  ->  200 OK (10 bytes)
+    GET /revocation/manifest.json  ->  200 OK (530 bytes)
+    GET /revocation/filter1.filter  ->  200 OK (11 bytes)
+    GET /revocation/filter2.delta  ->  200 OK (14 bytes)
+    GET /revocation/filter3.delta  ->  200 OK (10 bytes)
     ");
     assert_eq!(
         list_dir(&temp.path().join("revocation")),
@@ -206,7 +227,7 @@ fn full_fetch() {
 fn full_fetch_and_incremental_update() {
     let _filters = apply_common_filters();
     let (server, _filters) = http_server("tests/data/typical/");
-    let (temp, config_file, _filters) = temp_dir_and_config(server.url());
+    let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config);
 
     assert_cmd_snapshot!(
         upki()
@@ -223,10 +244,10 @@ fn full_fetch_and_incremental_update() {
     assert_snapshot!(
         server.into_log(),
         @r"
-    GET /manifest.json  ->  200 OK (530 bytes)
-    GET /filter1.filter  ->  200 OK (11 bytes)
-    GET /filter2.delta  ->  200 OK (14 bytes)
-    GET /filter3.delta  ->  200 OK (10 bytes)
+    GET /revocation/manifest.json  ->  200 OK (530 bytes)
+    GET /revocation/filter1.filter  ->  200 OK (11 bytes)
+    GET /revocation/filter2.delta  ->  200 OK (14 bytes)
+    GET /revocation/filter3.delta  ->  200 OK (10 bytes)
     ");
     assert_eq!(
         list_dir(&temp.path().join("revocation")),
@@ -257,8 +278,8 @@ fn full_fetch_and_incremental_update() {
     assert_snapshot!(
         server.into_log(),
         @r"
-    GET /manifest.json  ->  200 OK (545 bytes)
-    GET /filter4.delta  ->  200 OK (3 bytes)
+    GET /revocation/manifest.json  ->  200 OK (545 bytes)
+    GET /revocation/filter4.delta  ->  200 OK (3 bytes)
     ");
     // filter2 could be deleted, filter4 is new
     assert_eq!(
@@ -289,7 +310,7 @@ fn full_fetch_and_incremental_update() {
     ");
     assert_snapshot!(
         server.into_log(),
-        @"GET /manifest.json  ->  200 OK (545 bytes)");
+        @"GET /revocation/manifest.json  ->  200 OK (545 bytes)");
 
     // filter2 is now deleted
     assert_eq!(
@@ -307,7 +328,7 @@ fn full_fetch_and_incremental_update() {
 fn typical_incremental_fetch() {
     let _filters = apply_common_filters();
     let (server, _filters) = http_server("tests/data/typical/");
-    let (temp, config_file, _filters) = temp_dir_and_config(server.url());
+    let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config);
 
     fs::copy(
         "tests/data/typical/revocation/manifest.json",
@@ -345,8 +366,8 @@ fn typical_incremental_fetch() {
     assert_snapshot!(
         server.into_log(),
         @r"
-    GET /manifest.json  ->  200 OK (530 bytes)
-    GET /filter2.delta  ->  200 OK (14 bytes)
+    GET /revocation/manifest.json  ->  200 OK (530 bytes)
+    GET /revocation/filter2.delta  ->  200 OK (14 bytes)
     ");
 
     assert_eq!(list_dir(temp.path()), vec!["config.toml", "revocation",],);
@@ -366,7 +387,7 @@ fn typical_incremental_fetch() {
 fn typical_incremental_fetch_dry_run() {
     let _filters = apply_common_filters();
     let (server, _filters) = http_server("tests/data/typical/");
-    let (temp, config_file, _filters) = temp_dir_and_config(server.url());
+    let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config);
     fs::copy(
         "tests/data/typical/revocation/manifest.json",
         temp.path()
@@ -397,7 +418,7 @@ fn typical_incremental_fetch_dry_run() {
     exit_code: 0
     ----- stdout -----
     3 steps required (14 bytes to download)
-    - download 14 bytes from http://127.0.0.1:[PORT]/filter2.delta to "[TEMPDIR]/revocation/filter2.delta"
+    - download 14 bytes from http://127.0.0.1:[PORT]/revocation/filter2.delta to "[TEMPDIR]/revocation/filter2.delta"
     - build index from filters into "[TEMPDIR]/revocation"
     - save new manifest into "[TEMPDIR]/revocation"
 
@@ -429,11 +450,9 @@ fn http_server(root: &str) -> (TestHttpServer, SettingsBindDropGuard) {
     // add a filter eliding the (random) port in logs
     let mut current_filters = insta::Settings::clone_current();
     current_filters.add_filter(&format!(":{port}/"), ":[PORT]/");
-    let mut root = PathBuf::from(root);
-    root.push("revocation");
 
     (
-        TestHttpServer::new(("127.0.0.1", port), &root).unwrap(),
+        TestHttpServer::new(("127.0.0.1", port), Path::new(root)).unwrap(),
         current_filters.bind_to_scope(),
     )
 }
@@ -452,9 +471,12 @@ fn list_dir(path: &Path) -> Vec<String> {
     list
 }
 
-fn temp_dir_and_config(fetch_url: &str) -> (TempDir, PathBuf, SettingsBindDropGuard) {
+fn temp_dir_and_config(
+    fetch_url: &str,
+    config_write: impl FnOnce(&TempDir, &str),
+) -> (TempDir, PathBuf, SettingsBindDropGuard) {
     let temp = TempDir::new().unwrap();
-    write_config(&temp, fetch_url);
+    config_write(&temp, fetch_url);
 
     let mut settings = insta::Settings::clone_current();
     // remove tempdirs references
@@ -475,7 +497,7 @@ fn write_config(temp: &TempDir, fetch_url: &str) {
         format!(
             "cache-dir=\"{}\"\n\
             [revocation]\n\
-            fetch-url=\"{fetch_url}\"\n",
+            fetch-url=\"{fetch_url}revocation/\"\n",
             temp.path().display(),
         )
         .as_bytes(),

upki-mirror/src/bin/intermediates.rs

@@ -10,7 +10,7 @@ use eyre::{Context, Report, anyhow};
 use rustls_pki_types::CertificateDer;
 use rustls_pki_types::pem::PemObject;
 use serde::Deserialize;
-use upki::revocation::{Manifest, ManifestFile};
+use upki::data::{Manifest, ManifestFile};
 
 #[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<(), Report> {

upki-mirror/src/bin/mozilla-crlite.rs

@@ -7,7 +7,7 @@ use std::time::SystemTime;
 use aws_lc_rs::digest::{SHA256, digest};
 use clap::{Parser, ValueEnum};
 use eyre::{Context, Report, anyhow};
-use upki::revocation::{Manifest, ManifestFile};
+use upki::data::{Manifest, ManifestFile};
 
 #[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<(), Report> {

upki/src/data.rs

@@ -0,0 +1,76 @@
+#[cfg(feature = "__fetch")]
+use std::{fs::File, io::BufReader, path::PathBuf};
+
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use tracing::info;
+
+use crate::revocation::Error;
+
+/// The structure contained in a manifest.json
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Manifest {
+    /// When this file was generated.
+    ///
+    /// UNIX timestamp in seconds.
+    pub generated_at: u64,
+
+    /// Some human-readable text.
+    pub comment: String,
+
+    /// List of required files.
+    #[serde(alias = "filters")]
+    pub files: Vec<ManifestFile>,
+}
+
+impl Manifest {
+    #[cfg(feature = "__fetch")]
+    pub(crate) fn from_file(file_name: PathBuf) -> Result<Self, Error> {
+        let file = match File::open(&file_name) {
+            Ok(f) => f,
+            Err(error) => {
+                return Err(Error::FileRead {
+                    error,
+                    path: Some(file_name),
+                });
+            }
+        };
+
+        serde_json::from_reader(BufReader::new(file)).map_err(|error| Error::FileDecode {
+            error: Box::new(error),
+            path: Some(file_name),
+        })
+    }
+
+    /// Logs metadata fields in this manifest.
+    pub fn introduce(&self) -> Result<(), Error> {
+        let dt = match DateTime::<Utc>::from_timestamp(self.generated_at as i64, 0) {
+            Some(dt) => dt.to_rfc3339(),
+            None => {
+                return Err(Error::InvalidTimestamp {
+                    input: self.generated_at.to_string(),
+                    context: "manifest generated (in s)",
+                });
+            }
+        };
+
+        info!(comment = self.comment, date = dt, "parsed manifest");
+        Ok(())
+    }
+}
+
+/// Manifest data for a single manifest file.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct ManifestFile {
+    /// Relative filename.
+    ///
+    /// This is also the suggested local filename.
+    pub filename: String,
+
+    /// File size, indicative.  Allows a fetcher to predict data usage.
+    pub size: usize,
+
+    /// SHA256 hash of file contents.
+    #[serde(with = "hex::serde")]
+    pub hash: Vec<u8>,
+}