diff --git a/upki-cli/src/bin/upki.rs b/upki-cli/src/bin/upki.rs index 027ca09b..4fefe604 100644 --- a/upki-cli/src/bin/upki.rs +++ b/upki-cli/src/bin/upki.rs @@ -12,9 +12,11 @@ use tracing::level_filters::LevelFilter; use tracing_subscriber::EnvFilter; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; -use upki::revocation::{Index, RevocationCheckInput}; #[cfg(feature = "__fetch")] -use upki::revocation::{Manifest, fetch}; +use upki::intermediates; +#[cfg(feature = "__fetch")] +use upki::revocation; +use upki::revocation::{Index, RevocationCheckInput}; use upki::{Config, ConfigPath}; #[tokio::main(flavor = "current_thread")] @@ -47,9 +49,19 @@ async fn main() -> Result { Ok(match args.command { #[cfg(feature = "__fetch")] - Command::Fetch { dry_run } => fetch(dry_run, &config).await?, + Command::Fetch { dry_run } => { + revocation::fetch(dry_run, &config).await?; + intermediates::fetch(dry_run, &config).await?; + ExitCode::SUCCESS + } #[cfg(feature = "__fetch")] - Command::Verify => Manifest::from_config(&config)?.verify(&config)?, + Command::Verify => { + revocation::Manifest::from_config(&config)?.verify(&config)?; + if config.intermediates.enabled { + intermediates::Manifest::from_config(&config)?.verify(&config)?; + } + ExitCode::SUCCESS + } Command::ShowConfigPath => unreachable!(), Command::ShowConfig => { print!( diff --git a/upki-cli/tests/data/typical-intermediates/intermediates/01.pem b/upki-cli/tests/data/typical-intermediates/intermediates/01.pem new file mode 100644 index 00000000..ad7f8c2b --- /dev/null +++ b/upki-cli/tests/data/typical-intermediates/intermediates/01.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDejCCAwCgAwIBAgIQDhl+Y1ebEqZMU1NhDojs+zAKBggqhkjOPQQDAzCBiDELM +AkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNle +SBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJ +VVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMjMwODAyM +DAwMDAwWhcNMzMwODAxMjM1OTU5WjBhMQswCQYDVQQGEwJDTjEtMCsGA1UECgwk5 +bm/5Lic5pe25Luj5LqS6IGU56eR5oqA5pyJ6ZmQ5YWs5Y+4MSMwIQYDVQQDDBrml +7bku6PkupLogZQgRUNDIERWIFNTTCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA +0IABPM5xjioDj+MTMUvVcu+mvehklAx8lZtFx1mluI+1sJaQC1WunaAefAn1nyIg +QvmGX5MOSPkxJgxTaglH57UzI6jggFwMIIBbDAfBgNVHSMEGDAWgBQ64QmG1M8Zw +pZ2dEl23OA1xmNjmjAdBgNVHQ4EFgQUEatQraNKT0JKMgCCGyLYu5PmPREwDgYDV +R0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0lBBYwFAYIKwYBB +QUHAwEGCCsGAQUFBwMCMCIGA1UdIAQbMBkwDQYLKwYBBAGyMQECAnAwCAYGZ4EMA +QIBMFAGA1UdHwRJMEcwRaBDoEGGP2h0dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VU +0VSVHJ1c3RFQ0NDZXJ0aWZpY2F0aW9uQXV0aG9yaXR5LmNybDBxBggrBgEFBQcBA +QRlMGMwOgYIKwYBBQUHMAKGLmh0dHA6Ly9jcnQudXNlcnRydXN0LmNvbS9VU0VSV +HJ1c3RFQ0NBQUFDQS5jcnQwJQYIKwYBBQUHMAGGGWh0dHA6Ly9vY3NwLnVzZXJ0c +nVzdC5jb20wCgYIKoZIzj0EAwMDaAAwZQIwFQ10XkI2EJqgq/uimmsknSuJFSrPb +bpE0T/bZhNwpzYPMHMLvLPHphKVsdZRRl5zAjEA4yfOfGuscopZ5CKd4beKBBuyE +TD2L1n17Vz1m1MyYdRmDYcnSw1yq/CF14Ug0OtR +-----END CERTIFICATE----- diff --git a/upki-cli/tests/data/typical-intermediates/intermediates/02.pem b/upki-cli/tests/data/typical-intermediates/intermediates/02.pem new file mode 100644 index 00000000..73f2c7e1 --- /dev/null +++ b/upki-cli/tests/data/typical-intermediates/intermediates/02.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDZjCCAuygAwIBAgIUSqaZqiFHW6RplEGCjSLNztXVwnYwCgYIKoZIzj0EAwMw +WDELMAkGA1UEBhMCQ04xJTAjBgNVBAoTHFRydXN0QXNpYSBUZWNobm9sb2dpZXMs +IEluYy4xIjAgBgNVBAMTGVRydXN0QXNpYSBUTFMgRUNDIFJvb3QgQ0EwHhcNMjUx +MDIxMDYwNzQ2WhcNMzAxMDIxMDYwNzQ1WjBgMQswCQYDVQQGEwJDTjEtMCsGA1UE +Cgwk5bm/5Lic5aCh5aGU5a6J5YWo5oqA5pyv5pyJ6ZmQ5YWs5Y+4MSIwIAYDVQQD +DBnlrp3loZQgRFYgVExTIEVDQyBDQSAyMDI1MHYwEAYHKoZIzj0CAQYFK4EEACID +YgAEZCtNSjEF5wehqx/uvl42QbwTGy+EdS0QOwwJNowLTL3ij851T4RtpKTHomni +ZG93QY7pBX/PsrJxxfGcDcWiGsLKxl7kzbX+SKxiER3rQ4E839mUL/NAuEX55dt3 +YbCJo4IBbTCCAWkwEgYDVR0TAQH/BAgwBgEB/wIBADAfBgNVHSMEGDAWgBQshVO7 +sUPNMuqeo4f+opioppPpEDCBkQYIKwYBBQUHAQEEgYQwgYEwQAYIKwYBBQUHMAKG +NGh0dHA6Ly9pY2Eub2VtLnRydXN0Y2EubmV0L1RydXN0QXNpYVRMU0VDQ1Jvb3RD +QS5jcnQwPQYIKwYBBQUHMAGGMWh0dHA6Ly9vY3NwLm9lbS50cnVzdGNhLm5ldC9U +cnVzdEFzaWFUTFNFQ0NSb290Q0EwEwYDVR0gBAwwCjAIBgZngQwBAgEwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwRQYDVR0fBD4wPDA6oDigNoY0aHR0cDovL2NybC5vZW0u +dHJ1c3RjYS5uZXQvVHJ1c3RBc2lhVExTRUNDUm9vdENBLmNybDAdBgNVHQ4EFgQU +1/pOVkY1qoJckXd6cBui1ECX/hkwDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMD +A2gAMGUCMQD0Rc/yrS5puh71tMc5A5Zycj8uIJ3Vy7iv3bOzshO9uSKuSAXvPi+Z +hlkde+9n6qICMEhD3oLT2RQYvyQh07xOGJtaHSeKL58T2ZJJN9sGzta0/Tp6t3BM +rrzSTYUYmUJ56w== +-----END CERTIFICATE----- diff --git a/upki-cli/tests/data/typical-intermediates/intermediates/ff.pem b/upki-cli/tests/data/typical-intermediates/intermediates/ff.pem new file mode 100644 index 00000000..cc986264 --- /dev/null +++ b/upki-cli/tests/data/typical-intermediates/intermediates/ff.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDATCCAoegAwIBAgIQbqITfZL8UTBF2jH1eJ5SxTAKBggqhkjOPQQDAzBPMQsw +CQYDVQQGEwJVUzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSYwJAYDVQQDDB1T +U0wuY29tIFRMUyBUcmFuc2l0IEVDQyBDQSBSMjAeFw0yNTA1MDcxOTE1MjVaFw0z +NTA1MDUxOTE1MjRaMFIxCzAJBgNVBAYTAkNOMSIwIAYDVQQKDBlab1RydXMgVGVj +aG5vbG9neSBMaW1pdGVkMR8wHQYDVQQDDBZab1RydXMgRFYgVExTIEVDQyBDQSAx +MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE3csCtDZ+nT94AFGWZ5KYelpbAvmJ8pjM +hhI4TYf/2VCJVJgVmi/PQZAVZ6GRzFNUsUyyuTm09GPIyjFoxEtno2do62a6ACoe +DzZO4nwTX3oZ44mzpV8ouS5PLJ1RwnDyo4IBIzCCAR8wEgYDVR0TAQH/BAgwBgEB +/wIBADAfBgNVHSMEGDAWgBQyosfYWIv/f8A88lVpM+zOzB+8lzBIBggrBgEFBQcB +AQQ8MDowOAYIKwYBBQUHMAKGLGh0dHA6Ly9jZXJ0LnNzbC5jb20vU1NMLmNvbS1U +TFMtVC1FQ0MtUjIuY2VyMBEGA1UdIAQKMAgwBgYEVR0gADAdBgNVHSUEFjAUBggr +BgEFBQcDAgYIKwYBBQUHAwEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDovL2NybHMu +c3NsLmNvbS9TU0wuY29tLVRMUy1ULUVDQy1SMi5jcmwwHQYDVR0OBBYEFPYfkvjv +FR3ijLztfWArkrX5uSnHMA4GA1UdDwEB/wQEAwIBhjAKBggqhkjOPQQDAwNoADBl +AjEA8NMnFZKKerkWdIDnbA3I1YuZzsv0scy1YC0GPgDVHbldqZ6grtTbDNl2UHVO +9mLkAjAfz7nkpaDPiXVBVSeYz1kOFUlMh/09er0rK4AVlGSz5KFjattSiFtQZVjk +yi2JzP4= +-----END CERTIFICATE----- diff --git a/upki-cli/tests/data/typical-intermediates/intermediates/manifest.json b/upki-cli/tests/data/typical-intermediates/intermediates/manifest.json new file mode 100644 index 00000000..01ad098e --- /dev/null +++ b/upki-cli/tests/data/typical-intermediates/intermediates/manifest.json @@ -0,0 +1,21 @@ +{ + "generated_at": 1765445031, + "comment": "typical test manifest for intermediates", + "files": [ + { + "filename": "01.pem", + "size": 1265, + "hash": "e1b2101ca9bbe7efd8056da198db614d0af067c877b751c167d4f645a0b85621" + }, + { + "filename": "02.pem", + "size": 1241, + "hash": "2181939592dc7ec156e9d11e927733465346978e6d8b95bdd4d8195db3414118" + }, + { + "filename": "ff.pem", + "size": 1103, + "hash": "cf2ea11741268b27a36f90324a068b2e7c07d656c6455b31e9c986d080d2360d" + } + ] +} diff --git a/upki-cli/tests/data/typical-intermediates/revocation/manifest.json b/upki-cli/tests/data/typical-intermediates/revocation/manifest.json new file mode 100644 index 00000000..04372747 --- /dev/null +++ b/upki-cli/tests/data/typical-intermediates/revocation/manifest.json @@ -0,0 +1,5 @@ +{ + "generated_at": 1765445031, + "comment": "empty manifest", + "files": [] +} diff --git a/upki-cli/tests/data/verify_non_existent_dir/config.toml b/upki-cli/tests/data/verify_non_existent_dir/config.toml index cc453c82..bda41d54 100644 --- a/upki-cli/tests/data/verify_non_existent_dir/config.toml +++ b/upki-cli/tests/data/verify_non_existent_dir/config.toml @@ -2,3 +2,7 @@ cache-dir = "not-exist/" [revocation] fetch-url = "" + +[intermediates] +enabled = false +fetch-url = "" diff --git a/upki-cli/tests/data/verify_of_empty_intermediates_manifest/config.toml b/upki-cli/tests/data/verify_of_empty_intermediates_manifest/config.toml new file mode 100644 index 00000000..c664ea3e --- /dev/null +++ b/upki-cli/tests/data/verify_of_empty_intermediates_manifest/config.toml @@ -0,0 +1,8 @@ +cache-dir = "tests/data/verify_of_empty_intermediates_manifest/" + +[revocation] +fetch-url = "" + +[intermediates] +enabled = true +fetch-url = "" diff --git a/upki-cli/tests/data/verify_of_empty_intermediates_manifest/intermediates/manifest.json b/upki-cli/tests/data/verify_of_empty_intermediates_manifest/intermediates/manifest.json new file mode 100644 index 00000000..04372747 --- /dev/null +++ b/upki-cli/tests/data/verify_of_empty_intermediates_manifest/intermediates/manifest.json @@ -0,0 +1,5 @@ +{ + "generated_at": 1765445031, + "comment": "empty manifest", + "files": [] +} diff --git a/upki-cli/tests/data/verify_of_empty_intermediates_manifest/revocation/manifest.json b/upki-cli/tests/data/verify_of_empty_intermediates_manifest/revocation/manifest.json new file mode 100644 index 00000000..04372747 --- /dev/null +++ b/upki-cli/tests/data/verify_of_empty_intermediates_manifest/revocation/manifest.json @@ -0,0 +1,5 @@ +{ + "generated_at": 1765445031, + "comment": "empty manifest", + "files": [] +} diff --git a/upki-cli/tests/integration.rs b/upki-cli/tests/integration.rs index 9b823313..4293aa06 100644 --- a/upki-cli/tests/integration.rs +++ b/upki-cli/tests/integration.rs @@ -49,7 +49,7 @@ fn config_unknown_fields() { | 1 | cache_dir = "tests/data/config_unknown_fields/" | ^^^^^^^^^ - unknown field `cache_dir`, expected `cache-dir` or `revocation` + unknown field `cache_dir`, expected one of `cache-dir`, `revocation`, `intermediates` Location: @@ -92,6 +92,10 @@ fn show_config_fixpoint() { [revocation] fetch-url = "" + [intermediates] + enabled = false + fetch-url = "" + ----- stderr ----- "#); } @@ -137,11 +141,28 @@ fn verify_of_empty_manifest() { "); } +#[test] +fn verify_of_empty_intermediates_manifest() { + let _filters = apply_common_filters(); + assert_cmd_snapshot!( + upki() + .arg("--config-file") + .arg("tests/data/verify_of_empty_intermediates_manifest/config.toml") + .arg("verify"), + @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + "); +} + #[test] fn fetch_of_empty_manifest() { let _filters = apply_common_filters(); let (server, _filters) = http_server("tests/data/verify_of_empty_manifest/"); - let (temp, config_file, _filters) = temp_dir_and_config(server.url()); + let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config); assert_cmd_snapshot!( upki() @@ -157,7 +178,7 @@ fn fetch_of_empty_manifest() { "); assert_snapshot!( server.into_log(), - @"GET /manifest.json -> 200 OK (79 bytes)" + @"GET /revocation/manifest.json -> 200 OK (79 bytes)" ); assert_eq!( list_dir(&temp.path().join("revocation")), @@ -169,7 +190,7 @@ fn fetch_of_empty_manifest() { fn full_fetch() { let _filters = apply_common_filters(); let (server, _filters) = http_server("tests/data/typical/"); - let (temp, config_file, _filters) = temp_dir_and_config(server.url()); + let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config); assert_cmd_snapshot!( upki() @@ -186,10 +207,10 @@ fn full_fetch() { assert_snapshot!( server.into_log(), @r" - GET /manifest.json -> 200 OK (530 bytes) - GET /filter1.filter -> 200 OK (11 bytes) - GET /filter2.delta -> 200 OK (14 bytes) - GET /filter3.delta -> 200 OK (10 bytes) + GET /revocation/manifest.json -> 200 OK (530 bytes) + GET /revocation/filter1.filter -> 200 OK (11 bytes) + GET /revocation/filter2.delta -> 200 OK (14 bytes) + GET /revocation/filter3.delta -> 200 OK (10 bytes) "); assert_eq!( list_dir(&temp.path().join("revocation")), @@ -202,11 +223,45 @@ fn full_fetch() { ); } +#[test] +fn full_fetch_of_intermediates() { + let _filters = apply_common_filters(); + let (server, _filters) = http_server("tests/data/typical-intermediates/"); + let (temp, config_file, _filters) = + temp_dir_and_config(server.url(), write_config_with_intermediates); + + assert_cmd_snapshot!( + upki() + .arg("--config-file") + .arg(config_file) + .arg("fetch"), + @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + "); + assert_snapshot!( + server.into_log(), + @r" + GET /revocation/manifest.json -> 200 OK (79 bytes) + GET /intermediates/manifest.json -> 200 OK (532 bytes) + GET /intermediates/01.pem -> 200 OK (1265 bytes) + GET /intermediates/02.pem -> 200 OK (1241 bytes) + GET /intermediates/ff.pem -> 200 OK (1103 bytes) + "); + assert_eq!( + list_dir(&temp.path().join("intermediates")), + vec!["01.pem", "02.pem", "ff.pem", "manifest.json"] + ); +} + #[test] fn full_fetch_and_incremental_update() { let _filters = apply_common_filters(); let (server, _filters) = http_server("tests/data/typical/"); - let (temp, config_file, _filters) = temp_dir_and_config(server.url()); + let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config); assert_cmd_snapshot!( upki() @@ -223,10 +278,10 @@ fn full_fetch_and_incremental_update() { assert_snapshot!( server.into_log(), @r" - GET /manifest.json -> 200 OK (530 bytes) - GET /filter1.filter -> 200 OK (11 bytes) - GET /filter2.delta -> 200 OK (14 bytes) - GET /filter3.delta -> 200 OK (10 bytes) + GET /revocation/manifest.json -> 200 OK (530 bytes) + GET /revocation/filter1.filter -> 200 OK (11 bytes) + GET /revocation/filter2.delta -> 200 OK (14 bytes) + GET /revocation/filter3.delta -> 200 OK (10 bytes) "); assert_eq!( list_dir(&temp.path().join("revocation")), @@ -257,8 +312,8 @@ fn full_fetch_and_incremental_update() { assert_snapshot!( server.into_log(), @r" - GET /manifest.json -> 200 OK (545 bytes) - GET /filter4.delta -> 200 OK (3 bytes) + GET /revocation/manifest.json -> 200 OK (545 bytes) + GET /revocation/filter4.delta -> 200 OK (3 bytes) "); // filter2 could be deleted, filter4 is new assert_eq!( @@ -289,7 +344,7 @@ fn full_fetch_and_incremental_update() { "); assert_snapshot!( server.into_log(), - @"GET /manifest.json -> 200 OK (545 bytes)"); + @"GET /revocation/manifest.json -> 200 OK (545 bytes)"); // filter2 is now deleted assert_eq!( @@ -307,7 +362,7 @@ fn full_fetch_and_incremental_update() { fn typical_incremental_fetch() { let _filters = apply_common_filters(); let (server, _filters) = http_server("tests/data/typical/"); - let (temp, config_file, _filters) = temp_dir_and_config(server.url()); + let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config); fs::copy( "tests/data/typical/revocation/manifest.json", @@ -345,8 +400,8 @@ fn typical_incremental_fetch() { assert_snapshot!( server.into_log(), @r" - GET /manifest.json -> 200 OK (530 bytes) - GET /filter2.delta -> 200 OK (14 bytes) + GET /revocation/manifest.json -> 200 OK (530 bytes) + GET /revocation/filter2.delta -> 200 OK (14 bytes) "); assert_eq!(list_dir(temp.path()), vec!["config.toml", "revocation",],); @@ -366,7 +421,7 @@ fn typical_incremental_fetch() { fn typical_incremental_fetch_dry_run() { let _filters = apply_common_filters(); let (server, _filters) = http_server("tests/data/typical/"); - let (temp, config_file, _filters) = temp_dir_and_config(server.url()); + let (temp, config_file, _filters) = temp_dir_and_config(server.url(), write_config); fs::copy( "tests/data/typical/revocation/manifest.json", temp.path() @@ -397,7 +452,7 @@ fn typical_incremental_fetch_dry_run() { exit_code: 0 ----- stdout ----- 3 steps required (14 bytes to download) - - download 14 bytes from http://127.0.0.1:[PORT]/filter2.delta to "[TEMPDIR]/revocation/filter2.delta" + - download 14 bytes from http://127.0.0.1:[PORT]/revocation/filter2.delta to "[TEMPDIR]/revocation/filter2.delta" - build index from filters into "[TEMPDIR]/revocation" - save new manifest into "[TEMPDIR]/revocation" @@ -429,11 +484,9 @@ fn http_server(root: &str) -> (TestHttpServer, SettingsBindDropGuard) { // add a filter eliding the (random) port in logs let mut current_filters = insta::Settings::clone_current(); current_filters.add_filter(&format!(":{port}/"), ":[PORT]/"); - let mut root = PathBuf::from(root); - root.push("revocation"); ( - TestHttpServer::new(("127.0.0.1", port), &root).unwrap(), + TestHttpServer::new(("127.0.0.1", port), Path::new(root)).unwrap(), current_filters.bind_to_scope(), ) } @@ -452,9 +505,12 @@ fn list_dir(path: &Path) -> Vec { list } -fn temp_dir_and_config(fetch_url: &str) -> (TempDir, PathBuf, SettingsBindDropGuard) { +fn temp_dir_and_config( + fetch_url: &str, + config_write: impl FnOnce(&TempDir, &str), +) -> (TempDir, PathBuf, SettingsBindDropGuard) { let temp = TempDir::new().unwrap(); - write_config(&temp, fetch_url); + config_write(&temp, fetch_url); let mut settings = insta::Settings::clone_current(); // remove tempdirs references @@ -475,7 +531,24 @@ fn write_config(temp: &TempDir, fetch_url: &str) { format!( "cache-dir=\"{}\"\n\ [revocation]\n\ - fetch-url=\"{fetch_url}\"\n", + fetch-url=\"{fetch_url}revocation/\"\n", + temp.path().display(), + ) + .as_bytes(), + ) + .unwrap(); +} + +fn write_config_with_intermediates(temp: &TempDir, fetch_url: &str) { + fs::write( + temp.path().join("config.toml"), + format!( + "cache-dir=\"{}\"\n\ + [revocation]\n\ + fetch-url=\"{fetch_url}revocation/\"\n\ + [intermediates]\n\ + enabled=true\n\ + fetch-url=\"{fetch_url}intermediates/\"\n", temp.path().display(), ) .as_bytes(), diff --git a/upki-mirror/src/bin/intermediates.rs b/upki-mirror/src/bin/intermediates.rs index d4cf38fa..52d0ecf4 100644 --- a/upki-mirror/src/bin/intermediates.rs +++ b/upki-mirror/src/bin/intermediates.rs @@ -10,7 +10,7 @@ use eyre::{Context, Report, anyhow}; use rustls_pki_types::CertificateDer; use rustls_pki_types::pem::PemObject; use serde::Deserialize; -use upki::revocation::{Manifest, ManifestFile}; +use upki::data::{Manifest, ManifestFile}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), Report> { diff --git a/upki-mirror/src/bin/mozilla-crlite.rs b/upki-mirror/src/bin/mozilla-crlite.rs index dca4619f..b493ad27 100644 --- a/upki-mirror/src/bin/mozilla-crlite.rs +++ b/upki-mirror/src/bin/mozilla-crlite.rs @@ -7,7 +7,7 @@ use std::time::SystemTime; use aws_lc_rs::digest::{SHA256, digest}; use clap::{Parser, ValueEnum}; use eyre::{Context, Report, anyhow}; -use upki::revocation::{Manifest, ManifestFile}; +use upki::data::{Manifest, ManifestFile}; #[tokio::main(flavor = "current_thread")] async fn main() -> Result<(), Report> { diff --git a/upki/src/data.rs b/upki/src/data.rs new file mode 100644 index 00000000..9c78c893 --- /dev/null +++ b/upki/src/data.rs @@ -0,0 +1,76 @@ +#[cfg(feature = "__fetch")] +use std::{fs::File, io::BufReader, path::PathBuf}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use tracing::info; + +use crate::revocation::Error; + +/// The structure contained in a manifest.json +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Manifest { + /// When this file was generated. + /// + /// UNIX timestamp in seconds. + pub generated_at: u64, + + /// Some human-readable text. + pub comment: String, + + /// List of required files. + #[serde(alias = "filters")] + pub files: Vec, +} + +impl Manifest { + #[cfg(feature = "__fetch")] + pub(crate) fn from_file(file_name: PathBuf) -> Result { + let file = match File::open(&file_name) { + Ok(f) => f, + Err(error) => { + return Err(Error::FileRead { + error, + path: Some(file_name), + }); + } + }; + + serde_json::from_reader(BufReader::new(file)).map_err(|error| Error::FileDecode { + error: Box::new(error), + path: Some(file_name), + }) + } + + /// Logs metadata fields in this manifest. + pub fn introduce(&self) -> Result<(), Error> { + let dt = match DateTime::::from_timestamp(self.generated_at as i64, 0) { + Some(dt) => dt.to_rfc3339(), + None => { + return Err(Error::InvalidTimestamp { + input: self.generated_at.to_string(), + context: "manifest generated (in s)", + }); + } + }; + + info!(comment = self.comment, date = dt, "parsed manifest"); + Ok(()) + } +} + +/// Manifest data for a single manifest file. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct ManifestFile { + /// Relative filename. + /// + /// This is also the suggested local filename. + pub filename: String, + + /// File size, indicative. Allows a fetcher to predict data usage. + pub size: usize, + + /// SHA256 hash of file contents. + #[serde(with = "hex::serde")] + pub hash: Vec, +} diff --git a/upki/src/intermediates.rs b/upki/src/intermediates.rs new file mode 100644 index 00000000..9673f4a2 --- /dev/null +++ b/upki/src/intermediates.rs @@ -0,0 +1,102 @@ +use core::ops::Deref; + +use serde::{Deserialize, Serialize}; +#[cfg(feature = "__fetch")] +use tracing::info; + +#[cfg(feature = "__fetch")] +use crate::Config; +use crate::data; +#[cfg(feature = "__fetch")] +use crate::revocation::{Error, FetchContext, FetchType, Plan}; + +/// Details about intermediate preloading. +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields, default)] +pub struct IntermediatesConfig { + /// Whether to fetch things at all. + pub enabled: bool, + /// Where to fetch intermediate certificates. + pub fetch_url: String, +} + +impl Default for IntermediatesConfig { + fn default() -> Self { + Self { + enabled: false, + fetch_url: "https://upki.rustls.dev/intermediates/".into(), + } + } +} + +/// Update the local intermediates cache by fetching updates over the network. +/// +/// `dry_run` means this call fetches the new manifest, but does not fetch any +/// required files; but the necessary files are printed to stdout. +#[cfg(feature = "__fetch")] +pub async fn fetch(dry_run: bool, config: &Config) -> Result<(), Error> { + let IntermediatesConfig { + enabled: true, + fetch_url, + } = &config.intermediates + else { + return Ok(()); + }; + + let cache_dir = config.intermediates_cache_dir(); + info!("fetching intermediates from {fetch_url} into {cache_dir:?}...",); + + FetchContext { + cache_dir, + fetch_url, + old_manifest: Manifest::from_config(config) + .ok() + .as_deref(), + typ: FetchType::Intermediates, + } + .fetch(dry_run) + .await +} + +/// The structure contained in a manifest.json +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Manifest(data::Manifest); + +impl Manifest { + /// Load the intermediates manifest from the cache directory specified in the configuration. + #[cfg(feature = "__fetch")] + pub fn from_config(config: &Config) -> Result { + let mut file_name = config.intermediates_cache_dir(); + file_name.push("manifest.json"); + data::Manifest::from_file(file_name).map(Self) + } + + /// Verify the current contents of the cache against this manifest. + /// + /// This performs disk IO but does not perform network IO. + #[cfg(feature = "__fetch")] + pub fn verify(&self, config: &Config) -> Result<(), Error> { + self.introduce()?; + let plan = Plan::construct( + self, + &FetchContext { + cache_dir: config.intermediates_cache_dir(), + fetch_url: "https://.../", + old_manifest: None, + typ: FetchType::Intermediates, + }, + )?; + match plan.download_bytes() { + 0 => Ok(()), + bytes => Err(Error::Outdated(bytes)), + } + } +} + +impl Deref for Manifest { + type Target = data::Manifest; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/upki/src/lib.rs b/upki/src/lib.rs index 8d6881d1..e9c2d3a6 100644 --- a/upki/src/lib.rs +++ b/upki/src/lib.rs @@ -14,6 +14,14 @@ pub(crate) mod sha256; /// Determining revocation status of publicly trusted certificates. pub mod revocation; + +/// Fetching intermediate certificates to assist chain building. +pub mod intermediates; + +/// Common data storage formats. +pub mod data; + +use crate::intermediates::IntermediatesConfig; use crate::revocation::RevocationConfig; /// Foreign function interface. @@ -29,6 +37,10 @@ pub struct Config { /// Configuration for crlite-style revocation. pub revocation: RevocationConfig, + + /// Configuration for intermediate preloading. + #[serde(default)] + pub intermediates: IntermediatesConfig, } impl Config { @@ -71,12 +83,17 @@ impl Config { } }, revocation: RevocationConfig::default(), + intermediates: IntermediatesConfig::default(), }) } pub(crate) fn revocation_cache_dir(&self) -> PathBuf { self.cache_dir.join("revocation") } + + pub(crate) fn intermediates_cache_dir(&self) -> PathBuf { + self.cache_dir.join("intermediates") + } } /// How the path to a configuration file was decided upon. diff --git a/upki/src/revocation/fetch.rs b/upki/src/revocation/fetch.rs index ecf43c2a..e4921227 100644 --- a/upki/src/revocation/fetch.rs +++ b/upki/src/revocation/fetch.rs @@ -17,103 +17,133 @@ use std::io::{self, Read, Write}; #[cfg(target_family = "unix")] use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; -use std::process::ExitCode; use tracing::{debug, info}; use super::index::INDEX_BIN; -use super::{Error, Index, Manifest, ManifestFile}; -use crate::{Config, sha256}; +use super::{Error, Index, Manifest}; +use crate::{Config, data, sha256}; /// Update the local revocation cache by fetching updates over the network. /// /// `dry_run` means this call fetches the new manifest, but does not fetch any /// required files; but the necessary files are printed to stdout. Therefore /// such a call is not completely "dry" -- perhaps "moist". -pub async fn fetch(dry_run: bool, config: &Config) -> Result { +pub async fn fetch(dry_run: bool, config: &Config) -> Result<(), Error> { let cache_dir = config.revocation_cache_dir(); info!( "fetching {} into {:?}...", &config.revocation.fetch_url, &cache_dir, ); - let manifest_url = format!("{}{MANIFEST_JSON}", config.revocation.fetch_url); - #[cfg(feature = "fetch")] - let builder = reqwest::Client::builder().use_rustls_tls(); - #[cfg(all(feature = "fetch-native-tls", not(feature = "fetch")))] - let builder = reqwest::Client::builder().use_native_tls(); - - let client = builder - .timeout(Duration::from_secs(REQUEST_TIMEOUT)) - .user_agent(format!( - "{}/{} ({})", - env!("CARGO_PKG_NAME"), - env!("CARGO_PKG_VERSION"), - env!("CARGO_PKG_REPOSITORY") - )) - .build() - .map_err(|error| Error::HttpFetch { - error: Box::new(error), - url: manifest_url.clone(), - })?; - - let response = client - .get(&manifest_url) - .send() - .await - .map_err(|error| Error::HttpFetch { - error: Box::new(error), - url: manifest_url.clone(), - })? - .error_for_status() - .map_err(|error| Error::HttpFetch { - error: Box::new(error), - url: manifest_url.clone(), - })?; - - let manifest = response - .json::() - .await - .map_err(|error| Error::FileDecode { - error: Box::new(error), - path: None, - })?; - - manifest.introduce()?; - - let old_manifest = Manifest::from_config(config).ok(); - - let plan = Plan::construct( - &manifest, - &old_manifest, - &config.revocation.fetch_url, - &cache_dir, - )?; - - if dry_run { - println!( - "{} steps required ({} bytes to download)", + FetchContext { + cache_dir, + fetch_url: &config.revocation.fetch_url, + old_manifest: Manifest::from_config(config) + .ok() + .as_deref(), + typ: FetchType::Revocation, + } + .fetch(dry_run) + .await +} + +pub(crate) struct FetchContext<'a> { + pub(crate) cache_dir: PathBuf, + pub(crate) fetch_url: &'a str, + pub(crate) old_manifest: Option<&'a data::Manifest>, + pub(crate) typ: FetchType, +} + +impl FetchContext<'_> { + pub(crate) async fn fetch(&self, dry_run: bool) -> Result<(), Error> { + let manifest_url = format!("{}{MANIFEST_JSON}", self.fetch_url); + #[cfg(feature = "fetch")] + let builder = reqwest::Client::builder().use_rustls_tls(); + #[cfg(all(feature = "fetch-native-tls", not(feature = "fetch")))] + let builder = reqwest::Client::builder().use_native_tls(); + + let client = builder + .timeout(Duration::from_secs(REQUEST_TIMEOUT)) + .user_agent(format!( + "{}/{} ({})", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_VERSION"), + env!("CARGO_PKG_REPOSITORY") + )) + .build() + .map_err(|error| Error::HttpFetch { + error: Box::new(error), + url: manifest_url.clone(), + })?; + + let response = client + .get(&manifest_url) + .send() + .await + .map_err(|error| Error::HttpFetch { + error: Box::new(error), + url: manifest_url.clone(), + })? + .error_for_status() + .map_err(|error| Error::HttpFetch { + error: Box::new(error), + url: manifest_url.clone(), + })?; + + let manifest = response + .json::() + .await + .map_err(|error| Error::FileDecode { + error: Box::new(error), + path: None, + })?; + + manifest.introduce()?; + + let plan = Plan::construct(&manifest, self)?; + + if dry_run { + println!( + "{} steps required ({} bytes to download)", + plan.steps.len(), + plan.download_bytes() + ); + for step in plan.steps { + println!("- {step}"); + } + return Ok(()); + } + + info!( + "{} steps required ({} bytes to download).", plan.steps.len(), plan.download_bytes() ); + for step in plan.steps { - println!("- {step}"); + step.execute(&client).await?; } - return Ok(ExitCode::SUCCESS); + + info!("success"); + Ok(()) } - info!( - "{} steps required ({} bytes to download).", - plan.steps.len(), - plan.download_bytes() - ); + fn should_clean_up_file_name(&self, name: &str) -> bool { + match self.typ { + FetchType::Revocation => name.ends_with(".filter") || name.ends_with(".delta"), + FetchType::Intermediates => name.ends_with(".pem"), + } + } - for step in plan.steps { - step.execute(&client).await?; + fn requires_revocation_index(&self) -> bool { + matches!(self.typ, FetchType::Revocation) } +} - info!("success"); - Ok(ExitCode::SUCCESS) +pub(crate) enum FetchType { + Revocation, + Intermediates, } pub(crate) struct Plan { @@ -124,24 +154,19 @@ impl Plan { /// Form a plan of how to synchronize with the remote server. /// /// - `manifest` describes the contents of the remote server. - /// - `old_manifest` is an alleged current manifest, whose files are left alone. - /// - `remote_url` is the base URL. - /// - `local` is the path into which files are downloaded. The caller ensures this exists. pub(crate) fn construct( - manifest: &Manifest, - old_manifest: &Option, - remote_url: &str, - local: &Path, + manifest: &data::Manifest, + ctx: &FetchContext<'_>, ) -> Result { let mut steps = Vec::new(); // Collect unwanted files for deletion let mut unwanted_files = HashSet::new(); - if local.exists() { - let iter = fs::read_dir(local).map_err(|error| Error::CreateDirectory { + if ctx.cache_dir.exists() { + let iter = fs::read_dir(&ctx.cache_dir).map_err(|error| Error::CreateDirectory { error, - path: local.to_owned(), + path: ctx.cache_dir.to_owned(), })?; for entry in iter { @@ -152,44 +177,46 @@ impl Plan { let path = Path::new(&entry.file_name()).to_owned(); let name = path.to_string_lossy(); - if name.ends_with(".filter") || name.ends_with(".delta") { + if ctx.should_clean_up_file_name(&name) { unwanted_files.insert(path); } } } else { - steps.push(PlanStep::CreateDir(local.to_owned())); + steps.push(PlanStep::CreateDir(ctx.cache_dir.to_owned())); } for file in &manifest.files { unwanted_files.remove(Path::new(&file.filename)); - let path = local.join(&file.filename); + let path = ctx.cache_dir.join(&file.filename); match hash_file(&path) { Ok(digest) if digest.as_ref() == file.hash => continue, _ => {} } - steps.push(PlanStep::download(file, remote_url, local)); + steps.push(PlanStep::download(file, ctx.fetch_url, &ctx.cache_dir)); } - if let Some(old_manifest) = &old_manifest { + if let Some(old_manifest) = &ctx.old_manifest { for file in &old_manifest.files { unwanted_files.remove(Path::new(&file.filename)); } } - steps.push(PlanStep::SaveIndex { - manifest: manifest.clone(), - local_dir: local.to_owned(), - }); + if ctx.requires_revocation_index() { + steps.push(PlanStep::SaveIndex { + manifest: manifest.clone(), + local_dir: ctx.cache_dir.to_owned(), + }); + } steps.push(PlanStep::SaveManifest { manifest: manifest.clone(), - local_dir: local.to_owned(), + local_dir: ctx.cache_dir.to_owned(), }); for filename in unwanted_files { - steps.push(PlanStep::Delete(local.join(filename))); + steps.push(PlanStep::Delete(ctx.cache_dir.join(filename))); } Ok(Self { steps }) @@ -213,7 +240,7 @@ enum PlanStep { /// Download `file` from `remote` to `local` Download { - file: ManifestFile, + file: data::ManifestFile, /// URL. remote_url: String, /// Full path to output file. @@ -225,13 +252,13 @@ enum PlanStep { /// Build and save the index from filter universe metadata. SaveIndex { - manifest: Manifest, + manifest: data::Manifest, local_dir: PathBuf, }, /// Save the manifest structure SaveManifest { - manifest: Manifest, + manifest: data::Manifest, local_dir: PathBuf, }, } @@ -354,7 +381,7 @@ impl PlanStep { Ok(()) } - fn download(file: &ManifestFile, remote_url: &str, local: &Path) -> Self { + fn download(file: &data::ManifestFile, remote_url: &str, local: &Path) -> Self { Self::Download { file: file.clone(), remote_url: format!("{remote_url}{}", file.filename), diff --git a/upki/src/revocation/index.rs b/upki/src/revocation/index.rs index e019490d..1f16436e 100644 --- a/upki/src/revocation/index.rs +++ b/upki/src/revocation/index.rs @@ -12,10 +12,10 @@ use std::path::PathBuf; use clubcard_crlite::TimestampInterval; use clubcard_crlite::{CRLiteClubcard, CRLiteStatus, LogId, Timestamp}; -#[cfg(feature = "__fetch")] -use super::Manifest; use super::{Error, RevocationCheckInput, RevocationStatus}; use crate::Config; +#[cfg(feature = "__fetch")] +use crate::data::Manifest; /// Binary-encoded index of universe metadata for all filters in a manifest. /// @@ -426,6 +426,7 @@ mod tests { use clubcard_crlite::{CRLiteClubcard, CRLiteCoverage, CRLiteQuery, Encoding}; use super::*; + use crate::intermediates::IntermediatesConfig; use crate::revocation::{CertSerial, CtTimestamp, IssuerSpkiHash, RevocationConfig}; #[test] @@ -1208,6 +1209,7 @@ mod tests { Config { cache_dir: dir.to_owned(), revocation: RevocationConfig::default(), + intermediates: IntermediatesConfig::default(), } } diff --git a/upki/src/revocation/mod.rs b/upki/src/revocation/mod.rs index 77c77a67..3483fb68 100644 --- a/upki/src/revocation/mod.rs +++ b/upki/src/revocation/mod.rs @@ -1,52 +1,35 @@ use core::error::Error as StdError; +use core::ops::Deref; use core::str::FromStr; use core::{fmt, str}; -#[cfg(feature = "__fetch")] -use std::fs::File; use std::io; -#[cfg(feature = "__fetch")] -use std::io::BufReader; use std::path::PathBuf; use std::process::ExitCode; use base64::Engine; use base64::prelude::BASE64_STANDARD; -use chrono::{DateTime, Utc}; use clubcard_crlite::CRLiteKey; pub use clubcard_crlite::IssuerSpkiHash; use rustls_pki_types::{CertificateDer, TrustAnchor}; use serde::{Deserialize, Serialize}; -use tracing::info; #[cfg(feature = "__fetch")] use crate::Config; -use crate::sha256; +use crate::{data, sha256}; #[cfg(feature = "__fetch")] mod fetch; #[cfg(feature = "__fetch")] -use fetch::Plan; -#[cfg(feature = "__fetch")] pub use fetch::fetch; +#[cfg(feature = "__fetch")] +pub(crate) use fetch::{FetchContext, FetchType, Plan}; mod index; pub use index::Index; /// The structure contained in a manifest.json -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Manifest { - /// When this file was generated. - /// - /// UNIX timestamp in seconds. - pub generated_at: u64, - - /// Some human-readable text. - pub comment: String, - - /// List of required files. - #[serde(alias = "filters")] - pub files: Vec, -} +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Manifest(data::Manifest); impl Manifest { /// Load the revocation manifest from the cache directory specified in the configuration. @@ -54,67 +37,37 @@ impl Manifest { pub fn from_config(config: &Config) -> Result { let mut file_name = config.revocation_cache_dir(); file_name.push("manifest.json"); - - let file = match File::open(&file_name) { - Ok(f) => f, - Err(error) => { - return Err(Error::FileRead { - error, - path: Some(file_name), - }); - } - }; - - serde_json::from_reader(BufReader::new(file)).map_err(|error| Error::FileDecode { - error: Box::new(error), - path: Some(file_name), - }) + data::Manifest::from_file(file_name).map(Self) } /// Verify the current contents of the cache against this manifest. /// /// This performs disk IO but does not perform network IO. #[cfg(feature = "__fetch")] - pub fn verify(&self, config: &Config) -> Result { + pub fn verify(&self, config: &Config) -> Result<(), Error> { self.introduce()?; - let plan = Plan::construct(self, &None, "https://.../", &config.revocation_cache_dir())?; + let plan = Plan::construct( + self, + &FetchContext { + cache_dir: config.revocation_cache_dir(), + fetch_url: "https://.../", + old_manifest: None, + typ: FetchType::Revocation, + }, + )?; match plan.download_bytes() { - 0 => Ok(ExitCode::SUCCESS), + 0 => Ok(()), bytes => Err(Error::Outdated(bytes)), } } - - /// Logs metadata fields in this manifest. - pub fn introduce(&self) -> Result<(), Error> { - let dt = match DateTime::::from_timestamp(self.generated_at as i64, 0) { - Some(dt) => dt.to_rfc3339(), - None => { - return Err(Error::InvalidTimestamp { - input: self.generated_at.to_string(), - context: "manifest generated (in s)", - }); - } - }; - - info!(comment = self.comment, date = dt, "parsed manifest"); - Ok(()) - } } -/// Manifest data for a single crlite filter file. -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct ManifestFile { - /// Relative filename. - /// - /// This is also the suggested local filename. - pub filename: String, +impl Deref for Manifest { + type Target = data::Manifest; - /// File size, indicative. Allows a fetcher to predict data usage. - pub size: usize, - - /// SHA256 hash of file contents. - #[serde(with = "hex::serde")] - pub hash: Vec, + fn deref(&self) -> &Self::Target { + &self.0 + } } /// Input parameters for a revocation check.