diff --git a/Cargo.toml b/Cargo.toml index ebcada637..ef38fa615 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,3 +134,7 @@ harness = false [[bench]] name = "merge_archive" harness = false + +[[bench]] +name = "zip_file_list" +harness = false diff --git a/benches/read_entry.rs b/benches/read_entry.rs index f8f72cd59..69e82fa3c 100644 --- a/benches/read_entry.rs +++ b/benches/read_entry.rs @@ -1,9 +1,15 @@ +//! Test the read entry functions +//! Usage: +//! ```sh +//! cargo bench --bench read_entry +//! ``` + use bencher::{benchmark_group, benchmark_main}; use std::io::{Cursor, Read, Write}; use bencher::Bencher; -use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; +use zip::{ZipArchive, ZipReadOptions, ZipWriter, write::SimpleFileOptions}; fn generate_random_archive(size: usize) -> Result, std::io::Error> { let data = Vec::new(); @@ -41,5 +47,40 @@ fn read_entry(bench: &mut Bencher) { bench.bytes = size as u64; } -benchmark_group!(benches, read_entry); +fn read_entry_iterable(bench: &mut Bencher) { + use zip::read::Config; + use zip::unstable::read::ZipIterable; + let size = 1024 * 1024; + let bytes = generate_random_archive(size) + .expect("Failed to create a random archive for the bench read_entry()"); + let mut reader = Cursor::new(&bytes); + let mut archive = ZipIterable::try_new(reader.clone(), Config::default()).unwrap(); + + bench.iter(|| { + let file = archive + .files() + .unwrap() + .find(|f| { + let file = f.as_ref().unwrap(); + let filename = file.name().unwrap(); + filename == "random.dat" + }) + .unwrap() + .unwrap(); + let mut buf = [0u8; 1024]; + let mut zip_file = file + .with_reader(&mut reader, ZipReadOptions::new()) + .unwrap(); + loop { + let n = zip_file.read(&mut buf).unwrap(); + if n == 0 { + break; + } + } + }); + + bench.bytes = size as u64; +} + +benchmark_group!(benches, read_entry, read_entry_iterable); benchmark_main!(benches); diff --git a/benches/zip_file_list.rs b/benches/zip_file_list.rs new file mode 100644 index 000000000..c8e0de63f --- /dev/null +++ b/benches/zip_file_list.rs @@ -0,0 +1,116 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::io::{Cursor, Write}; + +use bencher::Bencher; +use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + +const NB_FILES: usize = 100; +const FILENAME: &str = "bench_file_listing.zip"; + +fn generate_random_archive(size: usize) -> Result, std::io::Error> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + for count in 0..NB_FILES { + writer.start_file(format!("random_{}.dat", count), options)?; + let mut bytes = vec![0u8; size]; + getrandom::fill(&mut bytes) + .map_err(|e| std::io::Error::other(format!("getrandom error: {}", e)))?; + writer.write_all(&bytes)?; + } + let w = writer.finish()?; + + Ok(w.into_inner()) +} + +fn generate_random_archive_to_file(size: usize) -> Result<(), std::io::Error> { + use std::fs::File; + + let bytes = generate_random_archive(size)?; + let mut file = File::create(FILENAME)?; + file.write_all(&bytes)?; + Ok(()) +} + +fn file_listing_memory(bench: &mut Bencher) { + let size = 1024 * 1024; + let bytes = generate_random_archive(size) + .expect("Failed to create a random archive for the bench read_entry()"); + + bench.iter(|| { + let mut archive = ZipArchive::new(Cursor::new(&bytes)).unwrap(); + let mut names = vec![]; + for idx in 0..archive.len() { + let file = archive.by_index(idx).unwrap(); + names.push(file.name().unwrap().to_string()); + } + }); +} + +fn file_listing_file(bench: &mut Bencher) { + use std::fs::File; + + let size = 1024 * 1024; + generate_random_archive_to_file(size) + .expect("Failed to create a random archive for the bench read_entry()"); + + bench.iter(|| { + let file = File::open(FILENAME).unwrap(); + let mut archive = ZipArchive::new(file).unwrap(); + let mut names = vec![]; + for idx in 0..archive.len() { + let file = archive.by_index(idx).unwrap(); + names.push(file.name().unwrap().to_string()); + } + }); + + std::fs::remove_file(FILENAME).unwrap(); +} + +fn file_listing_iterable_memory(bench: &mut Bencher) { + use zip::read::Config; + use zip::unstable::read::ZipIterable; + let size = 1024 * 1024; + let bytes = generate_random_archive(size) + .expect("Failed to create a random archive for the bench read_entry()"); + + bench.iter(|| { + let mut archive = ZipIterable::try_new(Cursor::new(&bytes), Config::default()).unwrap(); + let mut names = vec![]; + for file in archive.files().unwrap() { + let file = file.unwrap(); + names.push(file.name().unwrap().to_string()); + } + }); +} + +fn file_listing_iterable_file(bench: &mut Bencher) { + use std::fs::File; + use zip::read::Config; + use zip::unstable::read::ZipIterable; + + let size = 1024 * 1024; + generate_random_archive_to_file(size) + .expect("Failed to create a random archive for the bench read_entry()"); + + bench.iter(|| { + let file = File::open(FILENAME).unwrap(); + let mut archive = ZipIterable::try_new(file, Config::default()).unwrap(); + let mut names = vec![]; + for file in archive.files().unwrap() { + let file = file.unwrap(); + names.push(file.name().unwrap().to_string()); + } + }); + std::fs::remove_file(FILENAME).unwrap(); +} + +benchmark_group!( + benches, + file_listing_memory, + file_listing_iterable_memory, + file_listing_file, + file_listing_iterable_file +); +benchmark_main!(benches); diff --git a/src/read/mod.rs b/src/read/mod.rs index ff223b6e3..4153d74ea 100644 --- a/src/read/mod.rs +++ b/src/read/mod.rs @@ -27,6 +27,7 @@ pub use stream::{ pub(crate) mod magic_finder; pub(crate) mod readers; +pub(crate) mod zip_iterable; pub(crate) mod zipfile; pub use zipfile::{ZipFile, ZipFileEntry, ZipFileSeek}; diff --git a/src/read/zip_iterable.rs b/src/read/zip_iterable.rs new file mode 100644 index 000000000..abef33d76 --- /dev/null +++ b/src/read/zip_iterable.rs @@ -0,0 +1,137 @@ +//! Iterable zip reader + +use crate::format::blocks::{FixedSizeBlock, ZipCentralEntryBlock}; +use crate::format::find_central_directory_end; +use crate::read::{ZipFileEntry, central_header_to_zip_file_inner}; +use crate::{ + read::{CentralDirectoryInfo, Config}, + result::{ZipError, ZipResult}, +}; +use std::{ + borrow::Cow, + io::{Read, Seek, SeekFrom}, +}; + +/// Iterable version of ZipArchive +pub struct ZipIterable { + #[allow(unused)] + pub(crate) config: Config, + pub(crate) iterable_files: ZipIterableFiles, +} +impl ZipIterable { + /// Try to create a new zip archive + pub fn try_new(mut reader: R, config: Config) -> ZipResult> { + let file_len = reader.seek(SeekFrom::End(0))?; + let mut end_exclusive = file_len; + let mut last_err = None; + + let central_directory = loop { + let cde = match find_central_directory_end( + &mut reader, + config.archive_offset, + end_exclusive, + file_len, + ) { + Ok(cde) => cde, + Err(e) => return Err(last_err.unwrap_or(e)), + }; + + match CentralDirectoryInfo::try_from(&cde) { + Ok(info) => break info, + Err(e) => { + last_err = Some(e); + end_exclusive = cde.eocd.position; + } + } + }; + + // If the parsed number of files is greater than the offset then + // something fishy is going on and we shouldn't trust number_of_files. + if central_directory.number_of_files > central_directory.directory_start as usize { + return Err(ZipError::UnsupportedArchive("Fishy error :)")); + } + + if central_directory.disk_number != central_directory.disk_with_central_directory { + return Err(ZipError::UnsupportedArchive( + "Support for multi-disk files is not implemented", + )); + } + + let iterable_shared = ZipIterableFiles::try_new(reader, central_directory)?; + + Ok(Self { + config, + iterable_files: iterable_shared, + }) + } + + /// Get the file as an iterator + pub fn files(&mut self) -> ZipResult<&mut ZipIterableFiles> { + self.iterable_files.reset()?; + Ok(&mut self.iterable_files) + } +} + +/// Iterable Files +#[derive(Debug)] +pub struct ZipIterableFiles { + reader: R, + central_directory: CentralDirectoryInfo, + current_file: usize, +} + +impl ZipIterableFiles { + /// Try to create an iterable of files + pub(crate) fn try_new( + mut reader: R, + central_directory: CentralDirectoryInfo, + ) -> ZipResult { + reader.seek(SeekFrom::Start(central_directory.directory_start))?; + Ok(Self { + reader, + central_directory, + current_file: 0, + }) + } + + pub(crate) fn reset(&mut self) -> ZipResult<()> { + self.current_file = 0; + self.reader + .seek(SeekFrom::Start(self.central_directory.directory_start))?; + Ok(()) + } + + pub(crate) fn parse_entry(&mut self) -> ZipResult> { + let central_header_start = self.reader.stream_position()?; + + // Parse central header + let block = ZipCentralEntryBlock::parse(&mut self.reader)?; + let (data, file_name) = central_header_to_zip_file_inner( + &mut self.reader, + self.central_directory.archive_offset, + central_header_start, + block, + )?; + let file = ZipFileEntry { + file_name_raw: Cow::Owned(file_name), + data: Cow::Owned(data), + }; + let central_header_end = self.reader.stream_position()?; + + self.reader.seek(SeekFrom::Start(central_header_end))?; + Ok(file) + } +} + +impl Iterator for ZipIterableFiles { + type Item = ZipResult>; + + fn next(&mut self) -> Option { + if self.current_file >= self.central_directory.number_of_files { + return None; + } + self.current_file += 1; + let file = self.parse_entry(); + Some(file) + } +} diff --git a/src/unstable.rs b/src/unstable.rs index 4744fa823..2de460210 100644 --- a/src/unstable.rs +++ b/src/unstable.rs @@ -25,6 +25,12 @@ pub mod stream { pub mod format { pub use crate::format::*; } + +/// Iterable zip +pub mod read { + pub use crate::read::zip_iterable::ZipIterable; +} + /// Types for creating ZIP archives. pub mod write { use crate::result::{ZipError, ZipResult};