Draft: iterable zip

Cargo.toml

@@ -134,3 +134,7 @@ harness = false
 [[bench]]
 name = "merge_archive"
 harness = false
+
+[[bench]]
+name = "zip_file_list"
+harness = false

benches/read_entry.rs

@@ -1,9 +1,15 @@
+//! Test the read entry functions
+//! Usage:
+//! ```sh
+//! cargo bench --bench read_entry
+//! ```
+
 use bencher::{benchmark_group, benchmark_main};
 
 use std::io::{Cursor, Read, Write};
 
 use bencher::Bencher;
-use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
+use zip::{ZipArchive, ZipReadOptions, ZipWriter, write::SimpleFileOptions};
 
 fn generate_random_archive(size: usize) -> Result<Vec<u8>, std::io::Error> {
     let data = Vec::new();
@@ -41,5 +47,40 @@ fn read_entry(bench: &mut Bencher) {
     bench.bytes = size as u64;
 }
 
-benchmark_group!(benches, read_entry);
+fn read_entry_iterable(bench: &mut Bencher) {
+    use zip::read::Config;
+    use zip::unstable::read::ZipIterable;
+    let size = 1024 * 1024;
+    let bytes = generate_random_archive(size)
+        .expect("Failed to create a random archive for the bench read_entry()");
+    let mut reader = Cursor::new(&bytes);
+    let mut archive = ZipIterable::try_new(reader.clone(), Config::default()).unwrap();
+
+    bench.iter(|| {
+        let file = archive
+            .files()
+            .unwrap()
+            .find(|f| {
+                let file = f.as_ref().unwrap();
+                let filename = file.name().unwrap();
+                filename == "random.dat"
+            })
+            .unwrap()
+            .unwrap();
+        let mut buf = [0u8; 1024];
+        let mut zip_file = file
+            .with_reader(&mut reader, ZipReadOptions::new())
+            .unwrap();
+        loop {
+            let n = zip_file.read(&mut buf).unwrap();
+            if n == 0 {
+                break;
+            }
+        }
+    });
+
+    bench.bytes = size as u64;
+}
+
+benchmark_group!(benches, read_entry, read_entry_iterable);
 benchmark_main!(benches);

benches/zip_file_list.rs

@@ -0,0 +1,116 @@
+use bencher::{benchmark_group, benchmark_main};
+
+use std::io::{Cursor, Write};
+
+use bencher::Bencher;
+use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
+
+const NB_FILES: usize = 100;
+const FILENAME: &str = "bench_file_listing.zip";
+
+fn generate_random_archive(size: usize) -> Result<Vec<u8>, std::io::Error> {
+    let data = Vec::new();
+    let mut writer = ZipWriter::new(Cursor::new(data));
+    let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
+    for count in 0..NB_FILES {
+        writer.start_file(format!("random_{}.dat", count), options)?;
+        let mut bytes = vec![0u8; size];
+        getrandom::fill(&mut bytes)
+            .map_err(|e| std::io::Error::other(format!("getrandom error: {}", e)))?;
+        writer.write_all(&bytes)?;
+    }
+    let w = writer.finish()?;
+
+    Ok(w.into_inner())
+}
+
+fn generate_random_archive_to_file(size: usize) -> Result<(), std::io::Error> {
+    use std::fs::File;
+
+    let bytes = generate_random_archive(size)?;
+    let mut file = File::create(FILENAME)?;
+    file.write_all(&bytes)?;
+    Ok(())
+}
+
+fn file_listing_memory(bench: &mut Bencher) {
+    let size = 1024 * 1024;
+    let bytes = generate_random_archive(size)
+        .expect("Failed to create a random archive for the bench read_entry()");
+
+    bench.iter(|| {
+        let mut archive = ZipArchive::new(Cursor::new(&bytes)).unwrap();
+        let mut names = vec![];
+        for idx in 0..archive.len() {
+            let file = archive.by_index(idx).unwrap();
+            names.push(file.name().unwrap().to_string());
+        }
+    });
+}
+
+fn file_listing_file(bench: &mut Bencher) {
+    use std::fs::File;
+
+    let size = 1024 * 1024;
+    generate_random_archive_to_file(size)
+        .expect("Failed to create a random archive for the bench read_entry()");
+
+    bench.iter(|| {
+        let file = File::open(FILENAME).unwrap();
+        let mut archive = ZipArchive::new(file).unwrap();
+        let mut names = vec![];
+        for idx in 0..archive.len() {
+            let file = archive.by_index(idx).unwrap();
+            names.push(file.name().unwrap().to_string());
+        }
+    });
+
+    std::fs::remove_file(FILENAME).unwrap();
+}
+
+fn file_listing_iterable_memory(bench: &mut Bencher) {
+    use zip::read::Config;
+    use zip::unstable::read::ZipIterable;
+    let size = 1024 * 1024;
+    let bytes = generate_random_archive(size)
+        .expect("Failed to create a random archive for the bench read_entry()");
+
+    bench.iter(|| {
+        let mut archive = ZipIterable::try_new(Cursor::new(&bytes), Config::default()).unwrap();
+        let mut names = vec![];
+        for file in archive.files().unwrap() {
+            let file = file.unwrap();
+            names.push(file.name().unwrap().to_string());
+        }
+    });
+}
+
+fn file_listing_iterable_file(bench: &mut Bencher) {
+    use std::fs::File;
+    use zip::read::Config;
+    use zip::unstable::read::ZipIterable;
+
+    let size = 1024 * 1024;
+    generate_random_archive_to_file(size)
+        .expect("Failed to create a random archive for the bench read_entry()");
+
+    bench.iter(|| {
+        let file = File::open(FILENAME).unwrap();
+        let mut archive = ZipIterable::try_new(file, Config::default()).unwrap();
+        let mut names = vec![];
+        for file in archive.files().unwrap() {
+            let file = file.unwrap();
+            names.push(file.name().unwrap().to_string());
+        }
+    });
+    std::fs::remove_file(FILENAME).unwrap();
+}
+
+benchmark_group!(
+    benches,
+    file_listing_memory,
+    file_listing_iterable_memory,
+    file_listing_file,
+    file_listing_iterable_file
+);
+benchmark_main!(benches);

src/read/mod.rs

@@ -27,6 +27,7 @@ pub use stream::{
 
 pub(crate) mod magic_finder;
 pub(crate) mod readers;
+pub(crate) mod zip_iterable;
 
 pub(crate) mod zipfile;
 pub use zipfile::{ZipFile, ZipFileEntry, ZipFileSeek};

src/read/zip_iterable.rs

@@ -0,0 +1,137 @@
+//! Iterable zip reader
+
+use crate::format::blocks::{FixedSizeBlock, ZipCentralEntryBlock};
+use crate::format::find_central_directory_end;
+use crate::read::{ZipFileEntry, central_header_to_zip_file_inner};
+use crate::{
+    read::{CentralDirectoryInfo, Config},
+    result::{ZipError, ZipResult},
+};
+use std::{
+    borrow::Cow,
+    io::{Read, Seek, SeekFrom},
+};
+
+/// Iterable version of ZipArchive
+pub struct ZipIterable<R> {
+    #[allow(unused)]
+    pub(crate) config: Config,
+    pub(crate) iterable_files: ZipIterableFiles<R>,
+}
+impl<R: Read + Seek> ZipIterable<R> {
+    /// Try to create a new zip archive
+    pub fn try_new(mut reader: R, config: Config) -> ZipResult<ZipIterable<R>> {
+        let file_len = reader.seek(SeekFrom::End(0))?;
+        let mut end_exclusive = file_len;
+        let mut last_err = None;
+
+        let central_directory = loop {
+            let cde = match find_central_directory_end(
+                &mut reader,
+                config.archive_offset,
+                end_exclusive,
+                file_len,
+            ) {
+                Ok(cde) => cde,
+                Err(e) => return Err(last_err.unwrap_or(e)),
+            };
+
+            match CentralDirectoryInfo::try_from(&cde) {
+                Ok(info) => break info,
+                Err(e) => {
+                    last_err = Some(e);
+                    end_exclusive = cde.eocd.position;
+                }
+            }
+        };
+
+        // If the parsed number of files is greater than the offset then
+        // something fishy is going on and we shouldn't trust number_of_files.
+        if central_directory.number_of_files > central_directory.directory_start as usize {
+            return Err(ZipError::UnsupportedArchive("Fishy error :)"));
+        }
+
+        if central_directory.disk_number != central_directory.disk_with_central_directory {
+            return Err(ZipError::UnsupportedArchive(
+                "Support for multi-disk files is not implemented",
+            ));
+        }
+
+        let iterable_shared = ZipIterableFiles::try_new(reader, central_directory)?;
+
+        Ok(Self {
+            config,
+            iterable_files: iterable_shared,
+        })
+    }
+
+    /// Get the file as an iterator
+    pub fn files(&mut self) -> ZipResult<&mut ZipIterableFiles<R>> {
+        self.iterable_files.reset()?;
+        Ok(&mut self.iterable_files)
+    }
+}
+
+/// Iterable Files
+#[derive(Debug)]
+pub struct ZipIterableFiles<R> {
+    reader: R,
+    central_directory: CentralDirectoryInfo,
+    current_file: usize,
+}
+
+impl<R: Read + Seek> ZipIterableFiles<R> {
+    /// Try to create an iterable of files
+    pub(crate) fn try_new(
+        mut reader: R,
+        central_directory: CentralDirectoryInfo,
+    ) -> ZipResult<Self> {
+        reader.seek(SeekFrom::Start(central_directory.directory_start))?;
+        Ok(Self {
+            reader,
+            central_directory,
+            current_file: 0,
+        })
+    }
+
+    pub(crate) fn reset(&mut self) -> ZipResult<()> {
+        self.current_file = 0;
+        self.reader
+            .seek(SeekFrom::Start(self.central_directory.directory_start))?;
+        Ok(())
+    }
+
+    pub(crate) fn parse_entry(&mut self) -> ZipResult<ZipFileEntry<'static>> {
+        let central_header_start = self.reader.stream_position()?;
+
+        // Parse central header
+        let block = ZipCentralEntryBlock::parse(&mut self.reader)?;
+        let (data, file_name) = central_header_to_zip_file_inner(
+            &mut self.reader,
+            self.central_directory.archive_offset,
+            central_header_start,
+            block,
+        )?;
+        let file = ZipFileEntry {
+            file_name_raw: Cow::Owned(file_name),
+            data: Cow::Owned(data),
+        };
+        let central_header_end = self.reader.stream_position()?;
+
+        self.reader.seek(SeekFrom::Start(central_header_end))?;
+        Ok(file)
+    }
+}
+
+impl<R: Read + Seek> Iterator for ZipIterableFiles<R> {
+    type Item = ZipResult<ZipFileEntry<'static>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current_file >= self.central_directory.number_of_files {
+            return None;
+        }
+        self.current_file += 1;
+        let file = self.parse_entry();
+        Some(file)
+    }
+}

src/unstable.rs

@@ -25,6 +25,12 @@ pub mod stream {
 pub mod format {
     pub use crate::format::*;
 }
+
+/// Iterable zip
+pub mod read {
+    pub use crate::read::zip_iterable::ZipIterable;
+}
+
 /// Types for creating ZIP archives.
 pub mod write {
     use crate::result::{ZipError, ZipResult};