fix(data): honor Shapefile code-page metadata

Cargo.toml

@@ -37,7 +37,8 @@ web-time = "1"
 geo = "0.33"
 # A5, the equal-area pentagonal DGGS (opengeos/RustGIS#551)
 a5 = "1"
-shapefile = "0.9"
+shapefile = { version = "0.9", features = ["encoding_rs", "yore"] }
+encoding_rs = "0.8"
 csv = "1"
 tiny-skia = "0.12"
 # GPU map backend: vertex data shared with wgpu in ui-egui (derive only; unsafe stays forbidden).

THIRD-PARTY-LICENSES.txt

@@ -8769,7 +8769,7 @@ SOFTWARE.
 ====================================================================================================
 MIT License
 Used by: block2, bufrw, dispatch, dpi, i_key_sort, libm, objc-sys, objc2, objc2-app-kit, objc2-encode,
-objc2-foundation, rsqlite-vfs
+objc2-foundation, rsqlite-vfs, yore
 ====================================================================================================
 
 MIT License

apps/rustgis-python/THIRD-PARTY-LICENSES.txt

@@ -6913,7 +6913,7 @@ SOFTWARE.
 
 ====================================================================================================
 MIT License
-Used by: bufrw, i_key_sort, libm, rsqlite-vfs
+Used by: bufrw, i_key_sort, libm, rsqlite-vfs, yore
 ====================================================================================================
 
 MIT License

crates/data/Cargo.toml

@@ -15,6 +15,7 @@ serde_json = { workspace = true }
 thiserror = { workspace = true }
 csv = { workspace = true }
 shapefile = { workspace = true }
+encoding_rs = { workspace = true }
 # Zstandard, the usual compression of COGs, is C code that compiles to WebAssembly with clang
 # (on the CI runners), so the browser decodes it too.
 tiff = { workspace = true, features = ["zstd"] }

crates/data/src/dbf.rs

@@ -0,0 +1,145 @@
+//! Shared Shapefile character encoding selection for whole-file and streamed reads.
+
+use encoding_rs as rs;
+use shapefile::dbase::{CodePageMark, UnicodeLossy, encoding::DynEncoding};
+
+const MAX_CPG_BYTES: usize = 1024;
+
+/// Keep the selection thread-safe; dbase's dynamic decoder is created per read because it
+/// is `Send` but not `Sync`, whereas feature sources can be shared between readers.
+pub(crate) enum DbfEncoding {
+    Standard(&'static rs::Encoding),
+    Oem(CodePageMark),
+    Utf8,
+}
+
+impl DbfEncoding {
+    pub(crate) fn decoder(&self) -> DynEncoding {
+        match self {
+            Self::Standard(encoding) => shapefile::dbase::encoding::EncodingRs::from(*encoding).into(),
+            Self::Oem(mark) => mark.to_encoding().unwrap_or_else(|| UnicodeLossy.into()),
+            Self::Utf8 => UnicodeLossy.into(),
+        }
+    }
+
+    fn from_number(number: u16) -> Option<Self> {
+        let encoding = match number {
+            437 => return Some(Self::Oem(CodePageMark::CP437)),
+            850 => return Some(Self::Oem(CodePageMark::CP850)),
+            852 => return Some(Self::Oem(CodePageMark::CP852)),
+            861 => return Some(Self::Oem(CodePageMark::CP861)),
+            865 => return Some(Self::Oem(CodePageMark::CP865)),
+            866 => rs::IBM866,
+            874 => rs::WINDOWS_874,
+            932 => rs::SHIFT_JIS,
+            936 => rs::GBK,
+            949 => rs::EUC_KR,
+            950 => rs::BIG5,
+            1250..=1258 => return Self::from_label(&format!("windows-{number}")),
+            65001 => rs::UTF_8,
+            _ => return None,
+        };
+        Some(Self::Standard(encoding))
+    }
+
+    fn from_label(label: &str) -> Option<Self> {
+        // dbase's name lookup with both decoder features enabled falls back to CP1252
+        // for unknown labels. Resolve explicitly so unusable metadata is reported.
+        let encoding = rs::Encoding::for_label(label.as_bytes())?;
+        // UTF-16 and replacement encodings cannot represent DBF's ASCII padding and
+        // NUL-terminated field names. Treat these labels as unsupported metadata.
+        encoding.is_ascii_compatible().then_some(Self::Standard(encoding))
+    }
+
+    fn from_name(name: &str) -> Option<Self> {
+        let upper = name.to_ascii_uppercase();
+        if let Some(suffix) = upper.strip_prefix("8859") {
+            return Self::from_label(&format!("iso-8859-{}", suffix.strip_prefix('-').unwrap_or(suffix)));
+        }
+        let prefixed = ["CP", "IBM", "OEM", "WINDOWS", "ANSI"].into_iter().find_map(|prefix| upper.strip_prefix(prefix));
+        let number = match prefixed {
+            Some(suffix) => suffix.strip_prefix('-').unwrap_or(suffix).trim(),
+            None => &upper,
+        }
+        .parse::<u16>();
+        match number {
+            Ok(number) => Self::from_number(number),
+            Err(_) => Self::from_label(&upper),
+        }
+    }
+
+    fn from_mark(mark: CodePageMark) -> Option<Self> {
+        // Keep encoding_rs for its supported pages: dbase's yore adapter is strict for
+        // those encodings, which can discard records containing malformed text.
+        let encoding = match mark {
+            CodePageMark::CP437 | CodePageMark::CP850 | CodePageMark::CP852 | CodePageMark::CP861 | CodePageMark::CP865 => {
+                return Some(Self::Oem(mark));
+            }
+            CodePageMark::CP1250 => rs::WINDOWS_1250,
+            CodePageMark::CP1251 => rs::WINDOWS_1251,
+            CodePageMark::CP1252 => rs::WINDOWS_1252,
+            CodePageMark::CP1253 => rs::WINDOWS_1253,
+            CodePageMark::CP1254 => rs::WINDOWS_1254,
+            CodePageMark::CP1255 => rs::WINDOWS_1255,
+            CodePageMark::CP1256 => rs::WINDOWS_1256,
+            CodePageMark::CP866 => rs::IBM866,
+            CodePageMark::CP874 => rs::WINDOWS_874,
+            CodePageMark::CP932 => rs::SHIFT_JIS,
+            CodePageMark::CP936 => rs::GBK,
+            CodePageMark::CP949 => rs::EUC_KR,
+            CodePageMark::CP950 => rs::BIG5,
+            CodePageMark::Utf8 => rs::UTF_8,
+            CodePageMark::Undefined => return Some(Self::Utf8),
+            _ => return None,
+        };
+        Some(Self::Standard(encoding))
+    }
+}
+
+pub(crate) fn encoding(cpg: Option<&[u8]>, ldid: u8, notes: &mut Vec<String>) -> DbfEncoding {
+    if let Some(bytes) = cpg {
+        let label = if bytes.len() <= MAX_CPG_BYTES { std::str::from_utf8(bytes).unwrap_or("") } else { "" };
+        let label = label.trim().trim_start_matches('\u{feff}').trim();
+        if let Some(encoding) = DbfEncoding::from_name(label) {
+            return encoding;
+        }
+        notes.push("Shapefile: unsupported or empty .cpg; using the .dbf code page, or UTF-8 if unavailable".into());
+    }
+    // Esri's LDID table assigns 0x57 to ANSI 1252 (ArcPad, "Setting the code page
+    // for shapefiles"); dbase does not recognize that widely used marker.
+    let mark = if ldid == 0x57 { CodePageMark::CP1252 } else { CodePageMark::from(ldid) };
+    if let Some(encoding) = DbfEncoding::from_mark(mark) {
+        return encoding;
+    }
+    notes.push(format!("Shapefile: unsupported .dbf code page 0x{ldid:02X}; using UTF-8"));
+    DbfEncoding::Utf8
+}
+
+/// A code-page label should be a few bytes. Bound reads even for a hostile sidecar.
+pub(crate) fn read_cpg(path: &std::path::Path, notes: &mut Vec<String>) -> Option<Vec<u8>> {
+    match try_read_cpg(path) {
+        Ok(bytes) => bytes,
+        Err(e) => {
+            notes.push(format!("Shapefile: could not read .cpg ({e}); using the .dbf code page, or UTF-8 if unavailable"));
+            None
+        }
+    }
+}
+
+fn try_read_cpg(path: &std::path::Path) -> std::io::Result<Option<Vec<u8>>> {
+    use std::io::Read;
+    // Check every ASCII case combination without scanning a potentially large directory.
+    // Prefer the conventional spellings if multiple sidecars exist.
+    for ext in ["cpg", "CPG", "Cpg", "cPg", "cpG", "CPg", "CpG", "cPG"] {
+        match std::fs::File::open(path.with_extension(ext)) {
+            Ok(file) => {
+                let mut bytes = Vec::new();
+                file.take(MAX_CPG_BYTES as u64 + 1).read_to_end(&mut bytes)?;
+                return Ok(Some(bytes));
+            }
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+            Err(e) => return Err(e),
+        }
+    }
+    Ok(None)
+}

crates/data/src/io.rs

@@ -1314,22 +1314,25 @@ fn read_shapefile(path: &Path, name: &str) -> Result<(Dataset, ReadReport)> {
         return Err(DataError::Format(format!("Shapefile: {} is missing", dbf.display())));
     }
     let shx = path.with_extension("shx");
+    let mut notes = Vec::new();
     let parts = ShapefileParts {
         shp: std::fs::read(path)?,
         shx: if shx.is_file() { Some(std::fs::read(shx)?) } else { None },
         dbf: std::fs::read(dbf)?,
         prj: crate::raster::read_prj(path),
+        cpg: crate::dbf::read_cpg(path, &mut notes),
     };
-    read_shapefile_parts(parts, name)
+    let (ds, mut report) = read_shapefile_parts(parts, name)?;
+    report.messages.extend(notes);
+    Ok((ds, report))
 }
 
-/// The files of a Shapefile besides its `.shp` that [`gather_shapefiles`] zips with it. The `.cpg`
-/// isn't read (the `.dbf`'s text is decoded without it); it's taken along so it isn't refused on its own.
+/// The files of a Shapefile besides its `.shp` that [`gather_shapefiles`] zips with it.
 const SHAPEFILE_SIDECARS: [&str; 4] = ["dbf", "shx", "prj", "cpg"];
 
 /// Gathers each Shapefile among files given together (several picked at once in the browser, or
 /// dropped at once) into a zip that reads as a zipped Shapefile: a `.shp` with the `.dbf` of the
-/// same base name, ignoring case, and its `.shx`, `.prj` and `.cpg` when they are there (the `.cpg` only so it goes with the rest). The zip,
+/// same base name, ignoring case, and its `.shx`, `.prj` and `.cpg` when they are there. The zip,
 /// named after the `.shp`, takes its place; its parts go. Other files, and a `.shp` without its
 /// `.dbf`, pass through as they came. A Shapefile whose parts are too large to read together (more
 /// than [`MAX_BYTES`], or the zip reader's limit) is left out, and said why in the messages
@@ -1403,7 +1406,7 @@ pub fn read_zip(file_name: &str, bytes: &[u8]) -> Result<Vec<(Dataset, ReadRepor
     let wanted = |n: &str| {
         let e = ext_of(n);
         !is_fork(n)
-            && (matches!(e.as_str(), "shp" | "shx" | "dbf" | "prj" | "mid")
+            && (matches!(e.as_str(), "shp" | "shx" | "dbf" | "prj" | "cpg" | "mid")
                 || (e == "pbf" && lower(n).ends_with(".osm.pbf"))
                 || (!matches!(e.as_str(), "zip" | "txt" | "pbf") && READ_EXTENSIONS.contains(&e.as_str())))
     };
@@ -1429,11 +1432,12 @@ pub fn read_zip(file_name: &str, bytes: &[u8]) -> Result<Vec<(Dataset, ReadRepor
         let shp = take(&mut entries, &base, "shp").map(|(_, b)| b).unwrap_or_default();
         let shx = take(&mut entries, &base, "shx").map(|(_, b)| b);
         let prj = take(&mut entries, &base, "prj").map(|(_, b)| String::from_utf8_lossy(&b).into_owned());
+        let cpg = take(&mut entries, &base, "cpg").map(|(_, b)| b);
         let Some((_, dbf)) = take(&mut entries, &base, "dbf") else {
             skipped.push(format!("{n} (no .dbf beside it)"));
             continue;
         };
-        match read_shapefile_parts(ShapefileParts { shp, shx, dbf, prj }, &name) {
+        match read_shapefile_parts(ShapefileParts { shp, shx, dbf, prj, cpg }, &name) {
             Ok(mut read) => {
                 read.0.source = Some(format!("{file_name}/{n}"));
                 out.push(read);
@@ -1460,8 +1464,11 @@ pub fn read_zip(file_name: &str, bytes: &[u8]) -> Result<Vec<(Dataset, ReadRepor
             Err(e) => skipped.push(format!("{n} ({e})")),
         }
     }
-    let mut others: Vec<(String, Vec<u8>)> =
-        entries.into_iter().flatten().filter(|(n, _)| !matches!(ext_of(n).as_str(), "shp" | "shx" | "dbf" | "prj" | "mid")).collect();
+    let mut others: Vec<(String, Vec<u8>)> = entries
+        .into_iter()
+        .flatten()
+        .filter(|(n, _)| !matches!(ext_of(n).as_str(), "shp" | "shx" | "dbf" | "prj" | "cpg" | "mid"))
+        .collect();
     others.sort_by(|a, b| a.0.cmp(&b.0));
     for (n, b) in others {
         if out.len() >= MAX_ZIP_LAYERS {
@@ -1495,14 +1502,20 @@ struct ShapefileParts {
     shx: Option<Vec<u8>>,
     dbf: Vec<u8>,
     prj: Option<String>,
+    cpg: Option<Vec<u8>>,
 }
 
 fn read_shapefile_parts(parts: ShapefileParts, name: &str) -> Result<(Dataset, ReadReport)> {
     use shapefile::Shape;
     use shapefile::dbase::FieldValue;
     use std::io::Cursor;
     let fmt = |e: shapefile::Error| DataError::Format(format!("Shapefile: {e}"));
-    let dbf = shapefile::dbase::Reader::new(Cursor::new(parts.dbf)).map_err(|e| DataError::Format(format!("Shapefile: the .dbf: {e}")))?;
+    let mut encoding_notes = Vec::new();
+    let encoding = crate::dbf::encoding(parts.cpg.as_deref(), parts.dbf.get(29).copied().unwrap_or(0), &mut encoding_notes);
+    let dbf = shapefile::dbase::ReaderBuilder::new()
+        .with_encoding(encoding.decoder())
+        .build(Cursor::new(parts.dbf))
+        .map_err(|e| DataError::Format(format!("Shapefile: the .dbf: {e}")))?;
     // Field order comes from the .dbf header (records are unordered maps).
     let order: Vec<(String, FieldType)> = dbf
         .fields()
@@ -1528,6 +1541,7 @@ fn read_shapefile_parts(parts: ShapefileParts, name: &str) -> Result<(Dataset, R
     let fields: Vec<Field> = order.iter().map(|(n, t)| Field::new(n.clone(), *t)).collect();
     let mut ds = Dataset::new(name, GeometryType::None, fields);
     let mut report = ReadReport::default();
+    report.messages.extend(encoding_notes);
     let to_lonlat = shapefile_crs(parts.prj.as_deref(), &mut report).map(|(crs, t)| {
         ds.crs = Some(crs);
         t

crates/data/src/lib.rs

@@ -4,6 +4,7 @@
 //! bad features are skipped and counted in [`ReadReport`].
 #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
 
+mod dbf;
 pub mod dirs;
 pub mod dxf;
 pub mod filegdb;