feat(connections): browse and add datasets from the Connections tree

crates/engine/src/cmd/connections.rs

@@ -10,6 +10,9 @@ use crate::connections::{Connection, DATABASE_KINDS, DESKTOP_ONLY, KINDS};
 use crate::render::rustgis_tiles::{s3, stac};
 use crate::{CmdResult, CommandSpec as C, Session};
 
+#[path = "connection_browse.rs"]
+mod browse;
+
 pub fn commands() -> Vec<C> {
     vec![
         C::new("connection.list", "List Connections", "Home › Add › Data Sources", list).view().params(
@@ -29,6 +32,9 @@ pub fn commands() -> Vec<C> {
         C::new("connection.test", "Test Connection", "Home › Add › Data Sources", test).view().params(
             r#"{"id"?: "a connection's id or name", "kind"?: "postgres"|"mysql"|"sqlserver"|"service"|"wms"|"wmts"|"wfs"|"wcs"|"ogcapi"|"arcgis"|"csw"|"stac"|"s3", "name"?: text, "group"?: text, "address"?: text, "options"?: "an unsaved connection, as connection.new (when no id)", "password"?: text, "accessKeyId"?: text, "secretAccessKey"?: text, "sessionToken"?: "as connection.login (logs in first)", "probe"?: "true: only the address the test reads, {fetch} (null for databases, which their driver tests)", "answer"?: "non-database kinds only: what that address returned, when the app fetched it (no network request then); databases ignore answer and test through their driver"}"#,
         ),
+        C::new("connection.browse", "Browse Connection", "Home › Add › Data Sources", browse::browse).view().params(
+            r#"{"id": "a connection's id or name", "node"?: "a child's node, as a listing gives it (the connection's top level when absent)", "probe"?: "true: only the address this level reads, {fetch} (null: read without a fetch, or by the database driver)", "answer"?: "what that address returned, when the app fetched it", "documents"?: [{"url": "a further document a listing asked for (next)", "text": "its answer"}]} → {connection, node, children: [{key, name, what, detail?, node? (browse it), children? (listed already), add? {command, params}: the command that adds it}], truncated?, matched?, notes?, service? {url, type}, next? (fetch it, then ask again with answer and documents)}"#,
+        ),
     ]
 }
 
@@ -53,16 +59,7 @@ fn shown(c: &Connection) -> Value {
     } else {
         false
     };
-    let add_with = match c.kind.as_str() {
-        "postgres" | "mysql" | "sqlserver" => "info.databaseTables {database: address}, then layer.addDatabase {database: address, layer}",
-        "service" => "service.layers {url: address}, then service.add {url, layer}",
-        "wms" | "wmts" | "wfs" | "ogcapi" | "arcgis" => "service.layers {url: address, type: kind}, then service.add {url, type, layer}",
-        "wcs" => "service.layers {url: address, type: wcs}, then service.coverage {url, layer}",
-        "csw" => "catalog.search {url: address, query}",
-        "stac" => "stac.collections {url: address}, then stac.search {url, collections}",
-        "s3" => "connection.login {id} once, then s3.list {connection: name}, then layer.addUrl {url: s3://…}",
-        _ => "",
-    };
+    let add_with = "connection.browse {id}, then {id, node} for a child's node; run a dataset's add {command, params}";
     let mut out = serde_json::to_value(c).unwrap_or(Value::Null);
     out["available"] = json!(available);
     if !available {

crates/engine/src/cmd/layer.rs

@@ -1280,6 +1280,15 @@ pub(super) fn database_tables(_s: &mut Session, p: &Value) -> CmdResult {
     }
 }
 
+/// The geometry tables of a live database server, with the schema, name, column and type of
+/// each (`connection.browse` groups them by schema).
+#[cfg(not(target_arch = "wasm32"))]
+pub(super) fn database_table_details(p: &Value) -> Result<Vec<Value>, String> {
+    let db = database_param(p).ok_or("database is required")?;
+    let live = LiveDb::connect(&db).ok_or("not a database server")?.map_err(|e| format!("{}: {e}", db.short()))?;
+    Ok(live.tables().map_err(|e| format!("{}: {e}", db.short()))?.iter().map(LiveTable::detail).collect())
+}
+
 /// Adds the tables of a database (a PostgreSQL connection or a DuckDB file) that have a geometry
 /// column, or the one `layer` names.
 fn add_database(s: &mut Session, p: &Value) -> CmdResult {
@@ -1399,6 +1408,16 @@ impl LiveTable {
             LiveTable::Sqlserver(t) => t.label(),
         }
     }
+
+    /// The table as a browser lists it: its label for `layer.addDatabase`, and what it is made of.
+    fn detail(&self) -> Value {
+        let (schema, name, column, kind) = match self {
+            LiveTable::Postgis(t) => (&t.schema, &t.name, &t.column, &t.kind),
+            LiveTable::Mysql(t) => (&t.schema, &t.name, &t.column, &t.kind),
+            LiveTable::Sqlserver(t) => (&t.schema, &t.name, &t.column, &t.kind),
+        };
+        json!({"label": self.label(), "schema": schema, "name": name, "column": column, "kind": kind})
+    }
 }
 
 #[cfg(not(target_arch = "wasm32"))]

crates/engine/src/cmd/plugins.rs

@@ -276,6 +276,7 @@ const NEVER: &[&str] = &[
     "connection.remove",
     "connection.login",
     "connection.test",
+    "connection.browse",
     "s3.connect",
     "s3.disconnect",
     "earthdata.login",

crates/engine/src/tests.rs

@@ -1148,6 +1148,7 @@ fn hostile_params_never_panic() {
         json!({"id": "", "kind": "postgres", "address": "postgres:host=127.0.0.1 port=9 password=x", "options": "x"}),
         json!({"kind": "wms", "address": "http://127.0.0.1:9/wms", "probe": "x", "answer": 7}),
         json!({"kind": "stac", "address": "http://127.0.0.1:9/stac", "answer": "\u{0}{"}),
+        json!({"id": "c1", "node": "s3:/\u{7}", "probe": true, "answer": 7, "documents": "x"}),
     ];
     let ids: Vec<&str> = Session::new(Project::default()).registry.commands.iter().map(|c| c.id).collect();
     let streamed = grid_fgb("hostile", 20);
@@ -6800,6 +6801,18 @@ fn wcs_coverages_and_catalogue_records() {
     );
     let full = s.run("service.coverage", &json!({"url": url, "layer": "soc", "text": caps, "describe": describe, "bbox": "full"})).unwrap();
     assert_eq!(full["bbox"], json!([5.0, 49.2, 5.8, 50.0]));
+    // The same coverage browsed from a saved connection: the add it lists runs as it is.
+    s.run("connection.new", &json!({"kind": "wcs", "address": url, "name": "Soils"})).unwrap();
+    let browsed = s.run("connection.browse", &json!({"id": "Soils", "answer": caps})).unwrap();
+    let child = &browsed["children"][0];
+    assert_eq!((child["what"].as_str(), child["add"]["command"].as_str()), (Some("coverage"), Some("service.add")), "{browsed}");
+    let mut listed = child["add"]["params"].clone();
+    listed["text"] = json!(caps);
+    listed["describe"] = json!(describe);
+    listed["bbox"] = json!("full");
+    let layers = s.map().layers.len();
+    s.run("service.add", &listed).unwrap();
+    assert_eq!(s.map().layers.len(), layers + 1);
     let (err, _) = serve_file(b"<ows:ExceptionReport>no</ows:ExceptionReport>".to_vec(), false);
     let p = json!({"url": format!("{err}/wcs"), "layer": "soc", "text": caps, "describe": describe, "bbox": "full"});
     assert!(s.run("service.add", &p).unwrap_err().contains("error"));
@@ -8856,7 +8869,15 @@ fn connections_commands() {
     assert_eq!(renamed["name"], "wms (2)");
     assert_ne!(default["id"], renamed["id"]);
     assert!(crate::sources::holds_secret(&json!({"id":"x","password":"p"})));
-    for id in ["connection.new", "connection.edit", "connection.duplicate", "connection.remove", "connection.login", "connection.test"] {
+    for id in [
+        "connection.new",
+        "connection.edit",
+        "connection.duplicate",
+        "connection.remove",
+        "connection.login",
+        "connection.test",
+        "connection.browse",
+    ] {
         assert!(crate::cmd::plugins::user_only(id), "{id}");
     }
     assert!(!crate::cmd::plugins::user_only("connection.list"));
@@ -8967,6 +8988,132 @@ fn connection_test_probes_and_answers() {
     assert!(s.connections.connections.is_empty());
 }
 
+/// Browsing a saved connection lists what each kind offers, a level at a time, with the command
+/// that adds each dataset; offline, with the answers supplied. A node that isn't the
+/// connection's is refused, and nothing browsed is saved or undoable.
+#[test]
+fn connection_browse_levels_and_adds() {
+    use crate::render::rustgis_tiles::s3;
+    let mut s = Session::new(Project::default());
+    let new = |s: &mut Session, p: Value| s.run("connection.new", &p).unwrap();
+    let wms_caps = r#"<WMS_Capabilities version="1.3.0" xmlns="http://www.opengis.net/wms" xmlns:xlink="http://www.w3.org/1999/xlink"><Service><Title>Local</Title></Service><Capability><Request><GetMap><Format>image/png</Format><DCPType><HTTP><Get><OnlineResource xlink:href="https://a.example/wms?"/></Get></HTTP></DCPType></GetMap></Request><Layer><CRS>EPSG:3857</CRS><Layer><Name>a</Name><Title>A</Title></Layer><Layer><Name>b</Name><Title>B</Title></Layer></Layer></Capability></WMS_Capabilities>"#;
+
+    // WMS: the address read, its layers, and the add that adds one.
+    new(&mut s, json!({"kind": "wms", "address": "https://a.example/wms", "name": "W"}));
+    let v = s.run("connection.browse", &json!({"id": "W", "probe": true})).unwrap();
+    assert!(v["fetch"].as_str().is_some_and(|f| f.contains("REQUEST=GetCapabilities")), "{v}");
+    let v = s.run("connection.browse", &json!({"id": "W", "answer": wms_caps})).unwrap();
+    assert_eq!(v["service"], json!({"url": "https://a.example/wms", "type": "wms"}));
+    let first = v["children"][0].clone();
+    assert_eq!(first["what"], "map");
+    assert_eq!(first["name"], "A");
+    assert_eq!(first["add"]["command"], "service.add");
+    assert_eq!(first["add"]["params"]["url"], "https://a.example/wms");
+    assert_eq!(first["add"]["params"]["type"], "wms");
+    assert_eq!(first["add"]["params"]["layer"], "a");
+    assert!(first["add"]["params"].get("source_url").is_some());
+    assert_eq!(v["children"].as_array().unwrap().len(), 2);
+    let before = s.map().tiles.len();
+    let mut add = first["add"]["params"].clone();
+    add["text"] = json!(wms_caps);
+    s.run("service.add", &add).unwrap();
+    assert_eq!(s.map().tiles.len(), before + 1);
+    s.run("edit.undo", &Value::Null).unwrap();
+    assert_eq!(s.map().tiles.len(), before);
+
+    // ArcGIS: a directory's folder is a node of its own address.
+    new(&mut s, json!({"kind": "arcgis", "address": "https://x.example.org/arcgis/rest/services", "name": "Arc"}));
+    let dir = r#"{"currentVersion": 11, "folders": ["Hydro"], "services": [{"name": "Topo", "type": "MapServer"}]}"#;
+    let v = s.run("connection.browse", &json!({"id": "Arc", "answer": dir})).unwrap();
+    let folder = "service:arcgis:https://x.example.org/arcgis/rest/services/Hydro";
+    let child = v["children"].as_array().unwrap().iter().find(|c| c["node"] == folder).unwrap_or_else(|| panic!("{v}"));
+    assert_eq!(child["what"], "folder");
+    assert_eq!(child["name"], "Hydro");
+    let v = s.run("connection.browse", &json!({"id": "Arc", "node": folder, "probe": true})).unwrap();
+    assert!(v["fetch"].as_str().is_some_and(|f| f.starts_with("https://x.example.org/arcgis/rest/services/Hydro")), "{v}");
+
+    // CSW: a record, the service it links to as a node, and more records than listed.
+    new(&mut s, json!({"kind": "csw", "address": "https://cat.example.org/csw", "name": "Cat"}));
+    let records = r#"<csw:GetRecordsResponse xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" xmlns:dc="http://purl.org/dc/elements/1.1/">
+        <csw:SearchResults numberOfRecordsMatched="3" nextRecord="2"><csw:Record><dc:title>Forests</dc:title>
+          <dc:URI protocol="OGC:WMS">https://geo.example.org/wms</dc:URI><dc:URI protocol="OGC:WFS">http://192.168.1.2/wfs</dc:URI>
+        </csw:Record></csw:SearchResults></csw:GetRecordsResponse>"#;
+    let v = s.run("connection.browse", &json!({"id": "Cat", "answer": records})).unwrap();
+    assert_eq!(v["children"].as_array().unwrap().len(), 1, "{v}");
+    assert_eq!(v["children"][0]["what"], "record");
+    assert_eq!(v["children"][0]["children"][0]["node"], "service:wms:https://geo.example.org/wms");
+    assert_eq!((v["matched"].as_u64(), v["truncated"].as_bool()), (Some(3), Some(true)));
+    assert!(v.get("next").is_none());
+
+    // STAC: collections, a collection's search, and an item's asset with the add of it.
+    new(&mut s, json!({"kind": "stac", "address": "https://s.example/v1", "name": "Stac"}));
+    let v = s.run("connection.browse", &json!({"id": "Stac", "probe": true})).unwrap();
+    assert_eq!(v["fetch"], "https://s.example/v1/collections");
+    let v = s.run("connection.browse", &json!({"id": "Stac", "answer": r#"{"collections":[{"id":"c1","title":"C"}]}"#})).unwrap();
+    assert_eq!(v["children"][0]["node"], "collection:c1");
+    assert_eq!(v["children"][0]["options"], true);
+    let v = s.run("connection.browse", &json!({"id": "Stac", "node": "collection:c1", "probe": true})).unwrap();
+    let fetch = v["fetch"].as_str().unwrap();
+    assert!(fetch.contains("/search?") && fetch.contains("collections=c1") && fetch.contains("limit=20"), "{fetch}");
+    let self_link = "https://s.example/v1/collections/c1/items/i1";
+    let items = json!({"type": "FeatureCollection", "numberMatched": 1, "features": [{
+        "type": "Feature", "id": "i1", "collection": "c1", "bbox": [0, 0, 1, 1], "geometry": null,
+        "properties": {"datetime": "2024-01-01T00:00:00Z"},
+        "assets": {"visual": {"href": "https://s.example/a.tif", "type": "image/tiff; application=geotiff"}},
+        "links": [{"rel": "self", "href": self_link}]}]});
+    let v = s.run("connection.browse", &json!({"id": "Stac", "node": "collection:c1", "answer": items.to_string()})).unwrap();
+    assert_eq!(v["children"][0]["what"], "item");
+    assert_eq!(
+        v["children"][0]["children"][0]["add"],
+        json!({"command": "stac.addAsset", "params": {"url": self_link, "asset": "visual", "name": "i1 visual"}})
+    );
+    assert!(v.get("truncated").is_none());
+
+    // S3: buckets from the connection's options need no fetch; a bucket's level is read.
+    new(&mut s, json!({"kind": "s3", "name": "BrowseB", "options": {"credentials": "none", "buckets": "bkt1"}}));
+    let v = s.run("connection.browse", &json!({"id": "BrowseB", "probe": true})).unwrap();
+    assert!(v["fetch"].is_null(), "{v}");
+    let v = s.run("connection.browse", &json!({"id": "BrowseB"})).unwrap();
+    assert_eq!(v["children"][0]["node"], "s3:bkt1/");
+    let v = s.run("connection.browse", &json!({"id": "BrowseB", "node": "s3:bkt1/", "probe": true})).unwrap();
+    assert!(v["fetch"].as_str().is_some_and(|f| f.contains("bkt1")), "{v}");
+    let listing = "<ListBucketResult><CommonPrefixes><Prefix>p/</Prefix></CommonPrefixes><Contents><Key>p.tif</Key><Size>5</Size></Contents><Contents><Key>q/</Key><Size>0</Size></Contents></ListBucketResult>";
+    let v = s.run("connection.browse", &json!({"id": "BrowseB", "node": "s3:bkt1/", "answer": listing})).unwrap();
+    assert_eq!(v["children"][0]["node"], "s3:bkt1/p/");
+    assert_eq!(v["children"][0]["name"], "p/");
+    assert_eq!(v["children"][1]["add"]["params"]["url"], "s3://bkt1/p.tif");
+    assert_eq!(v["children"].as_array().unwrap().len(), 2, "{v}");
+    s3::disconnect("BrowseB");
+
+    // Nodes that aren't the connection's, and ones that are hostile.
+    let long = "x".repeat(5000);
+    for (id, node) in [
+        ("W", "collection:x"),
+        ("BrowseB", "s3:BAD_/"),
+        ("W", long.as_str()),
+        ("W", "service:ftp:x"),
+        ("W", "service:wms:http://127.0.0.1:9/wms"),
+        ("Stac", "catalog:http://127.0.0.1:9/c.json"),
+        ("Stac", "s3:bkt1/"),
+        ("W", "nope"),
+        ("W", "a\u{7}b"),
+    ] {
+        let e = s.run("connection.browse", &json!({"id": id, "node": node})).unwrap_err();
+        assert!(!e.is_empty());
+    }
+    assert!(s.run("connection.browse", &json!({"id": "missing"})).unwrap_err().contains("no connection"));
+    #[cfg(not(target_arch = "wasm32"))]
+    {
+        new(&mut s, json!({"kind": "postgres", "address": "postgres:host=127.0.0.1 port=9 dbname=x user=u", "name": "Pg"}));
+        assert!(!s.run("connection.browse", &json!({"id": "Pg"})).unwrap_err().is_empty());
+    }
+    let saved = s.connections.connections.len();
+    assert!(saved >= 5);
+    s.run("connection.browse", &json!({"id": "W", "answer": wms_caps})).unwrap();
+    assert_eq!(s.connections.connections.len(), saved);
+    assert!(!s.can_undo());
+}
+
 fn rustgis_engine_sources_roundtrip(text: &str) -> crate::sources::Sources {
     crate::sources::Sources::from_json(text)
 }