feat: support installing wally dependencies

lux-lib/Cargo.toml

@@ -70,7 +70,7 @@ tree-sitter-generate = { version = "0.27", default-features = false, features =
   "pathdiff",
 ] }
 tree-sitter-loader = "0.26"
-url = "2.5"
+url = { version = "2.5", features = ["serde"] }
 vfs = "0.13"
 
 darklua = { workspace = true }

lux-lib/resources/test/sample-projects/wally-signal/lux.toml

@@ -0,0 +1,9 @@
+package = "wally-signal"
+version = "0.1.0"
+lua = "luau"
+
+[dependencies.signal]
+wally = "sleitnick/signal@=2.0.3"
+
+[run]
+args = ["src/main.luau"]

lux-lib/resources/test/sample-projects/wally-signal/src/main.luau

@@ -0,0 +1,5 @@
+local signal = require("@signal")
+
+assert(type(signal) == "table", "signal package did not load")
+local instance = signal.new()
+assert(signal.Is(instance), "signal.new() returned an invalid signal")

lux-lib/src/config/mod.rs

@@ -44,6 +44,7 @@ pub struct Config {
     enable_development_packages: bool,
     server: Url,
     extra_servers: Vec<Url>,
+    extra_wally_registries: Vec<Url>,
     namespace: Option<String>,
     lua_dir: Option<PathBuf>,
     lua_version: Option<LuaVersion>,
@@ -148,6 +149,12 @@ impl Config {
         self.extra_servers.as_ref()
     }
 
+    /// Additional wally package registries (git-index URLs), in addition to the
+    /// official wally index.
+    pub fn extra_wally_registries(&self) -> &Vec<Url> {
+        self.extra_wally_registries.as_ref()
+    }
+
     /// Enabled luarocks repository servers that provide dev/scm rocks
     pub fn enabled_dev_servers(&self) -> Result<Vec<Url>, ConfigError> {
         let mut enabled_dev_servers = Vec::new();
@@ -405,6 +412,12 @@ pub struct ConfigBuilder {
         serialize_with = "serialize_url_vec"
     )]
     extra_servers: Option<Vec<Url>>,
+    #[serde(
+        default,
+        deserialize_with = "deserialize_url_vec",
+        serialize_with = "serialize_url_vec"
+    )]
+    extra_wally_registries: Option<Vec<Url>>,
     namespace: Option<String>,
     lua_version: Option<LuaVersion>,
     user_tree: Option<PathBuf>,
@@ -496,6 +509,14 @@ impl ConfigBuilder {
         }
     }
 
+    /// Additional wally package registries, in addition to the official wally index
+    pub fn extra_wally_registries(self, extra_wally_registries: Option<Vec<Url>>) -> Self {
+        Self {
+            extra_wally_registries: extra_wally_registries.or(self.extra_wally_registries),
+            ..self
+        }
+    }
+
     /// The luarocks server namespace to use
     pub fn namespace(self, namespace: Option<String>) -> Self {
         Self {
@@ -763,6 +784,7 @@ impl ConfigBuilder {
         Self {
             server: other.server.or(self.server),
             extra_servers: other.extra_servers.or(self.extra_servers),
+            extra_wally_registries: other.extra_wally_registries.or(self.extra_wally_registries),
             namespace: other.namespace.or(self.namespace),
             lua_version: other.lua_version.or(self.lua_version),
             user_tree: other.user_tree.or(self.user_tree),
@@ -812,6 +834,7 @@ impl ConfigBuilder {
                 Url::parse("https://luarocks.org/").unwrap_unchecked()
             }),
             extra_servers: self.extra_servers.unwrap_or_default(),
+            extra_wally_registries: self.extra_wally_registries.unwrap_or_default(),
             namespace: self.namespace,
             lua_dir: self.lua_dir,
             lua_version,
@@ -854,6 +877,7 @@ impl From<Config> for ConfigBuilder {
             enable_development_packages: Some(value.enable_development_packages),
             server: Some(value.server),
             extra_servers: Some(value.extra_servers),
+            extra_wally_registries: Some(value.extra_wally_registries),
             namespace: value.namespace,
             lua_dir: value.lua_dir,
             lua_version: value.lua_version,

lux-lib/src/lockfile/mod.rs

@@ -708,6 +708,7 @@ impl LocalPackageLock {
                             RemotePackageSource::LuarocksRockspec(_) => filter_spec.rockspec,
                             RemotePackageSource::LuarocksSrcRock(_) => filter_spec.src,
                             RemotePackageSource::LuarocksBinaryRock(_) => filter_spec.binary,
+                            RemotePackageSource::Wally(_) => true,
                             RemotePackageSource::RockspecContent(_) => true,
                             RemotePackageSource::Local => true,
                             #[cfg(test)]

lux-lib/src/lua_rockspec/mod.rs

@@ -444,6 +444,92 @@ impl RemoteLuaRockspec {
     pub fn from_package_and_source_spec(
         package_spec: PackageSpec,
         source_spec: RockSourceSpec,
+    ) -> Self {
+        Self::from_source_spec(package_spec, source_spec, Vec::new())
+    }
+
+    /// Build a rockspec for a wally package, whose contents are fetched from
+    /// `content_url`, whose dependencies come from the package's manifest
+    pub fn from_wally(
+        package_spec: PackageSpec,
+        content_url: Url,
+        dependencies: Vec<LuaDependencySpec>,
+        modules: HashMap<LuaModule, ModuleSpec>,
+    ) -> Self {
+        let version = package_spec.version().clone();
+        let rockspec_format = RockspecFormat::default();
+        let source_spec = RockSourceSpec::Url(content_url);
+
+        let mut module_entries: Vec<_> = modules.iter().collect();
+        module_entries.sort_by_key(|(module, _)| module.as_str());
+        let modules_lua = module_entries
+            .into_iter()
+            .map(|(module, spec)| {
+                let path = match spec {
+                    ModuleSpec::SourcePath(path) => path,
+                    _ => unreachable!(),
+                };
+                format!(
+                    "    [\"{module}\"] = \"{}\",",
+                    path.to_string_lossy().replace('\\', "/")
+                )
+            })
+            .collect::<Vec<_>>()
+            .join("\n");
+
+        let raw_content = format!(
+            r#"
+rockspec_format = "{}"
+package = "{}"
+version = "{}"
+{}
+build = {{
+  type = "builtin",
+  modules = {{
+{}
+  }}
+}}"#,
+            rockspec_format,
+            package_spec.name(),
+            version,
+            source_spec.display_lua(),
+            modules_lua,
+        );
+
+        let source: RemoteRockSource = source_spec.into();
+
+        let local = LocalLuaRockspec {
+            rockspec_format: Some(rockspec_format),
+            package: package_spec.name().clone(),
+            version,
+            description: RockDescription::default(),
+            supported_platforms: PlatformSupport::default(),
+            lua: PackageVersionReq::Any,
+            dependencies: PerPlatform::new(dependencies),
+            build_dependencies: PerPlatform::default(),
+            external_dependencies: PerPlatform::default(),
+            test_dependencies: PerPlatform::default(),
+            build: PerPlatform::new(BuildSpec {
+                build_backend: Some(BuildBackendSpec::Builtin(BuiltinBuildSpec { modules })),
+                install: InstallSpec::default(),
+                copy_directories: Vec::new(),
+                patches: HashMap::new(),
+            }),
+            source: PerPlatform::new(source.clone()),
+            test: PerPlatform::default(),
+            deploy: PerPlatform::default(),
+            raw_content,
+        };
+        Self {
+            local,
+            source: PerPlatform::new(source),
+        }
+    }
+
+    fn from_source_spec(
+        package_spec: PackageSpec,
+        source_spec: RockSourceSpec,
+        dependencies: Vec<LuaDependencySpec>,
     ) -> Self {
         let version = package_spec.version().clone();
         let rockspec_format = RockspecFormat::default();
@@ -471,7 +557,7 @@ build = {{
             description: RockDescription::default(),
             supported_platforms: PlatformSupport::default(),
             lua: PackageVersionReq::Any,
-            dependencies: PerPlatform::default(),
+            dependencies: PerPlatform::new(dependencies),
             build_dependencies: PerPlatform::default(),
             external_dependencies: PerPlatform::default(),
             test_dependencies: PerPlatform::default(),

lux-lib/src/lua_rockspec/rock_source.rs

@@ -4,6 +4,7 @@ use crate::{
         GitSource,
     },
     lua_rockspec::per_platform_from_intermediate,
+    package::wally::PackageReq as WallyPackageReq,
 };
 
 pub use crate::git::GitRef;
@@ -112,6 +113,7 @@ pub enum RockSourceSpec {
     Git(GitSource),
     File(PathBuf),
     Url(Url),
+    Wally(WallyPackageReq),
 }
 
 impl RockSourceSpec {
@@ -130,6 +132,7 @@ impl Display for RockSourceSpec {
             RockSourceSpec::Git(git_source) => git_source.fmt(f),
             RockSourceSpec::File(path_buf) => path_buf.display().fmt(f),
             RockSourceSpec::Url(url) => url.fmt(f),
+            RockSourceSpec::Wally(req) => req.fmt(f),
         }
     }
 }
@@ -172,6 +175,7 @@ impl DisplayAsLuaKV for RockSourceSpec {
                     value: DisplayLuaValue::Table(source_tbl),
                 }
             }
+            RockSourceSpec::Wally(_) => unimplemented!(),
         }
     }
 }

lux-lib/src/operations/download.rs

@@ -14,17 +14,20 @@ use crate::{
     config::Config,
     fs,
     git::{GitRef, GitSource},
-    lockfile::RemotePackageSourceUrl,
+    lockfile::{OptState, PinnedState, RemotePackageSourceUrl},
     lua_rockspec::{LuaRockspecError, RemoteLuaRockspec, RockSourceSpec},
     luarocks,
     package::{
+        wally::{
+            self, PackageReq as WallyPackageReq, WallyIndex, WallyIndexError, WallyModulesError,
+        },
         PackageName, PackageReq, PackageSpec, PackageSpecFromPackageReqError, PackageVersion,
-        RemotePackageTypeFilterSpec,
+        PackageVersionParseError, PackageVersionReq, RemotePackageTypeFilterSpec,
     },
     remote_package_db::{RemotePackageDB, RemotePackageDBError, SearchError},
     remote_package_source::RemotePackageSource,
     reqwest::{RequestBuilderExt, RequestError},
-    rockspec::Rockspec,
+    rockspec::{lua_dependency::LuaDependencySpec, Rockspec},
 };
 
 /// Builder for a rock downloader.
@@ -183,6 +186,7 @@ impl RemoteRockDownload {
             }
             RockSourceSpec::File(path) => RemotePackageSourceUrl::File { path: path.clone() },
             RockSourceSpec::Url(url) => RemotePackageSourceUrl::Url { url: url.clone() },
+            RockSourceSpec::Wally(_) => unimplemented!(),
         });
         let rockspec = RemoteLuaRockspec::from_package_and_source_spec(package_spec, source_spec);
         let rockspec_content = rockspec
@@ -197,6 +201,83 @@ impl RemoteRockDownload {
     }
 }
 
+/// Resolve and download a wally package from the configured registries.
+pub(crate) async fn download_wally_rock(
+    package_req: &PackageReq,
+    wally_req: &WallyPackageReq,
+    config: &Config,
+) -> Result<RemoteRockDownload, SearchAndDownloadError> {
+    let mut registries = vec![unsafe { Url::parse(wally::DEFAULT_INDEX_URL).unwrap_unchecked() }];
+    registries.extend(config.extra_wally_registries().iter().cloned());
+    download_wally_rock_from_registries(package_req, wally_req, &registries, config).await
+}
+
+pub(crate) async fn download_wally_rock_from_registries(
+    package_req: &PackageReq,
+    wally_req: &WallyPackageReq,
+    registries: &[Url],
+    config: &Config,
+) -> Result<RemoteRockDownload, SearchAndDownloadError> {
+    for registry in registries {
+        let index = WallyIndex::open(registry, config.cache_dir())?;
+        let Some(manifest) = index.find(wally_req)? else {
+            continue;
+        };
+        let version = PackageVersion::parse(&manifest.package.version.to_string())?;
+        let package_spec = PackageSpec::new(package_req.name().clone(), version);
+        let content_url = index
+            .config()
+            .api
+            .join(&format!(
+                "/v1/package-contents/{}/{}/{}",
+                wally_req.name().scope(),
+                wally_req.name().name(),
+                manifest.package.version
+            ))
+            .map_err(|source| SearchAndDownloadError::Parse {
+                source,
+                url: index.config().api.to_string(),
+            })?;
+        let contents = fetch_wally_contents(&content_url, config).await?;
+        let modules = wally::modules_from_zip(&contents)?;
+        let dependencies = manifest
+            .dependencies
+            .into_iter()
+            .map(|(alias, dep_req)| LuaDependencySpec {
+                package_req: PackageReq {
+                    name: PackageName::new(alias),
+                    version_req: PackageVersionReq::SemVer(dep_req.version_req().clone()),
+                },
+                pin: PinnedState::default(),
+                opt: OptState::default(),
+                source: Some(RockSourceSpec::Wally(dep_req)),
+            })
+            .collect();
+        let rockspec =
+            RemoteLuaRockspec::from_wally(package_spec, content_url.clone(), dependencies, modules);
+        let rockspec_download = DownloadedRockspec {
+            rockspec,
+            source: RemotePackageSource::Wally(registry.clone()),
+            source_url: Some(RemotePackageSourceUrl::Url { url: content_url }),
+        };
+        return Ok(RemoteRockDownload::RockspecOnly { rockspec_download });
+    }
+
+    Err(SearchAndDownloadError::WallyNotFound(wally_req.clone()))
+}
+
+async fn fetch_wally_contents(url: &Url, config: &Config) -> Result<Bytes, SearchAndDownloadError> {
+    let response = crate::reqwest::http_client(config)?
+        .get(url.clone())
+        .header("Wally-Version", wally::WALLY_VERSION)
+        .send()
+        .await?
+        .error_for_status()?
+        .bytes()
+        .await?;
+    Ok(response)
+}
+
 #[derive(Error, Debug, Diagnostic)]
 pub enum DownloadRockspecError {
     #[error("failed to download rockspec")]
@@ -336,6 +417,7 @@ async fn download_remote_rock(
                 source_url: RemotePackageSourceUrl::Url { url },
             })
         }
+        RemotePackageSource::Wally(_) => unimplemented!("wally packages"),
         RemotePackageSource::Local => Err(SearchAndDownloadError::LocalSource),
         #[cfg(test)]
         RemotePackageSource::Test => unimplemented!(),
@@ -420,6 +502,14 @@ for local dependencies, use `path` in your lux.toml."#
         url("https://lux.lumen-labs.org/reference/lux-toml#local-dependencies")
     )]
     NonURLSource,
+    #[error("failed to resolve wally package")]
+    WallyIndex(#[from] WallyIndexError),
+    #[error("failed to inspect wally package contents")]
+    WallyModules(#[from] WallyModulesError),
+    #[error("failed to parse wally package version")]
+    WallyVersion(#[from] PackageVersionParseError),
+    #[error("wally package '{0}' was not found in any registry")]
+    WallyNotFound(WallyPackageReq),
     #[error("client error")]
     #[diagnostic(transparent)]
     Request(#[from] RequestError),