-
Notifications
You must be signed in to change notification settings - Fork 23
feat: Add support for SOURCE packages #244
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 26 commits
a532262
a9b0110
0847e0d
1377324
482bd9e
8d7d436
4f94186
177e276
456da43
3506c08
2f23e6e
bea3f62
b6b2895
0384d5c
e7eac88
eaf607c
5540214
f26f503
61c64d1
6728477
420b29f
30a922b
e212351
fb84a3d
579fd5c
4d703ac
d2ab537
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| cmake_minimum_required(VERSION 3.22) | ||
|
|
||
| project(local_build_main_pkg C) | ||
|
|
||
| add_executable(hello-main hello.c) | ||
| install(TARGETS hello-main DESTINATION bin) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| #include <stdio.h> | ||
| int main(void) { | ||
| printf("Hello from local-build-main-pkg!\n"); | ||
| return 0; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| cmake_minimum_required(VERSION 3.22) | ||
|
|
||
| project(local_build_local_pkg C) | ||
|
|
||
| add_executable(hello-pkg hello.c) | ||
| install(TARGETS hello-pkg DESTINATION bin) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| #include <stdio.h> | ||
| int main(void) { | ||
| printf("Hello from local-build-local-pkg!\n"); | ||
| return 0; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| [package] | ||
| name = "local-build-local-pkg" | ||
| version = "0.1.0" | ||
|
|
||
| [package.build] | ||
| # xref: https://github.com/prefix-dev/pixi/issues/6524 | ||
| backend = { name = "pixi-build-cmake", version = "==0.4.3" } | ||
|
|
||
| [package.run-dependencies] | ||
| curl = "==8.17.0" | ||
| local-build-local-pkg-2 = { path = "../local-pkg2" } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| cmake_minimum_required(VERSION 3.22) | ||
|
|
||
| project(local_build_local_pkg2 C) | ||
|
|
||
| add_executable(hello-pkg2 hello.c) | ||
| install(TARGETS hello-pkg2 DESTINATION bin) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| #include <stdio.h> | ||
| int main(void) { | ||
| printf("Hello from local-build-local-pkg-2!\n"); | ||
| return 0; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| [package] | ||
| name = "local-build-local-pkg-2" | ||
| version = "0.1.0" | ||
|
|
||
| [package.build] | ||
| # xref: https://github.com/prefix-dev/pixi/issues/6524 | ||
| backend = { name = "pixi-build-cmake", version = "==0.4.3" } | ||
|
|
||
| [package.run-dependencies] | ||
| curl = "==8.17.0" |
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| [workspace] | ||
| preview = ["pixi-build"] | ||
| channels = ["conda-forge"] | ||
| platforms = ["linux-64", "linux-aarch64", "win-64", "osx-64", "osx-arm64"] | ||
|
|
||
| [dependencies] | ||
| local-build-main-pkg = { path = "." } | ||
|
|
||
| [package] | ||
| name = "local-build-main-pkg" | ||
| version = "0.1.0" | ||
|
|
||
| [package.build] | ||
| # xref: https://github.com/prefix-dev/pixi/issues/6524 | ||
| backend = { name = "pixi-build-cmake", version = "==0.4.3" } | ||
|
|
||
| [package.run-dependencies] | ||
| local-build-local-pkg = { path = "./local-pkg" } | ||
|
|
||
| [tool.update] | ||
| autoupdate-schedule = "never" |
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,11 +10,12 @@ use std::{ | |
| #[cfg(not(target_os = "windows"))] | ||
| use std::os::unix::fs::PermissionsExt as _; | ||
|
|
||
| use indicatif::HumanBytes; | ||
| use indicatif::{HumanBytes, ProgressBar}; | ||
| use rattler_index::{package_record_from_conda, package_record_from_tar_bz2}; | ||
| use tokio::{ | ||
| fs::{self, File, create_dir_all}, | ||
| io::AsyncWriteExt, | ||
| process::Command as TokioCommand, | ||
| }; | ||
|
|
||
| use anyhow::Result; | ||
|
|
@@ -33,6 +34,7 @@ use rattler_networking::{ | |
| AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, MirrorMiddleware, S3Middleware, | ||
| authentication_storage, mirror_middleware::Mirror, | ||
| }; | ||
|
|
||
| use reqwest_middleware::ClientWithMiddleware; | ||
| use tar::{Builder, HeaderMode}; | ||
| use tokio::io::AsyncReadExt; | ||
|
|
@@ -126,6 +128,77 @@ fn load_lockfile(manifest_path: &Path) -> Result<LockFile> { | |
| }) | ||
| } | ||
|
|
||
| async fn build_local_package( | ||
| manifest_path: &Path, | ||
| output_dir: &Path, | ||
| platform: &str, | ||
| ) -> Result<()> { | ||
| let manifest_dir = manifest_path | ||
| .parent() | ||
| .expect("manifest_path should have a parent directory"); | ||
| let mut command = TokioCommand::new("pixi"); | ||
| command | ||
| .arg("publish") | ||
| .arg("--target-dir") | ||
| .arg(output_dir) | ||
| .arg("--build-platform") | ||
| .arg(platform) | ||
| .arg("--path") | ||
| .arg(manifest_path); | ||
|
|
||
| let is_verbose = tracing::enabled!(tracing::Level::WARN); | ||
|
|
||
| if is_verbose { | ||
| eprintln!( | ||
| "🔧 Running: pixi publish --target-dir {} --build-platform {} (in {})", | ||
| output_dir.display(), | ||
| platform, | ||
| manifest_dir.display() | ||
| ); | ||
|
|
||
| let status = command | ||
| .stdout(std::process::Stdio::inherit()) | ||
| .stderr(std::process::Stdio::inherit()) | ||
| .status() | ||
| .await?; | ||
|
|
||
| if !status.success() { | ||
| anyhow::bail!("Failed to publish package in {}", manifest_dir.display()); | ||
| } | ||
|
Comment on lines
+165
to
+167
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. in a follow-up we could stream the logs like
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| } else { | ||
| let output = command.output().await?; | ||
|
|
||
| if !output.status.success() { | ||
| let stdout = String::from_utf8_lossy(&output.stdout); | ||
| let stderr = String::from_utf8_lossy(&output.stderr); | ||
|
|
||
| eprintln!( | ||
| "❌ Publish failed with exit code: {:?}", | ||
| output.status.code() | ||
| ); | ||
| eprintln!( | ||
| "Command: pixi publish --target-dir {} --build-platform {} (in {})", | ||
| output_dir.display(), | ||
| platform, | ||
| manifest_dir.display() | ||
| ); | ||
|
|
||
| if !stdout.is_empty() { | ||
| eprintln!("📤 stdout:"); | ||
| eprintln!("{}", stdout); | ||
| } | ||
| if !stderr.is_empty() { | ||
| eprintln!("📤 stderr:"); | ||
| eprintln!("{}", stderr); | ||
| } | ||
|
|
||
| anyhow::bail!("Failed to publish package in {}", manifest_dir.display()); | ||
| } | ||
| } | ||
|
|
||
| Ok(()) | ||
| } | ||
|
|
||
| /// Pack a pixi environment. | ||
| pub async fn pack(options: PackOptions) -> Result<()> { | ||
| let lockfile = load_lockfile(&options.manifest_path)?; | ||
|
|
@@ -179,14 +252,78 @@ pub async fn pack(options: PackOptions) -> Result<()> { | |
|
|
||
| let mut conda_packages_from_lockfile: Vec<CondaBinaryData> = Vec::new(); | ||
| let mut pypi_packages_from_lockfile: Vec<PypiPackageData> = Vec::new(); | ||
| let mut built_source_packages: Vec<PathBuf> = Vec::new(); | ||
| let mut _temp_dirs: Vec<tempfile::TempDir> = Vec::new(); | ||
|
pavelzw marked this conversation as resolved.
|
||
|
|
||
| for package in packages { | ||
| match package { | ||
| LockedPackage::Conda(CondaPackageData::Binary(binary_data)) => { | ||
| conda_packages_from_lockfile.push((**binary_data).clone()) | ||
| } | ||
| LockedPackage::Conda(CondaPackageData::Source(_)) => { | ||
| anyhow::bail!("Conda source packages are not yet supported by pixi-pack") | ||
| LockedPackage::Conda(CondaPackageData::Source(source_data)) => { | ||
| let base_dir = if options.manifest_path.is_file() { | ||
| options.manifest_path.parent().unwrap_or(Path::new(".")) | ||
| } else { | ||
| &options.manifest_path | ||
| }; | ||
| let relative_path = match &source_data.location { | ||
| UrlOrPath::Path(p) => PathBuf::from(p.to_path().as_str()), | ||
| UrlOrPath::Url(u) => { | ||
| anyhow::bail!("Unexpected URL for local package source: {}", u) | ||
|
delsner marked this conversation as resolved.
|
||
| } | ||
|
Comment on lines
+271
to
+273
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this could be useful as well in a follow-up to support things like this: [workspace]
preview = ["pixi-build"]
channels = ["conda-forge"]
platforms = ["linux-64", "linux-aarch64", "win-64", "osx-64", "osx-arm64"]
[dependencies]
agent-skill-conda-forge = { git = "https://github.com/pavelzw/skill-forge", rev = "main", subdirectory = "recipes/conda-forge" }
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| }; | ||
|
|
||
| let pkg_dir = base_dir.join(&relative_path); | ||
| let manifest_path = pkg_dir.join("pixi.toml"); | ||
| let canonical_manifest_path = manifest_path.canonicalize().map_err(|e| { | ||
| anyhow!( | ||
| "Cannot find manifest file {}: {}", | ||
| manifest_path.display(), | ||
| e | ||
| ) | ||
| })?; | ||
| let build_temp_dir = tempfile::tempdir() | ||
| .map_err(|e| anyhow!("could not create temporary build directory: {}", e))?; | ||
| let bar = ProgressBar::new_spinner(); | ||
|
|
||
| bar.set_message(format!("Publishing {}", source_data.name().as_source())); | ||
| bar.enable_steady_tick(std::time::Duration::from_millis(100)); | ||
| build_local_package( | ||
| &canonical_manifest_path, | ||
| build_temp_dir.path(), | ||
| options.platform.as_str(), | ||
| ) | ||
| .await?; | ||
|
delsner marked this conversation as resolved.
|
||
| bar.finish_with_message(format!( | ||
| "✅ Built and published {}", | ||
| source_data.name().as_source() | ||
| )); | ||
|
|
||
| let built_package = std::fs::read_dir(build_temp_dir.path()) | ||
| .map_err(|e| anyhow!("Failed to read build directory: {}", e))? | ||
| .next() | ||
| .ok_or_else(|| anyhow!("Build directory is empty"))? | ||
| .map_err(|e| anyhow!("Failed to read directory entry: {}", e))? | ||
| .path(); | ||
|
|
||
| anyhow::ensure!( | ||
| built_package.extension().and_then(|s| s.to_str()) == Some("conda"), | ||
| "Expected built package to be a .conda file, got: {}", | ||
| built_package.to_string_lossy() | ||
| ); | ||
| anyhow::ensure!( | ||
| built_package | ||
| .file_name() | ||
| .and_then(|s| s.to_str()) | ||
| .map(|s| s.starts_with(source_data.name().as_source())) | ||
| .unwrap_or(false), | ||
| "Expected built package to start with {}, got: {}", | ||
| source_data.name().as_source(), | ||
| built_package.to_string_lossy() | ||
| ); | ||
|
|
||
| built_source_packages.push(built_package); | ||
| _temp_dirs.push(build_temp_dir); | ||
| } | ||
| LockedPackage::Pypi(pypi_data) => { | ||
| let package_name = pypi_data.name().clone(); | ||
|
|
@@ -239,9 +376,9 @@ pub async fn pack(options: PackOptions) -> Result<()> { | |
| conda_packages.push((filename, package.package_record)); | ||
| } | ||
|
|
||
| let injected_packages: Vec<(PathBuf, CondaArchiveType)> = options | ||
| .injected_packages | ||
| let injected_packages: Vec<(PathBuf, CondaArchiveType)> = built_source_packages | ||
| .iter() | ||
| .chain(options.injected_packages.iter()) | ||
| .filter_map(|e| { | ||
| CondaArchiveType::split_str(e.as_path().to_string_lossy().as_ref()) | ||
| .map(|(p, t)| (PathBuf::from(format!("{}{}", p, t.extension())), t)) | ||
|
|
@@ -273,6 +410,7 @@ pub async fn pack(options: PackOptions) -> Result<()> { | |
|
|
||
| conda_packages.push((filename, package_record)); | ||
| } | ||
| drop(_temp_dirs); | ||
|
|
||
| // In case we injected packages, we need to validate that these packages are solvable with the | ||
| // environment (i.e., that each packages dependencies and run constraints are still satisfied). | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.