Skip to content
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
a532262
feat: update pixi-pack to be able to pack packages from source
Nov 21, 2025
a9b0110
feat: add integration tests for local build packages
Nov 24, 2025
0847e0d
fix: remove recursive search and use packages from lock file
Nov 29, 2025
1377324
fix: fix after review
Dec 1, 2025
482bd9e
refactor local-build structure
pavelzw Dec 1, 2025
8d7d436
explicitly drop
pavelzw Dec 1, 2025
4f94186
reword
pavelzw Dec 1, 2025
177e276
fix: fix manifest_dir
Dec 2, 2025
456da43
chore: Update shasums after release (#242)
pavelzw Nov 21, 2025
3506c08
ci: bump the gh-actions group with 5 updates (#247)
dependabot[bot] Dec 7, 2025
2f23e6e
build: Bump rattler, use uv from crates.io (#243)
pavelzw Dec 8, 2025
bea3f62
fix: fix tests to match pixi 0.6.1
Dec 18, 2025
b6b2895
Merge branch 'main' into add_source_pkg_handling
toprinse Dec 18, 2025
0384d5c
fix: fix cargo lock
Dec 18, 2025
e7eac88
merge main into add_source_pkg_handling
Mar 30, 2026
eaf607c
feat: add source package handling with cross-platform fixes
Mar 30, 2026
5540214
Merge remote-tracking branch 'upstream/main' into add_source_pkg_hand…
May 27, 2026
f26f503
fix: fix typo
May 27, 2026
61c64d1
Merge branch 'main' of github.com:Quantco/pixi-pack into add_source_p…
delsner Jul 3, 2026
6728477
Bump pixi
delsner Jul 3, 2026
420b29f
Adjust implementation for most recent pixi publish
delsner Jul 3, 2026
30a922b
Fix
delsner Jul 3, 2026
e212351
nit
delsner Jul 3, 2026
fb84a3d
fix?
delsner Jul 3, 2026
579fd5c
Disable test again
delsner Jul 3, 2026
4d703ac
Merge branch 'main' into add_source_pkg_handling
pavelzw Jul 3, 2026
d2ab537
bump
pavelzw Jul 3, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions examples/local-build/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
cmake_minimum_required(VERSION 3.22)

project(local_build_main_pkg C)

add_executable(hello-main hello.c)
install(TARGETS hello-main DESTINATION bin)
5 changes: 5 additions & 0 deletions examples/local-build/hello.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#include <stdio.h>
int main(void) {
printf("Hello from local-build-main-pkg!\n");
return 0;
}
6 changes: 6 additions & 0 deletions examples/local-build/local-pkg/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
cmake_minimum_required(VERSION 3.22)

project(local_build_local_pkg C)

add_executable(hello-pkg hello.c)
install(TARGETS hello-pkg DESTINATION bin)
5 changes: 5 additions & 0 deletions examples/local-build/local-pkg/hello.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#include <stdio.h>
int main(void) {
printf("Hello from local-build-local-pkg!\n");
return 0;
}
11 changes: 11 additions & 0 deletions examples/local-build/local-pkg/pixi.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
[package]
name = "local-build-local-pkg"
version = "0.1.0"

[package.build]
# xref: https://github.com/prefix-dev/pixi/issues/6524
backend = { name = "pixi-build-cmake", version = "==0.4.3" }

[package.run-dependencies]
curl = "==8.17.0"
local-build-local-pkg-2 = { path = "../local-pkg2" }
6 changes: 6 additions & 0 deletions examples/local-build/local-pkg2/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
cmake_minimum_required(VERSION 3.22)

project(local_build_local_pkg2 C)

add_executable(hello-pkg2 hello.c)
install(TARGETS hello-pkg2 DESTINATION bin)
5 changes: 5 additions & 0 deletions examples/local-build/local-pkg2/hello.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#include <stdio.h>
int main(void) {
printf("Hello from local-build-local-pkg-2!\n");
return 0;
}
10 changes: 10 additions & 0 deletions examples/local-build/local-pkg2/pixi.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[package]
name = "local-build-local-pkg-2"
version = "0.1.0"

[package.build]
# xref: https://github.com/prefix-dev/pixi/issues/6524
backend = { name = "pixi-build-cmake", version = "==0.4.3" }

[package.run-dependencies]
curl = "==8.17.0"
3,963 changes: 3,963 additions & 0 deletions examples/local-build/pixi.lock

Large diffs are not rendered by default.

21 changes: 21 additions & 0 deletions examples/local-build/pixi.toml
Comment thread
pavelzw marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
[workspace]
preview = ["pixi-build"]
channels = ["conda-forge"]
platforms = ["linux-64", "linux-aarch64", "win-64", "osx-64", "osx-arm64"]

[dependencies]
local-build-main-pkg = { path = "." }

[package]
name = "local-build-main-pkg"
version = "0.1.0"

[package.build]
# xref: https://github.com/prefix-dev/pixi/issues/6524
backend = { name = "pixi-build-cmake", version = "==0.4.3" }

[package.run-dependencies]
local-build-local-pkg = { path = "./local-pkg" }

[tool.update]
autoupdate-schedule = "never"
1,382 changes: 718 additions & 664 deletions pixi.lock

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ update-snapshots = "INSTA_UPDATE=always cargo test -p pixi-pack -- test_reproduc
[dependencies]
rust = "==1.95.0"
openssl = "3.*"
pixi = ">=0.72.0"
pkg-config = "*"

[target.linux.dependencies]
Expand Down
148 changes: 143 additions & 5 deletions src/pack.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,12 @@ use std::{
#[cfg(not(target_os = "windows"))]
use std::os::unix::fs::PermissionsExt as _;

use indicatif::HumanBytes;
use indicatif::{HumanBytes, ProgressBar};
use rattler_index::{package_record_from_conda, package_record_from_tar_bz2};
use tokio::{
fs::{self, File, create_dir_all},
io::AsyncWriteExt,
process::Command as TokioCommand,
};

use anyhow::Result;
Expand All @@ -33,6 +34,7 @@ use rattler_networking::{
AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, MirrorMiddleware, S3Middleware,
authentication_storage, mirror_middleware::Mirror,
};

use reqwest_middleware::ClientWithMiddleware;
use tar::{Builder, HeaderMode};
use tokio::io::AsyncReadExt;
Expand Down Expand Up @@ -100,6 +102,77 @@ fn load_lockfile(manifest_path: &Path) -> Result<LockFile> {
})
}

async fn build_local_package(
manifest_path: &Path,
output_dir: &Path,
platform: &str,
) -> Result<()> {
let manifest_dir = manifest_path
.parent()
.expect("manifest_path should have a parent directory");
let mut command = TokioCommand::new("pixi");
command
.arg("publish")
.arg("--target-dir")
.arg(output_dir)
.arg("--build-platform")
.arg(platform)
.arg("--path")
.arg(manifest_path);

let is_verbose = tracing::enabled!(tracing::Level::WARN);

if is_verbose {
eprintln!(
"🔧 Running: pixi publish --target-dir {} --build-platform {} (in {})",
output_dir.display(),
platform,
manifest_dir.display()
);

let status = command
.stdout(std::process::Stdio::inherit())
.stderr(std::process::Stdio::inherit())
.status()
.await?;

if !status.success() {
anyhow::bail!("Failed to publish package in {}", manifest_dir.display());
}
Comment on lines +165 to +167

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in a follow-up we could stream the logs like pixi install

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

} else {
let output = command.output().await?;

if !output.status.success() {
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);

eprintln!(
"❌ Publish failed with exit code: {:?}",
output.status.code()
);
eprintln!(
"Command: pixi publish --target-dir {} --build-platform {} (in {})",
output_dir.display(),
platform,
manifest_dir.display()
);

if !stdout.is_empty() {
eprintln!("📤 stdout:");
eprintln!("{}", stdout);
}
if !stderr.is_empty() {
eprintln!("📤 stderr:");
eprintln!("{}", stderr);
}

anyhow::bail!("Failed to publish package in {}", manifest_dir.display());
}
}

Ok(())
}

/// Pack a pixi environment.
pub async fn pack(options: PackOptions) -> Result<()> {
let lockfile = load_lockfile(&options.manifest_path)?;
Expand Down Expand Up @@ -147,14 +220,78 @@ pub async fn pack(options: PackOptions) -> Result<()> {

let mut conda_packages_from_lockfile: Vec<CondaBinaryData> = Vec::new();
let mut pypi_packages_from_lockfile: Vec<PypiPackageData> = Vec::new();
let mut built_source_packages: Vec<PathBuf> = Vec::new();
let mut _temp_dirs: Vec<tempfile::TempDir> = Vec::new();
Comment thread
pavelzw marked this conversation as resolved.

for package in packages {
match package {
LockedPackage::Conda(CondaPackageData::Binary(binary_data)) => {
conda_packages_from_lockfile.push((**binary_data).clone())
}
LockedPackage::Conda(CondaPackageData::Source(_)) => {
anyhow::bail!("Conda source packages are not yet supported by pixi-pack")
LockedPackage::Conda(CondaPackageData::Source(source_data)) => {
let base_dir = if options.manifest_path.is_file() {
options.manifest_path.parent().unwrap_or(Path::new("."))
} else {
&options.manifest_path
};
let relative_path = match &source_data.location {
UrlOrPath::Path(p) => PathBuf::from(p.to_path().as_str()),
UrlOrPath::Url(u) => {
anyhow::bail!("Unexpected URL for local package source: {}", u)
Comment thread
delsner marked this conversation as resolved.
}
Comment on lines +271 to +273

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this could be useful as well in a follow-up to support things like this:

[workspace]
preview = ["pixi-build"]
channels = ["conda-forge"]
platforms = ["linux-64", "linux-aarch64", "win-64", "osx-64", "osx-arm64"]

[dependencies]
agent-skill-conda-forge = { git = "https://github.com/pavelzw/skill-forge", rev = "main", subdirectory = "recipes/conda-forge" }

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

};

let pkg_dir = base_dir.join(&relative_path);
let manifest_path = pkg_dir.join("pixi.toml");
let canonical_manifest_path = manifest_path.canonicalize().map_err(|e| {
anyhow!(
"Cannot find manifest file {}: {}",
manifest_path.display(),
e
)
})?;
let build_temp_dir = tempfile::tempdir()
.map_err(|e| anyhow!("could not create temporary build directory: {}", e))?;
let bar = ProgressBar::new_spinner();

bar.set_message(format!("Publishing {}", source_data.name().as_source()));
bar.enable_steady_tick(std::time::Duration::from_millis(100));
build_local_package(
&canonical_manifest_path,
build_temp_dir.path(),
options.platform.as_str(),
)
.await?;
Comment thread
delsner marked this conversation as resolved.
bar.finish_with_message(format!(
"✅ Built and published {}",
source_data.name().as_source()
));

let built_package = std::fs::read_dir(build_temp_dir.path())
.map_err(|e| anyhow!("Failed to read build directory: {}", e))?
.next()
.ok_or_else(|| anyhow!("Build directory is empty"))?
.map_err(|e| anyhow!("Failed to read directory entry: {}", e))?
.path();

anyhow::ensure!(
built_package.extension().and_then(|s| s.to_str()) == Some("conda"),
"Expected built package to be a .conda file, got: {}",
built_package.to_string_lossy()
);
anyhow::ensure!(
built_package
.file_name()
.and_then(|s| s.to_str())
.map(|s| s.starts_with(source_data.name().as_source()))
.unwrap_or(false),
"Expected built package to start with {}, got: {}",
source_data.name().as_source(),
built_package.to_string_lossy()
);

built_source_packages.push(built_package);
_temp_dirs.push(build_temp_dir);
}
LockedPackage::Pypi(pypi_data) => {
let package_name = pypi_data.name().clone();
Expand Down Expand Up @@ -207,9 +344,9 @@ pub async fn pack(options: PackOptions) -> Result<()> {
conda_packages.push((filename, package.package_record));
}

let injected_packages: Vec<(PathBuf, CondaArchiveType)> = options
.injected_packages
let injected_packages: Vec<(PathBuf, CondaArchiveType)> = built_source_packages
.iter()
.chain(options.injected_packages.iter())
.filter_map(|e| {
CondaArchiveType::split_str(e.as_path().to_string_lossy().as_ref())
.map(|(p, t)| (PathBuf::from(format!("{}{}", p, t.extension())), t))
Expand Down Expand Up @@ -241,6 +378,7 @@ pub async fn pack(options: PackOptions) -> Result<()> {

conda_packages.push((filename, package_record));
}
drop(_temp_dirs);

// In case we injected packages, we need to validate that these packages are solvable with the
// environment (i.e., that each packages dependencies and run constraints are still satisfied).
Expand Down
64 changes: 64 additions & 0 deletions tests/integration_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -928,3 +928,67 @@ async fn test_local_channel(
.exists()
);
}

#[rstest]
#[tokio::test]
async fn test_local_build_package(options: Options) {
if cfg!(windows) {
// pixi-build hits Windows PATH length limits in GitHub Actions CI
// (vcvars64.bat fails with "input line too long" on deeply nested paths).
// This could be fixed upstream by moving the build folder to a short stable path.
return;
}

let mut pack_options = options.pack_options;
pack_options.manifest_path = PathBuf::from("examples/local-build/pixi.toml");

let unpack_options = options.unpack_options;
let pack_file = unpack_options.pack_file.clone();

let pack_result = pixi_pack::pack(pack_options).await;
assert!(pack_result.is_ok(), "{pack_result:?}");
assert!(pack_file.is_file());

let env_dir = unpack_options.output_directory.join("env");
let unpack_result = pixi_pack::unpack(unpack_options).await;
assert!(unpack_result.is_ok(), "{unpack_result:?}");

let curl_json = match Platform::current() {
Platform::Linux64 => env_dir.join("conda-meta/curl-8.17.0-h4e3cde8_1.json"),
Platform::LinuxAarch64 => env_dir.join("conda-meta/curl-8.17.0-h7bfdcfb_1.json"),
Platform::OsxArm64 => env_dir.join("conda-meta/curl-8.17.0-hdece5d2_1.json"),
Platform::Osx64 => env_dir.join("conda-meta/curl-8.17.0-h7dd4100_1.json"),
_ => panic!("Unsupported platform"),
};
assert!(curl_json.exists(), "curl not found in conda-meta");

let conda_meta = env_dir.join("conda-meta");
let has_pkg = |prefix: &str| {
fs::read_dir(&conda_meta).unwrap().flatten().any(|e| {
let name = e.file_name();
let name = name.to_string_lossy();
name.starts_with(prefix) && name.ends_with(".json")
})
};
assert!(
has_pkg("local-build-main-pkg-0.1.0-"),
"main-pkg not found in conda-meta"
);
assert!(
has_pkg("local-build-local-pkg-0.1.0-"),
"local-pkg not found in conda-meta"
);

assert!(
env_dir.join("bin/hello-main").exists(),
"hello-main binary not found"
);
assert!(
env_dir.join("bin/hello-pkg").exists(),
"hello-pkg binary not found"
);
assert!(
env_dir.join("bin/hello-pkg2").exists(),
"hello-pkg2 binary not found"
);
}
Loading