Skip to content

Misc. Improvements Draft PR #2294

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
antheas wants to merge 5 commits into
base: main
Choose a base branch
from
+504 −59
Draft

Misc. Improvements Draft PR #2294

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b950604
bootc compiles three times for specs, make it compile once
antheas Jun 17, 2026
b08e289
Drop reserved package layer
antheas Jun 19, 2026
b49c268
add parallel writer
antheas Jun 18, 2026
dfad00c
Preserve image version in merge commits
antheas Jun 28, 2026
69807fe
Optimize pure OSTree image imports
antheas Jun 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,42 @@ prefix ?= /usr
CARGO_FEATURES_DEFAULT ?= $(shell . /usr/lib/os-release; if echo "$$ID_LIKE" |grep -qF rhel; then echo rhsm; fi)
# You can set this to override all cargo features, including the defaults
CARGO_FEATURES ?= $(CARGO_FEATURES_DEFAULT)
BOOTC_PROFILE ?= $(if $(wildcard target/rpm/bootc),rpm,release)
BOOTC_TARGET_DIR = target/$(BOOTC_PROFILE)

# Build all binaries
.PHONY: bin
bin: manpages
cargo build --release --features "$(CARGO_FEATURES)" --bins
if [ ! -x "$(BOOTC_TARGET_DIR)/bootc" ]; then \

@cgwalters cgwalters Jul 9, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This completely breaks the buildsystem, no? We'd just silently be reusing old binaries...

@antheas antheas Jul 9, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, that patch is not good and not designed for merge. But it did its job for me.

It is very hard to iterate with bootc while having it compile three times to deploy

cargo build --release --features "$(CARGO_FEATURES)" --bins; \
fi

# Note this cargo build is run without features (such as rhsm)
.PHONY: manpages
manpages:
cargo run --release --package xtask -- manpages
if [ -x "$(BOOTC_TARGET_DIR)/bootc" ]; then \

@cgwalters cgwalters Jul 9, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But yes, this is really really messy how we have our documentation depend on running the binary.

Maybe what is easier is to try to move instead to where we just validate the generated docs as a separate phase, but don't default to linking the two together.

It's effectively what we did for the tmt stuff

rm -rf target/man; \
if [ -x "$(BOOTC_TARGET_DIR)/xtask" ]; then \
BOOTC_DOCGEN_BIN="$(BOOTC_TARGET_DIR)/bootc" "$(BOOTC_TARGET_DIR)/xtask" manpages; \
else \
BOOTC_DOCGEN_BIN="$(BOOTC_TARGET_DIR)/bootc" cargo run --release --package xtask -- manpages; \
fi; \
else \
mkdir -p target/man; \
printf '.TH BOOTC 8\n.SH NAME\nbootc \\- bootable container system\n' > target/man/bootc.8; \
printf '.TH BOOTC-SETUP-ROOT-CONF 5\n.SH NAME\nbootc-setup-root.conf \\- bootc setup-root configuration\n' > target/man/bootc-setup-root-conf.5; \
fi

.PHONY: completion
completion: bin
mkdir -p target/completion
for shell in bash elvish fish powershell zsh; do \
target/release/bootc completion $$shell > target/completion/bootc.$$shell; \
$(BOOTC_TARGET_DIR)/bootc completion $$shell > target/completion/bootc.$$shell; \
done

STORAGE_RELATIVE_PATH ?= $(shell realpath -m -s --relative-to="$(prefix)/lib/bootc/storage" /sysroot/ostree/bootc/storage)
install: completion
install -D -m 0755 -t $(DESTDIR)$(prefix)/bin target/release/bootc
install -D -m 0755 -t $(DESTDIR)$(prefix)/bin target/release/system-reinstall-bootc
install -D -m 0755 -t $(DESTDIR)$(prefix)/bin $(BOOTC_TARGET_DIR)/bootc
install -D -m 0755 -t $(DESTDIR)$(prefix)/bin $(BOOTC_TARGET_DIR)/system-reinstall-bootc
install -d -m 0755 $(DESTDIR)$(prefix)/lib/bootc/bound-images.d
install -d -m 0755 $(DESTDIR)$(prefix)/lib/bootc/kargs.d
ln -s "$(STORAGE_RELATIVE_PATH)" "$(DESTDIR)$(prefix)/lib/bootc/storage"
Expand Down Expand Up @@ -83,7 +97,7 @@ install: completion
install -D -m 0755 -t $(DESTDIR)/$(prefix)/lib/bootc contrib/scripts/fedora-bootc-destructive-cleanup; \
fi
install -D -m 0644 -t $(DESTDIR)/usr/lib/systemd/system crates/initramfs/*.service
install -D -m 0755 target/release/bootc-initramfs-setup $(DESTDIR)/usr/lib/bootc/initramfs-setup
install -D -m 0755 $(BOOTC_TARGET_DIR)/bootc-initramfs-setup $(DESTDIR)/usr/lib/bootc/initramfs-setup
install -D -m 0755 -t $(DESTDIR)/usr/lib/dracut/modules.d/51bootc crates/initramfs/dracut/module-setup.sh

# Run this to also take over the functionality of `ostree container` for example.
Expand Down
2 changes: 1 addition & 1 deletion crates/cli/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ platforms = ["*-unknown-linux-gnu"]
[dependencies]
# Internal crates
bootc-lib = { version = "1.16", path = "../lib" }
bootc-utils = { package = "bootc-internal-utils", path = "../utils", version = "1.16.3" }
bootc-utils = { package = "bootc-internal-utils", path = "../utils", version = "1.16.0" }

# Workspace dependencies
anstream = { workspace = true }
Expand Down
8 changes: 1 addition & 7 deletions crates/ostree-ext/src/chunking.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -748,11 +748,7 @@ fn basic_packing<'a>(
// If there are fewer packages/components than there are bins, then we don't need to do
// any "bin packing" at all; just assign a single component to each and we're done.
if before_processing_pkgs_len < bin_size.get() as usize {
let mut r = components.iter().map(|pkg| vec![pkg]).collect::<Vec<_>>();
if before_processing_pkgs_len > 0 {
let new_pkgs_bin: Vec<&ObjectSourceMetaSized> = Vec::new();
r.push(new_pkgs_bin);
}
let r = components.iter().map(|pkg| vec![pkg]).collect::<Vec<_>>();
return Ok(r);
}

Expand Down Expand Up @@ -855,8 +851,6 @@ fn basic_packing<'a>(
r.push(max_freq_components);
}

// Allocate an empty bin for new packages

@cgwalters cgwalters Jul 9, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree we never used this in practice, and we can probably drop it now. OTOH, this code is basically obsoleted by https://github.com/coreos/chunkah anyways?

@antheas antheas Jul 9, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that is a nice tool. I will keep it in mind for non-ostree images

ostree-ext provides a fast deploy path for ostree backends so it will have to stay around until the composefs backend is stable with similar performance to the fast ostree deploy path, then add 2-3 more years for migration or a way to specify to bootc interim migration paths so e.g., version 2027.3 switches to composefs and points ostree instances to 2027.2 that has the machinery to migrate from ostree to composefs while being built for ostree. Ah.

r.push(Vec::new());
let after_processing_pkgs_len = r.iter().map(|b| b.len()).sum::<usize>();
assert_eq!(after_processing_pkgs_len, before_processing_pkgs_len);
assert!(r.len() <= bin_size.get() as usize);
Expand Down
91 changes: 88 additions & 3 deletions crates/ostree-ext/src/cli.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,17 @@ use std::collections::BTreeMap;
use std::ffi::OsString;
use std::fs::File;
use std::io::{BufReader, BufWriter, Write};
use std::num::NonZeroU32;
use std::num::{NonZeroU32, NonZeroUsize};
use std::path::PathBuf;
use std::process::Command;
use std::sync::mpsc;
use tokio::sync::mpsc::Receiver;

use crate::chunking::{ObjectMetaSized, ObjectSourceMetaSized};
use crate::commit::container_commit;
use crate::container::store::{ExportToOCIOpts, ImportProgress, LayerProgress, PreparedImport};
use crate::container::{self as ostree_container, ManifestDiff};
use crate::container::{Config, ImageReference, OstreeImageReference};
use crate::container::{Config, ExportProgress, ImageReference, OstreeImageReference};
use crate::objectsource::ObjectSourceMeta;
use crate::sysroot::SysrootLock;
use ostree_container::store::{ImageImporter, PrepareResult};
Expand Down Expand Up @@ -172,6 +173,14 @@ pub(crate) enum ContainerOpts {
#[clap(long)]
compression_fast: bool,

/// Number of parallel layer export workers
#[clap(long)]
jobs: Option<NonZeroUsize>,

/// Don't display progress
#[clap(long)]
quiet: bool,

/// Path to a JSON-formatted content meta object.
#[clap(long)]
contentmeta: Option<Utf8PathBuf>,
Expand Down Expand Up @@ -575,6 +584,50 @@ impl Into<ostree_container::store::ImageProxyConfig> for ContainerProxyOpts {
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn parse_container_encapsulate_jobs() {
let opt = Opt::parse_from([
"ostree-ext",
"container",
"encapsulate",
"--repo",
"/repo",
"--jobs",
"2",
"exampleos",
"oci:/tmp/exampleos:latest",
]);
let Opt::Container(ContainerOpts::Encapsulate { jobs, quiet, .. }) = opt else {
panic!("expected container encapsulate");
};
assert_eq!(jobs, NonZeroUsize::new(2));
assert!(!quiet);
}

#[test]
fn parse_container_encapsulate_quiet() {
let opt = Opt::parse_from([
"ostree-ext",
"container",
"encapsulate",
"--repo",
"/repo",
"--quiet",
"exampleos",
"oci:/tmp/exampleos:latest",
]);
let Opt::Container(ContainerOpts::Encapsulate { jobs, quiet, .. }) = opt else {
panic!("expected container encapsulate");
};
assert_eq!(jobs, None);
assert!(quiet);
}
}

/// Import a tar archive containing an ostree commit.
async fn tar_import(opts: &ImportOpts) -> Result<()> {
let repo = parse_repo(&opts.repo)?;
Expand Down Expand Up @@ -769,6 +822,8 @@ async fn container_export(
container_config: Option<Utf8PathBuf>,
cmd: Option<Vec<String>>,
compression_fast: bool,
jobs: Option<NonZeroUsize>,
quiet: bool,
package_contentmeta: Option<Utf8PathBuf>,
) -> Result<()> {
let container_config = if let Some(container_config) = container_config {
Expand Down Expand Up @@ -839,6 +894,16 @@ async fn container_export(
cmd,
};

let (progress, progress_thread) = if quiet {
(None, None)
} else {
let (tx, rx) = mpsc::channel();
(
Some(tx),
Some(std::thread::spawn(move || print_export_progress(rx))),
)
};

let opts = crate::container::ExportOpts {
copy_meta_keys,
copy_meta_opt_keys,
Expand All @@ -848,13 +913,29 @@ async fn container_export(
package_contentmeta: contentmeta_data.as_ref(),
max_layers,
created,
jobs,
progress,
..Default::default()
};
let pushed = crate::container::encapsulate(repo, rev, &config, Some(opts), imgref).await?;
let pushed = crate::container::encapsulate(repo, rev, &config, Some(opts), imgref).await;
if let Some(progress_thread) = progress_thread {
progress_thread
.join()
.map_err(|_| anyhow::anyhow!("export progress thread panicked"))?;
}
let pushed = pushed?;
println!("{pushed}");
Ok(())
}

fn print_export_progress(rx: mpsc::Receiver<ExportProgress>) {
for event in rx {
if let ExportProgress::Finished { index, total, name } = event {
eprintln!("Exported OCI layer {index}/{total}: {name}");
}
}
}

/// Load metadata for a container image with an encapsulated ostree commit.
async fn container_info(imgref: &OstreeImageReference) -> Result<()> {
let (_, digest) = crate::container::fetch_manifest(imgref).await?;
Expand Down Expand Up @@ -1071,6 +1152,8 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
config,
cmd,
compression_fast,
jobs,
quiet,
contentmeta,
} => {
let labels: Result<BTreeMap<_, _>> = labels
Expand All @@ -1094,6 +1177,8 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
config,
cmd,
compression_fast,
jobs,
quiet,
contentmeta,
)
.await
Expand Down
Loading
Loading