aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNikolay Govorov <me@govorov.online>2026-07-12 23:31:45 +0100
committerNikolay Govorov <me@govorov.online>2026-07-12 23:32:02 +0100
commit97becb033efb285cb27f866d198781ec6ddb5403 (patch)
treebac56a8310bb193d3b763d7a3b5ac0e7b4ba8b63
parent53f3149e1d632f044c28162244304fd3b6b8ea9b (diff)
downloadtar
tar.gz
tar.bz2
tar.lz
tar.xz
tar.zst
zip
Split PoC cli and hule-vmm package
Diffstat
-rw-r--r--Cargo.lock12+7 −5
-rw-r--r--crates/hule-oci/Cargo.toml5+5 −0
-rw-r--r--crates/hule-oci/src/lib.rs457+456 −1
-rw-r--r--crates/hule-vmm/tests/qemu_lifecycle.rs118+118 −0
-rw-r--r--crates/hule/Cargo.toml5+0 −5
-rw-r--r--crates/hule/src/main.rs503+22 −481
6 files changed, 608 insertions, 492 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 08e65ee..f9ecd44 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -570,16 +570,11 @@ checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
name = "hule"
version = "0.1.0"
dependencies = [
- "futures-util",
"hule-image",
"hule-oci",
"hule-vmm",
- "oci-client",
- "serde_json",
- "sha2 0.10.9",
"tokio",
"uuid",
- "zstd",
]
[[package]]
@@ -593,6 +588,13 @@ dependencies = [
[[package]]
name = "hule-oci"
version = "0.1.0"
+dependencies = [
+ "hule-image",
+ "oci-client",
+ "serde_json",
+ "sha2 0.10.9",
+ "zstd",
+]
[[package]]
name = "hule-vmm"
diff --git a/crates/hule-oci/Cargo.toml b/crates/hule-oci/Cargo.toml
index 071f9c5..76ba6ed 100644
--- a/crates/hule-oci/Cargo.toml
+++ b/crates/hule-oci/Cargo.toml
@@ -13,3 +13,8 @@ authors.workspace = true
repository.workspace = true
[dependencies]
+hule-image.workspace = true
+serde_json.workspace = true
+oci-client = "0.17"
+zstd = "0.13"
+sha2 = "0.10"
diff --git a/crates/hule-oci/src/lib.rs b/crates/hule-oci/src/lib.rs
index 306f146..a63f448 100644
--- a/crates/hule-oci/src/lib.rs
+++ b/crates/hule-oci/src/lib.rs
@@ -1,4 +1,459 @@
// SPDX-FileCopyrightText: 2026 Nikolay Govorov
// SPDX-License-Identifier: Apache-2.0
-pub const NAME: &str = "I'm a oci library";
+//! OCI image storage and registry operations for Hule machine images.
+
+use hule_image::MachineImage;
+use oci_client::annotations::{ORG_OPENCONTAINERS_IMAGE_REF_NAME, ORG_OPENCONTAINERS_IMAGE_TITLE};
+use oci_client::client::{ClientConfig, ClientProtocol, Config, ImageLayer};
+use oci_client::manifest::{
+ ImageIndexEntry, OCI_IMAGE_MEDIA_TYPE, OciDescriptor, OciImageIndex, OciImageManifest,
+};
+use oci_client::secrets::RegistryAuth;
+use oci_client::{Client, Reference};
+use std::collections::{BTreeMap, BTreeSet};
+use std::io::Read;
+use std::path::{Path, PathBuf};
+pub type Result<T> = std::result::Result<T, String>;
+type R<T> = Result<T>;
+
+// ---- local OCI-layout store -------------------------------------------
+
+const CHUNK_SIZE: usize = 256 * 1024 * 1024;
+const HULE_CHUNK_MEDIA_TYPE: &str = "application/vnd.hule.disk.chunk.v1";
+const HULE_CONFIG_MEDIA_TYPE: &str = "application/vnd.hule.machine.config.v1+json";
+const ANNOTATION_CHUNK_OFFSET: &str = "io.hule.chunk.offset";
+const ANNOTATION_CHUNK_LENGTH: &str = "io.hule.chunk.length";
+
+fn store_root() -> R<PathBuf> {
+ let home = std::env::var("HOME").map_err(|_| "HOME is not set".to_string())?;
+ Ok(PathBuf::from(home).join(".hule").join("store"))
+}
+
+fn blobs_dir() -> R<PathBuf> {
+ Ok(store_root()?.join("blobs").join("sha256"))
+}
+
+fn materialized_dir(manifest_digest: &str) -> R<PathBuf> {
+ Ok(store_root()?
+ .join("materialized")
+ .join(strip_sha256(manifest_digest)))
+}
+
+fn index_path() -> R<PathBuf> {
+ Ok(store_root()?.join("index.json"))
+}
+
+fn strip_sha256(digest: &str) -> &str {
+ digest.strip_prefix("sha256:").unwrap_or(digest)
+}
+
+fn sha256_hex(data: &[u8]) -> String {
+ use sha2::{Digest, Sha256};
+ let mut hasher = Sha256::new();
+ hasher.update(data);
+ format!("sha256:{:x}", hasher.finalize())
+}
+
+fn ensure_store() -> R<()> {
+ std::fs::create_dir_all(blobs_dir()?).map_err(|e| e.to_string())?;
+ std::fs::create_dir_all(store_root()?.join("materialized")).map_err(|e| e.to_string())?;
+ let layout = store_root()?.join("oci-layout");
+ if !layout.exists() {
+ std::fs::write(&layout, br#"{"imageLayoutVersion":"1.0.0"}"#).map_err(|e| e.to_string())?;
+ }
+ Ok(())
+}
+
+/// Writes raw bytes as a content-addressed blob, no-op if already present.
+fn write_blob(data: &[u8]) -> R<String> {
+ let digest = sha256_hex(data);
+ let path = blobs_dir()?.join(strip_sha256(&digest));
+ if !path.exists() {
+ let tmp = path.with_extension("tmp");
+ std::fs::write(&tmp, data).map_err(|e| e.to_string())?;
+ std::fs::rename(&tmp, &path).map_err(|e| e.to_string())?;
+ }
+ Ok(digest)
+}
+
+fn read_blob(digest: &str) -> R<Vec<u8>> {
+ std::fs::read(blobs_dir()?.join(strip_sha256(digest))).map_err(|e| e.to_string())
+}
+
+fn read_index() -> R<OciImageIndex> {
+ let path = index_path()?;
+ if !path.exists() {
+ return Ok(OciImageIndex {
+ schema_version: 2,
+ media_type: None,
+ manifests: vec![],
+ artifact_type: None,
+ annotations: None,
+ });
+ }
+ let data = std::fs::read(&path).map_err(|e| e.to_string())?;
+ serde_json::from_slice(&data).map_err(|e| e.to_string())
+}
+
+fn write_index(index: &OciImageIndex) -> R<()> {
+ let data = serde_json::to_vec_pretty(index).map_err(|e| e.to_string())?;
+ std::fs::write(index_path()?, data).map_err(|e| e.to_string())
+}
+
+fn index_lookup(index: &OciImageIndex, reference: &str) -> Option<String> {
+ index
+ .manifests
+ .iter()
+ .find(|m| {
+ m.annotations
+ .as_ref()
+ .and_then(|a| a.get(ORG_OPENCONTAINERS_IMAGE_REF_NAME))
+ .map(String::as_str)
+ == Some(reference)
+ })
+ .map(|m| m.digest.clone())
+}
+
+fn index_set(index: &mut OciImageIndex, reference: &str, digest: &str, size: u64) {
+ index.manifests.retain(|m| {
+ m.annotations
+ .as_ref()
+ .and_then(|a| a.get(ORG_OPENCONTAINERS_IMAGE_REF_NAME))
+ .map(String::as_str)
+ != Some(reference)
+ });
+ let mut annotations = BTreeMap::new();
+ annotations.insert(
+ ORG_OPENCONTAINERS_IMAGE_REF_NAME.to_string(),
+ reference.to_string(),
+ );
+ index.manifests.push(ImageIndexEntry {
+ media_type: OCI_IMAGE_MEDIA_TYPE.to_string(),
+ digest: digest.to_string(),
+ size: size as i64,
+ platform: None,
+ annotations: Some(annotations),
+ artifact_type: None,
+ });
+}
+
+fn hardlink_or_copy(src: &Path, dest: &Path) -> R<()> {
+ if std::fs::hard_link(src, dest).is_err() {
+ std::fs::copy(src, dest).map_err(|e| e.to_string())?;
+ }
+ Ok(())
+}
+
+fn write_at(file: &std::fs::File, offset: u64, data: &[u8]) -> R<()> {
+ use std::os::unix::fs::FileExt;
+ let mut written = 0usize;
+ while written < data.len() {
+ let n = file
+ .write_at(&data[written..], offset + written as u64)
+ .map_err(|e| e.to_string())?;
+ if n == 0 {
+ return Err("write_at wrote 0 bytes".to_string());
+ }
+ written += n;
+ }
+ Ok(())
+}
+
+/// Splits `path` into fixed-size chunks, compresses each independently
+/// (own zstd frame, no state shared between chunks -- see the design notes
+/// in the project plan on why this must not be done the other way around),
+/// and writes each as its own blob. Returns one descriptor per chunk, all
+/// sharing a title annotation plus a chunk offset/length in terms of the
+/// *uncompressed* file so pull can reassemble it.
+fn write_file_chunks(path: &Path) -> R<Vec<OciDescriptor>> {
+ let title = path
+ .file_name()
+ .ok_or_else(|| format!("{} has no file name", path.display()))?
+ .to_string_lossy()
+ .to_string();
+ let mut file = std::fs::File::open(path).map_err(|e| e.to_string())?;
+ let mut descriptors = Vec::new();
+ let mut offset: u64 = 0;
+ loop {
+ let mut buf = Vec::with_capacity(CHUNK_SIZE);
+ (&mut file)
+ .take(CHUNK_SIZE as u64)
+ .read_to_end(&mut buf)
+ .map_err(|e| e.to_string())?;
+ if buf.is_empty() {
+ break;
+ }
+ let length = buf.len() as u64;
+ let compressed = zstd::stream::encode_all(&buf[..], 0).map_err(|e| e.to_string())?;
+ let digest = write_blob(&compressed)?;
+
+ let mut annotations = BTreeMap::new();
+ annotations.insert(ORG_OPENCONTAINERS_IMAGE_TITLE.to_string(), title.clone());
+ annotations.insert(ANNOTATION_CHUNK_OFFSET.to_string(), offset.to_string());
+ annotations.insert(ANNOTATION_CHUNK_LENGTH.to_string(), length.to_string());
+
+ descriptors.push(OciDescriptor {
+ media_type: HULE_CHUNK_MEDIA_TYPE.to_string(),
+ digest,
+ size: compressed.len() as i64,
+ urls: None,
+ annotations: Some(annotations),
+ artifact_type: None,
+ });
+
+ offset += length;
+ if length < CHUNK_SIZE as u64 {
+ break;
+ }
+ }
+ if descriptors.is_empty() {
+ return Err(format!("{} is empty", path.display()));
+ }
+ Ok(descriptors)
+}
+
+fn make_client(reference: &Reference) -> Client {
+ let registry = reference.resolve_registry();
+ let host = registry.split(':').next().unwrap_or(registry);
+ let protocol = if host == "localhost" || host == "127.0.0.1" {
+ ClientProtocol::Http
+ } else {
+ ClientProtocol::Https
+ };
+ Client::new(ClientConfig {
+ protocol,
+ ..Default::default()
+ })
+}
+
+fn parse_reference(s: &str) -> R<Reference> {
+ s.parse()
+ .map_err(|e| format!("invalid reference '{s}': {e}"))
+}
+
+// ---- import / push / pull / run ---------------------------------------
+
+/// Imports an image directory into the local OCI layout and optionally tags it.
+pub async fn import(path: &Path, reference: Option<&str>) -> R<String> {
+ ensure_store()?;
+ let config_path = path.join("config.json");
+ if !config_path.is_file() {
+ return Err(format!("{} does not contain config.json", path.display()));
+ }
+ let config_bytes = std::fs::read(&config_path).map_err(|e| e.to_string())?;
+ MachineImage::from_json(&config_bytes).map_err(|e| e.to_string())?;
+
+ let mut entries: Vec<_> = std::fs::read_dir(path)
+ .map_err(|e| e.to_string())?
+ .collect::<std::result::Result<Vec<_>, _>>()
+ .map_err(|e| e.to_string())?;
+ entries.sort_by_key(|e| e.file_name());
+
+ let mut layers = Vec::new();
+ let mut sources: Vec<(String, PathBuf)> = Vec::new();
+ for entry in entries {
+ if !entry.file_type().map_err(|e| e.to_string())?.is_file() {
+ continue;
+ }
+ let name = entry.file_name().to_string_lossy().to_string();
+ if name == "config.json" {
+ continue;
+ }
+ let file_path = entry.path();
+ layers.extend(write_file_chunks(&file_path)?);
+ sources.push((name, file_path));
+ }
+
+ let config_digest = write_blob(&config_bytes)?;
+ let config = OciDescriptor {
+ media_type: HULE_CONFIG_MEDIA_TYPE.to_string(),
+ digest: config_digest,
+ size: config_bytes.len() as i64,
+ urls: None,
+ annotations: None,
+ artifact_type: None,
+ };
+
+ let manifest = OciImageManifest {
+ schema_version: 2,
+ media_type: Some(OCI_IMAGE_MEDIA_TYPE.to_string()),
+ config,
+ layers,
+ subject: None,
+ artifact_type: None,
+ annotations: None,
+ };
+ let manifest_bytes = serde_json::to_vec(&manifest).map_err(|e| e.to_string())?;
+ let manifest_digest = write_blob(&manifest_bytes)?;
+
+ let materialized = materialized_dir(&manifest_digest)?;
+ std::fs::create_dir_all(&materialized).map_err(|e| e.to_string())?;
+ for (title, source) in &sources {
+ let dest = materialized.join(title);
+ if !dest.exists() {
+ hardlink_or_copy(source, &dest)?;
+ }
+ }
+
+ if let Some(reference) = reference {
+ let mut index = read_index()?;
+ index_set(
+ &mut index,
+ reference,
+ &manifest_digest,
+ manifest_bytes.len() as u64,
+ );
+ write_index(&index)?;
+ }
+ Ok(manifest_digest)
+}
+
+/// Pushes a locally tagged image to its registry reference.
+pub async fn push(reference_str: &str) -> R<()> {
+ ensure_store()?;
+ let index = read_index()?;
+ let manifest_digest = index_lookup(&index, reference_str)
+ .ok_or_else(|| format!("no local image tagged '{reference_str}'"))?;
+ let manifest_bytes = read_blob(&manifest_digest)?;
+ let manifest: OciImageManifest =
+ serde_json::from_slice(&manifest_bytes).map_err(|e| e.to_string())?;
+
+ let reference = parse_reference(reference_str)?;
+ let client = make_client(&reference);
+ let auth = RegistryAuth::Anonymous;
+
+ let mut layers = Vec::new();
+ for descriptor in &manifest.layers {
+ let data = read_blob(&descriptor.digest)?;
+ layers.push(ImageLayer::new(
+ data,
+ descriptor.media_type.clone(),
+ descriptor.annotations.clone(),
+ ));
+ }
+ let config_bytes = read_blob(&manifest.config.digest)?;
+ let config = Config::new(
+ config_bytes,
+ manifest.config.media_type.clone(),
+ manifest.config.annotations.clone(),
+ );
+
+ client
+ .push(&reference, &layers, config, &auth, Some(manifest))
+ .await
+ .map_err(|e| e.to_string())?;
+
+ Ok(())
+}
+
+/// Pulls an image into the local OCI layout and materializes its files.
+pub async fn pull(reference_str: &str) -> R<String> {
+ ensure_store()?;
+ let reference = parse_reference(reference_str)?;
+ let client = make_client(&reference);
+ let auth = RegistryAuth::Anonymous;
+
+ let image_data = client
+ .pull(&reference, &auth, vec![HULE_CHUNK_MEDIA_TYPE])
+ .await
+ .map_err(|e| e.to_string())?;
+ let manifest = image_data
+ .manifest
+ .ok_or("registry returned no image manifest")?;
+
+ // `client.pull()` fetches layers via `buffer_unordered`, so `image_data.layers` is in
+ // completion order, NOT `manifest.layers` order -- do not zip the two. Each `ImageLayer`
+ // already carries its own annotations, so it's self-describing without cross-referencing.
+ for layer in &image_data.layers {
+ write_blob(&layer.data)?;
+ }
+ write_blob(&image_data.config.data)?;
+
+ let manifest_bytes = serde_json::to_vec(&manifest).map_err(|e| e.to_string())?;
+ let manifest_digest = write_blob(&manifest_bytes)?;
+
+ let materialized = materialized_dir(&manifest_digest)?;
+ std::fs::create_dir_all(&materialized).map_err(|e| e.to_string())?;
+
+ let mut by_title: BTreeMap<String, Vec<(u64, &[u8])>> = BTreeMap::new();
+ for layer in &image_data.layers {
+ let annotations = layer
+ .annotations
+ .as_ref()
+ .ok_or("chunk layer missing annotations")?;
+ let title = annotations
+ .get(ORG_OPENCONTAINERS_IMAGE_TITLE)
+ .ok_or("chunk layer missing title annotation")?;
+ let offset: u64 = annotations
+ .get(ANNOTATION_CHUNK_OFFSET)
+ .ok_or("chunk layer missing offset annotation")?
+ .parse()
+ .map_err(|_| "invalid chunk offset annotation".to_string())?;
+ by_title
+ .entry(title.clone())
+ .or_default()
+ .push((offset, &layer.data[..]));
+ }
+
+ for (title, mut chunks) in by_title {
+ chunks.sort_by_key(|(offset, _)| *offset);
+ let file = std::fs::File::create(materialized.join(&title)).map_err(|e| e.to_string())?;
+ for (offset, compressed) in chunks {
+ let decompressed = zstd::stream::decode_all(compressed).map_err(|e| e.to_string())?;
+ write_at(&file, offset, &decompressed)?;
+ }
+ }
+
+ let mut index = read_index()?;
+ index_set(
+ &mut index,
+ reference_str,
+ &manifest_digest,
+ manifest_bytes.len() as u64,
+ );
+ write_index(&index)?;
+
+ Ok(manifest_digest)
+}
+
+/// Returns whether a reference is present in the local OCI layout.
+pub fn contains(reference_str: &str) -> R<bool> {
+ ensure_store()?;
+ Ok(index_lookup(&read_index()?, reference_str).is_some())
+}
+
+/// Materializes a locally tagged image into `destination` and returns config.json.
+pub fn materialize(reference_str: &str, destination: &Path) -> R<Vec<u8>> {
+ ensure_store()?;
+ let manifest_digest = index_lookup(&read_index()?, reference_str)
+ .ok_or_else(|| format!("no local image tagged '{reference_str}'"))?;
+
+ let manifest_bytes = read_blob(&manifest_digest)?;
+ let manifest: OciImageManifest =
+ serde_json::from_slice(&manifest_bytes).map_err(|e| e.to_string())?;
+ let config_bytes = read_blob(&manifest.config.digest)?;
+
+ std::fs::create_dir_all(destination).map_err(|e| e.to_string())?;
+ std::fs::write(destination.join("config.json"), &config_bytes).map_err(|e| e.to_string())?;
+
+ let materialized = materialized_dir(&manifest_digest)?;
+ let mut titles: BTreeSet<String> = BTreeSet::new();
+ for descriptor in &manifest.layers {
+ if let Some(title) = descriptor
+ .annotations
+ .as_ref()
+ .and_then(|a| a.get(ORG_OPENCONTAINERS_IMAGE_TITLE))
+ {
+ titles.insert(title.clone());
+ }
+ }
+ for title in titles {
+ let dest = destination.join(&title);
+ if !dest.exists() {
+ hardlink_or_copy(&materialized.join(&title), &dest)?;
+ }
+ }
+
+ Ok(config_bytes)
+}
diff --git a/crates/hule-vmm/tests/qemu_lifecycle.rs b/crates/hule-vmm/tests/qemu_lifecycle.rs
new file mode 100644
--- /dev/null
+++ b/crates/hule-vmm/tests/qemu_lifecycle.rs
@@ -0,0 +1,118 @@
+// SPDX-FileCopyrightText: 2026 Nikolay Govorov
+// SPDX-License-Identifier: Apache-2.0
+
+//! Boots a real qemu VM against the repo's alpine fixture and drives it
+//! through QMP-backed lifecycle methods. Needs qemu-system-x86_64 and
+//! `images/alpine/3.24/amd64` on disk; `#[ignore]`d so a normal `cargo
+//! test` run doesn't depend on either.
+
+use std::path::Path;
+use std::time::Duration;
+
+use hule_image::MachineImage;
+use hule_vmm::backend::qemu::QemuHypervisor;
+use hule_vmm::{Hypervisor, MachineId, Settings, State};
+use uuid::Uuid;
+
+#[tokio::test]
+#[ignore = "needs qemu-system-x86_64 and the alpine fixture on disk"]
+async fn pause_unpause_restart_and_kill_track_a_real_qemu() {
+ let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../images/alpine/3.24/amd64");
+ let config = std::fs::read(fixture.join("config.json")).expect("read fixture config.json");
+ let image = MachineImage::from_json(&config).expect("valid fixture config.json");
+
+ let scratch = std::env::temp_dir().join(format!("hule-vmm-test-{}", std::process::id()));
+ std::fs::create_dir_all(&scratch).unwrap();
+ for disk in &image.disks {
+ let src = fixture.join(&disk.path);
+ let dest = scratch.join(&disk.path);
+ std::fs::hard_link(&src, &dest)
+ .or_else(|_| std::fs::copy(&src, &dest).map(|_| ()))
+ .expect("stage disk into scratch dir");
+ }
+
+ let hv = QemuHypervisor;
+ let boot = image
+ .machine
+ .boot
+ .iter()
+ .find(|b| hv.supports(b))
+ .expect("fixture declares a boot protocol qemu supports");
+ let settings = Settings {
+ cpu: 1,
+ ram: 512,
+ port_forwards: vec![],
+ };
+
+ let mut machine = hv
+ .create(
+ MachineId(Uuid::now_v7()),
+ None,
+ &image,
+ &scratch,
+ boot,
+ &settings,
+ )
+ .await
+ .expect("create");
+
+ machine.start().await.expect("start");
+
+ assert_eq!(machine.state().await, State::Running);
+
+ machine.pause().await.expect("pause");
+ assert_eq!(machine.state().await, State::Paused);
+
+ machine.unpause().await.expect("unpause");
+ assert_eq!(machine.state().await, State::Running);
+
+ machine.restart().await.expect("restart");
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ assert_eq!(machine.state().await, State::Running);
+
+ let mut new_settings = machine.settings();
+ new_settings.ram = 384;
+ machine
+ .update(new_settings)
+ .await
+ .expect("update (balloon)");
+ assert_eq!(machine.settings().ram, 384);
+
+ machine.kill().await.expect("kill");
+ assert!(matches!(machine.state().await, State::Exited(_)));
+
+ let _ = std::fs::remove_dir_all(&scratch);
+}
+
+/// QMP is internal to the qemu backend, so a black-box image is valid.
+#[tokio::test]
+async fn create_accepts_an_image_without_access() {
+ let fixture = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../images/alpine/3.24/amd64");
+ let config = std::fs::read(fixture.join("config.json")).expect("read fixture config.json");
+ let mut image = MachineImage::from_json(&config).expect("valid fixture config.json");
+ image.machine.access.clear();
+
+ let hv = QemuHypervisor;
+ let boot = image
+ .machine
+ .boot
+ .iter()
+ .find(|b| hv.supports(b))
+ .expect("fixture declares a boot protocol qemu supports");
+ let settings = Settings {
+ cpu: 1,
+ ram: 512,
+ port_forwards: vec![],
+ };
+
+ hv.create(
+ MachineId(Uuid::now_v7()),
+ None,
+ &image,
+ &fixture,
+ boot,
+ &settings,
+ )
+ .await
+ .expect("black-box images don't need to declare qemu's QMP channel");
+}
diff --git a/crates/hule/Cargo.toml b/crates/hule/Cargo.toml
index cf0d99a..a783fda 100644
--- a/crates/hule/Cargo.toml
+++ b/crates/hule/Cargo.toml
@@ -16,10 +16,5 @@ repository.workspace = true
hule-oci.workspace = true
hule-vmm.workspace = true
hule-image.workspace = true
-serde_json.workspace = true
-oci-client = "0.17"
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "fs"] }
-futures-util = "0.3"
-zstd = "0.13"
-sha2 = "0.10"
uuid = { version = "1", features = ["v7"] }
diff --git a/crates/hule/src/main.rs b/crates/hule/src/main.rs
index 82ff039..6af3d54 100644
--- a/crates/hule/src/main.rs
+++ b/crates/hule/src/main.rs
@@ -1,33 +1,12 @@
// SPDX-FileCopyrightText: 2026 Nikolay Govorov
// SPDX-License-Identifier: Apache-2.0
-//! Manifest-driven VM harness and OCI image lifecycle.
-//!
-//! Reads a Hule machine configuration (`config.json`) from an image
-//! directory and boots the guest via `hule-vmm`'s qemu backend according to
-//! the boot protocol it declares. No per-distro logic lives here --
-//! everything comes from the manifest.
-//!
-//! Images live in a local content-addressed store (`$HOME/.hule/store`, a
-//! plain OCI Image Layout) and move in and out of OCI registries via
-//! `import`/`push`/`pull`; `run` resolves a reference against that store
-//! instead of taking a raw directory.
-
-use std::collections::{BTreeMap, BTreeSet};
-use std::io::Read;
-use std::path::{Path, PathBuf};
+use std::path::Path;
use std::process::exit;
use hule_image::{Access, MachineImage};
use hule_vmm::backend::qemu::QemuHypervisor;
use hule_vmm::{Hypervisor, MachineId, Settings};
-use oci_client::annotations::{ORG_OPENCONTAINERS_IMAGE_REF_NAME, ORG_OPENCONTAINERS_IMAGE_TITLE};
-use oci_client::client::{ClientConfig, ClientProtocol, Config, ImageLayer};
-use oci_client::manifest::{
- ImageIndexEntry, OCI_IMAGE_MEDIA_TYPE, OciDescriptor, OciImageIndex, OciImageManifest,
-};
-use oci_client::secrets::RegistryAuth;
-use oci_client::{Client, Reference};
use uuid::Uuid;
type R<T> = std::result::Result<T, String>;
@@ -37,461 +16,17 @@ fn die(msg: impl AsRef<str>) -> ! {
exit(1);
}
-// ---- local OCI-layout store -------------------------------------------
-
-const CHUNK_SIZE: usize = 256 * 1024 * 1024;
-const HULE_CHUNK_MEDIA_TYPE: &str = "application/vnd.hule.disk.chunk.v1";
-const HULE_CONFIG_MEDIA_TYPE: &str = "application/vnd.hule.machine.config.v1+json";
-const ANNOTATION_CHUNK_OFFSET: &str = "io.hule.chunk.offset";
-const ANNOTATION_CHUNK_LENGTH: &str = "io.hule.chunk.length";
-
-fn store_root() -> PathBuf {
- let home = std::env::var("HOME").unwrap_or_else(|_| die("HOME is not set"));
- PathBuf::from(home).join(".hule").join("store")
-}
-
-fn blobs_dir() -> PathBuf {
- store_root().join("blobs").join("sha256")
-}
-
-fn materialized_dir(manifest_digest: &str) -> PathBuf {
- store_root()
- .join("materialized")
- .join(strip_sha256(manifest_digest))
-}
-
-fn index_path() -> PathBuf {
- store_root().join("index.json")
-}
-
-fn strip_sha256(digest: &str) -> &str {
- digest.strip_prefix("sha256:").unwrap_or(digest)
-}
-
-fn sha256_hex(data: &[u8]) -> String {
- use sha2::{Digest, Sha256};
- let mut hasher = Sha256::new();
- hasher.update(data);
- format!("sha256:{:x}", hasher.finalize())
-}
-
-fn ensure_store() -> R<()> {
- std::fs::create_dir_all(blobs_dir()).map_err(|e| e.to_string())?;
- std::fs::create_dir_all(store_root().join("materialized")).map_err(|e| e.to_string())?;
- let layout = store_root().join("oci-layout");
- if !layout.exists() {
- std::fs::write(&layout, br#"{"imageLayoutVersion":"1.0.0"}"#).map_err(|e| e.to_string())?;
- }
- Ok(())
-}
-
-/// Writes raw bytes as a content-addressed blob, no-op if already present.
-fn write_blob(data: &[u8]) -> R<String> {
- let digest = sha256_hex(data);
- let path = blobs_dir().join(strip_sha256(&digest));
- if !path.exists() {
- let tmp = path.with_extension("tmp");
- std::fs::write(&tmp, data).map_err(|e| e.to_string())?;
- std::fs::rename(&tmp, &path).map_err(|e| e.to_string())?;
- }
- Ok(digest)
-}
-
-fn read_blob(digest: &str) -> R<Vec<u8>> {
- std::fs::read(blobs_dir().join(strip_sha256(digest))).map_err(|e| e.to_string())
-}
-
-fn read_index() -> R<OciImageIndex> {
- let path = index_path();
- if !path.exists() {
- return Ok(OciImageIndex {
- schema_version: 2,
- media_type: None,
- manifests: vec![],
- artifact_type: None,
- annotations: None,
- });
- }
- let data = std::fs::read(&path).map_err(|e| e.to_string())?;
- serde_json::from_slice(&data).map_err(|e| e.to_string())
-}
-
-fn write_index(index: &OciImageIndex) -> R<()> {
- let data = serde_json::to_vec_pretty(index).map_err(|e| e.to_string())?;
- std::fs::write(index_path(), data).map_err(|e| e.to_string())
-}
-
-fn index_lookup(index: &OciImageIndex, reference: &str) -> Option<String> {
- index
- .manifests
- .iter()
- .find(|m| {
- m.annotations
- .as_ref()
- .and_then(|a| a.get(ORG_OPENCONTAINERS_IMAGE_REF_NAME))
- .map(String::as_str)
- == Some(reference)
- })
- .map(|m| m.digest.clone())
-}
-
-fn index_set(index: &mut OciImageIndex, reference: &str, digest: &str, size: u64) {
- index.manifests.retain(|m| {
- m.annotations
- .as_ref()
- .and_then(|a| a.get(ORG_OPENCONTAINERS_IMAGE_REF_NAME))
- .map(String::as_str)
- != Some(reference)
- });
- let mut annotations = BTreeMap::new();
- annotations.insert(
- ORG_OPENCONTAINERS_IMAGE_REF_NAME.to_string(),
- reference.to_string(),
- );
- index.manifests.push(ImageIndexEntry {
- media_type: OCI_IMAGE_MEDIA_TYPE.to_string(),
- digest: digest.to_string(),
- size: size as i64,
- platform: None,
- annotations: Some(annotations),
- artifact_type: None,
- });
-}
-
-fn hardlink_or_copy(src: &Path, dest: &Path) -> R<()> {
- if std::fs::hard_link(src, dest).is_err() {
- std::fs::copy(src, dest).map_err(|e| e.to_string())?;
- }
- Ok(())
-}
-
-fn write_at(file: &std::fs::File, offset: u64, data: &[u8]) -> R<()> {
- use std::os::unix::fs::FileExt;
- let mut written = 0usize;
- while written < data.len() {
- let n = file
- .write_at(&data[written..], offset + written as u64)
- .map_err(|e| e.to_string())?;
- if n == 0 {
- return Err("write_at wrote 0 bytes".to_string());
- }
- written += n;
- }
- Ok(())
-}
-
-/// Splits `path` into fixed-size chunks, compresses each independently
-/// (own zstd frame, no state shared between chunks -- see the design notes
-/// in the project plan on why this must not be done the other way around),
-/// and writes each as its own blob. Returns one descriptor per chunk, all
-/// sharing a title annotation plus a chunk offset/length in terms of the
-/// *uncompressed* file so pull can reassemble it.
-fn write_file_chunks(path: &Path) -> R<Vec<OciDescriptor>> {
- let title = path
- .file_name()
- .ok_or_else(|| format!("{} has no file name", path.display()))?
- .to_string_lossy()
- .to_string();
- let mut file = std::fs::File::open(path).map_err(|e| e.to_string())?;
- let mut descriptors = Vec::new();
- let mut offset: u64 = 0;
- loop {
- let mut buf = Vec::with_capacity(CHUNK_SIZE);
- (&mut file)
- .take(CHUNK_SIZE as u64)
- .read_to_end(&mut buf)
- .map_err(|e| e.to_string())?;
- if buf.is_empty() {
- break;
- }
- let length = buf.len() as u64;
- let compressed = zstd::stream::encode_all(&buf[..], 0).map_err(|e| e.to_string())?;
- let digest = write_blob(&compressed)?;
-
- let mut annotations = BTreeMap::new();
- annotations.insert(ORG_OPENCONTAINERS_IMAGE_TITLE.to_string(), title.clone());
- annotations.insert(ANNOTATION_CHUNK_OFFSET.to_string(), offset.to_string());
- annotations.insert(ANNOTATION_CHUNK_LENGTH.to_string(), length.to_string());
-
- descriptors.push(OciDescriptor {
- media_type: HULE_CHUNK_MEDIA_TYPE.to_string(),
- digest,
- size: compressed.len() as i64,
- urls: None,
- annotations: Some(annotations),
- artifact_type: None,
- });
-
- offset += length;
- if length < CHUNK_SIZE as u64 {
- break;
- }
- }
- if descriptors.is_empty() {
- return Err(format!("{} is empty", path.display()));
- }
- Ok(descriptors)
-}
-
-fn make_client(reference: &Reference) -> Client {
- let registry = reference.resolve_registry();
- let host = registry.split(':').next().unwrap_or(registry);
- let protocol = if host == "localhost" || host == "127.0.0.1" {
- ClientProtocol::Http
- } else {
- ClientProtocol::Https
- };
- Client::new(ClientConfig {
- protocol,
- ..Default::default()
- })
-}
-
-fn parse_reference(s: &str) -> R<Reference> {
- s.parse()
- .map_err(|e| format!("invalid reference '{s}': {e}"))
-}
-
-// ---- import / push / pull / run ---------------------------------------
-
-async fn cmd_import(path: &Path, reference: Option<&str>) -> R<()> {
- ensure_store()?;
- let config_path = path.join("config.json");
- if !config_path.is_file() {
- return Err(format!("{} does not contain config.json", path.display()));
- }
- let config_bytes = std::fs::read(&config_path).map_err(|e| e.to_string())?;
- MachineImage::from_json(&config_bytes).map_err(|e| e.to_string())?;
-
- let mut entries: Vec<_> = std::fs::read_dir(path)
- .map_err(|e| e.to_string())?
- .collect::<std::result::Result<Vec<_>, _>>()
- .map_err(|e| e.to_string())?;
- entries.sort_by_key(|e| e.file_name());
-
- let mut layers = Vec::new();
- let mut sources: Vec<(String, PathBuf)> = Vec::new();
- for entry in entries {
- if !entry.file_type().map_err(|e| e.to_string())?.is_file() {
- continue;
- }
- let name = entry.file_name().to_string_lossy().to_string();
- if name == "config.json" {
- continue;
- }
- let file_path = entry.path();
- layers.extend(write_file_chunks(&file_path)?);
- sources.push((name, file_path));
- }
-
- let config_digest = write_blob(&config_bytes)?;
- let config = OciDescriptor {
- media_type: HULE_CONFIG_MEDIA_TYPE.to_string(),
- digest: config_digest,
- size: config_bytes.len() as i64,
- urls: None,
- annotations: None,
- artifact_type: None,
- };
-
- let manifest = OciImageManifest {
- schema_version: 2,
- media_type: Some(OCI_IMAGE_MEDIA_TYPE.to_string()),
- config,
- layers,
- subject: None,
- artifact_type: None,
- annotations: None,
- };
- let manifest_bytes = serde_json::to_vec(&manifest).map_err(|e| e.to_string())?;
- let manifest_digest = write_blob(&manifest_bytes)?;
-
- let materialized = materialized_dir(&manifest_digest);
- std::fs::create_dir_all(&materialized).map_err(|e| e.to_string())?;
- for (title, source) in &sources {
- let dest = materialized.join(title);
- if !dest.exists() {
- hardlink_or_copy(source, &dest)?;
- }
- }
-
- if let Some(reference) = reference {
- let mut index = read_index()?;
- index_set(
- &mut index,
- reference,
- &manifest_digest,
- manifest_bytes.len() as u64,
- );
- write_index(&index)?;
- eprintln!(
- "hule: imported {} as {reference} ({manifest_digest})",
- path.display()
- );
- } else {
- eprintln!(
- "hule: imported {} ({manifest_digest}, untagged)",
- path.display()
- );
- }
- Ok(())
-}
-
-async fn cmd_push(reference_str: &str) -> R<()> {
- ensure_store()?;
- let index = read_index()?;
- let manifest_digest = index_lookup(&index, reference_str)
- .ok_or_else(|| format!("no local image tagged '{reference_str}'"))?;
- let manifest_bytes = read_blob(&manifest_digest)?;
- let manifest: OciImageManifest =
- serde_json::from_slice(&manifest_bytes).map_err(|e| e.to_string())?;
-
- let reference = parse_reference(reference_str)?;
- let client = make_client(&reference);
- let auth = RegistryAuth::Anonymous;
-
- let mut layers = Vec::new();
- for descriptor in &manifest.layers {
- let data = read_blob(&descriptor.digest)?;
- layers.push(ImageLayer::new(
- data,
- descriptor.media_type.clone(),
- descriptor.annotations.clone(),
- ));
- }
- let config_bytes = read_blob(&manifest.config.digest)?;
- let config = Config::new(
- config_bytes,
- manifest.config.media_type.clone(),
- manifest.config.annotations.clone(),
- );
-
- client
- .push(&reference, &layers, config, &auth, Some(manifest))
- .await
- .map_err(|e| e.to_string())?;
-
- eprintln!("hule: pushed {reference_str}");
- Ok(())
-}
-
-async fn cmd_pull(reference_str: &str) -> R<()> {
- ensure_store()?;
- let reference = parse_reference(reference_str)?;
- let client = make_client(&reference);
- let auth = RegistryAuth::Anonymous;
-
- let image_data = client
- .pull(&reference, &auth, vec![HULE_CHUNK_MEDIA_TYPE])
- .await
- .map_err(|e| e.to_string())?;
- let manifest = image_data
- .manifest
- .ok_or("registry returned no image manifest")?;
-
- // `client.pull()` fetches layers via `buffer_unordered`, so `image_data.layers` is in
- // completion order, NOT `manifest.layers` order -- do not zip the two. Each `ImageLayer`
- // already carries its own annotations, so it's self-describing without cross-referencing.
- for layer in &image_data.layers {
- write_blob(&layer.data)?;
- }
- write_blob(&image_data.config.data)?;
-
- let manifest_bytes = serde_json::to_vec(&manifest).map_err(|e| e.to_string())?;
- let manifest_digest = write_blob(&manifest_bytes)?;
-
- let materialized = materialized_dir(&manifest_digest);
- std::fs::create_dir_all(&materialized).map_err(|e| e.to_string())?;
-
- let mut by_title: BTreeMap<String, Vec<(u64, &[u8])>> = BTreeMap::new();
- for layer in &image_data.layers {
- let annotations = layer
- .annotations
- .as_ref()
- .ok_or("chunk layer missing annotations")?;
- let title = annotations
- .get(ORG_OPENCONTAINERS_IMAGE_TITLE)
- .ok_or("chunk layer missing title annotation")?;
- let offset: u64 = annotations
- .get(ANNOTATION_CHUNK_OFFSET)
- .ok_or("chunk layer missing offset annotation")?
- .parse()
- .map_err(|_| "invalid chunk offset annotation".to_string())?;
- by_title
- .entry(title.clone())
- .or_default()
- .push((offset, &layer.data[..]));
+async fn cmd_run(reference: &str, port: u16) -> R<()> {
+ if !hule_oci::contains(reference)? {
+ eprintln!("hule: {reference} not found locally, pulling...");
+ hule_oci::pull(reference).await?;
}
- for (title, mut chunks) in by_title {
- chunks.sort_by_key(|(offset, _)| *offset);
- let file = std::fs::File::create(materialized.join(&title)).map_err(|e| e.to_string())?;
- for (offset, compressed) in chunks {
- let decompressed = zstd::stream::decode_all(compressed).map_err(|e| e.to_string())?;
- write_at(&file, offset, &decompressed)?;
- }
- }
-
- let mut index = read_index()?;
- index_set(
- &mut index,
- reference_str,
- &manifest_digest,
- manifest_bytes.len() as u64,
- );
- write_index(&index)?;
-
- eprintln!("hule: pulled {reference_str} ({manifest_digest})");
- Ok(())
-}
-
-async fn cmd_run(reference_str: &str, port: u16) -> R<()> {
- ensure_store()?;
- let mut index = read_index()?;
- let manifest_digest = match index_lookup(&index, reference_str) {
- Some(d) => d,
- None => {
- eprintln!("hule: {reference_str} not found locally, pulling...");
- cmd_pull(reference_str).await?;
- index = read_index()?;
- index_lookup(&index, reference_str)
- .ok_or_else(|| format!("pull of '{reference_str}' did not produce a local tag"))?
- }
- };
-
- let manifest_bytes = read_blob(&manifest_digest)?;
- let manifest: OciImageManifest =
- serde_json::from_slice(&manifest_bytes).map_err(|e| e.to_string())?;
- let config_bytes = read_blob(&manifest.config.digest)?;
-
let scratch = std::env::temp_dir().join(format!("hule-run-{}", std::process::id()));
- std::fs::create_dir_all(&scratch).map_err(|e| e.to_string())?;
- std::fs::write(scratch.join("config.json"), &config_bytes).map_err(|e| e.to_string())?;
-
- let materialized = materialized_dir(&manifest_digest);
- let mut titles: BTreeSet<String> = BTreeSet::new();
- for descriptor in &manifest.layers {
- if let Some(title) = descriptor
- .annotations
- .as_ref()
- .and_then(|a| a.get(ORG_OPENCONTAINERS_IMAGE_TITLE))
- {
- titles.insert(title.clone());
- }
- }
- for title in titles {
- let dest = scratch.join(&title);
- if !dest.exists() {
- hardlink_or_copy(&materialized.join(&title), &dest)?;
- }
- }
-
- let image = MachineImage::from_json(&config_bytes).map_err(|e| e.to_string())?;
+ let config = hule_oci::materialize(reference, &scratch)?;
+ let image = MachineImage::from_json(&config).map_err(|e| e.to_string())?;
let hv = QemuHypervisor;
- // Negotiation: prefer direct-kernel (skips firmware+bootloader) over
- // firmware-disk when both are declared and supported; no Monitor yet,
- // so this stays hand-rolled instead of `Monitor::create`.
let boot = image
.machine
.boot
@@ -537,8 +72,6 @@ async fn cmd_run(reference_str: &str, port: u16) -> R<()> {
exit(status.code().unwrap_or(1));
}
-// ---- CLI dispatch -------------------------------------------------------
-
#[tokio::main]
async fn main() {
let args: Vec<String> = std::env::args().collect();
@@ -561,30 +94,38 @@ async fn main() {
usage(&args[0]);
}
let reference = args.get(3).map(String::as_str);
- cmd_import(Path::new(&args[2]), reference)
+ let digest = hule_oci::import(Path::new(&args[2]), reference)
.await
.unwrap_or_else(|e| die(e));
+ match reference {
+ Some(reference) => {
+ eprintln!("hule: imported {} as {reference} ({digest})", args[2])
+ }
+ None => eprintln!("hule: imported {} ({digest}, untagged)", args[2]),
+ }
}
"push" => {
if args.len() < 3 {
usage(&args[0]);
}
- cmd_push(&args[2]).await.unwrap_or_else(|e| die(e));
+ hule_oci::push(&args[2]).await.unwrap_or_else(|e| die(e));
+ eprintln!("hule: pushed {}", args[2]);
}
"pull" => {
if args.len() < 3 {
usage(&args[0]);
}
- cmd_pull(&args[2]).await.unwrap_or_else(|e| die(e));
+ let digest = hule_oci::pull(&args[2]).await.unwrap_or_else(|e| die(e));
+ eprintln!("hule: pulled {} ({digest})", args[2]);
}
"run" => {
if args.len() < 3 {
usage(&args[0]);
}
- let port: u16 = args.get(3).map_or(8022, |s| {
- s.parse()
- .unwrap_or_else(|_| die(format!("invalid port '{s}'")))
- });
+ let port = args
+ .get(3)
+ .map_or(Ok(8022), |s| s.parse::<u16>())
+ .unwrap_or_else(|_| die(format!("invalid port '{}'", args[3])));
cmd_run(&args[2], port).await.unwrap_or_else(|e| die(e));
}
_ => usage(&args[0]),
generated by Gilti