From c4b18c33797bd1bbe5d614be1767cb17df5a5b55 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 16 Dec 2024 16:20:29 +0200 Subject: [PATCH] xtask: LLVM clone, clone progress --- .cargo/config.toml | 2 +- Cargo.lock | 61 ++++++++++++++++++++++++++++++++++ xtask/Cargo.toml | 1 + xtask/src/build/c.rs | 2 +- xtask/src/build/mod.rs | 2 +- xtask/src/build/toolchain.rs | 6 +--- xtask/src/build/toolchain_c.rs | 10 ++++-- xtask/src/util.rs | 33 +++++++++++++++++- 8 files changed, 105 insertions(+), 12 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index d8c2032b..5e376525 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,2 @@ [alias] -xtask = "run --manifest-path ./xtask/Cargo.toml --" +xtask = "run --manifest-path ./xtask/Cargo.toml --release --" diff --git a/Cargo.lock b/Cargo.lock index 9c4036cc..4662d5f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -391,6 +391,19 @@ dependencies = [ "rustc-std-workspace-core", ] +[[package]] +name = "console" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -482,6 +495,12 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4445909572dbd556c457c849c4ca58623d84b27c8fff1e74b0b4227d8b90d17b" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "endian-type-rs" version = "0.1.4" @@ -824,6 +843,19 @@ dependencies = [ "hashbrown 0.15.0", ] +[[package]] +name = "indicatif" +version = "0.17.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -1227,6 +1259,12 @@ dependencies = [ "libm 0.2.11", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.36.5" @@ -1300,6 +1338,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -1811,6 +1855,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unsafe_unwrap" version = "0.1.0" @@ -1945,6 +1995,16 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "6.0.3" @@ -2114,6 +2174,7 @@ dependencies = [ "fatfs", "fscommon", "git2", + "indicatif", "log", "qemu", "semver 1.0.23", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index c4ce8a06..f2407263 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -21,3 +21,4 @@ dependency-graph = "0.1.5" semver = { version = "1.0.22", features = ["serde"] } fatfs = "0.3.6" fscommon = "0.1.1" +indicatif = "0.17.9" diff --git a/xtask/src/build/c.rs b/xtask/src/build/c.rs index 7446992b..1a6f02c8 100644 --- a/xtask/src/build/c.rs +++ b/xtask/src/build/c.rs @@ -185,7 +185,7 @@ fn build_openlibm(env: &BuildEnv, llvm: &Llvm) -> Result { .workspace_root .join(format!("toolchain-c/libs/openlibm-{}", env.arch.name())); - git_clone(&libm_path, OPENLIBM_URL, "alnyan/yggdrasil")?; + git_clone(&libm_path, OPENLIBM_URL, "alnyan/yggdrasil", false)?; log::info!("Building openlibm"); diff --git a/xtask/src/build/mod.rs b/xtask/src/build/mod.rs index c5c9f717..fcf02c55 100644 --- a/xtask/src/build/mod.rs +++ b/xtask/src/build/mod.rs @@ -160,7 +160,7 @@ pub fn clean_all(env: &BuildEnv, clean_toolchain: bool, repos: bool) -> Result<( CargoBuilder::Host(env.verbose).run(&env.workspace_root, "clean")?; clean_userspace(env)?; - fs::remove_dir_all(&env.llvm_sysroot)?; + fs::remove_dir_all(env.workspace_root.join("toolchain-c/sysroot"))?; // Clean libm for target in Arch::all() { Command::new("make") diff --git a/xtask/src/build/toolchain.rs b/xtask/src/build/toolchain.rs index c883a516..b14c325f 100644 --- a/xtask/src/build/toolchain.rs +++ b/xtask/src/build/toolchain.rs @@ -34,8 +34,6 @@ struct RustRustConfig { #[serde(rename_all = "kebab-case")] struct RustToolchainConfig { profile: String, - changelog_seen: u32, - change_id: u32, build: RustBuildConfig, target: HashMap, @@ -45,8 +43,6 @@ struct RustToolchainConfig { fn configure>(env: &BuildEnv, config_toml_path: P) -> Result<(), Error> { let config = RustToolchainConfig { profile: "user".into(), - changelog_seen: 2, - change_id: 102579, build: RustBuildConfig { host: vec![env.host_triple.clone()], @@ -143,7 +139,7 @@ fn install_shims>(env: &BuildEnv, toolchain_root: P) -> Result<() pub fn fetch(env: &BuildEnv, branch: &str) -> Result<(), Error> { let path = env.workspace_root.join("toolchain"); - git_clone(path, TOOLCHAIN_URL, branch) + git_clone(path, TOOLCHAIN_URL, branch, false) } pub fn build(env: &BuildEnv) -> Result<(), Error> { diff --git a/xtask/src/build/toolchain_c.rs b/xtask/src/build/toolchain_c.rs index 23f2169e..6dab02ba 100644 --- a/xtask/src/build/toolchain_c.rs +++ b/xtask/src/build/toolchain_c.rs @@ -7,8 +7,11 @@ use std::{ use crate::{ env::BuildEnv, error::{CommandExt, Error}, + util, }; +pub const LLVM_URL: &str = "https://git.alnyan.me/yggdrasil/llvm-project.git"; + pub struct Llvm { root: PathBuf, } @@ -180,12 +183,13 @@ pub fn install_llvm_cxx_runtime(env: &BuildEnv, llvm: &Llvm) -> Result<(), Error pub fn install_llvm_compiler(env: &BuildEnv) -> Result { let root = env.workspace_root.join("toolchain-c/prefix/host"); - let build_dir = env.workspace_root.join("toolchain-c/llvm-project/build"); + let src_dir = env.workspace_root.join("toolchain-c/llvm-project"); + let build_dir = src_dir.join("build"); let makefile = build_dir.join("build.ninja"); - fs::create_dir_all(&build_dir)?; + util::git_clone(&src_dir, LLVM_URL, "alnyan/yggdrasil", true)?; - // TODO clone LLVM + fs::create_dir_all(&build_dir)?; // Configure LLVM // TODO option to force reconfig diff --git a/xtask/src/util.rs b/xtask/src/util.rs index 3f352823..4fa699de 100644 --- a/xtask/src/util.rs +++ b/xtask/src/util.rs @@ -1,10 +1,12 @@ use std::{ ffi::OsStr, fs, + io::{stderr, Write}, path::Path, process::{Command, Stdio}, }; +use indicatif::ProgressBar; use walkdir::WalkDir; use crate::error::Error; @@ -59,16 +61,45 @@ pub fn copy_dir_recursive, D: AsRef>(src: S, dst: D) -> Res Ok(()) } -pub fn git_clone(dst: impl AsRef, url: &str, branch: &str) -> Result<(), Error> { +pub fn git_clone( + dst: impl AsRef, + url: &str, + branch: &str, + shallow: bool, +) -> Result<(), Error> { let path = dst.as_ref(); match git2::Repository::open(path) { Ok(_) => {} Err(err) if err.code() == git2::ErrorCode::NotFound => { log::info!("Fetching: src={}, branch={}", url, branch); + let progress_bar = ProgressBar::no_length(); + + let mut remote_cbs = git2::RemoteCallbacks::new(); + remote_cbs.sideband_progress(|bytes| { + stderr().write_all(bytes).ok(); + true + }); + remote_cbs.transfer_progress(|progress| { + progress_bar.set_message("Receiving objects"); + if progress_bar.length().is_none() { + progress_bar.set_length(progress.total_objects() as _); + } + progress_bar.set_position(progress.received_objects() as _); + true + }); + + let mut fetch_opts = git2::FetchOptions::new(); + if shallow { + fetch_opts.depth(1); + } + fetch_opts.remote_callbacks(remote_cbs); git2::build::RepoBuilder::new() .branch(branch) + .fetch_options(fetch_opts) .clone(url, &path)?; + + progress_bar.finish_and_clear(); } Err(err) => return Err(err.into()), }