From 8983863ffaae9aeb62b26f1f8b837381301583c7 Mon Sep 17 00:00:00 2001 From: Arthur Pinheiro Date: Wed, 12 Mar 2025 13:26:31 +0100 Subject: [PATCH 1/3] fix: allow relative paths in allowlist and blocklist --- src/args.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/args.rs b/src/args.rs index af165ae..7a00d3b 100644 --- a/src/args.rs +++ b/src/args.rs @@ -3,7 +3,29 @@ use std::path::PathBuf; -use clap::Parser; +use clap::{builder::{PathBufValueParser, TypedValueParser}, Parser}; + +#[derive(Clone, Default)] +struct AbsolutePathBufValueParser; + +impl TypedValueParser for AbsolutePathBufValueParser { + type Value = PathBuf; + + fn parse_ref( + &self, + cmd: &clap::Command, + arg: Option<&clap::Arg>, + value: &std::ffi::OsStr, + ) -> Result { + let pathbuf_parser = PathBufValueParser::new(); + let v = pathbuf_parser.parse_ref(cmd, arg, value)?; + + v.canonicalize().map_err(|e| clap::Error::raw( + clap::error::ErrorKind::Io, + e, + )) + } +} /// A tool to clean up sysroots for Linux embedded devices to save storage space. #[derive(Parser, Debug)] @@ -20,7 +42,7 @@ pub struct Args { /// An allowlist of files to keep, in .gitignore format. /// Note: this will take precedence over all other removal decisions. - #[arg(long)] + #[arg(long, value_parser = AbsolutePathBufValueParser::default())] pub allowlist: Option, /// A blocklist of files to remove, in .gitignore format. @@ -35,3 +57,4 @@ pub struct Args { /// The location of the sysroot to clean up pub sysroot_location: PathBuf, } + From 266a00d98303b245908acbda3a0ed3d2d83f27f0 Mon Sep 17 00:00:00 2001 From: Arthur Pinheiro Date: Wed, 12 Mar 2025 14:18:40 +0100 Subject: [PATCH 2/3] feat: use vector for --allowlist and --blocklist for passing multiples --- src/args.rs | 6 ++++-- src/cleaners.rs | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/args.rs b/src/args.rs index 7a00d3b..a1db7e6 100644 --- a/src/args.rs +++ b/src/args.rs @@ -41,13 +41,15 @@ pub struct Args { pub split_to: Option, /// An allowlist of files to keep, in .gitignore format. + /// Can be passed multiple times. /// Note: this will take precedence over all other removal decisions. #[arg(long, value_parser = AbsolutePathBufValueParser::default())] - pub allowlist: Option, + pub allowlist: Vec, /// A blocklist of files to remove, in .gitignore format. + /// Can be passed multiple times. #[arg(long)] - pub blocklist: Option, + pub blocklist: Vec, /// An optional path to save the file graph of the DSO cleaner /// in GraphViz format. Useful for debugging. diff --git a/src/cleaners.rs b/src/cleaners.rs index 925937e..3e2860a 100644 --- a/src/cleaners.rs +++ b/src/cleaners.rs @@ -41,11 +41,11 @@ impl Runner { let removal_fn = Self::new_removal_fn(&args); let mut cleaners: Cleaners = vec![]; - if let Some(wl) = args.allowlist { + for wl in args.allowlist { cleaners.push(Box::new(ListCleaner::new(list::ListType::Allow, wl))); } - if let Some(bl) = args.blocklist { + for bl in args.blocklist { cleaners.push(Box::new(ListCleaner::new(list::ListType::Block, bl))); } From 6825d63aaa92e3ae4dbae998dbcf44d730d6b67d Mon Sep 17 00:00:00 2001 From: Arthur Pinheiro Date: Wed, 12 Mar 2025 16:52:03 +0100 Subject: [PATCH 3/3] feat: display size of each file and total removed size --- Cargo.lock | 7 +++++ Cargo.toml | 1 + src/cleaners.rs | 70 +++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 64 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 080ae12..a5ea2e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -136,6 +136,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" +[[package]] +name = "bytesize" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3c8f83209414aacf0eeae3cf730b18d6981697fba62f200fcfb92b9f082acba" + [[package]] name = "cfg-if" version = "1.0.0" @@ -634,6 +640,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "bytesize", "clap", "env_logger", "goblin", diff --git a/Cargo.toml b/Cargo.toml index 53bc72a..3fe8e90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,4 @@ nix = { version = "0.29", features = ["fs"] } petgraph = { version = "0.7" } tokio = { version = "1", features = ["full"] } walkdir = { version = "2" } +bytesize = { version = "2.0" } \ No newline at end of file diff --git a/src/cleaners.rs b/src/cleaners.rs index 3e2860a..d431716 100644 --- a/src/cleaners.rs +++ b/src/cleaners.rs @@ -10,10 +10,11 @@ use crate::{ }; use anyhow::{Error, Result}; use async_trait::async_trait; +use bytesize::ByteSize; use dso::DsoCleaner; use list::ListCleaner; use nix::libc::EXDEV; -use std::{collections::HashMap, io, path::Path}; +use std::{collections::HashMap, fmt, io, ops::AddAssign, path::Path}; use tokio::{sync::mpsc, task::JoinSet}; use walkdir::{DirEntry, WalkDir}; @@ -26,8 +27,22 @@ pub trait Cleaner { ) -> Result<()>; } +struct FileSize(u64); + +impl fmt::Display for FileSize { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", ByteSize(self.0)) + } +} + +impl AddAssign for FileSize { + fn add_assign(&mut self, rhs: Self) { + self.0.add_assign(rhs.0); + } +} + type Cleaners = Vec>; -type RemovalFn = Box io::Result<()>>; +type RemovalFn = Box io::Result>; pub struct Runner { cleaners: Cleaners, @@ -129,6 +144,8 @@ impl Runner { async fn final_decision(removal_fn: RemovalFn, mut output_rx: mpsc::Receiver) { let mut final_decisions = HashMap::new(); + let mut total_removed_size = FileSize(0); + while let Some(input_decision) = output_rx.recv().await { if input_decision.action == Action::Undecided { continue; @@ -147,28 +164,38 @@ impl Runner { for (file, action) in final_decisions { if action == Action::Remove { - if let Err(err) = (removal_fn)(&file) { - log::error!("{}: {}", file.display(), err); - } + let removed_size = match (removal_fn)(&file) { + Ok(size) => size, + Err(err) => { + log::error!("{}: {}", file.display(), err); + FileSize(0) + } + }; + total_removed_size += removed_size; } } + log::info!("Total space removed: {}", total_removed_size); } fn new_removal_fn(args: &Args) -> RemovalFn { if let Some(dest) = args.split_to.clone() { if args.dry_run { Box::new(move |path| { + let size = Self::get_file_size(path)?; log::info!( - "(dry-run) would move {} to {}", + "(dry-run) would move {} to {} ({})", path.display(), - dest.display() + dest.display(), + size ); - Ok(()) + Ok(size) }) } else { Box::new(move |path| { - log::info!("moving {} to {}", path.display(), dest.display()); - Self::move_preserve(&path, &dest) + let size = Self::get_file_size(path)?; + log::info!("moving {} to {} ({})", path.display(), dest.display(), size); + Self::move_preserve(&path, &dest)?; + Ok(size) }) } } else { @@ -179,18 +206,33 @@ impl Runner { } else { "regular file" }; - log::info!("(dry-run) would remove {} {}", ty, path.display()); - Ok(()) + let size = Self::get_file_size(path)?; + log::info!("(dry-run) would remove {} {} ({})", ty, path.display(), size); + Ok(size) }) } else { Box::new(move |path| { - log::info!("removing {}", path.display()); - std::fs::remove_file(&path) + let size = Self::get_file_size(path)?; + log::info!("removing {} ({})", path.display(), size); + std::fs::remove_file(&path)?; + Ok(size) }) } } } + fn get_file_size(file: &Path) -> io::Result { + let lstat = nix::sys::stat::lstat(file); + let size = match lstat { + Err(err) => { + log::error!("failed to get metadata from: {}, {}", file.display(), err); + FileSize(0) + }, + Ok(lstat) => FileSize(lstat.st_size as u64) + }; + Ok(size) + } + fn move_preserve(src: &Path, dest: &Path) -> io::Result<()> { assert!(src.is_relative()); let abs_dest = dest.join(src);