Merge pull request 'Extra features and fixes' (#1) from arthurep/sysroot-cleaner:extra-features into main

Reviewed-on: matteo/sysroot-cleaner#1
Reviewed-by: Matteo Settenvini <matteo.settenvini@montecristosoftware.eu>

Cargo.lock

@@ -136,6 +136,12 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9"
 
+[[package]]
+name = "bytesize"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3c8f83209414aacf0eeae3cf730b18d6981697fba62f200fcfb92b9f082acba"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -634,6 +640,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
+ "bytesize",
  "clap",
  "env_logger",
  "goblin",

Cargo.toml

@@ -23,3 +23,4 @@ nix = { version = "0.29", features = ["fs"] }
 petgraph = { version = "0.7" }
 tokio = { version = "1", features = ["full"] }
 walkdir = { version = "2" }
+bytesize = { version = "2.0" }

src/args.rs

@@ -3,7 +3,29 @@
 
 use std::path::PathBuf;
 
-use clap::Parser;
+use clap::{builder::{PathBufValueParser, TypedValueParser}, Parser};
+
+#[derive(Clone, Default)]
+struct AbsolutePathBufValueParser;
+
+impl TypedValueParser for AbsolutePathBufValueParser {
+    type Value = PathBuf;
+
+    fn parse_ref(
+        &self,
+        cmd: &clap::Command,
+        arg: Option<&clap::Arg>,
+        value: &std::ffi::OsStr,
+    ) -> Result<Self::Value, clap::Error> {
+        let pathbuf_parser = PathBufValueParser::new();
+        let v = pathbuf_parser.parse_ref(cmd, arg, value)?;
+
+        v.canonicalize().map_err(|e| clap::Error::raw(
+            clap::error::ErrorKind::Io,
+            e,
+        ))
+    }
+}
 
 /// A tool to clean up sysroots for Linux embedded devices to save storage space.
 #[derive(Parser, Debug)]
@@ -19,13 +41,15 @@ pub struct Args {
     pub split_to: Option<PathBuf>,
 
     /// An allowlist of files to keep, in .gitignore format.
+    /// Can be passed multiple times.
     /// Note: this will take precedence over all other removal decisions.
-    #[arg(long)]
+    #[arg(long, value_parser = AbsolutePathBufValueParser::default())]
-    pub allowlist: Option<PathBuf>,
+    pub allowlist: Vec<PathBuf>,
 
     /// A blocklist of files to remove, in .gitignore format.
+    /// Can be passed multiple times.
     #[arg(long)]
-    pub blocklist: Option<PathBuf>,
+    pub blocklist: Vec<PathBuf>,
 
     /// An optional path to save the file graph of the DSO cleaner
     /// in GraphViz format. Useful for debugging.
@@ -35,3 +59,4 @@ pub struct Args {
     /// The location of the sysroot to clean up
     pub sysroot_location: PathBuf,
 }
+

src/cleaners.rs

@@ -10,10 +10,11 @@ use crate::{
 };
 use anyhow::{Error, Result};
 use async_trait::async_trait;
+use bytesize::ByteSize;
 use dso::DsoCleaner;
 use list::ListCleaner;
 use nix::libc::EXDEV;
-use std::{collections::HashMap, io, path::Path};
+use std::{collections::HashMap, fmt, io, ops::AddAssign, path::Path};
 use tokio::{sync::mpsc, task::JoinSet};
 use walkdir::{DirEntry, WalkDir};
 
@@ -26,8 +27,22 @@ pub trait Cleaner {
     ) -> Result<()>;
 }
 
+struct FileSize(u64);
+
+impl fmt::Display for FileSize {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", ByteSize(self.0))
+    }
+}
+
+impl AddAssign for FileSize {
+    fn add_assign(&mut self, rhs: Self) {
+        self.0.add_assign(rhs.0);
+    }
+}
+
 type Cleaners = Vec<Box<dyn Cleaner + Send>>;
-type RemovalFn = Box<dyn Fn(&Path) -> io::Result<()>>;
+type RemovalFn = Box<dyn Fn(&Path) -> io::Result<FileSize>>;
 
 pub struct Runner {
     cleaners: Cleaners,
@@ -41,11 +56,11 @@ impl Runner {
         let removal_fn = Self::new_removal_fn(&args);
         let mut cleaners: Cleaners = vec![];
 
-        if let Some(wl) = args.allowlist {
+        for wl in args.allowlist {
             cleaners.push(Box::new(ListCleaner::new(list::ListType::Allow, wl)));
         }
 
-        if let Some(bl) = args.blocklist {
+        for bl in args.blocklist {
             cleaners.push(Box::new(ListCleaner::new(list::ListType::Block, bl)));
         }
 
@@ -129,6 +144,8 @@ impl Runner {
 
     async fn final_decision(removal_fn: RemovalFn, mut output_rx: mpsc::Receiver<Decision>) {
         let mut final_decisions = HashMap::new();
+        let mut total_removed_size = FileSize(0);
+
         while let Some(input_decision) = output_rx.recv().await {
             if input_decision.action == Action::Undecided {
                 continue;
@@ -147,28 +164,38 @@ impl Runner {
 
         for (file, action) in final_decisions {
             if action == Action::Remove {
-                if let Err(err) = (removal_fn)(&file) {
+                let removed_size = match (removal_fn)(&file) {
-                    log::error!("{}: {}", file.display(), err);
+                    Ok(size) => size,
-                }
+                    Err(err) => {
+                        log::error!("{}: {}", file.display(), err);
+                        FileSize(0)
+                    }
+                };
+                total_removed_size += removed_size;
             }
         }
+        log::info!("Total space removed: {}", total_removed_size);
     }
 
     fn new_removal_fn(args: &Args) -> RemovalFn {
         if let Some(dest) = args.split_to.clone() {
             if args.dry_run {
                 Box::new(move |path| {
+                    let size = Self::get_file_size(path)?;
                     log::info!(
-                        "(dry-run) would move {} to {}",
+                        "(dry-run) would move {} to {} ({})",
                         path.display(),
-                        dest.display()
+                        dest.display(),
+                        size
                     );
-                    Ok(())
+                    Ok(size)
                 })
             } else {
                 Box::new(move |path| {
-                    log::info!("moving {} to {}", path.display(), dest.display());
+                    let size = Self::get_file_size(path)?;
-                    Self::move_preserve(&path, &dest)
+                    log::info!("moving {} to {} ({})", path.display(), dest.display(), size);
+                    Self::move_preserve(&path, &dest)?;
+                    Ok(size)
                 })
             }
         } else {
@@ -179,18 +206,33 @@ impl Runner {
                     } else {
                         "regular file"
                     };
-                    log::info!("(dry-run) would remove {} {}", ty, path.display());
+                    let size = Self::get_file_size(path)?;
-                    Ok(())
+                    log::info!("(dry-run) would remove {} {} ({})", ty, path.display(), size);
+                    Ok(size)
                 })
             } else {
                 Box::new(move |path| {
-                    log::info!("removing {}", path.display());
+                    let size = Self::get_file_size(path)?;
-                    std::fs::remove_file(&path)
+                    log::info!("removing {} ({})", path.display(), size);
+                    std::fs::remove_file(&path)?;
+                    Ok(size)
                 })
             }
         }
     }
 
+    fn get_file_size(file: &Path) -> io::Result<FileSize> {
+        let lstat = nix::sys::stat::lstat(file);
+        let size = match lstat {
+            Err(err) => {
+                log::error!("failed to get metadata from: {}, {}", file.display(), err);
+                FileSize(0)
+        },
+            Ok(lstat) => FileSize(lstat.st_size as u64)
+        };
+        Ok(size)
+    }
+
     fn move_preserve(src: &Path, dest: &Path) -> io::Result<()> {
         assert!(src.is_relative());
         let abs_dest = dest.join(src);