feat: gix odb stats to calculate statistics on the object database.

Byron · Byron · commit 8c9c243fc574 · 2022-12-18T21:10:49.000+01:00
This includes the amount of objects along with a listing of where they are
located.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/gitoxide-core/Cargo.toml b/gitoxide-core/Cargo.toml
@@ -31,7 +31,7 @@ async-client = ["git-repository/async-network-client-async-std", "git-transport-
 
 #! ### Other
 ## Data structures implement `serde::Serialize` and `serde::Deserialize`.
-serde1 = ["git-commitgraph/serde1", "git-repository/serde1", "serde_json", "serde"]
+serde1 = ["git-commitgraph/serde1", "git-repository/serde1", "serde_json", "serde", "bytesize/serde"]
 
 
 [dependencies]
diff --git a/gitoxide-core/src/repository/odb.rs b/gitoxide-core/src/repository/odb.rs
@@ -5,21 +5,6 @@ use git_repository as git;
 
 use crate::OutputFormat;
 
-mod info {
-    use std::path::PathBuf;
-
-    use git_repository::odb::store;
-
-    #[cfg_attr(feature = "serde1", derive(serde::Serialize))]
-    pub struct Statistics {
-        pub path: PathBuf,
-        pub object_hash: String,
-        pub use_multi_pack_index: bool,
-        pub structure: Vec<store::structure::Record>,
-        pub metrics: store::Metrics,
-    }
-}
-
 #[cfg_attr(not(feature = "serde1"), allow(unused_variables))]
 pub fn info(
     repo: git::Repository,
@@ -31,8 +16,17 @@ pub fn info(
         writeln!(err, "Only JSON is implemented - using that instead")?;
     }
 
+    #[cfg_attr(feature = "serde1", derive(serde::Serialize))]
+    pub struct Statistics {
+        pub path: std::path::PathBuf,
+        pub object_hash: String,
+        pub use_multi_pack_index: bool,
+        pub structure: Vec<git::odb::store::structure::Record>,
+        pub metrics: git::odb::store::Metrics,
+    }
+
     let store = repo.objects.store_ref();
-    let stats = info::Statistics {
+    let stats = Statistics {
         path: store.path().into(),
         object_hash: store.object_hash().to_string(),
         use_multi_pack_index: store.use_multi_pack_index(),
@@ -48,6 +42,169 @@ pub fn info(
     Ok(())
 }
 
+pub mod statistics {
+    use crate::OutputFormat;
+
+    pub const PROGRESS_RANGE: std::ops::RangeInclusive<u8> = 0..=3;
+
+    #[derive(Debug, Copy, Clone)]
+    pub struct Options {
+        pub format: OutputFormat,
+        pub thread_limit: Option<usize>,
+    }
+}
+
+#[cfg_attr(not(feature = "serde1"), allow(unused_variables))]
+pub fn statistics(
+    repo: git::Repository,
+    mut progress: impl git::Progress,
+    out: impl io::Write,
+    mut err: impl io::Write,
+    statistics::Options { format, thread_limit }: statistics::Options,
+) -> anyhow::Result<()> {
+    use bytesize::ByteSize;
+    use git::odb::find;
+    use git::odb::HeaderExt;
+
+    if format == OutputFormat::Human {
+        writeln!(err, "Only JSON is implemented - using that instead")?;
+    }
+
+    progress.init(None, git::progress::count("objects"));
+    progress.set_name("counting");
+    let counter = progress.counter();
+    let start = std::time::Instant::now();
+
+    #[cfg_attr(feature = "serde1", derive(serde::Serialize))]
+    #[derive(Default)]
+    struct Statistics {
+        total_objects: usize,
+        loose_objects: usize,
+        packed_objects: usize,
+        packed_delta_objects: usize,
+        total_delta_chain_length: u64,
+        trees: usize,
+        trees_size: ByteSize,
+        tags: usize,
+        tags_size: ByteSize,
+        commits: usize,
+        commits_size: ByteSize,
+        blobs: usize,
+        blobs_size: ByteSize,
+    }
+
+    impl Statistics {
+        fn count(&mut self, kind: git::object::Kind, size: u64) {
+            use git::object::Kind::*;
+            match kind {
+                Commit => {
+                    self.commits += 1;
+                    self.commits_size += size;
+                }
+                Tree => {
+                    self.trees += 1;
+                    self.trees_size += size;
+                }
+                Tag => {
+                    self.tags += 1;
+                    self.tags_size += size;
+                }
+                Blob => {
+                    self.blobs += 1;
+                    self.blobs_size += size;
+                }
+            }
+        }
+        fn consume(&mut self, item: git::odb::find::Header) {
+            match item {
+                find::Header::Loose { size, kind } => {
+                    self.loose_objects += 1;
+                    self.count(kind, size)
+                }
+                find::Header::Packed(packed) => {
+                    self.packed_objects += 1;
+                    self.packed_delta_objects += usize::from(packed.num_deltas > 0);
+                    self.total_delta_chain_length += packed.num_deltas as u64;
+                    self.count(packed.kind, packed.object_size);
+                }
+            }
+        }
+    }
+
+    #[derive(Default)]
+    struct Reduce {
+        stats: Statistics,
+    }
+
+    impl git::parallel::Reduce for Reduce {
+        type Input = Result<Vec<git::odb::find::Header>, anyhow::Error>;
+        type FeedProduce = ();
+        type Output = Statistics;
+        type Error = anyhow::Error;
+
+        fn feed(&mut self, items: Self::Input) -> Result<Self::FeedProduce, Self::Error> {
+            for item in items? {
+                self.stats.consume(item);
+            }
+            Ok(())
+        }
+
+        fn finalize(mut self) -> Result<Self::Output, Self::Error> {
+            self.stats.total_objects = self.stats.loose_objects + self.stats.packed_objects;
+            Ok(self.stats)
+        }
+    }
+
+    let cancelled = || anyhow::anyhow!("Cancelled by user");
+    let object_ids = repo.objects.store_ref().iter()?.filter_map(Result::ok);
+    let chunk_size = 1_000;
+    let stats = if git::parallel::num_threads(thread_limit) > 1 {
+        git::parallel::in_parallel(
+            git::interrupt::Iter::new(
+                git::features::iter::Chunks {
+                    inner: object_ids,
+                    size: chunk_size,
+                },
+                cancelled,
+            ),
+            thread_limit,
+            move |_| (repo.objects.clone().into_inner(), counter.clone()),
+            |ids, (handle, counter)| {
+                let ids = ids?;
+                if let Some(counter) = counter {
+                    counter.fetch_add(ids.len(), std::sync::atomic::Ordering::SeqCst);
+                }
+                let out = ids
+                    .into_iter()
+                    .map(|id| handle.header(id))
+                    .collect::<Result<Vec<_>, _>>()?;
+                Ok(out)
+            },
+            Reduce::default(),
+        )?
+    } else {
+        let mut stats = Statistics::default();
+
+        for (count, id) in object_ids.enumerate() {
+            if count % chunk_size == 0 && git::interrupt::is_triggered() {
+                return Err(cancelled());
+            }
+            stats.consume(repo.objects.header(id)?);
+            progress.inc();
+        }
+        stats
+    };
+
+    progress.show_throughput(start);
+
+    #[cfg(feature = "serde1")]
+    {
+        serde_json::to_writer_pretty(out, &stats)?;
+    }
+
+    Ok(())
+}
+
 pub fn entries(repo: git::Repository, format: OutputFormat, mut out: impl io::Write) -> anyhow::Result<()> {
     if format != OutputFormat::Human {
         bail!("Only human output format is supported at the moment");
diff --git a/src/plumbing/main.rs b/src/plumbing/main.rs
@@ -775,6 +775,22 @@ pub fn main() -> Result<()> {
             ),
         },
         Subcommands::Odb(cmd) => match cmd {
+            odb::Subcommands::Stats => prepare_and_run(
+                "odb-stats",
+                auto_verbose,
+                progress,
+                progress_keep_open,
+                core::repository::odb::statistics::PROGRESS_RANGE,
+                move |progress, out, err| {
+                    core::repository::odb::statistics(
+                        repository(Mode::Strict)?,
+                        progress,
+                        out,
+                        err,
+                        core::repository::odb::statistics::Options { format, thread_limit },
+                    )
+                },
+            ),
             odb::Subcommands::Entries => prepare_and_run(
                 "odb-entries",
                 verbose,
diff --git a/src/plumbing/options/mod.rs b/src/plumbing/options/mod.rs
@@ -239,6 +239,9 @@ pub mod odb {
         Entries,
         /// Provide general information about the object database.
         Info,
+        /// Count and obtain information on all, possibly duplicate, objects in the database.
+        #[clap(visible_alias = "statistics")]
+        Stats,
     }
 }
 

Original file line number	Diff line number	Diff line change
`@@ -239,6 +239,9 @@ pub mod odb {`
`239`	`239`	`Entries,`
`240`	`240`	`/// Provide general information about the object database.`
`241`	`241`	`Info,`
	`242`	`+ /// Count and obtain information on all, possibly duplicate, objects in the database.`
	`243`	`+ #[clap(visible_alias = "statistics")]`
	`244`	`+ Stats,`
`242`	`245`	`}`
`243`	`246`	`}`
`244`	`247`