Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 4aea9b0

Browse files
ByronByron
committed
feat: add thediff::resource_cache() low-level utility for rapid in-memory diffing of combinations of resources.
We also add the `object::tree::diff::Platform::for_each_to_obtain_tree_with_cache()` to pass a resource-cache for re-use between multiple invocation for significant savings.
1 parent feca5d0 commit 4aea9b0

12 files changed

Lines changed: 417 additions & 21 deletions

File tree

‎gix/src/config/cache/access.rs‎

Lines changed: 91 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,97 @@ impl Cache {
3939
.copied()
4040
}
4141

42+
#[cfg(feature = "blob-diff")]
43+
pub(crate) fn diff_drivers(&self) -> Result<Vec<gix_diff::blob::Driver>, config::diff::drivers::Error> {
44+
use crate::config::cache::util::ApplyLeniencyDefault;
45+
let mut out = Vec::<gix_diff::blob::Driver>::new();
46+
for section in self
47+
.resolved
48+
.sections_by_name("diff")
49+
.into_iter()
50+
.flatten()
51+
.filter(|s| (self.filter_config_section)(s.meta()))
52+
{
53+
let Some(name) = section.header().subsection_name().filter(|n| !n.is_empty()) else {
54+
continue;
55+
};
56+
57+
let driver = match out.iter_mut().find(|d| d.name == name) {
58+
Some(existing) => existing,
59+
None => {
60+
out.push(gix_diff::blob::Driver {
61+
name: name.into(),
62+
..Default::default()
63+
});
64+
out.last_mut().expect("just pushed")
65+
}
66+
};
67+
68+
if let Some(binary) = section.value_implicit("binary") {
69+
driver.is_binary = config::tree::Diff::DRIVER_BINARY
70+
.try_into_binary(binary)
71+
.with_leniency(self.lenient_config)
72+
.map_err(|err| config::diff::drivers::Error {
73+
name: driver.name.clone(),
74+
attribute: "binary",
75+
source: Box::new(err),
76+
})?;
77+
}
78+
if let Some(command) = section.value(config::tree::Diff::DRIVER_COMMAND.name) {
79+
driver.command = command.into_owned().into();
80+
}
81+
if let Some(textconv) = section.value(config::tree::Diff::DRIVER_TEXTCONV.name) {
82+
driver.binary_to_text_command = textconv.into_owned().into();
83+
}
84+
if let Some(algorithm) = section.value("algorithm") {
85+
driver.algorithm = config::tree::Diff::DRIVER_ALGORITHM
86+
.try_into_algorithm(algorithm)
87+
.or_else(|err| match err {
88+
config::diff::algorithm::Error::Unimplemented { .. } if self.lenient_config => {
89+
Ok(gix_diff::blob::Algorithm::Histogram)
90+
}
91+
err => Err(err),
92+
})
93+
.with_lenient_default(self.lenient_config)
94+
.map_err(|err| config::diff::drivers::Error {
95+
name: driver.name.clone(),
96+
attribute: "algorithm",
97+
source: Box::new(err),
98+
})?
99+
.into();
100+
}
101+
}
102+
Ok(out)
103+
}
104+
105+
#[cfg(feature = "blob-diff")]
106+
pub(crate) fn diff_pipeline_options(
107+
&self,
108+
) -> Result<gix_diff::blob::pipeline::Options, config::diff::pipeline_options::Error> {
109+
Ok(gix_diff::blob::pipeline::Options {
110+
large_file_threshold_bytes: self.big_file_threshold()?,
111+
fs: self.fs_capabilities()?,
112+
})
113+
}
114+
115+
#[cfg(feature = "blob-diff")]
116+
pub(crate) fn diff_renames(&self) -> Result<Option<crate::diff::Rewrites>, crate::diff::new_rewrites::Error> {
117+
self.diff_renames
118+
.get_or_try_init(|| crate::diff::new_rewrites(&self.resolved, self.lenient_config))
119+
.copied()
120+
}
121+
122+
#[cfg(feature = "blob-diff")]
123+
pub(crate) fn big_file_threshold(&self) -> Result<u64, config::unsigned_integer::Error> {
124+
Ok(self
125+
.resolved
126+
.integer_by_key("core.bigFileThreshold")
127+
.map(|number| Core::BIG_FILE_THRESHOLD.try_into_u64(number))
128+
.transpose()
129+
.with_leniency(self.lenient_config)?
130+
.unwrap_or(512 * 1024 * 1024))
131+
}
132+
42133
/// Returns a user agent for use with servers.
43134
#[cfg(any(feature = "async-network-client", feature = "blocking-network-client"))]
44135
pub(crate) fn user_agent_tuple(&self) -> (&'static str, Option<Cow<'static, str>>) {
@@ -92,13 +183,6 @@ impl Cache {
92183
})
93184
}
94185

95-
#[cfg(feature = "blob-diff")]
96-
pub(crate) fn diff_renames(&self) -> Result<Option<crate::diff::Rewrites>, crate::diff::new_rewrites::Error> {
97-
self.diff_renames
98-
.get_or_try_init(|| crate::diff::new_rewrites(&self.resolved, self.lenient_config))
99-
.copied()
100-
}
101-
102186
/// Returns (file-timeout, pack-refs timeout)
103187
pub(crate) fn lock_timeout(
104188
&self,

‎gix/src/config/mod.rs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,36 @@ pub mod diff {
122122
Unimplemented { name: BString },
123123
}
124124
}
125+
126+
///
127+
pub mod pipeline_options {
128+
/// The error produced when obtaining options needed to fill in [gix_diff::blob::pipeline::Options].
129+
#[derive(Debug, thiserror::Error)]
130+
#[allow(missing_docs)]
131+
pub enum Error {
132+
#[error(transparent)]
133+
FilesystemCapabilities(#[from] crate::config::boolean::Error),
134+
#[error(transparent)]
135+
BigFileThreshold(#[from] crate::config::unsigned_integer::Error),
136+
}
137+
}
138+
139+
///
140+
pub mod drivers {
141+
use crate::bstr::BString;
142+
143+
/// The error produced when obtaining a list of [Drivers](gix_diff::blob::Driver).
144+
#[derive(Debug, thiserror::Error)]
145+
#[error("Failed to parse value of 'diff.{name}.{attribute}'")]
146+
pub struct Error {
147+
/// The name fo the driver.
148+
pub name: BString,
149+
/// The name of the attribute we tried to parse.
150+
pub attribute: &'static str,
151+
/// The actual error that occurred.
152+
pub source: Box<dyn std::error::Error + Send + Sync + 'static>,
153+
}
154+
}
125155
}
126156

127157
///

‎gix/src/config/tree/sections/core.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ impl Core {
88
pub const ABBREV: Abbrev = Abbrev::new_with_validate("abbrev", &config::Tree::CORE, validate::Abbrev);
99
/// The `core.bare` key.
1010
pub const BARE: keys::Boolean = keys::Boolean::new_boolean("bare", &config::Tree::CORE);
11+
/// The `core.bigFileThreshold` key.
12+
pub const BIG_FILE_THRESHOLD: keys::UnsignedInteger =
13+
keys::UnsignedInteger::new_unsigned_integer("bigFileThreshold", &config::Tree::CORE);
1114
/// The `core.checkStat` key.
1215
pub const CHECK_STAT: CheckStat =
1316
CheckStat::new_with_validate("checkStat", &config::Tree::CORE, validate::CheckStat);
@@ -95,6 +98,7 @@ impl Section for Core {
9598
&[
9699
&Self::ABBREV,
97100
&Self::BARE,
101+
&Self::BIG_FILE_THRESHOLD,
98102
&Self::CHECK_STAT,
99103
&Self::DELTA_BASE_CACHE_LIMIT,
100104
&Self::DISAMBIGUATE,

‎gix/src/config/tree/sections/diff.rs‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::config::tree::SubSectionRequirement;
12
use crate::{
23
config,
34
config::tree::{keys, Diff, Key, Section},
@@ -17,6 +18,20 @@ impl Diff {
1718
);
1819
/// The `diff.renames` key.
1920
pub const RENAMES: Renames = Renames::new_renames("renames", &config::Tree::DIFF);
21+
22+
/// The `diff.<driver>.command` key.
23+
pub const DRIVER_COMMAND: keys::String = keys::String::new_string("command", &config::Tree::DIFF)
24+
.with_subsection_requirement(Some(SubSectionRequirement::Parameter("driver")));
25+
/// The `diff.<driver>.textconv` key.
26+
pub const DRIVER_TEXTCONV: keys::String = keys::String::new_string("textconv", &config::Tree::DIFF)
27+
.with_subsection_requirement(Some(SubSectionRequirement::Parameter("driver")));
28+
/// The `diff.<driver>.algorithm` key.
29+
pub const DRIVER_ALGORITHM: Algorithm =
30+
Algorithm::new_with_validate("algorithm", &config::Tree::DIFF, validate::Algorithm)
31+
.with_subsection_requirement(Some(SubSectionRequirement::Parameter("driver")));
32+
/// The `diff.<driver>.binary` key.
33+
pub const DRIVER_BINARY: Binary = Binary::new_with_validate("binary", &config::Tree::DIFF, validate::Binary)
34+
.with_subsection_requirement(Some(SubSectionRequirement::Parameter("driver")));
2035
}
2136

2237
impl Section for Diff {
@@ -25,7 +40,15 @@ impl Section for Diff {
2540
}
2641

2742
fn keys(&self) -> &[&dyn Key] {
28-
&[&Self::ALGORITHM, &Self::RENAME_LIMIT, &Self::RENAMES]
43+
&[
44+
&Self::ALGORITHM,
45+
&Self::RENAME_LIMIT,
46+
&Self::RENAMES,
47+
&Self::DRIVER_COMMAND,
48+
&Self::DRIVER_TEXTCONV,
49+
&Self::DRIVER_ALGORITHM,
50+
&Self::DRIVER_BINARY,
51+
]
2952
}
3053
}
3154

@@ -35,6 +58,9 @@ pub type Algorithm = keys::Any<validate::Algorithm>;
3558
/// The `diff.renames` key.
3659
pub type Renames = keys::Any<validate::Renames>;
3760

61+
/// The `diff.<driver>.binary` key.
62+
pub type Binary = keys::Any<validate::Binary>;
63+
3864
mod algorithm {
3965
use std::borrow::Cow;
4066

@@ -67,6 +93,38 @@ mod algorithm {
6793
}
6894
}
6995

96+
mod binary {
97+
use crate::config::tree::diff::Binary;
98+
99+
impl Binary {
100+
/// Convert `value` into a tri-state boolean that can take the special value `auto`, resulting in `None`, or is a boolean.
101+
/// If `None` is given, it's treated as implicit boolean `true`, as this method is made to be used
102+
/// with [`gix_config::file::section::Body::value_implicit()`].
103+
pub fn try_into_binary(
104+
&'static self,
105+
value: Option<std::borrow::Cow<'_, crate::bstr::BStr>>,
106+
) -> Result<Option<bool>, crate::config::key::GenericErrorWithValue> {
107+
Ok(match value {
108+
None => Some(true),
109+
Some(value) => {
110+
if value.as_ref() == "auto" {
111+
None
112+
} else {
113+
Some(
114+
gix_config::Boolean::try_from(value.as_ref())
115+
.map(|b| b.0)
116+
.map_err(|err| {
117+
crate::config::key::GenericErrorWithValue::from_value(self, value.into_owned())
118+
.with_source(err)
119+
})?,
120+
)
121+
}
122+
}
123+
})
124+
}
125+
}
126+
}
127+
70128
mod renames {
71129
use crate::{
72130
bstr::ByteSlice,
@@ -125,4 +183,12 @@ mod validate {
125183
Ok(())
126184
}
127185
}
186+
187+
pub struct Binary;
188+
impl keys::Validate for Binary {
189+
fn validate(&self, value: &BStr) -> Result<(), Box<dyn std::error::Error + Send + Sync + 'static>> {
190+
Diff::DRIVER_BINARY.try_into_binary(Some(value.into()))?;
191+
Ok(())
192+
}
193+
}
128194
}

‎gix/src/diff.rs‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ mod utils {
2222
use crate::config::cache::util::ApplyLeniency;
2323
use crate::config::tree::Diff;
2424
use crate::diff::rename::Tracking;
25+
use crate::Repository;
2526
use gix_diff::rewrites::Copies;
2627
use gix_diff::Rewrites;
2728

@@ -38,6 +39,27 @@ mod utils {
3839
}
3940
}
4041

42+
///
43+
pub mod resource_cache {
44+
/// The error returned by [`resource_cache()`](super::resource_cache()).
45+
#[derive(Debug, thiserror::Error)]
46+
#[allow(missing_docs)]
47+
pub enum Error {
48+
#[error(transparent)]
49+
DiffAlgorithm(#[from] crate::config::diff::algorithm::Error),
50+
#[error(transparent)]
51+
WorktreeFilterOptions(#[from] crate::filter::pipeline::options::Error),
52+
#[error(transparent)]
53+
DiffDrivers(#[from] crate::config::diff::drivers::Error),
54+
#[error(transparent)]
55+
DiffPipelineOptions(#[from] crate::config::diff::pipeline_options::Error),
56+
#[error(transparent)]
57+
CommandContext(#[from] crate::config::command_context::Error),
58+
#[error(transparent)]
59+
AttributeStack(#[from] crate::config::attribute_stack::Error),
60+
}
61+
}
62+
4163
/// Create an instance by reading all relevant information from the `config`uration, while being `lenient` or not.
4264
/// Returns `Ok(None)` if nothing is configured.
4365
///
@@ -75,6 +97,46 @@ mod utils {
7597
}
7698
.into())
7799
}
100+
101+
/// Return a low-level utility to efficiently prepare a the blob-level diff operation between two resources,
102+
/// and cache these diffable versions so that matrix-like MxN diffs are efficient.
103+
///
104+
/// `repo` is used to obtain the needed configuration values, and `index` is used to potentially read `.gitattributes`
105+
/// files from which may affect the diff operation.
106+
/// `mode` determines how the diffable files will look like, and also how fast, in average, these conversions are.
107+
/// `roots` provide information about where to get diffable data from, so source and destination can either be sourced from
108+
/// a worktree, or from the object database, or both.
109+
pub fn resource_cache(
110+
repo: &Repository,
111+
index: &gix_index::State,
112+
mode: gix_diff::blob::pipeline::Mode,
113+
roots: gix_diff::blob::pipeline::WorktreeRoots,
114+
) -> Result<gix_diff::blob::Platform, resource_cache::Error> {
115+
let diff_algo = repo.config.diff_algorithm()?;
116+
let diff_cache = gix_diff::blob::Platform::new(
117+
gix_diff::blob::platform::Options {
118+
algorithm: Some(diff_algo),
119+
skip_internal_diff_if_external_is_configured: false,
120+
},
121+
gix_diff::blob::Pipeline::new(
122+
roots,
123+
gix_filter::Pipeline::new(repo.command_context()?, crate::filter::Pipeline::options(repo)?),
124+
repo.config.diff_drivers()?,
125+
repo.config.diff_pipeline_options()?,
126+
),
127+
mode,
128+
repo.attributes_only(
129+
// TODO(perf): this could benefit from not having to build an intermediate index,
130+
// and traverse the a tree directly.
131+
index,
132+
// This is an optimization, as we avoid reading files from the working tree, which also
133+
// might not match the index at all depending on what the user passed.
134+
gix_worktree::stack::state::attributes::Source::IdMapping,
135+
)?
136+
.inner,
137+
);
138+
Ok(diff_cache)
139+
}
78140
}
79141
#[cfg(feature = "blob-diff")]
80-
pub use utils::new_rewrites;
142+
pub use utils::{new_rewrites, resource_cache};

0 commit comments

Comments
 (0)