Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 8812e91

Browse files
ByronByron
committed
remove memory mode entirely (and some complexity with it)
1 parent 657aa2c commit 8812e91

8 files changed

Lines changed: 47 additions & 222 deletions

File tree

‎git-odb/src/pack/bundle/write/mod.rs‎

Lines changed: 17 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,14 @@ use error::Error;
1111

1212
mod types;
1313
use filebuffer::FileBuffer;
14+
pub use types::Outcome;
1415
use types::PassThrough;
15-
pub use types::{MemoryMode, Outcome};
1616

1717
#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]
1818
#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
1919
pub struct Options {
2020
pub thread_limit: Option<usize>,
2121
pub iteration_mode: pack::data::iter::Mode,
22-
pub memory_mode: MemoryMode,
2322
pub index_kind: pack::index::Kind,
2423
}
2524

@@ -33,7 +32,6 @@ impl pack::Bundle {
3332
Options {
3433
thread_limit,
3534
iteration_mode,
36-
memory_mode,
3735
index_kind,
3836
}: Options,
3937
) -> Result<Outcome, Error>
@@ -47,12 +45,10 @@ impl pack::Bundle {
4745
reader: pack,
4846
progress: read_progress,
4947
};
50-
let is_in_memory_completely = memory_mode.is_in_memory();
5148
let indexing_progress = progress.add_child("create index file");
5249

5350
let data_file = match directory.as_ref() {
5451
Some(directory) => Some(NamedTempFile::new_in(directory.as_ref())?),
55-
None if is_in_memory_completely => None,
5652
None => Some(NamedTempFile::new()?),
5753
};
5854
let data_path: Option<PathBuf> = data_file.as_ref().map(|f| f.as_ref().into());
@@ -68,27 +64,14 @@ impl pack::Bundle {
6864
let directory = directory.as_ref();
6965
let mut index_file = io::BufWriter::with_capacity(4096 * 8, NamedTempFile::new_in(directory)?);
7066

71-
let outcome = if is_in_memory_completely {
72-
pack::index::File::write_data_iter_to_stream(
73-
index_kind,
74-
pack::index::write::Mode::noop_resolver,
75-
memory_mode,
76-
pack_entries_iter,
77-
thread_limit,
78-
indexing_progress,
79-
&mut index_file,
80-
)
81-
} else {
82-
pack::index::File::write_data_iter_to_stream(
83-
index_kind,
84-
move || new_pack_file_resolver(data_path),
85-
memory_mode,
86-
pack_entries_iter,
87-
thread_limit,
88-
indexing_progress,
89-
&mut index_file,
90-
)
91-
}?;
67+
let outcome = pack::index::File::write_data_iter_to_stream(
68+
index_kind,
69+
move || new_pack_file_resolver(data_path),
70+
pack_entries_iter,
71+
thread_limit,
72+
indexing_progress,
73+
&mut index_file,
74+
)?;
9275

9376
let data_file = pack.writer.expect("data file to always be set in write mode");
9477
let index_path = directory.join(format!("{}.idx", outcome.index_hash.to_sha1_hex_string()));
@@ -108,29 +91,14 @@ impl pack::Bundle {
10891
})?;
10992
outcome
11093
}
111-
None => {
112-
if is_in_memory_completely {
113-
pack::index::File::write_data_iter_to_stream(
114-
index_kind,
115-
pack::index::write::Mode::noop_resolver,
116-
memory_mode,
117-
pack_entries_iter,
118-
thread_limit,
119-
indexing_progress,
120-
io::sink(),
121-
)
122-
} else {
123-
pack::index::File::write_data_iter_to_stream(
124-
index_kind,
125-
move || new_pack_file_resolver(data_path),
126-
memory_mode,
127-
pack_entries_iter,
128-
thread_limit,
129-
indexing_progress,
130-
io::sink(),
131-
)
132-
}?
133-
}
94+
None => pack::index::File::write_data_iter_to_stream(
95+
index_kind,
96+
move || new_pack_file_resolver(data_path),
97+
pack_entries_iter,
98+
thread_limit,
99+
indexing_progress,
100+
io::sink(),
101+
)?,
134102
};
135103

136104
Ok(Outcome {

‎git-odb/src/pack/bundle/write/types.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ pub struct Outcome {
99
pub pack_kind: pack::data::Kind,
1010
}
1111

12-
pub type MemoryMode = pack::index::write::Mode;
13-
1412
pub(crate) struct PassThrough<R> {
1513
pub reader: R,
1614
pub writer: Option<NamedTempFile>,

‎git-odb/src/pack/index/write/consume.rs‎

Lines changed: 27 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,60 @@
11
use crate::{
22
hash, loose, pack,
3-
pack::index::write::types::{Cache, EntrySlice, Mode, ObjectKind},
3+
pack::index::write::types::{EntrySlice, ObjectKind},
44
pack::index::write::Error,
55
zlib,
66
};
77
use git_features::progress::Progress;
88
use git_object::{owned, HashKind};
9-
use std::{cell::RefCell, io};
9+
use std::{cell::RefCell, collections::BTreeMap, io};
1010

1111
pub(crate) fn apply_deltas<F, P>(
12-
mut nodes: Vec<pack::tree::Node<pack::index::write::types::TreeEntry>>,
12+
nodes: Vec<pack::tree::Node<pack::index::write::types::TreeEntry>>,
1313
(bytes_buf, progress): &mut (Vec<u8>, P),
14-
mode: Mode,
1514
resolve: F,
1615
hash_kind: HashKind,
1716
) -> Result<usize, Error>
1817
where
1918
F: for<'r> Fn(EntrySlice, &'r mut Vec<u8>) -> Option<()> + Send + Sync,
2019
P: Progress,
2120
{
21+
let mut decompressed_bytes_by_pack_offset = BTreeMap::new();
2222
let bytes_buf = RefCell::new(bytes_buf);
2323
let mut num_objects = 0;
24-
let decompress_from_cache = |cache: Cache, pack_offset: u64, entry_size: usize| -> Result<Vec<u8>, Error> {
25-
Ok(match cache {
26-
Cache::Unset => {
27-
let mut bytes_buf = bytes_buf.borrow_mut();
28-
bytes_buf.resize(entry_size, 0);
29-
match mode {
30-
Mode::ResolveDeltas | Mode::ResolveBasesAndDeltas => {
31-
resolve(pack_offset..pack_offset + entry_size as u64, &mut bytes_buf)
32-
.ok_or_else(|| Error::ConsumeResolveFailed(pack_offset))?;
33-
let entry = pack::data::Entry::from_bytes(&bytes_buf, pack_offset);
34-
decompress_all_at_once(
35-
&bytes_buf[entry.header_size() as usize..],
36-
entry.decompressed_size as usize,
37-
)?
38-
}
39-
Mode::InMemory => unreachable!("BUG: If there is no cache, we always need a resolver"),
40-
}
41-
}
42-
Cache::Decompressed(bytes) => bytes,
43-
})
24+
let decompress_from_resolver = |pack_offset: u64, entry_size: usize| -> Result<Vec<u8>, Error> {
25+
let mut bytes_buf = bytes_buf.borrow_mut();
26+
bytes_buf.resize(entry_size, 0);
27+
resolve(pack_offset..pack_offset + entry_size as u64, &mut bytes_buf)
28+
.ok_or_else(|| Error::ConsumeResolveFailed(pack_offset))?;
29+
let entry = pack::data::Entry::from_bytes(&bytes_buf, pack_offset);
30+
decompress_all_at_once(
31+
&bytes_buf[entry.header_size() as usize..],
32+
entry.decompressed_size as usize,
33+
)
4434
};
4535

4636
// Traverse the tree breadth first and loose the data produced for the base as it won't be needed anymore.
4737
progress.init(None, Some("objects"));
4838

4939
// each node is a base, and its children always start out as deltas which become a base after applying them.
5040
// These will be pushed onto our stack until all are processed
51-
while let Some(mut base) = nodes.pop() {
52-
let base_bytes = decompress_from_cache(
53-
std::mem::take(&mut base.data.cache),
54-
base.data.pack_offset,
55-
base.data.entry_len,
56-
)?;
41+
let root_level = 0;
42+
let mut nodes: Vec<_> = nodes.into_iter().map(|n| (root_level, n)).collect();
43+
while let Some((level, mut base)) = nodes.pop() {
44+
let base_bytes = if level == root_level {
45+
decompress_from_resolver(base.data.pack_offset, base.data.entry_len)?
46+
} else {
47+
decompressed_bytes_by_pack_offset
48+
.remove(&base.data.pack_offset)
49+
.expect("we store the resolved delta buffer when done")
50+
};
5751
let base_kind = base.data.kind.to_kind().expect("base object as source of iteration");
5852
let id = compute_hash(base_kind, &base_bytes, hash_kind);
5953
num_objects += 1;
6054

6155
base.data.id = id;
6256
for mut child in base.store_changes_then_into_child_iter() {
63-
let delta_bytes = decompress_from_cache(
64-
std::mem::take(&mut child.data.cache),
65-
child.data.pack_offset,
66-
child.data.entry_len,
67-
)?;
57+
let delta_bytes = decompress_from_resolver(child.data.pack_offset, child.data.entry_len)?;
6858
let (base_size, consumed) = pack::data::decode::delta_header_size_ofs(&delta_bytes);
6959
let mut header_ofs = consumed;
7060
assert_eq!(
@@ -79,9 +69,9 @@ where
7969
fully_resolved_delta_bytes.resize(result_size as usize, 0);
8070
pack::data::decode::apply_delta(&base_bytes, &mut fully_resolved_delta_bytes, &delta_bytes[header_ofs..]);
8171

82-
child.data.cache = Cache::Decompressed(fully_resolved_delta_bytes.to_owned());
72+
decompressed_bytes_by_pack_offset.insert(child.data.pack_offset, fully_resolved_delta_bytes.to_owned());
8373
child.data.kind = ObjectKind::Base(base_kind);
84-
nodes.push(child);
74+
nodes.push((level + 1, child));
8575
}
8676
}
8777

‎git-odb/src/pack/index/write/mod.rs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ mod error;
88
pub use error::Error;
99

1010
mod types;
11-
pub use types::{EntrySlice, Mode, Outcome};
11+
pub use types::{EntrySlice, Outcome};
1212
use types::{ObjectKind, Reducer, TreeEntry};
1313

1414
mod consume;
@@ -22,7 +22,6 @@ impl pack::index::File {
2222
pub fn write_data_iter_to_stream<F, F2, P>(
2323
kind: pack::index::Kind,
2424
make_resolver: F,
25-
mode: Mode,
2625
entries: impl Iterator<Item = Result<pack::data::iter::Entry, pack::data::iter::Error>>,
2726
thread_limit: Option<usize>,
2827
mut root_progress: P,
@@ -81,7 +80,6 @@ impl pack::index::File {
8180
entry_len,
8281
kind: ObjectKind::Base(header.to_kind().expect("a base object")),
8382
crc32,
84-
cache: mode.base_cache(decompressed),
8583
},
8684
)?;
8785
}
@@ -98,7 +96,6 @@ impl pack::index::File {
9896
entry_len,
9997
kind: ObjectKind::OfsDelta,
10098
crc32,
101-
cache: mode.delta_cache(decompressed),
10299
},
103100
)?;
104101
}
@@ -131,7 +128,7 @@ impl pack::index::File {
131128
reduce_progress.lock().add_child(format!("thread {}", thread_index)),
132129
)
133130
},
134-
|root_nodes, state| apply_deltas(root_nodes, state, mode, &resolver, kind.hash()),
131+
|root_nodes, state| apply_deltas(root_nodes, state, &resolver, kind.hash()),
135132
Reducer::new(num_objects, &reduce_progress),
136133
)?;
137134
let mut items = tree.into_items();

‎git-odb/src/pack/index/write/types.rs‎

Lines changed: 0 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
use crate::pack;
22
use git_features::{parallel, progress::Progress};
33
use git_object::owned;
4-
use std::io;
54

65
#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)]
76
#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
@@ -12,17 +11,6 @@ pub struct Outcome {
1211
pub num_objects: u32,
1312
}
1413

15-
pub(crate) enum Cache {
16-
Unset,
17-
Decompressed(Vec<u8>),
18-
}
19-
20-
impl Default for Cache {
21-
fn default() -> Self {
22-
Cache::Unset
23-
}
24-
}
25-
2614
#[derive(Clone)]
2715
pub(crate) enum ObjectKind {
2816
Base(git_object::Kind),
@@ -44,7 +32,6 @@ pub(crate) struct TreeEntry {
4432
pub entry_len: usize,
4533
pub kind: ObjectKind,
4634
pub crc32: u32,
47-
pub cache: Cache,
4835
}
4936

5037
impl Default for TreeEntry {
@@ -55,55 +42,12 @@ impl Default for TreeEntry {
5542
entry_len: 0,
5643
kind: ObjectKind::OfsDelta,
5744
crc32: 0,
58-
cache: Cache::Unset,
5945
}
6046
}
6147
}
6248

6349
pub type EntrySlice = std::ops::Range<u64>;
6450

65-
#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
66-
#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
67-
pub enum Mode {
68-
/// Base + deltas in memory, decompressed
69-
InMemory,
70-
/// Bases in memory, decompressed
71-
ResolveDeltas,
72-
ResolveBasesAndDeltas,
73-
}
74-
75-
impl Mode {
76-
pub(crate) fn base_cache(&self, decompressed: Vec<u8>) -> Cache {
77-
match self {
78-
Mode::InMemory | Mode::ResolveDeltas => Cache::Decompressed(decompressed),
79-
Mode::ResolveBasesAndDeltas => Cache::Unset,
80-
}
81-
}
82-
pub(crate) fn delta_cache(&self, decompressed: Vec<u8>) -> Cache {
83-
match self {
84-
Mode::InMemory => Cache::Decompressed(decompressed),
85-
Mode::ResolveDeltas | Mode::ResolveBasesAndDeltas => Cache::Unset,
86-
}
87-
}
88-
pub(crate) fn is_in_memory(&self) -> bool {
89-
match self {
90-
Mode::InMemory => true,
91-
Mode::ResolveDeltas | Mode::ResolveBasesAndDeltas => false,
92-
}
93-
}
94-
}
95-
96-
pub type ResolverFn = fn(EntrySlice, &mut Vec<u8>) -> Option<()>;
97-
98-
impl Mode {
99-
pub fn noop_resolver() -> io::Result<ResolverFn> {
100-
fn noop(_: EntrySlice, _: &mut Vec<u8>) -> Option<()> {
101-
None
102-
};
103-
Ok(noop)
104-
}
105-
}
106-
10751
pub(crate) struct Reducer<'a, P> {
10852
item_count: usize,
10953
progress: &'a parking_lot::Mutex<P>,

‎git-odb/tests/pack/index.rs‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -85,20 +85,7 @@ mod method {
8585
.map(|slice| out.copy_from_slice(slice))
8686
}
8787
};
88-
assert_index_write(
89-
mode,
90-
index_path,
91-
data_path,
92-
resolve,
93-
pack::index::write::Mode::ResolveBasesAndDeltas,
94-
)?;
95-
assert_index_write(
96-
mode,
97-
index_path,
98-
data_path,
99-
pack::index::write::Mode::noop_resolver()?,
100-
pack::index::write::Mode::InMemory,
101-
)?;
88+
assert_index_write(mode, index_path, data_path, resolve)?;
10289
}
10390
}
10491
Ok(())
@@ -109,7 +96,6 @@ mod method {
10996
index_path: &&str,
11097
data_path: &&str,
11198
resolve: F,
112-
memory_mode: pack::index::write::Mode,
11399
) -> Result<(), Box<dyn std::error::Error>>
114100
where
115101
F: Fn(pack::index::write::EntrySlice, &mut Vec<u8>) -> Option<()> + Send + Sync,
@@ -123,7 +109,6 @@ mod method {
123109
let outcome = pack::index::File::write_data_iter_to_stream(
124110
desired_kind,
125111
|| Ok(resolve),
126-
memory_mode,
127112
pack_iter,
128113
None,
129114
progress::Discard,

0 commit comments

Comments
 (0)