Thanks to visit codestin.com
Credit goes to Github.com

Skip to content

Commit 7c75631

Browse files
gitstergitster
committed
Merge branch 'ps/repack-avoid-noop-midx-rewrite' into jch
Even when there is no changes in the packfile and no need to recompute bitmaps, "git repack" recomputed and updated the MIDX file, which has been corrected. * ps/repack-avoid-noop-midx-rewrite: midx-write: skip rewriting MIDX with `--stdin-packs` unless needed midx-write: extract function to test whether MIDX needs updating midx: fix `BUG()` when getting preferred pack without a reverse index
2 parents edc623c + 6ce9d55 commit 7c75631

File tree

5 files changed

+195
-22
lines changed

5 files changed

+195
-22
lines changed

‎midx-write.c‎

Lines changed: 93 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,6 +1014,65 @@ static void clear_midx_files(struct odb_source *source,
10141014
strbuf_release(&buf);
10151015
}
10161016

1017+
static bool midx_needs_update(struct multi_pack_index *midx, struct write_midx_context *ctx)
1018+
{
1019+
struct strset packs = STRSET_INIT;
1020+
struct strbuf buf = STRBUF_INIT;
1021+
bool needed = true;
1022+
1023+
/*
1024+
* Ignore incremental updates for now. The assumption is that any
1025+
* incremental update would be either empty (in which case we will bail
1026+
* out later) or it would actually cover at least one new pack.
1027+
*/
1028+
if (ctx->incremental)
1029+
goto out;
1030+
1031+
/*
1032+
* Otherwise, we need to verify that the packs covered by the existing
1033+
* MIDX match the packs that we already have. The logic to do so is way
1034+
* more complicated than it has any right to be. This is because:
1035+
*
1036+
* - We cannot assume any ordering.
1037+
*
1038+
* - The MIDX packs may not be loaded at all, and loading them would
1039+
* be wasteful. So we need to use the pack names tracked by the
1040+
* MIDX itself.
1041+
*
1042+
* - The MIDX pack names are tracking the ".idx" files, whereas the
1043+
* packs themselves are tracking the ".pack" files. So we need to
1044+
* strip suffixes.
1045+
*/
1046+
if (ctx->nr != midx->num_packs + midx->num_packs_in_base)
1047+
goto out;
1048+
1049+
for (uint32_t i = 0; i < ctx->nr; i++) {
1050+
strbuf_reset(&buf);
1051+
strbuf_addstr(&buf, pack_basename(ctx->info[i].p));
1052+
strbuf_strip_suffix(&buf, ".pack");
1053+
1054+
if (!strset_add(&packs, buf.buf))
1055+
BUG("same pack added twice?");
1056+
}
1057+
1058+
for (uint32_t i = 0; i < ctx->nr; i++) {
1059+
strbuf_reset(&buf);
1060+
strbuf_addstr(&buf, midx->pack_names[i]);
1061+
strbuf_strip_suffix(&buf, ".idx");
1062+
1063+
if (!strset_contains(&packs, buf.buf))
1064+
goto out;
1065+
strset_remove(&packs, buf.buf);
1066+
}
1067+
1068+
needed = false;
1069+
1070+
out:
1071+
strbuf_release(&buf);
1072+
strset_clear(&packs);
1073+
return needed;
1074+
}
1075+
10171076
static int write_midx_internal(struct odb_source *source,
10181077
struct string_list *packs_to_include,
10191078
struct string_list *packs_to_drop,
@@ -1031,6 +1090,7 @@ static int write_midx_internal(struct odb_source *source,
10311090
struct write_midx_context ctx = {
10321091
.preferred_pack_idx = NO_PREFERRED_PACK,
10331092
};
1093+
struct multi_pack_index *midx_to_free = NULL;
10341094
int bitmapped_packs_concat_len = 0;
10351095
int pack_name_concat_len = 0;
10361096
int dropped_packs = 0;
@@ -1111,27 +1171,39 @@ static int write_midx_internal(struct odb_source *source,
11111171
for_each_file_in_pack_dir(source->path, add_pack_to_midx, &ctx);
11121172
stop_progress(&ctx.progress);
11131173

1114-
if ((ctx.m && ctx.nr == ctx.m->num_packs + ctx.m->num_packs_in_base) &&
1115-
!ctx.incremental &&
1116-
!(packs_to_include || packs_to_drop)) {
1117-
struct bitmap_index *bitmap_git;
1118-
int bitmap_exists;
1119-
int want_bitmap = flags & MIDX_WRITE_BITMAP;
1120-
1121-
bitmap_git = prepare_midx_bitmap_git(ctx.m);
1122-
bitmap_exists = bitmap_git && bitmap_is_midx(bitmap_git);
1123-
free_bitmap_index(bitmap_git);
1124-
1125-
if (bitmap_exists || !want_bitmap) {
1126-
/*
1127-
* The correct MIDX already exists, and so does a
1128-
* corresponding bitmap (or one wasn't requested).
1129-
*/
1130-
if (!want_bitmap)
1131-
clear_midx_files_ext(source, "bitmap", NULL);
1132-
result = 0;
1133-
goto cleanup;
1174+
if (!packs_to_drop) {
1175+
/*
1176+
* If there is no MIDX then either it doesn't exist, or we're
1177+
* doing a geometric repack. Try to load it from the source to
1178+
* tell these two cases apart.
1179+
*/
1180+
struct multi_pack_index *midx = ctx.m;
1181+
if (!midx)
1182+
midx = midx_to_free = load_multi_pack_index(ctx.source);
1183+
1184+
if (midx && !midx_needs_update(midx, &ctx)) {
1185+
struct bitmap_index *bitmap_git;
1186+
int bitmap_exists;
1187+
int want_bitmap = flags & MIDX_WRITE_BITMAP;
1188+
1189+
bitmap_git = prepare_midx_bitmap_git(midx);
1190+
bitmap_exists = bitmap_git && bitmap_is_midx(bitmap_git);
1191+
free_bitmap_index(bitmap_git);
1192+
1193+
if (bitmap_exists || !want_bitmap) {
1194+
/*
1195+
* The correct MIDX already exists, and so does a
1196+
* corresponding bitmap (or one wasn't requested).
1197+
*/
1198+
if (!want_bitmap)
1199+
clear_midx_files_ext(source, "bitmap", NULL);
1200+
result = 0;
1201+
goto cleanup;
1202+
}
11341203
}
1204+
1205+
close_midx(midx_to_free);
1206+
midx_to_free = NULL;
11351207
}
11361208

11371209
if (ctx.incremental && !ctx.nr) {
@@ -1487,6 +1559,7 @@ static int write_midx_internal(struct odb_source *source,
14871559
free(keep_hashes);
14881560
}
14891561
strbuf_release(&midx_name);
1562+
close_midx(midx_to_free);
14901563

14911564
trace2_region_leave("midx", "write_midx_internal", r);
14921565

‎midx.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -686,7 +686,7 @@ int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id)
686686
{
687687
if (m->preferred_pack_idx == -1) {
688688
uint32_t midx_pos;
689-
if (load_midx_revindex(m) < 0) {
689+
if (load_midx_revindex(m)) {
690690
m->preferred_pack_idx = -2;
691691
return -1;
692692
}

‎pack-revindex.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,8 @@ int verify_pack_revindex(struct packed_git *p);
7272
* multi-pack index by mmap-ing it and assigning pointers in the
7373
* multi_pack_index to point at it.
7474
*
75-
* A negative number is returned on error.
75+
* A negative number is returned on error. A positive number is returned in
76+
* case the multi-pack-index does not have a reverse index.
7677
*/
7778
int load_midx_revindex(struct multi_pack_index *m);
7879

‎t/t5319-multi-pack-index.sh‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,9 +350,73 @@ test_expect_success 'preferred pack from existing MIDX without bitmaps' '
350350
# the new MIDX
351351
git multi-pack-index write --preferred-pack=pack-$pack.pack
352352
)
353+
'
354+
355+
test_expect_success 'preferred pack cannot be determined without bitmap' '
356+
test_when_finished "rm -fr preferred-can-be-queried" &&
357+
git init preferred-can-be-queried &&
358+
(
359+
cd preferred-can-be-queried &&
360+
test_commit initial &&
361+
git repack -Adl --write-midx --no-write-bitmap-index &&
362+
test_must_fail test-tool read-midx --preferred-pack .git/objects 2>err &&
363+
test_grep "could not determine MIDX preferred pack" err &&
364+
git repack -Adl --write-midx --write-bitmap-index &&
365+
test-tool read-midx --preferred-pack .git/objects
366+
)
367+
'
368+
369+
test_midx_is_retained () {
370+
test-tool chmtime =0 .git/objects/pack/multi-pack-index &&
371+
ls -l .git/objects/pack/multi-pack-index >expect &&
372+
git multi-pack-index write "$@" &&
373+
ls -l .git/objects/pack/multi-pack-index >actual &&
374+
test_cmp expect actual
375+
}
376+
377+
test_midx_is_rewritten () {
378+
test-tool chmtime =0 .git/objects/pack/multi-pack-index &&
379+
ls -l .git/objects/pack/multi-pack-index >expect &&
380+
git multi-pack-index write "$@" &&
381+
ls -l .git/objects/pack/multi-pack-index >actual &&
382+
! test_cmp expect actual
383+
}
384+
385+
test_expect_success 'up-to-date multi-pack-index is retained' '
386+
test_when_finished "rm -fr midx-up-to-date" &&
387+
git init midx-up-to-date &&
388+
(
389+
cd midx-up-to-date &&
390+
391+
# Write the initial pack that contains the most objects.
392+
test_commit first &&
393+
test_commit second &&
394+
git repack -Ad --write-midx &&
395+
test_midx_is_retained &&
396+
397+
# Writing a new bitmap index should cause us to regenerate the MIDX.
398+
test_midx_is_rewritten --bitmap &&
399+
test_midx_is_retained --bitmap &&
353400
401+
# Ensure that writing a new packfile causes us to rewrite the index.
402+
test_commit incremental &&
403+
git repack -d &&
404+
test_midx_is_rewritten &&
405+
test_midx_is_retained &&
406+
407+
for pack in .git/objects/pack/*.idx
408+
do
409+
basename "$pack" || exit 1
410+
done >stdin &&
411+
test_line_count = 2 stdin &&
412+
test_midx_is_retained --stdin-packs <stdin &&
413+
head -n1 stdin >stdin.trimmed &&
414+
test_midx_is_rewritten --stdin-packs <stdin.trimmed
415+
)
354416
'
355417

418+
test_done
419+
356420
test_expect_success 'verify multi-pack-index success' '
357421
git multi-pack-index verify --object-dir=$objdir
358422
'

‎t/t7703-repack-geometric.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,41 @@ test_expect_success '--geometric with pack.packSizeLimit' '
287287
)
288288
'
289289

290+
test_expect_success '--geometric --write-midx retains up-to-date MIDX without bitmap index' '
291+
test_when_finished "rm -fr repo" &&
292+
git init repo &&
293+
(
294+
cd repo &&
295+
test_commit initial &&
296+
297+
test_path_is_missing .git/objects/pack/multi-pack-index &&
298+
git repack --geometric=2 --write-midx --no-write-bitmap-index &&
299+
test_path_is_file .git/objects/pack/multi-pack-index &&
300+
test-tool chmtime =0 .git/objects/pack/multi-pack-index &&
301+
302+
ls -l .git/objects/pack/ >expect &&
303+
git repack --geometric=2 --write-midx --no-write-bitmap-index &&
304+
ls -l .git/objects/pack/ >actual &&
305+
test_cmp expect actual
306+
)
307+
'
308+
309+
test_expect_success '--geometric --write-midx retains up-to-date MIDX with bitmap index' '
310+
test_when_finished "rm -fr repo" &&
311+
git init repo &&
312+
test_commit -C repo initial &&
313+
314+
test_path_is_missing repo/.git/objects/pack/multi-pack-index &&
315+
git -C repo repack --geometric=2 --write-midx --write-bitmap-index &&
316+
test_path_is_file repo/.git/objects/pack/multi-pack-index &&
317+
test-tool chmtime =0 repo/.git/objects/pack/multi-pack-index &&
318+
319+
ls -l repo/.git/objects/pack/ >expect &&
320+
git -C repo repack --geometric=2 --write-midx --write-bitmap-index &&
321+
ls -l repo/.git/objects/pack/ >actual &&
322+
test_cmp expect actual
323+
'
324+
290325
test_expect_success '--geometric --write-midx with packfiles in main and alternate ODB' '
291326
test_when_finished "rm -fr shared member" &&
292327

0 commit comments

Comments
 (0)