xetdata · hoytak · Jun 18, 2024 · Jun 15, 2024 · Jun 17, 2024 · Jun 17, 2024
diff --git a/gitxet/Cargo.lock b/gitxet/Cargo.lock
diff --git a/gitxet/tests/integration_tests.rs b/gitxet/tests/integration_tests.rs
@@ -300,6 +300,11 @@ mod git_integration_tests {
     fn test_xet_lazy() -> anyhow::Result<()> {
         IntegrationTest::new(include_str!("integration_tests/test_xet_lazy.sh")).run()
     }
+
+    #[test]
+    fn test_repo_migration() -> anyhow::Result<()> {
+        IntegrationTest::new(include_str!("integration_tests/test_repo_migration.sh")).run()
+    }
 }
 
 #[cfg(test)]

diff --git a/gitxet/tests/integration_tests/test_repo_migration.sh b/gitxet/tests/integration_tests/test_repo_migration.sh
@@ -0,0 +1,147 @@
+#!/bin/bash
+
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )";
+. "$SCRIPT_DIR/initialize.sh"
+setup_basic_run_environment
+
+mkdir src_repo
+
+mkdir main
+mkdir br1
+mkdir br2
+mkdir br3
+mkdir br4
+
+pushd src_repo
+git init
+echo "Testing" > test.txt
+git add *
+git commit -a -m "Initial commit."
+
+create_data_file data.dat 100
+git add *
+git commit -a -m "main."
+cp * ../main/
+
+git checkout -b br1
+create_data_file data2.dat 100
+cp data2.dat data2b.dat
+git add *
+git commit -a -m "br1"
+cp * ../br1/
+
+# Replace the original data file.
+git checkout -b br2
+create_data_file data.dat 100
+git add *
+git commit -a -m "br2"
+cp * ../br2/
+
+# Replace the second data file and duplicate the data2.dat
+git checkout -b br3
+create_data_file data3.dat 100
+git add *
+git commit -a -m "br3"
+cp * ../br3/
+
+git checkout main
+git checkout -b br4 
+git merge br1 --no-edit
+git merge br2 --no-edit
+git merge br3 --no-edit
+cp data.dat data3b.dat
+create_data_file data3.dat 100
+git add *
+git commit -a -m "br4"
+cp * ../br4/
+
+
+# Add a bunch of notes there too. 
+add_note_to_branch_head() {
+    local repo=$1
+    local branch=$2
+    local note_content=$3
+
+    git -C "$repo" checkout "$branch"
+}
+
+# Step 1: Add test notes to SRC_REPO
+echo "Adding test notes to SRC_REPO..."
+
+note_branches="main br1 br2 br3"
+
+for branch in $note_branches ; do 
+
+    git checkout $branch
+    head_commit=$(git rev-parse HEAD)
+
+    git notes add -m "Test note: $branch" $head_commit
+done
+popd
+
+xet_remote=$(create_bare_repo)
+pushd $xet_remote
+git xet init --force
+popd
+
+ls ./src_repo
+git xet repo migrate --src=./src_repo --dest=$xet_remote --working-dir=./migration_working_dir --no-cleanup 
+
+# Now, do a clone from the local folder.
+git xet clone $xet_remote migrated_repo
+
+pushd ./migrated_repo
+git fetch origin "refs/notes/*:refs/notes/*"
+
+# Verify all the data files are correct.
+git checkout main
+assert_stored_as_pointer_file data.dat
+for f in * ; do 
+    assert_files_equal $f ../main/$f
+done 
+
+# Verify all the data files are correct.
+git checkout br1
+assert_stored_as_pointer_file data.dat
+assert_stored_as_pointer_file data2.dat
+for f in * ; do 
+    assert_files_equal $f ../br1/$f
+done 
+
+git checkout br2
+assert_stored_as_pointer_file data.dat
+assert_stored_as_pointer_file data2.dat
+assert_stored_as_pointer_file data2b.dat
+for f in * ; do 
+    assert_files_equal $f ../br2/$f
+done 
+
+git checkout br3
+assert_stored_as_pointer_file data.dat
+assert_stored_as_pointer_file data2.dat
+assert_stored_as_pointer_file data2b.dat
+assert_stored_as_pointer_file data3.dat
+for f in * ; do 
+    assert_files_equal $f ../br3/$f
+done 
+
+git checkout br4
+assert_stored_as_pointer_file data.dat
+assert_stored_as_pointer_file data2.dat
+assert_stored_as_pointer_file data3.dat
+assert_stored_as_pointer_file data3b.dat
+for f in * ; do 
+    assert_files_equal $f ../br4/$f
+done 
+
+
+for branch in $note_branches ; do 
+    git checkout $branch
+    head_commit=$(git rev-parse HEAD)
+
+    note_content="$(git notes show $head_commit || echo "NOT FOUND")"
+    if [[ $note_content != "Test note: $branch" ]]; then
+        die "Note for branch $branch not found or incorrect: ($note_content)"
+    fi
+done
+
diff --git a/libxet/Cargo.lock b/libxet/Cargo.lock
diff --git a/rust/gitxetcore/Cargo.toml b/rust/gitxetcore/Cargo.toml
@@ -125,6 +125,8 @@ prometheus = "0.13.0"
 prometheus_dict_encoder = { path = "../prometheus_dict_encoder" }
 utime = "0.3.1"
 
+# Other hashers for migration
+hashers = "1.0.1"
 
 # Need to specify this as optional to allow the openssl/vendored option below
 openssl = { version = "0.10", features = [], optional = true }

diff --git a/rust/gitxetcore/src/command/repo.rs b/rust/gitxetcore/src/command/repo.rs
@@ -180,7 +180,7 @@ async fn migrate_command(config: XetConfig, args: &MigrateArgs) -> Result<()> {
     let xet_repo = GitXetRepo::open_and_verify_setup(config.clone()).await?;
 
     // Now do the actual migration process.
-    let branch_list = migrate_repo(&source_dir, &xet_repo).await?;
+    let (branch_list, ref_list) = migrate_repo(&source_dir, &xet_repo).await?;
 
     eprintln!("Migration complete; packing repository at {dest_dir:?}.");
     run_git_passthrough(
@@ -228,7 +228,7 @@ async fn migrate_command(config: XetConfig, args: &MigrateArgs) -> Result<()> {
         ) {
             if slice_size == 1 {
                 eprintln!("Error updating remote branch {}.", branches_this_push[0]);
-                eprintln!("Please go to directory {dest_dir:?} and run `git push --force --set-upstream {}` to push manually.", branches_this_push[0]);
+                eprintln!("Please go to directory {dest_dir:?} and run `git push --force --set-upstream origin {}` to push manually.", branches_this_push[0]);
 
                 Err(e)?;
                 unreachable!();
@@ -239,13 +239,49 @@ async fn migrate_command(config: XetConfig, args: &MigrateArgs) -> Result<()> {
             }
         }
         // Success, now loop.
-        start_idx += slice_size;
+        start_idx += branches_this_push.len();
         eprintln!(
             "Synced {start_idx} / {} branches.",
             remaining_branches.len()
         );
     }
 
+    eprintln!("Syncing tags.");
+    run_git_passthrough(
+        Some(&dest_dir),
+        "push",
+        &["origin", "--force", "--tags", "--follow-tags"],
+        true,
+        false,
+        Some(&[("XET_DISABLE_HOOKS", "0")]),
+    ).map_err(|e| {
+        eprintln!("Error pushing to remote.");
+        eprintln!("Please go to directory {dest_dir:?} and run `git push origin --force --tags --follow-tags --all` to push manually.");
+        e
+    })?;
+
+    if !ref_list.is_empty() {
+        eprintln!("Syncing remaining references.");
+        let mut args = vec!["origin".to_owned()];
+
+        for r in ref_list {
+            args.push(format!("+{r}:{r}"));
+        }
+
+        run_git_passthrough(
+        Some(&dest_dir),
+        "push",
+        &args.iter().map(|s| s.as_str()).collect::<Vec<&str>>(),
+        true,
+        false,
+        Some(&[("XET_DISABLE_HOOKS", "0")]),
+    ).map_err(|e| {
+        eprintln!("Error pushing to remote.");
+        eprintln!("Please go to directory {dest_dir:?} and run `git push origin +refs/*:refs/*` to push manually.");
+        e
+    })?;
+    }
+
     if !args.no_cleanup {
         eprintln!("Cleaning up.");
         let _ = std::fs::remove_dir_all(&source_dir).map_err(|e| {

diff --git a/rust/gitxetcore/src/diff/csv.rs b/rust/gitxetcore/src/diff/csv.rs
@@ -1,5 +1,5 @@
-use std::borrow::Cow;
 use serde::{Deserialize, Serialize};
+use std::borrow::Cow;
 use tracing::warn;
 
 use crate::diff::error::DiffError;

diff --git a/rust/gitxetcore/src/diff/processor.rs b/rust/gitxetcore/src/diff/processor.rs
@@ -1,8 +1,8 @@
-use std::borrow::Cow;
 use crate::diff::error::DiffError;
 use crate::diff::output::{SummaryDiff, SummaryDiffData};
 use crate::summaries::analysis::FileSummary;
 use crate::summaries::summary_type::SummaryType;
+use std::borrow::Cow;
 
 /// Trait that different types of summaries can implement to generate diffs.
 ///

diff --git a/rust/gitxetcore/src/diff/tds.rs b/rust/gitxetcore/src/diff/tds.rs
@@ -1,8 +1,8 @@
-use std::borrow::Cow;
 use crate::diff::output::SummaryDiffData;
 use crate::diff::output::SummaryDiffData::Tds;
 use crate::diff::{DiffError, SummaryDiffProcessor};
 use crate::summaries::{FileSummary, SummaryType};
+use std::borrow::Cow;
 use tableau_summary::tds::diff::schema::TdsSummaryDiffContent;
 use tableau_summary::tds::TdsSummary;
 use tableau_summary::twb::diff::util::DiffProducer;

diff --git a/rust/gitxetcore/src/git_integration/git_process_wrapping.rs b/rust/gitxetcore/src/git_integration/git_process_wrapping.rs
@@ -50,20 +50,22 @@ fn spawn_git_command(
         }
     }
 
-    if let Some(dir) = base_directory {
-        debug_assert!(dir.exists());
+    let cwd = std::env::current_dir()?;
+    let run_dir = base_directory.unwrap_or(&cwd);
+
+    debug_assert!(run_dir.exists());
+    cmd.current_dir(run_dir);
 
-        cmd.current_dir(dir);
-    }
     debug!(
-        "Calling git, working dir={:?}, command = git {:?} {:?}, env = {:?}",
-        match base_directory {
-            Some(bd) => bd.to_owned(),
-            None => std::env::current_dir()?,
-        },
-        &command,
-        &args,
-        &env
+        "Calling git, base_dir={}, run_dir={run_dir:?}, command = git {command:?} {}, env: {}",
+        base_directory
+            .map(|d| d.to_str().unwrap_or("bad"))
+            .unwrap_or("None"),
+        args.iter().map(|a| format!("\"{a}\"")).join(" "),
+        env.unwrap_or_default()
+            .iter()
+            .map(|(k, v)| format!("{k}=\"{v}\""))
+            .join(" "),
     );
 
     // Using idea from https://stackoverflow.com/questions/30776520/closing-stdout-or-stdin

diff --git a/rust/gitxetcore/src/git_integration/mod.rs b/rust/gitxetcore/src/git_integration/mod.rs
@@ -8,7 +8,6 @@ mod git_repo_plumbing;
 pub mod git_repo_salt;
 mod git_xet_repo;
 pub mod hook_command_entry;
-pub mod repo_migration;
 
 pub mod git_url;
 pub mod git_user_config;
@@ -21,3 +20,4 @@ pub use git_process_wrapping::*;
 pub use git_repo_paths::*;
 pub use git_repo_plumbing::*;
 pub use git_xet_repo::GitXetRepo;
+pub mod repo_migration;