Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Implemented a rebase feature #635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dabutvin merged 2 commits into from
Aug 8, 2020
Merged

Implemented a rebase feature #635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7900250
Implemented a user-triggered rebase feature
javierron Apr 2, 2020
c922eae
Address "rebase" PR review comments:
javierron May 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Common/Messages/OpenPrMessage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,7 @@ public class OpenPrMessage
public string RepoName { get; set; }

public string CloneUrl { get; set; }

public bool Update { get; set; }
}
}
2 changes: 2 additions & 0 deletions Common/Messages/RouterMessage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,7 @@ public class RouterMessage
public string CloneUrl { get; set; }

public string Owner { get; set; }

public bool IsRebase { get; set; }
}
}
43 changes: 43 additions & 0 deletions CompressImagesFunction/CommitMessage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Common;

namespace CompressImagesFunction
Expand Down Expand Up @@ -53,6 +55,47 @@ public static string Create(CompressionResult[] optimizedImages)
return commitMessage.ToString();
}

public static CompressionResult[] Parse(string commitBody)
{
try
{
var compressionResults = new List<CompressionResult>();

var commitLines = commitBody.Split(new[] { '\r', '\n' });
for (var i = 0; i < commitLines.Length; i++)
{
if (i == 0 || string.IsNullOrWhiteSpace(commitLines[i]))
{
// skip the first line and blank lines
continue;
}

if (commitLines[i].StartsWith("Signed-off-by:") || commitLines[i].StartsWith("*Total --"))
{
// skip the DCO line
continue;
}

var pattern = @"\*?(.*) -- (.*)kb -> (.*)kb \((.*)%\)";
var capture = Regex.Matches(commitLines[i], pattern)[0];

compressionResults.Add(new CompressionResult
{
Title = capture.Groups[1].Value,
SizeBefore = Convert.ToDouble(capture.Groups[2].Value),
SizeAfter = Convert.ToDouble(capture.Groups[3].Value),
});
}

return compressionResults.ToArray();
}
catch
{
// commit messages can be out of our control
return null;
}
}

public static int ToSecondsSinceEpoch(this DateTimeOffset date)
{
var utcDate = date.ToUniversalTime();
Expand Down
97 changes: 87 additions & 10 deletions CompressImagesFunction/CompressImages.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ public static bool Run(CompressimagesParameters parameters, ICollector<CompressI
{
CredentialsProvider = credentialsProvider,
};

Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions);

var repo = new Repository(parameters.LocalPath);
Expand All @@ -52,19 +51,20 @@ public static bool Run(CompressimagesParameters parameters, ICollector<CompressI
logger.LogInformation("CompressImagesFunction: no references found for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return false;
}

if (repo.Network.ListReferences(remote, credentialsProvider).Any(x => x.CanonicalName == $"refs/heads/{KnownGitHubs.BranchName}"))
{
logger.LogInformation("CompressImagesFunction: branch already exists for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return false;
}
}
catch (Exception e)
{
// log + ignore
logger.LogWarning(e, "CompressImagesFunction: issue checking for existing branch or empty repo for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
}

// check if the branch exists and has been modified by the user
if (parameters.IsRebase && repo.Branches[$"refs/remotes/origin/{KnownGitHubs.BranchName}"].Tip.Author.Name != KnownGitHubs.ImgBotLogin)
{
logger.LogInformation("CompressImagesFunction: imgbot branch has been modified by the user.");
return false;
}

// check if we should switch away from the default branch
if (!isWikiCompress && parameters.Settings != null && !string.IsNullOrEmpty(parameters.Settings.DefaultBranchOverride))
{
Expand Down Expand Up @@ -123,19 +123,26 @@ public static bool Run(CompressimagesParameters parameters, ICollector<CompressI
return false;
}

// Should not create branch if we are compressing Wiki
if (isWikiCompress == false)
// Should not create branch if we are compressing Wiki or performing rebase
if (isWikiCompress == false && !parameters.IsRebase)
{
// check out the branch
repo.CreateBranch(KnownGitHubs.BranchName);
var branch = Commands.Checkout(repo, KnownGitHubs.BranchName);
}
else if (parameters.IsRebase)
{
// if rebasing, fetch the branch
var refspec = string.Format("{0}:{0}", KnownGitHubs.BranchName);
Commands.Fetch(repo, "origin", new List<string> { refspec }, null, "fetch");
}

// reset any mean files
repo.Reset(ResetMode.Mixed, repo.Head.Tip);

// optimize images
var imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration);

var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths, logger, repoConfiguration.AggressiveCompression);
if (optimizedImages.Length == 0)
return false;
Expand All @@ -156,6 +163,71 @@ public static bool Run(CompressimagesParameters parameters, ICollector<CompressI
var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now);
repo.Commit(commitMessage, signature, signature);

if (parameters.IsRebase)
{
var baseBranch = repo.Head;
var newCommit = baseBranch.Tip;

// we need to reset the default branch so that we can
// rebase to it later.
repo.Reset(ResetMode.Hard, repo.Head.Commits.ElementAt(1));

Commands.Checkout(repo, KnownGitHubs.BranchName);

// reset imgbot branch removing old commit
repo.Reset(ResetMode.Hard, repo.Head.Commits.ElementAt(1));

// cherry-pick
var cherryPickOptions = new CherryPickOptions()
{
MergeFileFavor = MergeFileFavor.Theirs,
};
var cherryPickResult = repo.CherryPick(newCommit, signature, cherryPickOptions);

if (cherryPickResult.Status == CherryPickStatus.Conflicts)
{
var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions() { });
foreach (var item in status)
{
if (item.State == FileStatus.Conflicted)
{
Commands.Stage(repo, item.FilePath);
}
}

repo.Commit(commitMessage, signature, signature);
}
Copy link
Contributor

@dabutvin dabutvin May 9, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Really interesting thought about optimizing the process to only compress the new or modified images 👍
This is something I've thought about in the past, and it could really reduce the cost of running the service.

Having this as part of the rebase is one small part of the overall optimization, because as it stands, every time a new image is added, when we are not rebasing, we run the optimizer over every single image anyway.
A huge win would be to figure out how to do this outside of the rebase for all ongoing optimization runs.

Because of the complexity here, I'd like to pull this functionality into a separate PR. Let's get the rebase feature all finished up and shipped and then come back to focus on optimizing the compute resources looking at the whole system. For the first iteration, when we rebase, we can trigger a full optimization run.

Copy link
Contributor Author

@javierron javierron May 13, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, removing this for now 👍

For optimizing the whole thing, what do you think about a checksum-based solution? We could store all (filename,checksum) entries on a table, and compare every time the compression function is called, therefore compressing only the images in which the checksum does not match.

In terms of operation cost, this solution would mean a storage/processing trade-off. I believe in the long run, and specially in repositories with hundreds/thousands of images the trade-off would be justified.

Copy link
Contributor

@dabutvin dabutvin May 13, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that's a cool idea 👍 and i think you're right, could be a huge win in the long run.
Not sure how familiar you with azure storage, but ideally we could pull this off with using either blobs or tables https://docs.microsoft.com/en-us/azure/storage/common/storage-introduction in a performant way.


// rebase
var rebaseOptions = new RebaseOptions()
{
FileConflictStrategy = CheckoutFileConflictStrategy.Theirs,
};

var rebaseResult = repo.Rebase.Start(repo.Head, baseBranch, null, new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions);

while (rebaseResult.Status == RebaseStatus.Conflicts)
{
var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions() { });
foreach (var item in status)
{
if (item.State == FileStatus.Conflicted)
{
if (imagePaths.Contains(Path.Combine(parameters.LocalPath, item.FilePath)))
{
Commands.Stage(repo, item.FilePath);
}
else
{
Commands.Remove(repo, item.FilePath);
}
}
}

rebaseResult = repo.Rebase.Continue(new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions);
}
}

// We just made a normal commit, now we are going to capture all the values generated from that commit
// then rewind and make a signed commit
var commitBuffer = Commit.CreateBuffer(
Expand Down Expand Up @@ -217,7 +289,12 @@ public static bool Run(CompressimagesParameters parameters, ICollector<CompressI
}
else
{
repo.Network.Push(remote, $"refs/heads/{KnownGitHubs.BranchName}", new PushOptions
var refs = $"refs/heads/{KnownGitHubs.BranchName}";
if (parameters.IsRebase)
refs = refs.Insert(0, "+");
logger.LogInformation("refs: {refs}", refs);

repo.Network.Push(remote, refs, new PushOptions
{
CredentialsProvider = credentialsProvider,
});
Expand Down
10 changes: 3 additions & 7 deletions CompressImagesFunction/CompressImagesFunction.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ public static async Task RunAsync(
AccessTokensUrl = string.Format(KnownGitHubs.AccessTokensUrlFormat, compressImagesMessage.InstallationId),
AppId = KnownGitHubs.AppId,
};

var installationToken = await installationTokenProvider.GenerateAsync(
installationTokenParameters,
KnownEnvironmentVariables.APP_PRIVATE_KEY);
Expand All @@ -103,19 +102,14 @@ public static async Task RunAsync(
RepoOwner = compressImagesMessage.Owner,
});

if (branchExists)
{
logger.LogInformation("CompressImagesFunction: skipping repo {Owner}/{RepoName} as branch exists", compressImagesMessage.Owner, compressImagesMessage.RepoName);
return;
}

var compressImagesParameters = new CompressimagesParameters
{
CloneUrl = compressImagesMessage.CloneUrl,
LocalPath = LocalPath.CloneDir(KnownEnvironmentVariables.TMP ?? "/private/tmp/", compressImagesMessage.RepoName),
Password = installationToken.Token,
RepoName = compressImagesMessage.RepoName,
RepoOwner = compressImagesMessage.Owner,
IsRebase = branchExists,
PgpPrivateKey = KnownEnvironmentVariables.PGP_PRIVATE_KEY,
PgPPassword = KnownEnvironmentVariables.PGP_PASSWORD,
CompressImagesMessage = compressImagesMessage,
Expand All @@ -130,12 +124,14 @@ public static async Task RunAsync(
}
else if (didCompress)
{
var update = compressImagesParameters.IsRebase;
logger.LogInformation("CompressImagesFunction: Successfully compressed images for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName);
openPrMessages.Add(new OpenPrMessage
{
InstallationId = compressImagesMessage.InstallationId,
RepoName = compressImagesMessage.RepoName,
CloneUrl = compressImagesMessage.CloneUrl,
Update = compressImagesParameters.IsRebase,
});
}

Expand Down
2 changes: 2 additions & 0 deletions CompressImagesFunction/CompressimagesParameters.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ public class CompressimagesParameters

public string PgPPassword { get; set; }

public bool IsRebase { get; set; }

public Common.TableModels.Settings Settings { get; set; }

public CompressImagesMessage CompressImagesMessage { get; set; }
Expand Down
2 changes: 1 addition & 1 deletion CompressImagesFunction/CompressionResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace CompressImagesFunction
{
public class CompressionResult
public partial class CompressionResult
{
public string Title { get; set; }

Expand Down
29 changes: 29 additions & 0 deletions CompressImagesFunction/CompressionResultUtil.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;

namespace CompressImagesFunction
{
public partial class CompressionResult
{
// If items with the same "Title" are found, the merge will ignore the entry from array2
public static CompressionResult[] Merge(CompressionResult[] array1, CompressionResult[] array2)
{
List<CompressionResult> list = new List<CompressionResult>();
list.AddRange(array1);
list.AddRange(array2);

var nonRepeat = list.GroupBy(x => x.Title).Select(y => y.First());

return nonRepeat.ToArray();
}

public static CompressionResult[] Filter(CompressionResult[] optimizedImages, string[] toRemove)
{
var relativePaths = toRemove.Select(path => Path.GetFileName(path));
var filtered = optimizedImages.Where(r => !relativePaths.Contains(r.Title));
return filtered.ToArray();
}
}
}
9 changes: 7 additions & 2 deletions CompressImagesFunction/ImageQuery.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,21 @@ public static string[] FindImages(string localPath, RepoConfiguration repoConfig
.Where(x => KnownImgPatterns.ImgExtensions.Contains(Path.GetExtension(x).ToLower()))
.Select(x => x.Replace("\\", "/"));

return FilterOutIgnoredFiles(images, repoConfiguration);
}

public static string[] FilterOutIgnoredFiles(IEnumerable<string> imagePaths, RepoConfiguration repoConfiguration)
{
if (repoConfiguration.IgnoredFiles != null)
{
foreach (var ignorePattern in repoConfiguration.IgnoredFiles)
{
var pattern = new Regex(NormalizePattern(ignorePattern), RegexOptions.IgnoreCase);
images = images.Where(x => !pattern.IsMatch(x));
imagePaths = imagePaths.Where(x => !pattern.IsMatch(x));
}
}

return images.ToArray();
return imagePaths.ToArray();
}

// this is to provide backwards compatibility with the previous globbing
Expand Down
2 changes: 1 addition & 1 deletion OpenPrFunction/IPullRequest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ namespace OpenPrFunction
{
public interface IPullRequest
{
Task<Pr> OpenAsync(GitHubClientParameters parameters, Settings settings = null);
Task<Pr> OpenAsync(GitHubClientParameters parameters, bool update, Settings settings = null);
}
}
1 change: 1 addition & 0 deletions OpenPrFunction/OpenPr.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public static async Task RunAsync(
RepoName = installation.RepoName,
RepoOwner = installation.Owner,
},
openPrMessage.Update,
settings);

if (result?.Id > 0)
Expand Down
Loading