From 12425c1af098b8de07a1c96a65581b1b402bd079 Mon Sep 17 00:00:00 2001 From: pvtsec Date: Thu, 4 Dec 2025 19:10:16 +0530 Subject: [PATCH] Feat: Show release upload progress --- go.mod | 2 +- pkg/cmd/release/create/create.go | 11 +- pkg/cmd/release/create/create_test.go | 2 +- pkg/cmd/release/shared/progress.go | 393 ++++++++++++++++++++++++++ pkg/cmd/release/shared/upload.go | 80 +++++- pkg/cmd/release/shared/upload_test.go | 70 +++++ pkg/cmd/release/upload/upload.go | 11 +- pkg/iostreams/iostreams.go | 4 + 8 files changed, 561 insertions(+), 12 deletions(-) create mode 100644 pkg/cmd/release/shared/progress.go diff --git a/go.mod b/go.mod index ab8ba56572d..73be112a9bf 100644 --- a/go.mod +++ b/go.mod @@ -109,7 +109,7 @@ require ( github.com/docker/cli v29.0.3+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect + github.com/dustin/go-humanize v1.0.1 github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect diff --git a/pkg/cmd/release/create/create.go b/pkg/cmd/release/create/create.go index 8771b2477a8..d2fe4fcd814 100644 --- a/pkg/cmd/release/create/create.go +++ b/pkg/cmd/release/create/create.go @@ -546,9 +546,14 @@ func createRun(opts *CreateOptions) error { uploadURL = uploadURL[:idx] } - opts.IO.StartProgressIndicator() - err = shared.ConcurrentUpload(httpClient, uploadURL, opts.Concurrency, opts.Assets) - opts.IO.StopProgressIndicator() + progressPrinter := shared.NewUploadProgressPrinter(opts.IO, opts.Assets) + var callbacks *shared.UploadCallbacks + if progressPrinter != nil { + callbacks = progressPrinter.Callbacks() + defer progressPrinter.Finish() + } + + err = shared.ConcurrentUpload(httpClient, uploadURL, opts.Concurrency, opts.Assets, callbacks) if err != nil { return cleanupDraftRelease(err) } diff --git a/pkg/cmd/release/create/create_test.go b/pkg/cmd/release/create/create_test.go index a6e6cd7c68b..853ad06d543 100644 --- a/pkg/cmd/release/create/create_test.go +++ b/pkg/cmd/release/create/create_test.go @@ -977,7 +977,7 @@ func Test_createRun(t *testing.T) { reg.Register(httpmock.REST("DELETE", "releases/123"), httpmock.StatusStringResponse(204, ``)) }, wantStdout: ``, - wantStderr: ``, + wantStderr: "[1/1] Failed uploading ball.tgz: HTTP 422 (https://api.github.com/assets/upload?label=&name=ball.tgz)\n", wantErr: `HTTP 422 (https://api.github.com/assets/upload?label=&name=ball.tgz)`, }, { diff --git a/pkg/cmd/release/shared/progress.go b/pkg/cmd/release/shared/progress.go new file mode 100644 index 00000000000..c1f6de87782 --- /dev/null +++ b/pkg/cmd/release/shared/progress.go @@ -0,0 +1,393 @@ +package shared + +import ( + "fmt" + "sync" + "time" + + "github.com/briandowns/spinner" + "github.com/dustin/go-humanize" + + "github.com/cli/cli/v2/pkg/iostreams" +) + +// UploadProgressPrinter renders per-asset progress, updating each line in-place +// when stderr supports TTY control codes. Otherwise it falls back to textual logs. +type UploadProgressPrinter struct { + io *iostreams.IOStreams + total int + dynamic bool + textual bool + + mu sync.Mutex + lines []string + lineStates []*uploadLine + lastPercent map[int]int + lastUploaded map[int]int64 + renderedLines int + + spinnerTicker *time.Ticker + spinnerStop chan struct{} + spinnerActive bool +} + +type uploadLine struct { + asset AssetForUpload + uploaded int64 + status lineStatus + err error + spinnerFrame int +} + +type lineStatus int + +const ( + lineStatusUploading lineStatus = iota + lineStatusSuccess + lineStatusFailure +) + +var spinnerFrames = spinner.CharSets[11] + +const spinnerInterval = 120 * time.Millisecond + +func NewUploadProgressPrinter(io *iostreams.IOStreams, assets []*AssetForUpload) *UploadProgressPrinter { + if len(assets) == 0 { + return nil + } + + AssignAssetOrdinals(assets) + + p := &UploadProgressPrinter{ + io: io, + total: len(assets), + dynamic: io.ProgressIndicatorEnabled() && io.IsStderrTTY() && !io.GetSpinnerDisabled(), + textual: io.GetSpinnerDisabled(), + lineStates: make([]*uploadLine, len(assets)), + lines: make([]string, 0, len(assets)), + lastPercent: make(map[int]int), + lastUploaded: make(map[int]int64), + } + + return p +} + +func (p *UploadProgressPrinter) Callbacks() *UploadCallbacks { + if p == nil { + return nil + } + + callbacks := &UploadCallbacks{ + OnUploadStart: p.onStart, + OnUploadComplete: p.onComplete, + } + + if p.dynamic { + callbacks.OnUploadProgress = p.onProgress + } + + return callbacks +} + +func (p *UploadProgressPrinter) Finish() { + if p == nil { + return + } + p.mu.Lock() + defer p.mu.Unlock() + p.stopSpinnerLocked() + p.renderLocked() +} + +func (p *UploadProgressPrinter) onStart(asset AssetForUpload) { + p.mu.Lock() + defer p.mu.Unlock() + + ordinal := p.ordinal(asset) + p.lastPercent[ordinal] = -1 + p.lastUploaded[ordinal] = 0 + + if !p.dynamic { + if p.textual { + p.printLine("[%d/%d] Uploading %s%s", ordinal, p.total, asset.Name, p.sizeSuffix(asset)) + } + return + } + + idx := ordinal - 1 + if idx < 0 || idx >= len(p.lineStates) { + return + } + + p.lineStates[idx] = &uploadLine{ + asset: asset, + status: lineStatusUploading, + uploaded: 0, + } + p.ensureSpinnerLoopLocked() + p.updateLinesLocked() + p.renderLocked() +} + +func (p *UploadProgressPrinter) onProgress(asset AssetForUpload, uploaded int64) { + p.mu.Lock() + defer p.mu.Unlock() + + if !p.dynamic { + return + } + + ordinal := p.ordinal(asset) + idx := ordinal - 1 + if idx < 0 || idx >= len(p.lineStates) { + return + } + + line := p.lineStates[idx] + if line == nil { + return + } + + if uploaded < p.lastUploaded[ordinal] { + p.lastPercent[ordinal] = -1 + } + + p.lastUploaded[ordinal] = uploaded + + percent := p.percent(asset, uploaded) + if percent == p.lastPercent[ordinal] && !(asset.Size > 0 && uploaded >= asset.Size) { + return + } + + p.lastPercent[ordinal] = percent + line = p.lineStates[idx] + if line == nil { + return + } + line.uploaded = uploaded + p.updateLinesLocked() + p.renderLocked() +} + +func (p *UploadProgressPrinter) onComplete(asset AssetForUpload, err error) { + p.mu.Lock() + defer p.mu.Unlock() + + ordinal := p.ordinal(asset) + + if !p.dynamic { + if err != nil { + p.printLine("[%d/%d] Failed uploading %s: %v", ordinal, p.total, asset.Name, err) + } else if p.textual { + p.printLine("[%d/%d] Uploaded %s", ordinal, p.total, asset.Name) + } + return + } + + idx := ordinal - 1 + if idx < 0 || idx >= len(p.lineStates) { + return + } + + line := p.lineStates[idx] + if line == nil { + return + } + + if err != nil { + line.status = lineStatusFailure + line.err = err + } else { + line.status = lineStatusSuccess + } + + if line.status != lineStatusUploading && p.spinnerActive && !p.hasUploadingLocked() { + p.stopSpinnerLocked() + } + + p.updateLinesLocked() + p.renderLocked() +} + +func (p *UploadProgressPrinter) render() { + if !p.dynamic || len(p.lines) == 0 { + return + } + + if p.renderedLines == 0 { + for _, line := range p.lines { + fmt.Fprintf(p.io.ErrOut, "%s\n", line) + } + p.renderedLines = len(p.lines) + return + } + + if p.renderedLines > 0 { + fmt.Fprintf(p.io.ErrOut, "\x1b[%dA", p.renderedLines) + } + + for _, line := range p.lines { + fmt.Fprintf(p.io.ErrOut, "\x1b[2K\r%s\n", line) + } + + p.renderedLines = len(p.lines) +} + +func (p *UploadProgressPrinter) renderLocked() { + if !p.dynamic || len(p.lines) == 0 { + return + } + p.render() +} + +func (p *UploadProgressPrinter) progressLabel(asset AssetForUpload, uploaded int64) string { + ordinal := p.ordinal(asset) + prefix := fmt.Sprintf("[%d/%d]", ordinal, p.total) + + if asset.Size > 0 { + percent := p.percent(asset, uploaded) + if uploaded > asset.Size { + uploaded = asset.Size + } + return fmt.Sprintf("%s Uploading %s %3d%% (%s/%s)", + prefix, + asset.Name, + percent, + humanize.IBytes(uint64(max(uploaded, 0))), + humanize.IBytes(uint64(asset.Size))) + } + + return fmt.Sprintf("%s Uploading %s (%s)", prefix, asset.Name, humanize.IBytes(uint64(max(uploaded, 0)))) +} + +func (p *UploadProgressPrinter) sizeSuffix(asset AssetForUpload) string { + if asset.Size <= 0 { + return "" + } + return fmt.Sprintf(" (%s)", humanize.IBytes(uint64(asset.Size))) +} + +func (p *UploadProgressPrinter) percent(asset AssetForUpload, uploaded int64) int { + if asset.Size <= 0 { + return -1 + } + if uploaded < 0 { + uploaded = 0 + } + percent := int(uploaded * 100 / asset.Size) + if percent > 100 { + return 100 + } + return percent +} + +func (p *UploadProgressPrinter) printLine(format string, args ...any) { + fmt.Fprintf(p.io.ErrOut, format+"\n", args...) +} + +func (p *UploadProgressPrinter) ordinal(asset AssetForUpload) int { + if asset.Ordinal > 0 { + return asset.Ordinal + } + return 1 +} + +func max(a, b int64) int64 { + if a > b { + return a + } + return b +} + +func (p *UploadProgressPrinter) updateLinesLocked() { + if !p.dynamic { + return + } + + p.lines = p.lines[:0] + cs := p.io.ColorScheme() + + for _, line := range p.lineStates { + if line == nil { + continue + } + + var prefix string + switch line.status { + case lineStatusUploading: + prefix = spinnerFrames[line.spinnerFrame%len(spinnerFrames)] + case lineStatusSuccess: + prefix = cs.SuccessIcon() + case lineStatusFailure: + prefix = cs.FailureIcon() + } + + body := p.bodyForLine(line) + p.lines = append(p.lines, fmt.Sprintf("%s %s", prefix, body)) + } +} + +func (p *UploadProgressPrinter) bodyForLine(line *uploadLine) string { + switch line.status { + case lineStatusUploading: + return p.progressLabel(line.asset, line.uploaded) + case lineStatusSuccess: + return fmt.Sprintf("[%d/%d] Uploaded %s", p.ordinal(line.asset), p.total, line.asset.Name) + case lineStatusFailure: + return fmt.Sprintf("[%d/%d] Failed uploading %s: %v", p.ordinal(line.asset), p.total, line.asset.Name, line.err) + default: + return "" + } +} + +func (p *UploadProgressPrinter) ensureSpinnerLoopLocked() { + if !p.dynamic || p.spinnerActive { + return + } + p.spinnerTicker = time.NewTicker(spinnerInterval) + p.spinnerStop = make(chan struct{}) + p.spinnerActive = true + + go p.spin() +} + +func (p *UploadProgressPrinter) spin() { + for { + select { + case <-p.spinnerTicker.C: + p.mu.Lock() + updated := false + for _, line := range p.lineStates { + if line != nil && line.status == lineStatusUploading { + line.spinnerFrame = (line.spinnerFrame + 1) % len(spinnerFrames) + updated = true + } + } + if updated { + p.updateLinesLocked() + p.renderLocked() + } + p.mu.Unlock() + case <-p.spinnerStop: + p.spinnerTicker.Stop() + return + } + } +} + +func (p *UploadProgressPrinter) stopSpinnerLocked() { + if !p.spinnerActive { + return + } + close(p.spinnerStop) + p.spinnerActive = false +} + +func (p *UploadProgressPrinter) hasUploadingLocked() bool { + for _, line := range p.lineStates { + if line != nil && line.status == lineStatusUploading { + return true + } + } + return false +} diff --git a/pkg/cmd/release/shared/upload.go b/pkg/cmd/release/shared/upload.go index ab7533320e8..4156b378156 100644 --- a/pkg/cmd/release/shared/upload.go +++ b/pkg/cmd/release/shared/upload.go @@ -26,8 +26,9 @@ type httpDoer interface { type errNetwork struct{ error } type AssetForUpload struct { - Name string - Label string + Name string + Label string + Ordinal int Size int64 MIMEType string @@ -36,6 +37,12 @@ type AssetForUpload struct { ExistingURL string } +type UploadCallbacks struct { + OnUploadStart func(AssetForUpload) + OnUploadProgress func(AssetForUpload, int64) + OnUploadComplete func(AssetForUpload, error) +} + func AssetsFromArgs(args []string) (assets []*AssetForUpload, err error) { labeledArgs, unlabeledArgs := cmdutil.Partition(args, func(arg string) bool { return strings.Contains(arg, "#") @@ -72,9 +79,21 @@ func AssetsFromArgs(args []string) (assets []*AssetForUpload, err error) { MIMEType: typeForFilename(fi.Name()), }) } + + AssignAssetOrdinals(assets) + return } +func AssignAssetOrdinals(assets []*AssetForUpload) { + for i, asset := range assets { + if asset == nil { + continue + } + asset.Ordinal = i + 1 + } +} + func typeForFilename(fn string) string { ext := fileExt(fn) switch ext { @@ -111,7 +130,7 @@ func fileExt(fn string) string { return path.Ext(fn) } -func ConcurrentUpload(httpClient httpDoer, uploadURL string, numWorkers int, assets []*AssetForUpload) error { +func ConcurrentUpload(httpClient httpDoer, uploadURL string, numWorkers int, assets []*AssetForUpload, callbacks *UploadCallbacks) error { if numWorkers == 0 { return errors.New("the number of concurrent workers needs to be greater than 0") } @@ -123,13 +142,49 @@ func ConcurrentUpload(httpClient httpDoer, uploadURL string, numWorkers int, ass for _, a := range assets { asset := *a g.Go(func() error { - return uploadWithDelete(gctx, httpClient, uploadURL, asset) + return runUpload(gctx, httpClient, uploadURL, asset, callbacks) }) } return g.Wait() } +func runUpload(ctx context.Context, httpClient httpDoer, uploadURL string, asset AssetForUpload, callbacks *UploadCallbacks) error { + if callbacks != nil && callbacks.OnUploadProgress != nil && asset.Open != nil { + asset = wrapAssetOpenWithProgress(asset, callbacks.OnUploadProgress) + } + + if callbacks != nil && callbacks.OnUploadStart != nil { + callbacks.OnUploadStart(asset) + } + + err := uploadWithDelete(ctx, httpClient, uploadURL, asset) + + if callbacks != nil && callbacks.OnUploadComplete != nil { + callbacks.OnUploadComplete(asset, err) + } + + return err +} + +func wrapAssetOpenWithProgress(asset AssetForUpload, progressFn func(AssetForUpload, int64)) AssetForUpload { + assetCopy := asset + origOpen := asset.Open + asset.Open = func() (io.ReadCloser, error) { + rc, err := origOpen() + if err != nil { + return nil, err + } + return &progressReadCloser{ + ReadCloser: rc, + onProgress: func(uploaded int64) { + progressFn(assetCopy, uploaded) + }, + }, nil + } + return asset +} + func shouldRetry(err error) bool { var networkError errNetwork if errors.As(err, &networkError) { @@ -221,3 +276,20 @@ func deleteAsset(ctx context.Context, httpClient httpDoer, assetURL string) erro return nil } + +type progressReadCloser struct { + io.ReadCloser + onProgress func(int64) + totalRead int64 +} + +func (p *progressReadCloser) Read(b []byte) (int, error) { + n, err := p.ReadCloser.Read(b) + if n > 0 { + p.totalRead += int64(n) + if p.onProgress != nil { + p.onProgress(p.totalRead) + } + } + return n, err +} diff --git a/pkg/cmd/release/shared/upload_test.go b/pkg/cmd/release/shared/upload_test.go index 26bed11c017..b7a9efe654d 100644 --- a/pkg/cmd/release/shared/upload_test.go +++ b/pkg/cmd/release/shared/upload_test.go @@ -6,6 +6,7 @@ import ( "errors" "io" "net/http" + "sync" "testing" ) @@ -114,6 +115,75 @@ func Test_uploadWithDelete_retry(t *testing.T) { } } +func Test_runUpload_reportsProgress(t *testing.T) { + data := []byte("hello world") + asset := AssetForUpload{ + Name: "hello.txt", + Label: "", + Size: int64(len(data)), + MIMEType: "text/plain", + Open: func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil + }, + } + + var ( + mu sync.Mutex + progress []int64 + started bool + finished bool + ) + + callbacks := &UploadCallbacks{ + OnUploadStart: func(a AssetForUpload) { + started = true + }, + OnUploadProgress: func(a AssetForUpload, uploaded int64) { + mu.Lock() + defer mu.Unlock() + progress = append(progress, uploaded) + }, + OnUploadComplete: func(a AssetForUpload, err error) { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + finished = true + }, + } + + client := funcClient(func(req *http.Request) (*http.Response, error) { + if _, err := io.Copy(io.Discard, req.Body); err != nil { + return nil, err + } + _ = req.Body.Close() + return &http.Response{ + Request: req, + StatusCode: 201, + Body: io.NopCloser(bytes.NewBufferString(`{}`)), + }, nil + }) + + if err := runUpload(context.Background(), client, "https://example.com/upload", asset, callbacks); err != nil { + t.Fatalf("runUpload error: %v", err) + } + + if !started { + t.Fatalf("expected OnUploadStart to be called") + } + if !finished { + t.Fatalf("expected OnUploadComplete to be called") + } + + mu.Lock() + defer mu.Unlock() + if len(progress) == 0 { + t.Fatalf("expected progress updates") + } + if got, want := progress[len(progress)-1], int64(len(data)); got != want { + t.Fatalf("got final progress %d, want %d", got, want) + } +} + type funcClient func(*http.Request) (*http.Response, error) func (f funcClient) Do(req *http.Request) (*http.Response, error) { diff --git a/pkg/cmd/release/upload/upload.go b/pkg/cmd/release/upload/upload.go index 42419ccf399..990c49b3376 100644 --- a/pkg/cmd/release/upload/upload.go +++ b/pkg/cmd/release/upload/upload.go @@ -108,9 +108,14 @@ func uploadRun(opts *UploadOptions) error { return fmt.Errorf("asset under the same name already exists: %v", existingNames) } - opts.IO.StartProgressIndicator() - err = shared.ConcurrentUpload(httpClient, uploadURL, opts.Concurrency, opts.Assets) - opts.IO.StopProgressIndicator() + progressPrinter := shared.NewUploadProgressPrinter(opts.IO, opts.Assets) + var callbacks *shared.UploadCallbacks + if progressPrinter != nil { + callbacks = progressPrinter.Callbacks() + defer progressPrinter.Finish() + } + + err = shared.ConcurrentUpload(httpClient, uploadURL, opts.Concurrency, opts.Assets, callbacks) if err != nil { return err } diff --git a/pkg/iostreams/iostreams.go b/pkg/iostreams/iostreams.go index 22f966ac810..8b5245c70cf 100644 --- a/pkg/iostreams/iostreams.go +++ b/pkg/iostreams/iostreams.go @@ -283,6 +283,10 @@ func (s *IOStreams) SetSpinnerDisabled(v bool) { s.spinnerDisabled = v } +func (s *IOStreams) ProgressIndicatorEnabled() bool { + return s.progressIndicatorEnabled +} + func (s *IOStreams) StartProgressIndicator() { s.StartProgressIndicatorWithLabel("") }