Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .golangci.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
linters-settings:
funlen:
lines: 65
lines: 75
linters:
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
disable-all: true
Expand Down
8 changes: 2 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anchore/go-testutils v0.0.0-20200520222037-edc2bf1864fe h1:YMXe4RA3qy4Ri5fmGQii/Gn+Pxv3oBfiS/LqzeOVuwo=
github.com/anchore/go-testutils v0.0.0-20200520222037-edc2bf1864fe/go.mod h1:D3rc2L/q4Hcp9eeX6AIJH4Q+kPjOtJCFhG9za90j+nU=
github.com/anchore/go-testutils v0.0.0-20200624184116-66aa578126db h1:LWKezJnFTFxNkZ4MzajVf+YWvJS0+7hwFr59u6SS7cw=
github.com/anchore/go-testutils v0.0.0-20200624184116-66aa578126db/go.mod h1:D3rc2L/q4Hcp9eeX6AIJH4Q+kPjOtJCFhG9za90j+nU=
github.com/anchore/stereoscope v0.0.0-20200520221116-025e07f1c93e/go.mod h1:bkyLl5VITnrmgErv4S1vDfVz/TGAZ5il6161IQo7w2g=
Expand Down Expand Up @@ -203,6 +201,7 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
Expand Down Expand Up @@ -611,6 +610,7 @@ github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7A
github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A=
github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
Expand Down Expand Up @@ -707,10 +707,6 @@ github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJb
github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d h1:KOxOL6qpmqwoPloNwi+CEgc1ayjHNOFNrvoOmeDOjDg=
github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d/go.mod h1:JPirS5jde/CF5qIjcK4WX+eQmKXdPc6vcZkJ/P0hfPw=
github.com/wagoodman/go-progress v0.0.0-20200526224006-dd1404d54b0b h1:UDJoympq2F2QqhIu0wF6PtI+Apq1sW3TobBoZOrUTa8=
github.com/wagoodman/go-progress v0.0.0-20200526224006-dd1404d54b0b/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA=
github.com/wagoodman/go-progress v0.0.0-20200621015745-33e4eae271f6 h1:UlFz5LlVG0sWVLP+TlXg74cuYU4e8pi0AI1DtGN94hg=
github.com/wagoodman/go-progress v0.0.0-20200621015745-33e4eae271f6/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA=
github.com/wagoodman/go-progress v0.0.0-20200621122631-1a2120f0695a h1:lV3ioFpbqvfZ1bXSQfloLWzom1OPU/5UjyU0wmBlkNc=
github.com/wagoodman/go-progress v0.0.0-20200621122631-1a2120f0695a/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA=
github.com/xanzy/go-gitlab v0.31.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
Expand Down
7 changes: 4 additions & 3 deletions pkg/event/event.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import (
)

const (
FetchImage partybus.EventType = "fetch-image-event"
ReadImage partybus.EventType = "read-image-event"
ReadLayer partybus.EventType = "read-layer-event"
PullDockerImage partybus.EventType = "pull-docker-image-event"
FetchImage partybus.EventType = "fetch-image-event"
ReadImage partybus.EventType = "read-image-event"
ReadLayer partybus.EventType = "read-layer-event"
)
20 changes: 20 additions & 0 deletions pkg/event/parsers/parsers.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package parsers
import (
"fmt"

"github.com/anchore/stereoscope/pkg/image/docker"

"github.com/anchore/stereoscope/pkg/event"
"github.com/anchore/stereoscope/pkg/image"
"github.com/wagoodman/go-partybus"
Expand Down Expand Up @@ -34,6 +36,24 @@ func checkEventType(actual, expected partybus.EventType) error {
return nil
}

func ParsePullDockerImage(e partybus.Event) (string, *docker.PullStatus, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this is an exported lib function, it'd be good to write a short explanation of what the function does as a doc comment.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll follow up in general with better docs for all of these functions

if err := checkEventType(e.Type, event.PullDockerImage); err != nil {
return "", nil, err
}

imgName, ok := e.Source.(string)
if !ok {
return "", nil, newPayloadErr(e.Type, "Source", e.Source)
}

prog, ok := e.Value.(*docker.PullStatus)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider renaming prog, it's a bit unclear to me what's going on, since we're sourcing the value from a docker.PullStatus and returning it as a docker.PullStatus. Is this a status value?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it is a reference to a object which the consumer can poll for status continually

if !ok {
return "", nil, newPayloadErr(e.Type, "Value", e.Value)
}

return imgName, prog, nil
}

func ParseFetchImage(e partybus.Event) (string, progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FetchImage); err != nil {
return "", nil, err
Expand Down
91 changes: 91 additions & 0 deletions pkg/image/docker/daemon_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,24 @@ package docker

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"math"
"os"
"path"
"strings"
"time"

"github.com/anchore/stereoscope/internal/bus"
"github.com/anchore/stereoscope/internal/docker"
"github.com/anchore/stereoscope/internal/log"
"github.com/anchore/stereoscope/pkg/event"
"github.com/anchore/stereoscope/pkg/image"
"github.com/docker/cli/cli/config"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/wagoodman/go-partybus"
Expand Down Expand Up @@ -73,6 +79,78 @@ func (p *DaemonImageProvider) trackSaveProgress() (*progress.TimedProgress, *pro
return estimateSaveProgress, copyProgress, stage, nil
}

// pull a docker image
func (p *DaemonImageProvider) pull(ctx context.Context) error {
log.Debugf("pulling docker image=%q", p.ImageRef.String())

// note: this will search the default config dir and allow for a DOCKER_CONFIG override
cfg, err := config.Load("")
if err != nil {
return fmt.Errorf("failed to load docker config: %w", err)
}
log.Debugf("using docker config=%q", cfg.Filename)

hostname := p.ImageRef.Context().RegistryStr()
creds, err := cfg.GetAuthConfig(hostname)
if err != nil {
return fmt.Errorf("failed to fetch registry auth (hostname=%s): %w", hostname, err)
}

dockerClient, err := docker.GetClient()
if err != nil {
return fmt.Errorf("failed to load docker client: %w", err)
}

var opts types.ImagePullOptions

if creds.Username != "" {
log.Debugf("using docker credentials for %q", hostname)
jsonBytes, _ := json.Marshal(map[string]string{
"username": creds.Username,
"password": creds.Password,
})
opts.RegistryAuth = base64.StdEncoding.EncodeToString(jsonBytes)
}

var status = newPullStatus()
defer func() {
status.complete = true
}()

// publish a pull event on the bus, allowing for read-only consumption of status
bus.Publish(partybus.Event{
Type: event.PullDockerImage,
Source: p.ImageRef.Name(),
Value: status,
})

resp, err := dockerClient.ImagePull(ctx, p.ImageRef.Name(), opts)
if err != nil {
return fmt.Errorf("pull failed: %w", err)
}

var thePullEvent *pullEvent
decoder := json.NewDecoder(resp)
for {
if err := decoder.Decode(&thePullEvent); err != nil {
if err == io.EOF {
break
}

return fmt.Errorf("failed to pull image: %w", err)
}

// check for the last two events indicating the pull is complete
if strings.HasPrefix(thePullEvent.Status, "Digest:") || strings.HasPrefix(thePullEvent.Status, "Status:") {
continue
}

status.onEvent(thePullEvent)
}

return nil
}

// Provide an image object that represents the cached docker image tar fetched from a docker daemon.
func (p *DaemonImageProvider) Provide() (*image.Image, error) {
// create a file within the temp dir
Expand All @@ -93,6 +171,19 @@ func (p *DaemonImageProvider) Provide() (*image.Image, error) {
return nil, fmt.Errorf("unable to get docker client: %w", err)
}

// check if the image exists, if not pull it...
_, _, err = dockerClient.ImageInspectWithRaw(context.Background(), p.ImageRef.Name())
if err != nil {
if client.IsErrNotFound(err) {
if err = p.pull(context.Background()); err != nil {
return nil, err
}
} else {
return nil, fmt.Errorf("unable to inspect existing image: %w", err)
}
}

// save the image from the docker daemon to a tar file
estimateSaveProgress, copyProgress, stage, err := p.trackSaveProgress()
if err != nil {
return nil, fmt.Errorf("unable to trace image save progress: %w", err)
Expand Down
145 changes: 145 additions & 0 deletions pkg/image/docker/pull_status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
package docker

import (
"sync"

"github.com/wagoodman/go-progress"
)

const (
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see the case for int here, as opposed to string, since it looks like you use the values assigned via iota for marking progress later on. In this approach, you need the parsePhase function just to make sense of a given value. I think this is probably fine. At the same time, it's helping me to better think through my philosophy about this, and I might've approached it a bit differently. To me, the string values seem more directly associated with a phase's identity, whereas the progress values seem like a derivative use case. In that scenario, I think I'd define these values as strings, and then use a map or something similar to declare the associated progress values for a given phase (e.g. map[PullPhase]int64). Primary reason: this makes the identity of the value the first class citizen (i.e. doesn't require a specialized function like parsePhase) and the specific use within the progress/eventing system use one extra logic step. Secondary reason: this lets you decouple the scale of iota with how you want to scale progress. For example, maybe you want to make the WaitingPhase a lower value, since not much has been accomplished; maybe you want to me the PullingFsPhase take longer since it's I/O and represents more work; whatever, hopefully you get the idea 😄

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

to summarize our conversation: I'll leave the type of PullPhase alone and I'll translate the existing parse function for a map

UnknownPhase PullPhase = iota
WaitingPhase
PullingFsPhase
DownloadingPhase
DownloadCompletePhase
ExtractingPhase
VerifyingChecksumPhase
AlreadyExistsPhase
PullCompletePhase
)

type PullPhase int
type LayerID string

type pullEvent struct {
ID string `json:"id"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
Progress string `json:"progress,omitempty"`
ProgressDetail struct {
Current int `json:"current"`
Total int `json:"total"`
} `json:"progressDetail"`
}

type LayerState struct {
Phase PullPhase
PhaseProgress progress.Progressable
DownloadProgress progress.Progressable
}

type PullStatus struct {
phaseProgress map[LayerID]*progress.Manual
downloadProgress map[LayerID]*progress.Manual
phase map[LayerID]PullPhase
layers []LayerID
lock sync.Mutex
complete bool
}

func newPullStatus() *PullStatus {
return &PullStatus{
phaseProgress: make(map[LayerID]*progress.Manual),
downloadProgress: make(map[LayerID]*progress.Manual),
phase: make(map[LayerID]PullPhase),
}
}

func (p *PullStatus) Complete() bool {
return p.complete
}

func (p *PullStatus) Layers() []LayerID {
p.lock.Lock()
defer p.lock.Unlock()

return append([]LayerID{}, p.layers...)
}

func (p *PullStatus) Current(layer LayerID) LayerState {
p.lock.Lock()
defer p.lock.Unlock()

return LayerState{
Phase: p.phase[layer],
PhaseProgress: progress.Progressable(p.phaseProgress[layer]),
DownloadProgress: progress.Progressable(p.downloadProgress[layer]),
}
}

func (p *PullStatus) onEvent(event *pullEvent) {
p.lock.Lock()
defer p.lock.Unlock()

layer := LayerID(event.ID)
if layer == "" {
return
}

if _, ok := p.phaseProgress[layer]; !ok {
// ignore the first layer as it's the image id
if p.layers == nil {
p.layers = make([]LayerID, 0)
return
}

// this is a new layer, initialize tracking info
p.phaseProgress[layer] = &progress.Manual{}
p.downloadProgress[layer] = &progress.Manual{}
p.layers = append(p.layers, layer)
}

// capture latest event info
currentPhase := parsePhase(event.Status)
p.phase[layer] = currentPhase
phaseProgress := p.phaseProgress[layer]

if currentPhase >= AlreadyExistsPhase {
phaseProgress.SetCompleted()
} else {
phaseProgress.N = int64(event.ProgressDetail.Current)
phaseProgress.Total = int64(event.ProgressDetail.Total)
}

if currentPhase == DownloadingPhase {
dl := p.downloadProgress[layer]
dl.N = int64(event.ProgressDetail.Current)
dl.Total = int64(event.ProgressDetail.Total)
} else if currentPhase >= DownloadCompletePhase {
dl := p.downloadProgress[layer]
dl.N = dl.Total
dl.SetCompleted()
}
}

func parsePhase(inputStr string) PullPhase {
switch inputStr {
case "Waiting":
return WaitingPhase
case "Pulling fs layer":
return PullingFsPhase
case "Downloading":
return DownloadingPhase
case "Download complete":
return DownloadCompletePhase
case "Extracting":
return ExtractingPhase
case "Verifying Checksum":
return VerifyingChecksumPhase
case "Already exists":
return AlreadyExistsPhase
case "Pull complete":
return PullCompletePhase
}
return UnknownPhase
}