From 560aad60173b53827a9c2a1d22e482b498ab1dcc Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 18 Nov 2021 10:48:35 -0500 Subject: [PATCH 1/6] add first-level archive processing when input is a file Signed-off-by: Alex Goodman --- go.mod | 4 + go.sum | 24 +++ syft/distro/identify.go | 2 +- syft/source/source.go | 69 ++++++-- syft/source/source_test.go | 164 ++++++++++++++++++ .../generate-tar-fixture-from-source-dir.sh | 9 + 6 files changed, 256 insertions(+), 16 deletions(-) create mode 100755 syft/source/test-fixtures/generate-tar-fixture-from-source-dir.sh diff --git a/go.mod b/go.mod index 265fd3f6464..c287930b95a 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,9 @@ require ( github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 github.com/anchore/stereoscope v0.0.0-20211116152349-7e4e1b56a15d + // we are hinting brotli to latest due to warning when installing archiver v3: + // go: warning: github.com/andybalholm/brotli@v1.0.1: retracted by module author: occasional panics and data corruption + github.com/andybalholm/brotli v1.0.4 // indirect github.com/antihax/optional v1.0.0 github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/docker/docker v20.10.10+incompatible @@ -26,6 +29,7 @@ require ( github.com/hashicorp/go-version v1.2.0 github.com/jinzhu/copier v0.3.2 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/mholt/archiver/v3 v3.5.1 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/mitchellh/mapstructure v1.4.1 diff --git a/go.sum b/go.sum index 8421eb7260c..5cd1281b4e0 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,9 @@ github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc github.com/anchore/stereoscope v0.0.0-20211116152349-7e4e1b56a15d h1:ieKMmJk1IcmdviUv/KGO1DAk0yFxoGwDpwGq8ckVeVU= github.com/anchore/stereoscope v0.0.0-20211116152349-7e4e1b56a15d/go.mod h1:Zzs5pLx2ZtUctlER7bNDAucDmsc8RO2g/oL3for1tVA= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -302,6 +305,9 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -403,6 +409,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -530,10 +538,15 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -571,6 +584,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= +github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= @@ -606,6 +621,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -658,6 +675,8 @@ github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrap github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -782,6 +801,9 @@ github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -816,6 +838,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/syft/distro/identify.go b/syft/distro/identify.go index f4d894d642b..1756aa6f5b8 100644 --- a/syft/distro/identify.go +++ b/syft/distro/identify.go @@ -60,7 +60,7 @@ identifyLoop: } if len(locations) == 0 { - log.Debugf("No Refs found from path: %s", entry.path) + log.Debugf("path not found: %s", entry.path) continue } diff --git a/syft/source/source.go b/syft/source/source.go index 6f2ec4a864d..fb530068dc4 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -7,11 +7,14 @@ package source import ( "fmt" + "io/ioutil" + "os" "sync" "github.com/anchore/stereoscope" "github.com/anchore/stereoscope/pkg/image" "github.com/anchore/syft/internal/log" + "github.com/mholt/archiver/v3" "github.com/spf13/afero" ) @@ -19,9 +22,10 @@ import ( // in cataloging (based on the data source and configuration) type Source struct { Image *image.Image // the image object to be cataloged (image only) - DirectoryResolver *directoryResolver Metadata Metadata - Mutex *sync.Mutex + directoryResolver *directoryResolver + path string + mutex *sync.Mutex } type sourceDetector func(string) (image.Source, string, error) @@ -97,34 +101,54 @@ func generateFileSource(fs afero.Fs, location string) (*Source, func(), error) { return &Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err) } - s, err := NewFromFile(location) + s, cleanupFn, err := NewFromFile(location) if err != nil { return &Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err) } - return &s, func() {}, nil + return &s, cleanupFn, nil } // NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively. func NewFromDirectory(path string) (Source, error) { return Source{ - Mutex: &sync.Mutex{}, + mutex: &sync.Mutex{}, Metadata: Metadata{ Scheme: DirectoryScheme, Path: path, }, + path: path, }, nil } -// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively. -func NewFromFile(path string) (Source, error) { +// NewFromFile creates a new source object tailored to catalog a file. +func NewFromFile(path string) (Source, func(), error) { + var analysisPath = path + var cleanupFn func() + + // if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and + // use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is + // unarchived. + envelopedUnarchiver, err := archiver.ByExtension(path) + if unarchiver, ok := envelopedUnarchiver.(archiver.Unarchiver); err == nil && ok { + unarchivedPath, tmpCleanup, err := unarchiveToTmp(path, unarchiver) + if err != nil { + log.Warnf("file could not be unarchived: %+v", err) + } else { + log.Debugf("source path is an archive") + analysisPath = unarchivedPath + cleanupFn = tmpCleanup + } + } + return Source{ - Mutex: &sync.Mutex{}, + mutex: &sync.Mutex{}, Metadata: Metadata{ Scheme: FileScheme, Path: path, }, - }, nil + path: analysisPath, + }, cleanupFn, nil } // NewFromImage creates a new source object tailored to catalog a given container image, relative to the @@ -146,16 +170,16 @@ func NewFromImage(img *image.Image, userImageStr string) (Source, error) { func (s *Source) FileResolver(scope Scope) (FileResolver, error) { switch s.Metadata.Scheme { case DirectoryScheme, FileScheme: - s.Mutex.Lock() - defer s.Mutex.Unlock() - if s.DirectoryResolver == nil { - resolver, err := newDirectoryResolver(s.Metadata.Path) + s.mutex.Lock() + defer s.mutex.Unlock() + if s.directoryResolver == nil { + resolver, err := newDirectoryResolver(s.path) if err != nil { return nil, err } - s.DirectoryResolver = resolver + s.directoryResolver = resolver } - return s.DirectoryResolver, nil + return s.directoryResolver, nil case ImageScheme: switch scope { case SquashedScope: @@ -168,3 +192,18 @@ func (s *Source) FileResolver(scope Scope) (FileResolver, error) { } return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme) } + +func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func(), error) { + tempDir, err := ioutil.TempDir("", "syft-archive-contents-") + if err != nil { + return "", func() {}, fmt.Errorf("unable to create tempdir for archive processing: %w", err) + } + + cleanupFn := func() { + if err := os.RemoveAll(tempDir); err != nil { + log.Errorf("unable to cleanup archive tempdir: %+v", err) + } + } + + return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir) +} diff --git a/syft/source/source_test.go b/syft/source/source_test.go index fe096bf4322..3a191a9515b 100644 --- a/syft/source/source_test.go +++ b/syft/source/source_test.go @@ -1,8 +1,16 @@ package source import ( + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "syscall" "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" "github.com/anchore/stereoscope/pkg/image" @@ -89,6 +97,82 @@ func TestNewFromDirectory(t *testing.T) { } } +func TestNewFromFile(t *testing.T) { + testCases := []struct { + desc string + input string + expString string + inputPaths []string + expRefs int + }{ + { + desc: "path detected", + input: "test-fixtures/path-detected", + inputPaths: []string{"/.vimrc"}, + expRefs: 1, + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + src, cleanup, err := NewFromFile(test.input) + require.NoError(t, err) + if cleanup != nil { + t.Cleanup(cleanup) + } + + assert.Equal(t, test.input, src.Metadata.Path) + assert.Equal(t, src.Metadata.Path, src.path) + + resolver, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + refs, err := resolver.FilesByPath(test.inputPaths...) + require.NoError(t, err) + assert.Len(t, refs, test.expRefs) + + }) + } +} + +func TestNewFromFile_WithArchive(t *testing.T) { + testCases := []struct { + desc string + input string + expString string + inputPaths []string + expRefs int + }{ + { + desc: "path detected", + input: "test-fixtures/path-detected", + inputPaths: []string{"/.vimrc"}, + expRefs: 1, + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + archivePath := setupArchiveTest(t, test.input) + + src, cleanup, err := NewFromFile(archivePath) + require.NoError(t, err) + if cleanup != nil { + t.Cleanup(cleanup) + } + + assert.Equal(t, archivePath, src.Metadata.Path) + assert.NotEqual(t, src.Metadata.Path, src.path) + + resolver, err := src.FileResolver(SquashedScope) + require.NoError(t, err) + + refs, err := resolver.FilesByPath(test.inputPaths...) + require.NoError(t, err) + assert.Len(t, refs, test.expRefs) + + }) + } +} + func TestNewFromDirectoryShared(t *testing.T) { testCases := []struct { desc string @@ -232,3 +316,83 @@ func TestFilesByGlob(t *testing.T) { }) } } + +// createArchive creates a new archive file at destinationArchivePath based on the directory found at sourceDirPath. +func createArchive(t testing.TB, sourceDirPath, destinationArchivePath string) { + t.Helper() + + cwd, err := os.Getwd() + if err != nil { + t.Fatalf("unable to get cwd: %+v", err) + } + + cmd := exec.Command("./generate-tar-fixture-from-source-dir.sh", destinationArchivePath, path.Base(sourceDirPath)) + cmd.Dir = filepath.Join(cwd, "test-fixtures") + + if err := cmd.Start(); err != nil { + t.Fatalf("unable to start generate zip fixture script: %+v", err) + } + + if err := cmd.Wait(); err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + // The program has exited with an exit code != 0 + + // This works on both Unix and Windows. Although package + // syscall is generally platform dependent, WaitStatus is + // defined for both Unix and Windows and in both cases has + // an ExitStatus() method with the same signature. + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + if status.ExitStatus() != 0 { + t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus()) + } + } + } else { + t.Fatalf("unable to get generate fixture script result: %+v", err) + } + } + +} + +// setupArchiveTest encapsulates common test setup work for tar file tests. It returns a cleanup function, +// which should be called (typically deferred) by the caller, the path of the created tar archive, and an error, +// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil +// (even if there's an error), and it should always be called. +func setupArchiveTest(t testing.TB, sourceDirPath string) string { + t.Helper() + + archivePrefix, err := ioutil.TempFile("", "syft-archive-TEST-") + require.NoError(t, err) + + t.Cleanup( + assertNoError(t, + func() error { + return os.Remove(archivePrefix.Name()) + }, + ), + ) + + destinationArchiveFilePath := archivePrefix.Name() + ".tar" + t.Logf("archive path: %s", destinationArchiveFilePath) + createArchive(t, sourceDirPath, destinationArchiveFilePath) + + t.Cleanup( + assertNoError(t, + func() error { + return os.Remove(destinationArchiveFilePath) + }, + ), + ) + + cwd, err := os.Getwd() + require.NoError(t, err) + + t.Logf("running from: %s", cwd) + + return destinationArchiveFilePath +} + +func assertNoError(t testing.TB, fn func() error) func() { + return func() { + assert.NoError(t, fn()) + } +} diff --git a/syft/source/test-fixtures/generate-tar-fixture-from-source-dir.sh b/syft/source/test-fixtures/generate-tar-fixture-from-source-dir.sh new file mode 100755 index 00000000000..922941d36fb --- /dev/null +++ b/syft/source/test-fixtures/generate-tar-fixture-from-source-dir.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eux + +# $1 —— absolute path to destination file, should end with .tar +# $2 —— absolute path to directory from which to add entries to the archive + +pushd "$2" + tar -cvf "$1" . +popd From b26a95f5f54506f2282f0064d979493bd8b96438 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 18 Nov 2021 11:02:18 -0500 Subject: [PATCH 2/6] add license exception for github.com/xi2/xz Signed-off-by: Alex Goodman --- .bouncer.yaml | 5 +++++ Makefile | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.bouncer.yaml b/.bouncer.yaml index a8c89ba3430..c89d68f9806 100644 --- a/.bouncer.yaml +++ b/.bouncer.yaml @@ -15,3 +15,8 @@ ignore-packages: # * GNU General Public License, version 2.0 or later (GPL-2.0-or-later). # (we choose Apache-2.0) - github.com/spdx/tools-golang + + # from: https://github.com/xi2/xz/blob/master/LICENSE + # All these files have been put into the public domain. + # You can do whatever you want with these files. + - github.com/xi2/xz diff --git a/Makefile b/Makefile index 650b77f46dd..b5cdb3cd071 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ lint-fix: ## Auto-format all source code + run golangci lint fixers go mod tidy .PHONY: check-licenses -check-licenses: +check-licenses: ## Ensure transitive dependencies are compliant with the current license policy $(TEMPDIR)/bouncer check check-go-mod-tidy: From 621dac8fba68aa73c609f0f50de72704601896e3 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 18 Nov 2021 14:10:50 -0500 Subject: [PATCH 3/6] always return cleanup function Signed-off-by: Alex Goodman --- cmd/packages.go | 4 +++- cmd/power_user.go | 4 +++- syft/source/source.go | 11 ++++------- syft/source/source_test.go | 6 ++---- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/packages.go b/cmd/packages.go index a11c078b139..1f0140ab3d5 100644 --- a/cmd/packages.go +++ b/cmd/packages.go @@ -251,7 +251,9 @@ func packagesExecWorker(userInput string) <-chan error { errs <- fmt.Errorf("failed to determine image source: %w", err) return } - defer cleanup() + if cleanup != nil { + defer cleanup() + } catalog, relationships, d, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt) if err != nil { diff --git a/cmd/power_user.go b/cmd/power_user.go index a65e544152f..54a696f8816 100644 --- a/cmd/power_user.go +++ b/cmd/power_user.go @@ -108,7 +108,9 @@ func powerUserExecWorker(userInput string) <-chan error { errs <- err return } - defer cleanup() + if cleanup != nil { + defer cleanup() + } s := sbom.SBOM{ Source: src.Metadata, diff --git a/syft/source/source.go b/syft/source/source.go index fb530068dc4..f7f8cc9ef10 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -101,10 +101,7 @@ func generateFileSource(fs afero.Fs, location string) (*Source, func(), error) { return &Source{}, func() {}, fmt.Errorf("given path is not a directory (path=%q): %w", location, err) } - s, cleanupFn, err := NewFromFile(location) - if err != nil { - return &Source{}, func() {}, fmt.Errorf("could not populate source from path=%q: %w", location, err) - } + s, cleanupFn := NewFromFile(location) return &s, cleanupFn, nil } @@ -122,9 +119,9 @@ func NewFromDirectory(path string) (Source, error) { } // NewFromFile creates a new source object tailored to catalog a file. -func NewFromFile(path string) (Source, func(), error) { +func NewFromFile(path string) (Source, func()) { var analysisPath = path - var cleanupFn func() + var cleanupFn = func() {} // if the given file is an archive (as indicated by the file extension and not MIME type) then unarchive it and // use the contents as the source. Note: this does NOT recursively unarchive contents, only the given path is @@ -148,7 +145,7 @@ func NewFromFile(path string) (Source, func(), error) { Path: path, }, path: analysisPath, - }, cleanupFn, nil + }, cleanupFn } // NewFromImage creates a new source object tailored to catalog a given container image, relative to the diff --git a/syft/source/source_test.go b/syft/source/source_test.go index 3a191a9515b..1b3dbd4f97c 100644 --- a/syft/source/source_test.go +++ b/syft/source/source_test.go @@ -114,8 +114,7 @@ func TestNewFromFile(t *testing.T) { } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { - src, cleanup, err := NewFromFile(test.input) - require.NoError(t, err) + src, cleanup := NewFromFile(test.input) if cleanup != nil { t.Cleanup(cleanup) } @@ -153,8 +152,7 @@ func TestNewFromFile_WithArchive(t *testing.T) { t.Run(test.desc, func(t *testing.T) { archivePath := setupArchiveTest(t, test.input) - src, cleanup, err := NewFromFile(archivePath) - require.NoError(t, err) + src, cleanup := NewFromFile(archivePath) if cleanup != nil { t.Cleanup(cleanup) } From 6c232220f8f52ffedcbee7f9dd5f5a064dd50e3f Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 18 Nov 2021 14:14:52 -0500 Subject: [PATCH 4/6] change source.NewFromFile log entry to warn Signed-off-by: Alex Goodman --- syft/source/source.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syft/source/source.go b/syft/source/source.go index f7f8cc9ef10..43a3d7adcfe 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -198,7 +198,7 @@ func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func() cleanupFn := func() { if err := os.RemoveAll(tempDir); err != nil { - log.Errorf("unable to cleanup archive tempdir: %+v", err) + log.Warnf("unable to cleanup archive tempdir: %+v", err) } } From 6e29a1d19afa049288443bccb16a85c521d85da0 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 19 Nov 2021 06:50:52 -0500 Subject: [PATCH 5/6] ensure file source always has cleanup function Signed-off-by: Alex Goodman --- syft/source/source.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/syft/source/source.go b/syft/source/source.go index 43a3d7adcfe..2eeaf031649 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -120,6 +120,21 @@ func NewFromDirectory(path string) (Source, error) { // NewFromFile creates a new source object tailored to catalog a file. func NewFromFile(path string) (Source, func()) { + analysisPath, cleanupFn := fileAnalysisPath(path) + + return Source{ + mutex: &sync.Mutex{}, + Metadata: Metadata{ + Scheme: FileScheme, + Path: path, + }, + path: analysisPath, + }, cleanupFn +} + +// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive +// contents have been made available. A cleanup function is provided for any temp files created (if any). +func fileAnalysisPath(path string) (string, func()) { var analysisPath = path var cleanupFn = func() {} @@ -134,18 +149,13 @@ func NewFromFile(path string) (Source, func()) { } else { log.Debugf("source path is an archive") analysisPath = unarchivedPath - cleanupFn = tmpCleanup + if tmpCleanup != nil { + cleanupFn = tmpCleanup + } } } - return Source{ - mutex: &sync.Mutex{}, - Metadata: Metadata{ - Scheme: FileScheme, - Path: path, - }, - path: analysisPath, - }, cleanupFn + return analysisPath, cleanupFn } // NewFromImage creates a new source object tailored to catalog a given container image, relative to the From 1e5e9be5b4d4188f21b1cadb8419dc43d2b15148 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 19 Nov 2021 09:09:34 -0500 Subject: [PATCH 6/6] ensure we are always preferring the unarchive cleanup function for source Signed-off-by: Alex Goodman --- syft/source/source.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syft/source/source.go b/syft/source/source.go index 2eeaf031649..d01550e22fd 100644 --- a/syft/source/source.go +++ b/syft/source/source.go @@ -149,9 +149,9 @@ func fileAnalysisPath(path string) (string, func()) { } else { log.Debugf("source path is an archive") analysisPath = unarchivedPath - if tmpCleanup != nil { - cleanupFn = tmpCleanup - } + } + if tmpCleanup != nil { + cleanupFn = tmpCleanup } }