anchore · wagoodman · Aug 4, 2020 · Aug 3, 2020 · Aug 4, 2020
diff --git a/Makefile b/Makefile
@@ -140,6 +140,19 @@ fixtures:
 	$(call title,Generating test fixtures)
 	cd syft/cataloger/java/test-fixtures/java-builds && make
 
+.PHONY: generate-json-schema
+generate-json-schema: clean-json-schema-examples integration ## Generate a new json schema for the json presenter, derived from integration test cases
+	docker run \
+		-i \
+		--rm \
+		-v $(shell pwd)/json-schema:/work \
+		-w /work \
+		python:3.8 \
+			bash -x -c "\
+				pip install -r requirements.txt && \
+				python generate.py \
+			"
+
 .PHONY: clear-test-cache
 clear-test-cache: ## Delete all test cache (built docker image tars)
 	find . -type f -wholename "**/test-fixtures/tar-cache/*.tar" -delete
@@ -215,7 +228,7 @@ release: clean-dist ## Build and publish final binaries and packages
 	.github/scripts/update-version-file.sh "$(DISTDIR)" "$(VERSION)"
 
 .PHONY: clean
-clean: clean-dist clean-snapshot  ## Remove previous builds and result reports
+clean: clean-dist clean-snapshot clean-json-schema-examples ## Remove previous builds and result reports
 	rm -rf $(RESULTSDIR)/*
 
 .PHONY: clean-snapshot
@@ -225,3 +238,7 @@ clean-snapshot:
 .PHONY: clean-dist
 clean-dist:
 	rm -rf $(DISTDIR) $(TEMPDIR)/goreleaser.yaml
+
+.PHONY: clean-json-schema-examples
+clean-json-schema-examples:
+	rm json-schema/examples/*
diff --git a/go.mod b/go.mod
@@ -29,6 +29,7 @@ require (
 	github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b
 	github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2
+	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9
 	golang.org/x/sys v0.0.0-20200610111108-226ff32320da // indirect
 	google.golang.org/genproto v0.0.0-20200615140333-fd031eab31e7 // indirect

diff --git a/go.sum b/go.sum
@@ -841,6 +841,12 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V
 github.com/xanzy/go-gitlab v0.31.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
 github.com/xanzy/go-gitlab v0.32.0 h1:tBm+OXv1t+KBsqlXkSDFz+YUjRM0GFsjpOWYOod3Ebs=
 github.com/xanzy/go-gitlab v0.32.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=

diff --git a/json-schema/.gitignore b/json-schema/.gitignore
@@ -0,0 +1 @@
+examples/
diff --git a/json-schema/generate.py b/json-schema/generate.py
@@ -0,0 +1,30 @@
+#!/usr/env/bin python3
+import os
+import glob
+import json
+
+from genson import SchemaBuilder
+
+EXAMPLES_DIR = "examples/"
+OUTPUT = "schema.json"
+
+
+def main():
+    builder = SchemaBuilder()
+
+    print("Generating new Syft json schema...")
+    for filepath in glob.glob(os.path.join(EXAMPLES_DIR, '*.json')):
+        with open(filepath, 'r') as f:
+            print(f"  adding {filepath}")
+            builder.add_object(json.loads(f.read()))
+
+    print("Building schema...")
+    new_schema = builder.to_schema()
+    with open(OUTPUT, 'w') as f:
+        f.write(json.dumps(new_schema, sort_keys=True, indent=4))
+
+    print(f"New schema written to '{OUTPUT}'")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/json-schema/requirements.txt b/json-schema/requirements.txt
@@ -0,0 +1 @@
+genson
diff --git a/json-schema/schema.json b/json-schema/schema.json
diff --git a/syft/cataloger/apkdb/parse_apk_db.go b/syft/cataloger/apkdb/parse_apk_db.go
@@ -4,9 +4,11 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"path"
 	"strconv"
 	"strings"
 
+	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/mitchellh/mapstructure"
 )
@@ -52,10 +54,14 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
 	return packages, nil
 }
 
+// nolint:funlen
 func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
 	var entry pkg.ApkMetadata
 	pkgFields := make(map[string]interface{})
-	files := make([]string, 0)
+	files := make([]pkg.ApkFileRecord, 0)
+
+	var fileRecord *pkg.ApkFileRecord
+	lastFile := "/"
 
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
@@ -70,9 +76,33 @@ func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
 
 		switch key {
 		case "F":
-			// extract all file entries, don't store in map
-			files = append(files, value)
+			lastFile = "/" + value
 			continue
+		case "R":
+			newFileRecord := pkg.ApkFileRecord{
+				Path: path.Join(lastFile, value),
+			}
+			files = append(files, newFileRecord)
+			fileRecord = &files[len(files)-1]
+		case "a":
+			ownershipFields := strings.Split(value, ":")
+			if len(ownershipFields) != 3 {
+				log.Errorf("unexpected APK ownership field: %q", value)
+				continue
+			}
+			if fileRecord == nil {
+				log.Errorf("ownership field with no parent record: %q", value)
+				continue
+			}
+			fileRecord.OwnerUID = ownershipFields[0]
+			fileRecord.OwnerGUI = ownershipFields[1]
+			fileRecord.Permissions = ownershipFields[2]
+		case "Z":
+			if fileRecord == nil {
+				log.Errorf("checksum field with no parent record: %q", value)
+				continue
+			}
+			fileRecord.Checksum = value
 		case "I", "S":
 			// coerce to integer
 			iVal, err := strconv.Atoi(value)

diff --git a/syft/cataloger/apkdb/parse_apk_db_test.go b/syft/cataloger/apkdb/parse_apk_db_test.go
@@ -31,7 +31,43 @@ func TestSinglePackage(t *testing.T) {
 				PullDependencies: "scanelf so:libc.musl-x86_64.so.1",
 				PullChecksum:     "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
 				GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
-				Files:            []string{"sbin", "usr", "usr/bin"},
+				Files: []pkg.ApkFileRecord{
+					{
+						Path:        "/sbin/ldconfig",
+						OwnerUID:    "0",
+						OwnerGUI:    "0",
+						Permissions: "755",
+						Checksum:    "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
+					},
+					{
+						Path:        "/usr/bin/iconv",
+						OwnerUID:    "0",
+						OwnerGUI:    "0",
+						Permissions: "755",
+						Checksum:    "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
+					},
+					{
+						Path:        "/usr/bin/ldd",
+						OwnerUID:    "0",
+						OwnerGUI:    "0",
+						Permissions: "755",
+						Checksum:    "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
+					},
+					{
+						Path:        "/usr/bin/getconf",
+						OwnerUID:    "0",
+						OwnerGUI:    "0",
+						Permissions: "755",
+						Checksum:    "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
+					},
+					{
+						Path:        "/usr/bin/getent",
+						OwnerUID:    "0",
+						OwnerGUI:    "0",
+						Permissions: "755",
+						Checksum:    "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
+					},
+				},
 			},
 		},
 	}
@@ -92,7 +128,7 @@ func TestMultiplePackages(t *testing.T) {
 						PullChecksum:     "Q1p78yvTLG094tHE1+dToJGbmYzQE=",
 						GitCommitOfAport: "97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479",
 						PullDependencies: "musl-utils",
-						Files:            []string{},
+						Files:            []pkg.ApkFileRecord{},
 					},
 				},
 				{
@@ -114,7 +150,43 @@ func TestMultiplePackages(t *testing.T) {
 						PullDependencies: "scanelf so:libc.musl-x86_64.so.1",
 						PullChecksum:     "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
 						GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
-						Files:            []string{"sbin", "usr", "usr/bin"},
+						Files: []pkg.ApkFileRecord{
+							{
+								Path:        "/sbin/ldconfig",
+								OwnerUID:    "0",
+								OwnerGUI:    "0",
+								Permissions: "755",
+								Checksum:    "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
+							},
+							{
+								Path:        "/usr/bin/iconv",
+								OwnerUID:    "0",
+								OwnerGUI:    "0",
+								Permissions: "755",
+								Checksum:    "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
+							},
+							{
+								Path:        "/usr/bin/ldd",
+								OwnerUID:    "0",
+								OwnerGUI:    "0",
+								Permissions: "755",
+								Checksum:    "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
+							},
+							{
+								Path:        "/usr/bin/getconf",
+								OwnerUID:    "0",
+								OwnerGUI:    "0",
+								Permissions: "755",
+								Checksum:    "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
+							},
+							{
+								Path:        "/usr/bin/getent",
+								OwnerUID:    "0",
+								OwnerGUI:    "0",
+								Permissions: "755",
+								Checksum:    "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
+							},
+						},
 					},
 				},
 			},

diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go
@@ -1,61 +1,70 @@
 package pkg
 
-// TODO: consider keeping the remaining values as an embedded map
 // Available fields are described at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html
 // in the --showformat section
 type DpkgMetadata struct {
-	Package string `mapstructure:"Package"`
-	Source  string `mapstructure:"Source"`
-	Version string `mapstructure:"Version"`
+	Package string `mapstructure:"Package" json:"package"`
+	Source  string `mapstructure:"Source" json:"source"`
+	Version string `mapstructure:"Version" json:"version"`
+	// TODO: consider keeping the remaining values as an embedded map
 }
 
 type RpmMetadata struct {
-	Epoch   int    `mapstructure:"Epoch"`
-	Arch    string `mapstructure:"Arch"`
-	Release string `mapstructure:"Release"`
+	Epoch   int    `mapstructure:"Epoch" json:"epoch"`
+	Arch    string `mapstructure:"Arch" json:"architecture"`
+	Release string `mapstructure:"Release" json:"release"`
+	// TODO: consider keeping the remaining values as an embedded map
 }
 
 type JavaManifest struct {
-	Name            string            `mapstructure:"Name"`
-	ManifestVersion string            `mapstructure:"Manifest-Version"`
-	SpecTitle       string            `mapstructure:"Specification-Title"`
-	SpecVersion     string            `mapstructure:"Specification-Version"`
-	SpecVendor      string            `mapstructure:"Specification-Vendor"`
-	ImplTitle       string            `mapstructure:"Implementation-Title"`
-	ImplVersion     string            `mapstructure:"Implementation-Version"`
-	ImplVendor      string            `mapstructure:"Implementation-Vendor"`
-	Extra           map[string]string `mapstructure:",remain"`
+	Name            string            `mapstructure:"Name" json:"name"`
+	ManifestVersion string            `mapstructure:"Manifest-Version" json:"manifest-version"`
+	SpecTitle       string            `mapstructure:"Specification-Title" json:"specification-title"`
+	SpecVersion     string            `mapstructure:"Specification-Version" json:"specification-version"`
+	SpecVendor      string            `mapstructure:"Specification-Vendor" json:"specification-vendor"`
+	ImplTitle       string            `mapstructure:"Implementation-Title" json:"implementation-title"`
+	ImplVersion     string            `mapstructure:"Implementation-Version" json:"implementation-version"`
+	ImplVendor      string            `mapstructure:"Implementation-Vendor" json:"implementation-vendor"`
+	Extra           map[string]string `mapstructure:",remain" json:"extra-fields"`
 }
 
 type PomProperties struct {
 	Path       string
-	Name       string            `mapstructure:"name"`
-	GroupID    string            `mapstructure:"groupId"`
-	ArtifactID string            `mapstructure:"artifactId"`
-	Version    string            `mapstructure:"version"`
-	Extra      map[string]string `mapstructure:",remain"`
+	Name       string            `mapstructure:"name" json:"name"`
+	GroupID    string            `mapstructure:"groupId" json:"group-id"`
+	ArtifactID string            `mapstructure:"artifactId" json:"artifact-id"`
+	Version    string            `mapstructure:"version" json:"version"`
+	Extra      map[string]string `mapstructure:",remain" json:"extra-fields"`
 }
 
 type JavaMetadata struct {
-	Manifest      *JavaManifest  `mapstructure:"Manifest"`
-	PomProperties *PomProperties `mapstructure:"PomProperties"`
-	Parent        *Package
+	Manifest      *JavaManifest  `mapstructure:"Manifest" json:"manifest"`
+	PomProperties *PomProperties `mapstructure:"PomProperties" json:"pom-properties"`
+	Parent        *Package       `json:"parent-package"`
 }
 
 // source: https://wiki.alpinelinux.org/wiki/Apk_spec
 type ApkMetadata struct {
-	Package          string `mapstructure:"P"`
-	OriginPackage    string `mapstructure:"o"`
-	Maintainer       string `mapstructure:"m"`
-	Version          string `mapstructure:"V"`
-	License          string `mapstructure:"L"`
-	Architecture     string `mapstructure:"A"`
-	URL              string `mapstructure:"U"`
-	Description      string `mapstructure:"T"`
-	Size             int    `mapstructure:"S"`
-	InstalledSize    int    `mapstructure:"I"`
-	PullDependencies string `mapstructure:"D"`
-	PullChecksum     string `mapstructure:"C"`
-	GitCommitOfAport string `mapstructure:"c"`
-	Files            []string
+	Package          string          `mapstructure:"P" json:"package"`
+	OriginPackage    string          `mapstructure:"o" json:"origin-package"`
+	Maintainer       string          `mapstructure:"m" json:"maintainer"`
+	Version          string          `mapstructure:"V" json:"version"`
+	License          string          `mapstructure:"L" json:"license"`
+	Architecture     string          `mapstructure:"A" json:"architecture"`
+	URL              string          `mapstructure:"U" json:"url"`
+	Description      string          `mapstructure:"T" json:"description"`
+	Size             int             `mapstructure:"S" json:"size"`
+	InstalledSize    int             `mapstructure:"I" json:"installed-size"`
+	PullDependencies string          `mapstructure:"D" json:"pull-dependencies"`
+	PullChecksum     string          `mapstructure:"C" json:"pull-checksum"`
+	GitCommitOfAport string          `mapstructure:"c" json:"git-commit-of-apk-port"`
+	Files            []ApkFileRecord `json:"files"`
+}
+
+type ApkFileRecord struct {
+	Path        string `json:"path"`
+	OwnerUID    string `json:"owner-uid"`
+	OwnerGUI    string `json:"owner-gid"`
+	Permissions string `json:"permissions"`
+	Checksum    string `json:"checksum"`
 }
diff --git a/syft/pkg/package.go b/syft/pkg/package.go
@@ -12,15 +12,15 @@ type ID int64
 
 // Package represents an application or library that has been bundled into a distributable format
 type Package struct {
-	id       ID // this is set when a package is added to the catalog
-	Name     string
-	Version  string
-	FoundBy  string
-	Source   []file.Reference
-	Licenses []string
-	Language Language // TODO: should this support multiple languages as a slice?
-	Type     Type
-	Metadata interface{}
+	id       ID               // this is set when a package is added to the catalog
+	Name     string           `json:"manifest"`
+	Version  string           `json:"version"`
+	FoundBy  string           `json:"found-by"`
+	Source   []file.Reference `json:"sources"`
+	Licenses []string         `json:"licenses"` // TODO: should we move this into metadata?
+	Language Language         `json:"language"` // TODO: should this support multiple languages as a slice?
+	Type     Type             `json:"type"`
+	Metadata interface{}      `json:"metadata,omitempty"`
 }
 
 func (p Package) ID() ID {