diff --git a/README.md b/README.md index fd598d6..66aa141 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,6 @@ This package currently has several functionalities: - [Install Go](https://go.dev/doc/install) - Run `ELASTICSEARCH_URL="" go run index.go` -- Import `dataview.ndjson` to have a bunch of runtime fields analyzing the structure ## Code analyzer usage diff --git a/dataview.ndjson b/dataview.ndjson deleted file mode 100644 index 0c7e2dd..0000000 --- a/dataview.ndjson +++ /dev/null @@ -1,2 +0,0 @@ -{"attributes":{"fieldAttrs":"{\"doc\":{\"count\":3},\"vis_uses_split\":{\"count\":2},\"vis_type\":{\"count\":4},\"app\":{\"count\":2},\"dashboard\":{\"count\":4},\"vis_title\":{\"count\":3},\"vis_tsvb_time_range_mode\":{\"count\":1},\"vis_tsvb_aggs\":{\"count\":1}}","fields":"[]","runtimeFieldMap":"{\"vis_title\":{\"type\":\"keyword\",\"script\":{\"source\":\"if (doc['soType'].value != 'visualization') return;\\nif (params['_source']['doc']['attributes'] != null) {\\n emit(params['_source']['doc']['attributes']['title'])\\n} else if (params['_source']['doc']['title'] != null) {\\n emit(params['_source']['doc']['title'])\\n} else if (params['_source']['doc']['embeddableConfig'] != null && params['_source']['doc']['embeddableConfig']['savedVis']['title'] != null) {\\n emit(params['_source']['doc']['embeddableConfig']['savedVis']['title'])\\n}\\n\"}},\"vis_tsvb_time_range_mode\":{\"type\":\"keyword\",\"script\":{\"source\":\"if (doc['soType'].value != 'visualization') return;\\nString visType = '';\\nif (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['type'] != null) {\\n visType = params['_source']['doc']['attributes']['type'];\\n} else if (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['visState'] != null && params['_source']['doc']['attributes']['visState']['type'] != null) {\\n visType = params['_source']['doc']['attributes']['visState']['type'];\\n} else if (params['_source']['doc']['embeddableConfig'] != null && params['_source']['doc']['embeddableConfig']['savedVis']['type'] != null) {\\n visType = params['_source']['doc']['embeddableConfig']['savedVis']['type']\\n}\\n\\nif (visType == 'metrics') {\\n boolean explicit = false;\\n if (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['visState'] != null && params['_source']['doc']['attributes']['visState']['params']['time_range_mode'] != null) {\\n emit(params['_source']['doc']['attributes']['visState']['params']['time_range_mode']);\\n explicit = true;\\n}\\nif (params['_source']['doc']['embeddableConfig'] != null && params['_source']['doc']['embeddableConfig']['savedVis'] != null && params['_source']['doc']['embeddableConfig']['savedVis']['params']['time_range_mode'] != null) {\\n emit(params['_source']['doc']['embeddableConfig']['savedVis']['params']['time_range_mode']);explicit = true;\\n}\\nif(!explicit) {\\n emit('last_value_implicit')\\n}\\n}\"}},\"vis_tsvb_type\":{\"type\":\"keyword\",\"script\":{\"source\":\"if (doc['soType'].value != 'visualization') return;\\nString visType = '';\\nif (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['type'] != null) {\\n visType = params['_source']['doc']['attributes']['type'];\\n} else if (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['visState'] != null && params['_source']['doc']['attributes']['visState']['type'] != null) {\\n visType = params['_source']['doc']['attributes']['visState']['type'];\\n} else if (params['_source']['doc']['embeddableConfig'] != null && params['_source']['doc']['embeddableConfig']['savedVis']['type'] != null) {\\n visType = params['_source']['doc']['embeddableConfig']['savedVis']['type']\\n}\\n\\nif (visType == 'metrics') {\\n if (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['visState'] != null) {\\n emit(params['_source']['doc']['attributes']['visState']['params']['type'])\\n}\\nif (params['_source']['doc']['embeddableConfig'] != null && params['_source']['doc']['embeddableConfig']['savedVis'] != null) {\\n emit(params['_source']['doc']['embeddableConfig']['savedVis']['params']['type'])\\n}\\n}\"}},\"vis_type\":{\"type\":\"keyword\",\"script\":{\"source\":\"if (doc['soType'].value != 'visualization') return;\\nif (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['type'] != null) {\\n emit(params['_source']['doc']['attributes']['type'])\\n} else if (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['visState'] != null && params['_source']['doc']['attributes']['visState']['type'] != null) {\\n emit(params['_source']['doc']['attributes']['visState']['type'])\\n} else if (params['_source']['doc']['embeddableConfig'] != null && params['_source']['doc']['embeddableConfig']['savedVis']['type'] != null) {\\n emit(params['_source']['doc']['embeddableConfig']['savedVis']['type'])\\n}\\n\"}},\"vis_uses_split\":{\"type\":\"boolean\",\"script\":{\"source\":\"if (doc['soType'].value != 'visualization') return;\\nboolean split = false;\\n if (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['visState'] != null && params['_source']['doc']['attributes']['visState']['aggs'] != null) {\\n List arr = params['_source']['doc']['attributes']['visState']['aggs'];\\n for(int i = 0; i < arr.length; i++) {\\n if (arr[i]['schema'] == 'split') { split = true}\\n }\\n \\n}\\nif (params['_source']['doc']['embeddableConfig'] != null && params['_source']['doc']['embeddableConfig']['savedVis'] != null && params['_source']['doc']['embeddableConfig']['savedVis']['aggs'] != null) {\\n List arr = params['_source']['doc']['embeddableConfig']['savedVis']['aggs'];\\n for(int i = 0; i < arr.length; i++) {\\n if (arr[i]['schema'] == 'split') { split = true}\\n }\\n}\\nemit(split)\"}},\"vis_tsvb_aggs\":{\"type\":\"keyword\",\"script\":{\"source\":\"if (doc['soType'].value != 'visualization') return;\\n\\n if (params['_source']['doc']['attributes'] != null && params['_source']['doc']['attributes']['visState'] != null && params['_source']['doc']['attributes']['visState']['params'] != null && params['_source']['doc']['attributes']['visState']['params']['series']!= null) {\\n List arr = params['_source']['doc']['attributes']['visState']['params']['series'];\\n for(int i = 0; i < arr.length; i++) {\\n List metrics = arr[i]['metrics'];\\n for(int j = 0; j < metrics.length; j++) {\\n emit (metrics[j]['type']);\\n }\\n \\n }\\n \\n}\\nif (params['_source']['doc']['embeddableConfig'] != null && params['_source']['doc']['embeddableConfig']['savedVis'] != null && params['_source']['doc']['embeddableConfig']['savedVis']['params'] != null && params['_source']['doc']['embeddableConfig']['savedVis']['params']['series'] != null) {\\n\\n List arr = params['_source']['doc']['embeddableConfig']['savedVis']['params']['series'];\\n for(int i = 0; i < arr.length; i++) {\\n List metrics = arr[i]['metrics'];\\n for(int j = 0; j < metrics.length; j++) {\\n emit (metrics[j]['type']);\\n }\\n \\n }\\n}\"}}}","title":"legacy_vis*","typeMeta":"{}"},"coreMigrationVersion":"8.1.1","id":"679c81b0-af4f-11ec-a029-d32028b9cf73","migrationVersion":{"index-pattern":"8.0.0"},"references":[],"type":"index-pattern","updated_at":"2022-04-01T11:46:18.900Z","version":"WzIxMDgsMl0="} -{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":1,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/go.mod b/go.mod index 5bab611..e46cac6 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,24 @@ module github.com/elastic/visualizations_integrations_tools -go 1.17 +go 1.22 -require gopkg.in/yaml.v2 v2.4.0 // indirect +toolchain go1.23.4 require ( - github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6 // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/stretchr/objx v0.5.1 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect ) require ( - github.com/elastic/go-elasticsearch/v8 v8.10.0 - github.com/elastic/kbncontent v0.1.1 - sigs.k8s.io/yaml v1.3.0 + github.com/elastic/go-elasticsearch/v8 v8.16.0 + github.com/elastic/kbncontent v0.1.4 + sigs.k8s.io/yaml v1.4.0 ) replace github.com/elastic/kbncontent v0.0.0 => ../kbncontent diff --git a/go.sum b/go.sum index 7c6bc01..963daa2 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,20 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6 h1:1+44gxLdKRnR/Bx/iAtr+XqNcE4e0oODa63+FABNANI= -github.com/elastic/elastic-transport-go/v8 v8.0.0-20230329154755-1a3c63de0db6/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= -github.com/elastic/go-elasticsearch/v8 v8.10.0 h1:ALg3DMxSrx07YmeMNcfPf7cFh1Ep2+Qa19EOXTbwr2k= -github.com/elastic/go-elasticsearch/v8 v8.10.0/go.mod h1:NGmpvohKiRHXI0Sw4fuUGn6hYOmAXlyCphKpzVBiqDE= -github.com/elastic/kbncontent v0.1.1 h1:X6ZXKPTW3MwFZPLMZcPAbpUIsKQhFImhBGKOd83+UFo= -github.com/elastic/kbncontent v0.1.1/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= +github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= +github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v8 v8.16.0 h1:f7bR+iBz8GTAVhwyFO3hm4ixsz2eMaEy0QroYnXV3jE= +github.com/elastic/go-elasticsearch/v8 v8.16.0/go.mod h1:lGMlgKIbYoRvay3xWBeKahAiJOgmFDsjZC39nmO3H64= +github.com/elastic/kbncontent v0.1.4 h1:GoUkJkqkn2H6iJTnOHcxEqYVVYyjvcebLQVaSR1aSvU= +github.com/elastic/kbncontent v0.1.4/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -18,14 +26,23 @@ github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/index.go b/index.go index 74c0831..ff0f8cb 100644 --- a/index.go +++ b/index.go @@ -13,13 +13,14 @@ import ( "strings" "context" + "runtime" + "sync/atomic" + "time" + "github.com/elastic/go-elasticsearch/v8" "github.com/elastic/go-elasticsearch/v8/esutil" "github.com/elastic/kbncontent" - "runtime" "sigs.k8s.io/yaml" - "sync/atomic" - "time" ) type CommitData struct { @@ -45,19 +46,43 @@ func getCommitData(path string) (CommitData, error) { } type Visualization struct { - Doc map[string]interface{} `json:"doc"` - SoType string `json:"soType"` - App string `json:"app"` - Source string `json:"source"` - Link string `json:"link"` - Dashboard string `json:"dashboard"` - Path string `json:"path"` - Commit CommitData `json:"commit"` - Manifest map[string]interface{} `json:"manifest"` - VisType string `json:"vis_type,omitempty"` - TSVBType string `json:"vis_tsvb_type,omitempty"` - VisTitle string `json:"vis_title,omitempty"` - IsLegacy bool `json:"is_legacy"` + Doc map[string]interface{} `json:"doc"` + SoType string `json:"soType"` + App string `json:"app"` + Source string `json:"source"` + Link string `json:"link"` + Dashboard string `json:"dashboard"` + Path string `json:"path"` + Commit CommitData `json:"commit"` + Manifest map[string]interface{} `json:"manifest"` + GithubOwner string `json:"gh_owner"` + VisType string `json:"vis_type,omitempty"` + TSVBType string `json:"vis_tsvb_type,omitempty"` + VisTitle string `json:"vis_title,omitempty"` + IsLegacy bool `json:"is_legacy"` + OwningGroup string `json:"owning_group"` +} + +func getGithubOwner(manifest map[string]interface{}) string { + if githubOwner, ok := manifest["owner"].(map[string]interface{})["github"].(string); ok { + return githubOwner + } + + return "" +} + +func getOwningGroup(githubOwner string) string { + if githubOwner == "" { + return "" + } + + if githubOwner == "elastic/security-external-integrations" || githubOwner == "elastic/security-asset-management" { + return "security" + } else if githubOwner == "elastic/ml-ui" { + return "platform" + } else { + return "observability" + } } func collectVisualizationFolder(app, path, source string, dashboards map[string]string, folderName string) []Visualization { @@ -201,6 +226,8 @@ func CollectIntegrationsVisualizations(integrationsPath string) []Visualization for _, vis := range visualizations { vis.Manifest = manifest + vis.GithubOwner = getGithubOwner(manifest) + vis.OwningGroup = getOwningGroup(vis.GithubOwner) allVis = append(allVis, vis) } } @@ -208,6 +235,62 @@ func CollectIntegrationsVisualizations(integrationsPath string) []Visualization return allVis } +func CollectIntegrationsDataStreams(integrationsPath string) []map[string]interface{} { + var allDataStreams []map[string]interface{} + packages, err := os.ReadDir(filepath.Join(integrationsPath, "packages")) + fmt.Printf("Collecting integrations\n") + if err != nil { + fmt.Printf("Error: %v\n", err) + return allDataStreams + } + for _, packageInfo := range packages { + if packageInfo.IsDir() { + packagePath := filepath.Join(integrationsPath, "packages", packageInfo.Name()) + buildYmlPath := filepath.Join(packagePath, "_dev", "build", "build.yml") + dataStreamPackagePath := filepath.Join(integrationsPath, "packages", packageInfo.Name(), "data_stream") + dataStreams, err := os.ReadDir(dataStreamPackagePath) + if err != nil { + fmt.Printf("Error: %v\n", err) + } + integrationManifestPath := filepath.Join(packagePath, "manifest.yml") + integrationManifest := collectManifest(integrationManifestPath) + buildYml := collectManifest(buildYmlPath) + // check whther the integration has a _dev/benchmark folder + benchmarkPath := filepath.Join(packagePath, "_dev", "benchmark") + if _, err := os.Stat(benchmarkPath); err == nil { + // add flag to the integration manifest + integrationManifest["has_benchmark"] = true + } else { + integrationManifest["has_benchmark"] = false + } + for _, dataStream := range dataStreams { + manifestPath := filepath.Join(dataStreamPackagePath, dataStream.Name(), "manifest.yml") + dataStreamManifest := collectManifest(manifestPath) + // enrich data stream manifest with integration manifest + dataStreamManifest["integration"] = integrationManifest + dataStreamManifest["buildYml"] = buildYml + // check whether the data streams has a _dev/test/pipeline folder + pipelinePath := filepath.Join(dataStreamPackagePath, dataStream.Name(), "_dev", "test", "pipeline") + if _, err := os.Stat(pipelinePath); err == nil { + // add flag to the data stream manifest + dataStreamManifest["has_pipeline_test"] = true + } else { + dataStreamManifest["has_pipeline_test"] = false + } + // same for system test + systemTestPath := filepath.Join(dataStreamPackagePath, dataStream.Name(), "_dev", "test", "system") + if _, err := os.Stat(systemTestPath); err == nil { + dataStreamManifest["has_system_test"] = true + } else { + dataStreamManifest["has_system_test"] = false + } + allDataStreams = append(allDataStreams, dataStreamManifest) + } + } + } + return allDataStreams +} + // func collectBeats() []map[string]interface{} { // allVis := []map[string]interface{}{} // recurse := func(root string) { @@ -264,6 +347,8 @@ func uploadVisualizations(visualizations []Visualization) { "vis_type": { "type": "keyword" }, "vis_tsvb_type": { "type": "keyword" }, "vis_title": { "type": "keyword" }, + "gh_owner": { "type": "keyword" }, + "owning_group": { "type": "keyword" }, "is_legacy": { "type": "boolean" }, "commit": { "properties": { @@ -385,8 +470,162 @@ func saveVisualizationsToFile(visualizations []Visualization) { fmt.Printf("JSON data saved to %s\n", filePath) } +func saveDataStreamsToFile(datastreams []map[string]interface{}) { + // Marshal the data into a JSON string + jsonData, err := json.Marshal(datastreams) + if err != nil { + fmt.Println("Error marshaling JSON:", err) + return + } + + // Define the file path + filePath := "result_data_stream.json" + + // Create or open the file for writing + file, err := os.Create(filePath) + if err != nil { + fmt.Println("Error creating file:", err) + return + } + defer file.Close() + + // Write the JSON data to the file + _, err = file.Write(jsonData) + if err != nil { + fmt.Println("Error writing JSON to file:", err) + return + } + + fmt.Printf("JSON data saved to %s\n", filePath) +} + +func uploadDatastreams(datastreams []map[string]interface{}) { + indexName := "integration_data_stream" + + es, err := elasticsearch.NewClient(elasticsearch.Config{}) + + if err != nil { + fmt.Printf("problem generating elasticsearch client") + } + + log.Println(es.Info()) + + _, err = es.Indices.Delete([]string{indexName}, es.Indices.Delete.WithIgnoreUnavailable(true)) + + if err != nil { + fmt.Printf("Problem deleting old index %v", err) + } + + mapping := `{ + "mappings": { + "dynamic_templates": [ + { + "default_as_keyword": { + "match_mapping_type": "*", + "path_match": "*default", + "runtime": { + "type": "keyword" + } + } + }, + { + "dynamic_as_keyword": { + "match_mapping_type": "*", + "path_match": "*mappings.dynamic", + "runtime": { + "type": "keyword" + } + } + } + ] + } + }` + + _, err = es.Indices.Create(indexName, es.Indices.Create.WithBody(strings.NewReader(mapping))) + + if err != nil { + fmt.Printf("Problem creating index: %v\n", err) + } + + bi, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ + Index: indexName, + Client: es, + NumWorkers: runtime.NumCPU(), + FlushBytes: 5e+6, + FlushInterval: 30 * time.Second, // The periodic flush interval + }) + + if err != nil { + log.Fatalf("Error creating the indexer: %s", err) + } + + var countSuccessful uint64 + + for i, vis := range datastreams { + // Prepare the data payload: encode vis to JSON + + data, err := json.Marshal(vis) + if err != nil { + log.Fatalf("Cannot encode datastream %d: %s", i, err) + } + + // >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + // + // Add an item to the BulkIndexer + // + err = bi.Add( + context.Background(), + esutil.BulkIndexerItem{ + // Action field configures the operation to perform (index, create, delete, update) + Action: "index", + + // DocumentID is the (optional) document ID + // DocumentID: strconv.Itoa(a.ID), + + // Body is an `io.Reader` with the payload + Body: bytes.NewReader(data), + + // OnSuccess is called for each successful operation + OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) { + atomic.AddUint64(&countSuccessful, 1) + }, + + // OnFailure is called for each failed operation + OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) { + if err != nil { + log.Printf("ERROR: %s", err) + } else { + log.Printf("ERROR: %s: %s", res.Error.Type, res.Error.Reason) + } + }, + }, + ) + + if err != nil { + log.Fatalf("Unexpected error: %s", err) + } + } + + // Close the indexer + if err := bi.Close(context.Background()); err != nil { + log.Fatalf("Unexpected error: %s", err) + } + + biStats := bi.Stats() + + log.Printf("%v", biStats) + + // beatsData := collectBeats() + // for _, vis := range beatsData { + // fmt.Printf("%v\n", vis) + // } +} + func main() { visualizations := CollectIntegrationsVisualizations("./integrations") saveVisualizationsToFile(visualizations) uploadVisualizations(visualizations) + dataStreams := CollectIntegrationsDataStreams("./integrations") + saveDataStreamsToFile(dataStreams) + uploadDatastreams(dataStreams) } diff --git a/index_kibana.js b/index_kibana.js index c04fa5b..9fd08dc 100644 --- a/index_kibana.js +++ b/index_kibana.js @@ -145,7 +145,7 @@ function collectUsages() { const chunkSize = 250; for (let i = 0; i < usages.length; i += chunkSize) { - console.log(i); + console.log(`uploading #${i+1} out of ${Math.ceil(usages.length / chunkSize)}`); const chunk = usages.slice(i, i + chunkSize); const response = await client.bulk({ operations: chunk.flatMap((v) => [ diff --git a/inline.js b/inline.js index 6dee881..23489a9 100644 --- a/inline.js +++ b/inline.js @@ -312,7 +312,10 @@ async function migrateSavedObjects(subFolder) { JSON.parse(fs.readFileSync(`${visPath}/${vis}`, { encoding: "utf8" })) ); - const response = await axios.post( + let response; + + try { + response = await axios.post( `${baseUrl}/api/saved_objects/_bulk_create?overwrite=true`, visualizations.map( ({ type, id, attributes, references, migrationVersion }) => ({ @@ -330,6 +333,15 @@ async function migrateSavedObjects(subFolder) { }, } ); + } catch (e){ + // Some machines are converting localhost to IPv6 address that doesn't work with axios + // so providing some helpful message here to help debug the issue + // on the first axios call + if(/ECONNREFUSED ::1/.test(e.message)){ + console.log('Either Kibana is not running or try to use a IPv4 address (i.e. 127.0.0.1)'); + } + throw Error(e) + } if (response.data.saved_objects.some((s) => s.error)) { throw new Error(`error loading ${subFolder}`); } diff --git a/integrations b/integrations index c82487d..6533314 160000 --- a/integrations +++ b/integrations @@ -1 +1 @@ -Subproject commit c82487db8c2d788911823576207269103e960006 +Subproject commit 6533314298b3bf3fb08d31df23445a977239538b diff --git a/package.json b/package.json index 2fb974b..5ab8dbd 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,9 @@ "private": true, "dependencies": { "@elastic/elasticsearch": "^8.1.0", - "axios": "^0.26.1", - "glob": "^8.0.3", + "axios": "^1.0.0", + "glob": "8.1.0", "lodash": "^4.17.21", - "yaml": "^2.0.1" + "yaml": "2.6.1" } } diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..da77b35 --- /dev/null +++ b/renovate.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "local>elastic/renovate-config" + ], + "enabled": false +} diff --git a/yarn.lock b/yarn.lock index fb06e2c..4cc1ae9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3,31 +3,101 @@ "@elastic/elasticsearch@^8.1.0": - version "8.1.0" - resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.1.0.tgz#b84cd418e5bef4c13263c267866fa7ea742844b8" - integrity sha512-IiZ6u77C7oYYbUkx/YFgEJk6ZtP+QDI97VaUWiYD14xIdn/w9WJtmx/Y1sN8ov0nZzrWbqScB2Z7Pb8oxo7vqw== + version "8.16.2" + resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.16.2.tgz#67d75bf7f642cd7f8eea583186bb87da6e42359b" + integrity sha512-2ivc6uS97fbEeW4tNtg5mvh/Jy82ZLfcwQ1HhNhdYxyapNnQxIgZ83Zd8Ir+5jCPMDWKSYgwDb8t4GAINDDv2w== dependencies: - "@elastic/transport" "^8.0.2" - tslib "^2.3.0" + "@elastic/transport" "^8.9.1" + apache-arrow "^18.0.0" + tslib "^2.4.0" -"@elastic/transport@^8.0.2": - version "8.0.2" - resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.0.2.tgz#715f06c7739516867508108df30c33973ca8e81c" - integrity sha512-OlDz3WO3pKE9vSxW4wV/mn7rYCtBmSsDwxr64h/S1Uc/zrIBXb0iUsRMSkiybXugXhjwyjqG2n1Wc7jjFxrskQ== +"@elastic/transport@^8.9.1": + version "8.9.1" + resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.9.1.tgz#1acc090ac45903a3d5a8b7269f6ba6410108dc0a" + integrity sha512-jasKNQeOb1vNf9aEYg+8zXmetaFjApDTSCC4QTl6aTixvyiRiSLcCiB8P6Q0lY9JIII/BhqNl8WbpFnsKitntw== dependencies: - debug "^4.3.2" - hpagent "^0.1.2" + "@opentelemetry/api" "1.x" + debug "^4.3.4" + hpagent "^1.0.0" ms "^2.1.3" secure-json-parse "^2.4.0" - tslib "^2.3.0" - undici "^4.14.1" + tslib "^2.4.0" + undici "^6.12.0" + +"@opentelemetry/api@1.x": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + +"@swc/helpers@^0.5.11": + version "0.5.15" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.15.tgz#79efab344c5819ecf83a43f3f9f811fc84b516d7" + integrity sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g== + dependencies: + tslib "^2.8.0" + +"@types/command-line-args@^5.2.3": + version "5.2.3" + resolved "https://registry.yarnpkg.com/@types/command-line-args/-/command-line-args-5.2.3.tgz#553ce2fd5acf160b448d307649b38ffc60d39639" + integrity sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw== + +"@types/command-line-usage@^5.0.4": + version "5.0.4" + resolved "https://registry.yarnpkg.com/@types/command-line-usage/-/command-line-usage-5.0.4.tgz#374e4c62d78fbc5a670a0f36da10235af879a0d5" + integrity sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg== + +"@types/node@^20.13.0": + version "20.17.6" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.17.6.tgz#6e4073230c180d3579e8c60141f99efdf5df0081" + integrity sha512-VEI7OdvK2wP7XHnsuXbAJnEpEkF6NjSN45QJlL4VGqZSXsnicpesdTWsg9RISeSdYd3yeRj/y3k5KGjUXYnFwQ== + dependencies: + undici-types "~6.19.2" + +ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +apache-arrow@^18.0.0: + version "18.0.0" + resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-18.0.0.tgz#a187bd55318a7a68b6ff09835d05e89e019eba2e" + integrity sha512-gFlPaqN9osetbB83zC29AbbZqGiCuFH1vyyPseJ+B7SIbfBtESV62mMT/CkiIt77W6ykC/nTWFzTXFs0Uldg4g== + dependencies: + "@swc/helpers" "^0.5.11" + "@types/command-line-args" "^5.2.3" + "@types/command-line-usage" "^5.0.4" + "@types/node" "^20.13.0" + command-line-args "^5.2.1" + command-line-usage "^7.0.1" + flatbuffers "^24.3.25" + json-bignum "^0.0.3" + tslib "^2.6.2" + +array-back@^3.0.1, array-back@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/array-back/-/array-back-3.1.0.tgz#b8859d7a508871c9a7b2cf42f99428f65e96bfb0" + integrity sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q== + +array-back@^6.2.2: + version "6.2.2" + resolved "https://registry.yarnpkg.com/array-back/-/array-back-6.2.2.tgz#f567d99e9af88a6d3d2f9dfcc21db6f9ba9fd157" + integrity sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw== -axios@^0.26.1: - version "0.26.1" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9" - integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA== +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + +axios@^1.0.0: + version "1.7.9" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.9.tgz#d7d071380c132a24accda1b2cfc1535b79ec650a" + integrity sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw== dependencies: - follow-redirects "^1.14.8" + follow-redirects "^1.15.6" + form-data "^4.0.0" + proxy-from-env "^1.1.0" balanced-match@^1.0.0: version "1.0.2" @@ -41,27 +111,107 @@ brace-expansion@^2.0.1: dependencies: balanced-match "^1.0.0" -debug@^4.3.2: - version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== +chalk-template@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/chalk-template/-/chalk-template-0.4.0.tgz#692c034d0ed62436b9062c1707fadcd0f753204b" + integrity sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg== + dependencies: + chalk "^4.1.2" + +chalk@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + +command-line-args@^5.2.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/command-line-args/-/command-line-args-5.2.1.tgz#c44c32e437a57d7c51157696893c5909e9cec42e" + integrity sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg== + dependencies: + array-back "^3.1.0" + find-replace "^3.0.0" + lodash.camelcase "^4.3.0" + typical "^4.0.0" + +command-line-usage@^7.0.1: + version "7.0.3" + resolved "https://registry.yarnpkg.com/command-line-usage/-/command-line-usage-7.0.3.tgz#6bce992354f6af10ecea2b631bfdf0c8b3bfaea3" + integrity sha512-PqMLy5+YGwhMh1wS04mVG44oqDsgyLRSKJBdOo1bnYhMKBW65gZF1dRp2OZRhiTjgUHljy99qkO7bsctLaw35Q== + dependencies: + array-back "^6.2.2" + chalk-template "^0.4.0" + table-layout "^4.1.0" + typical "^7.1.1" + +debug@^4.3.4: + version "4.3.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" + integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== dependencies: - ms "2.1.2" + ms "^2.1.3" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + +find-replace@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/find-replace/-/find-replace-3.0.0.tgz#3e7e23d3b05167a76f770c9fbd5258b0def68c38" + integrity sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ== + dependencies: + array-back "^3.0.1" + +flatbuffers@^24.3.25: + version "24.3.25" + resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-24.3.25.tgz#e2f92259ba8aa53acd0af7844afb7c7eb95e7089" + integrity sha512-3HDgPbgiwWMI9zVB7VYBHaMrbOO7Gm0v+yD2FV/sCKj+9NDeVL7BOBYUuhWAQGKWOzBo8S9WdMvV0eixO233XQ== -follow-redirects@^1.14.8: - version "1.14.9" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7" - integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w== +follow-redirects@^1.15.6: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + +form-data@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.1.tgz#ba1076daaaa5bfd7e99c1a6cb02aa0a5cff90d48" + integrity sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -glob@^8.0.3: - version "8.0.3" - resolved "https://registry.yarnpkg.com/glob/-/glob-8.0.3.tgz#415c6eb2deed9e502c68fa44a272e6da6eeca42e" - integrity sha512-ull455NHSHI/Y1FqGaaYFaLGkNMMJbavMrEGFXG/PGrg6y7sutWHUHrz6gy6WEBH6akM1M414dWKCNs+IhKdiQ== +glob@8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/glob/-/glob-8.1.0.tgz#d388f656593ef708ee3e34640fdfb99a9fd1c33e" + integrity sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ== dependencies: fs.realpath "^1.0.0" inflight "^1.0.4" @@ -69,10 +219,15 @@ glob@^8.0.3: minimatch "^5.0.1" once "^1.3.0" -hpagent@^0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9" - integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ== +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + +hpagent@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-1.2.0.tgz#0ae417895430eb3770c03443456b8d90ca464903" + integrity sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA== inflight@^1.0.4: version "1.0.6" @@ -87,11 +242,33 @@ inherits@2: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== +json-bignum@^0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/json-bignum/-/json-bignum-0.0.3.tgz#41163b50436c773d82424dbc20ed70db7604b8d7" + integrity sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg== + +lodash.camelcase@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" + integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA== + lodash@^4.17.21: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + minimatch@^5.0.1: version "5.1.0" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.0.tgz#1717b464f4971b144f6aabe8f2d0b8e4511e09c7" @@ -99,11 +276,6 @@ minimatch@^5.0.1: dependencies: brace-expansion "^2.0.1" -ms@2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" - integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== - ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" @@ -116,27 +288,67 @@ once@^1.3.0: dependencies: wrappy "1" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + secure-json-parse@^2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85" - integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg== + version "2.7.0" + resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" + integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== + +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + +table-layout@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/table-layout/-/table-layout-4.1.1.tgz#0f72965de1a5c0c1419c9ba21cae4e73a2f73a42" + integrity sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA== + dependencies: + array-back "^6.2.2" + wordwrapjs "^5.1.0" + +tslib@^2.4.0, tslib@^2.6.2, tslib@^2.8.0: + version "2.8.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" + integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== + +typical@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/typical/-/typical-4.0.0.tgz#cbeaff3b9d7ae1e2bbfaf5a4e6f11eccfde94fc4" + integrity sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw== + +typical@^7.1.1: + version "7.3.0" + resolved "https://registry.yarnpkg.com/typical/-/typical-7.3.0.tgz#930376be344228709f134613911fa22aa09617a4" + integrity sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw== -tslib@^2.3.0: - version "2.3.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" - integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw== +undici-types@~6.19.2: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" + integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== -undici@^4.14.1: - version "4.16.0" - resolved "https://registry.yarnpkg.com/undici/-/undici-4.16.0.tgz#469bb87b3b918818d3d7843d91a1d08da357d5ff" - integrity sha512-tkZSECUYi+/T1i4u+4+lwZmQgLXd4BLGlrc7KZPcLIW7Jpq99+Xpc30ONv7nS6F5UNOxp/HBZSSL9MafUrvJbw== +undici@^6.12.0: + version "6.21.0" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.21.0.tgz#4b3d3afaef984e07b48e7620c34ed8a285ed4cd4" + integrity sha512-BUgJXc752Kou3oOIuU1i+yZZypyZRqNPW0vqoMPl8VaoalSfeR0D8/t4iAS3yirs79SSMTxTag+ZC86uswv+Cw== + +wordwrapjs@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-5.1.0.tgz#4c4d20446dcc670b14fa115ef4f8fd9947af2b3a" + integrity sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg== wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== -yaml@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.0.1.tgz#71886d6021f3da28169dbefde78d4dd0f8d83650" - integrity sha512-1NpAYQ3wjzIlMs0mgdBmYzLkFgWBIWrzYVDYfrixhoFNNgJ444/jT2kUT2sicRbJES3oQYRZugjB6Ro8SjKeFg== +yaml@2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.6.1.tgz#42f2b1ba89203f374609572d5349fb8686500773" + integrity sha512-7r0XPzioN/Q9kXBro/XPnA6kznR73DHq+GXh5ON7ZozRO6aMjbmiBuKste2wslTFkC5d1dw0GooOCepZXJ2SAg==