Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5598d3b

Browse files
Linu-EliasLinu-Elias
authored
Merge branch 'main' into processor-package1
2 parents c2d4372 + c44dba8 commit 5598d3b

File tree

452 files changed

+112145
-6885
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

452 files changed

+112145
-6885
lines changed

‎.buildkite/hooks/post-checkout‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
11
#!/bin/bash
22

3+
# ******************************* WARNING ******************************************
4+
# This post-checkout hook is not the same as in the rest of repositories e.g. beats
5+
# because some steps in this pipeline (in PR context) take a very long time and we
6+
# want to make sure that THE SAME COMMIT FROM TARGET BRANCH gets merged in every
7+
# pipeline step. Otherwise, HEAD (target branch) may have changed in the meantime
8+
# and therefore, some steps (e.g. sonarqube) may end up testing a different commit.
9+
#
10+
# Running builds from branches or tags (out of PR context) maintains the same behavior
11+
# as in the rest of the repositories.
12+
#
13+
# Reference: https://github.com/elastic/integrations/pull/10397
14+
# **********************************************************************************
15+
316
set -euo pipefail
417

518
checkout_merge() {
@@ -13,8 +26,15 @@ checkout_merge() {
1326
fi
1427

1528
git fetch -v origin "${target_branch}"
16-
git checkout FETCH_HEAD
17-
echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
29+
if [[ ${REPOSITORY_TARGET_BRANCH_COMMIT} == "" ]]; then
30+
git checkout FETCH_HEAD
31+
echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
32+
else
33+
# Use the same commit from target branch as in the other steps.
34+
echo "Retrieved commit from meta-data: ${REPOSITORY_TARGET_BRANCH_COMMIT}"
35+
git checkout "${REPOSITORY_TARGET_BRANCH_COMMIT}"
36+
echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
37+
fi
1838

1939
# create temporal branch to merge the PR with the target branch
2040
git checkout -b ${merge_branch}
@@ -44,6 +64,9 @@ PR_COMMIT="${BUILDKITE_COMMIT}"
4464
PR_ID=${BUILDKITE_PULL_REQUEST}
4565
MERGE_BRANCH="pr_merge_${PR_ID}"
4666

67+
# This meta-data field is populated in the pre-command hook
68+
REPOSITORY_TARGET_BRANCH_COMMIT=$(buildkite-agent meta-data get "REPOSITORY_TARGET_BRANCH_COMMIT" --default "")
69+
4770
checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
4871

4972
echo "Commit information"

‎.buildkite/hooks/pre-command‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
2929
REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
3030
export REPO_BUILD_TAG
3131

32-
AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_account_auth
32+
AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
3333
PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
3434

3535
BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
@@ -74,6 +74,17 @@ if [ -n "${ELASTIC_PACKAGE_LINKS_FILE_PATH+x}" ]; then
7474
export ELASTIC_PACKAGE_LINKS_FILE_PATH=${BASE_DIR}/${ELASTIC_PACKAGE_LINKS_FILE_PATH}
7575
fi
7676

77+
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" && "${BUILDKITE_STEP_KEY}" == "reference-target-branch" ]]; then
78+
# Get the commit from target branch in the first step (reference-target-branch).
79+
# This step MUST be the first one and not run in parallel with any other step to ensure
80+
# that there is just one value for this variable
81+
if is_pr ; then
82+
git fetch -v origin ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
83+
commit_main=$(git rev-parse --verify FETCH_HEAD)
84+
buildkite-agent meta-data set "REPOSITORY_TARGET_BRANCH_COMMIT" "${commit_main}"
85+
fi
86+
fi
87+
7788
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then
7889
if [[ "${BUILDKITE_STEP_KEY}" == "trigger-publish" ]]; then
7990
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token ${BUILDKITE_API_TOKEN_PATH})
@@ -115,19 +126,13 @@ fi
115126

116127
if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
117128
if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
118-
ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
119-
export ELASTIC_PACKAGE_AWS_SECRET_KEY
120-
ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
121-
export ELASTIC_PACKAGE_AWS_ACCESS_KEY
129+
# Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
130+
# add the AWS credentials as in the integrations pipeline.
122131

123132
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
124133
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
125134
export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
126135

127-
# Environment variables required by the service deployer
128-
export AWS_SECRET_ACCESS_KEY=${ELASTIC_PACKAGE_AWS_SECRET_KEY}
129-
export AWS_ACCESS_KEY_ID=${ELASTIC_PACKAGE_AWS_ACCESS_KEY}
130-
131136
BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
132137
export BUILDKITE_API_TOKEN
133138

‎.buildkite/pipeline.schedule-daily.yml‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,8 @@ steps:
2020
build:
2121
env:
2222
SERVERLESS: "false"
23-
SKIP_PUBLISHING: "true"
2423
FORCE_CHECK_ALL: "true"
25-
# STACK_VERSION: 7.17-SNAPSHOT # Using 7.17.19 till https://github.com/elastic/fleet-server/issues/3435 is solved.
26-
STACK_VERSION: 7.17.19
24+
STACK_VERSION: 7.17.23-SNAPSHOT
2725
depends_on:
2826
- step: "check"
2927
allow_failure: false
@@ -33,7 +31,6 @@ steps:
3331
build:
3432
env:
3533
SERVERLESS: "false"
36-
SKIP_PUBLISHING: "true"
3734
FORCE_CHECK_ALL: "true"
3835
STACK_VERSION: 8.15.0-SNAPSHOT
3936
PUBLISH_COVERAGE_REPORTS: "true"

‎.buildkite/pipeline.serverless.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ env:
2222
ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
2323
# Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
2424
ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
25+
# Enable independent Elastic Agents for all packages
26+
ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
27+
# Set maximum number of parallel tests to run if package allows it
28+
ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
2529

2630
steps:
2731
- input: "Input values for the variables"

‎.buildkite/pipeline.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,28 @@ env:
2020
ELASTIC_PACKAGE_LINKS_FILE_PATH: "links_table.yml"
2121
# Disable comparison of results in pipeline tests to avoid errors related to GeoIP fields
2222
ELASTIC_PACKAGE_SERVERLESS_PIPELINE_TEST_DISABLE_COMPARE_RESULTS: "true"
23+
# Enable independent Elastic Agents for all packages
24+
ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
25+
# Set maximum number of parallel tests to run if package allows it
26+
ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
2327

2428
steps:
29+
- label: "Get reference from target branch"
30+
key: "reference-target-branch"
31+
command: "echo 'Get reference from main'"
32+
agents:
33+
image: "${LINUX_AGENT_IMAGE}"
34+
2535
- label: ":white_check_mark: Check go sources"
2636
key: "check"
2737
command: ".buildkite/scripts/check_sources.sh"
2838
agents:
2939
image: "${LINUX_AGENT_IMAGE}"
3040
cpu: "8"
3141
memory: "4G"
42+
depends_on:
43+
- step: "reference-target-branch"
44+
allow_failure: false
3245

3346
- label: "Trigger integrations"
3447
key: "test-integrations"

‎.buildkite/scripts/build_packages.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ build_packages() {
8282
}
8383

8484
if [ "${SKIP_PUBLISHING}" == "true" ] ; then
85-
echo "packageStoragePublish: skipping because skip_publishing param is ${SKIP_PUBLISHING}"
85+
echo "packageStoragePublish: skipping because SKIP_PUBLISHING environment variable is ${SKIP_PUBLISHING}"
8686
exit 0
8787
fi
8888

‎.buildkite/scripts/common.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ is_pr_affected() {
692692

693693
echo "[${package}] git-diff: check non-package files"
694694
commit_merge=$(git merge-base "${from}" "${to}")
695-
if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|.github/CODEOWNERS|docs/)' ; then
695+
if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|.github/CODEOWNERS|README.md|docs/)' ; then
696696
echo "[${package}] PR is affected: found non-package files"
697697
return 0
698698
fi
@@ -819,6 +819,9 @@ test_package_in_serverless() {
819819
if ! ${ELASTIC_PACKAGE_BIN} test pipeline ${TEST_OPTIONS} ; then
820820
return 1
821821
fi
822+
if ! ${ELASTIC_PACKAGE_BIN} test policy ${TEST_OPTIONS} ${COVERAGE_OPTIONS}; then
823+
return 1
824+
fi
822825
echo ""
823826
return 0
824827
}

‎.github/CODEOWNERS‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
/packages/aws/changelog.yml @elastic/obs-ds-hosted-services @elastic/security-service-integrations @elastic/obs-infraobs-integrations
2828
/packages/aws/data_stream/apigateway_logs @elastic/obs-infraobs-integrations
2929
/packages/aws/data_stream/apigateway_metrics @elastic/obs-infraobs-integrations
30+
/packages/aws/data_stream/awshealth @elastic/obs-infraobs-integrations
3031
/packages/aws/data_stream/billing @elastic/obs-infraobs-integrations
3132
/packages/aws/data_stream/cloudfront_logs @elastic/obs-infraobs-integrations
3233
/packages/aws/data_stream/cloudtrail @elastic/obs-infraobs-integrations
@@ -50,14 +51,22 @@
5051
/packages/aws/data_stream/s3_daily_storage @elastic/obs-ds-hosted-services
5152
/packages/aws/data_stream/s3_request @elastic/obs-ds-hosted-services
5253
/packages/aws/data_stream/s3_storage_lens @elastic/obs-infraobs-integrations
53-
/packages/aws/data_stream/s3access @elastic/obs-ds-hosted-services @elastic/security-service-integrations
54+
/packages/aws/data_stream/s3access @elastic/obs-ds-hosted-services
5455
/packages/aws/data_stream/securityhub_findings @elastic/security-service-integrations
5556
/packages/aws/data_stream/securityhub_insights @elastic/security-service-integrations
5657
/packages/aws/data_stream/sns @elastic/obs-infraobs-integrations
5758
/packages/aws/data_stream/sqs @elastic/obs-infraobs-integrations
5859
/packages/aws/data_stream/transitgateway @elastic/obs-infraobs-integrations
5960
/packages/aws/data_stream/usage @elastic/obs-infraobs-integrations
6061
/packages/aws/data_stream/vpn @elastic/obs-infraobs-integrations
62+
/packages/aws/data_stream/firewall_logs @elastic/security-service-integrations
63+
/packages/aws/data_stream/firewall_metrics @elastic/security-service-integrations
64+
/packages/aws/data_stream/inspector @elastic/security-service-integrations
65+
/packages/aws/data_stream/kafka_metrics @elastic/obs-infraobs-integrations
66+
/packages/aws/data_stream/route53_public_logs @elastic/security-service-integrations
67+
/packages/aws/data_stream/route53_resolver_logs @elastic/security-service-integrations
68+
/packages/aws/data_stream/vpcflow @elastic/security-service-integrations
69+
/packages/aws/data_stream/waf @elastic/security-service-integrations
6170
/packages/aws/kibana @elastic/obs-ds-hosted-services @elastic/kibana-visualizations @elastic/obs-infraobs-integrations
6271
/packages/aws/manifest.yml @elastic/obs-ds-hosted-services @elastic/security-service-integrations @elastic/obs-infraobs-integrations
6372
/packages/aws_bedrock @elastic/security-service-integrations
@@ -75,6 +84,7 @@
7584
/packages/azure/data_stream/springcloudlogs @elastic/obs-infraobs-integrations
7685
/packages/azure/data_stream/application_gateway @elastic/security-service-integrations
7786
/packages/azure/data_stream/firewall_logs @elastic/security-service-integrations
87+
/packages/azure/data_stream/graphactivitylogs @elastic/security-service-integrations
7888
/packages/azure_app_service @elastic/obs-infraobs-integrations
7989
/packages/azure_app_service/data_stream/app_service_logs @elastic/obs-infraobs-integrations
8090
/packages/azure_application_insights @elastic/obs-infraobs-integrations
@@ -278,6 +288,8 @@
278288
/packages/problemchild @elastic/ml-ui @elastic/sec-applied-ml
279289
/packages/prometheus @elastic/obs-infraobs-integrations
280290
/packages/prometheus/data_stream/remote_write @elastic/obs-cloudnative-monitoring
291+
/packages/prometheus/data_stream/collector @elastic/obs-infraobs-integrations
292+
/packages/prometheus/data_stream/query @elastic/obs-infraobs-integrations
281293
/packages/prometheus_input @elastic/obs-infraobs-integrations
282294
/packages/proofpoint_tap @elastic/security-service-integrations
283295
/packages/pulse_connect_secure @elastic/security-service-integrations
@@ -315,7 +327,22 @@
315327
/packages/system @elastic/obs-infraobs-integrations
316328
/packages/system/changelog.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
317329
/packages/system/data_stream/auth @elastic/sec-windows-platform
318-
/packages/system/data_stream/security @elastic/sec-linux-platform @elastic/sec-windows-platform
330+
/packages/system/data_stream/security @elastic/sec-windows-platform
331+
/packages/system/data_stream/application @elastic/sec-windows-platform
332+
/packages/system/data_stream/core @elastic/obs-infraobs-integrations
333+
/packages/system/data_stream/cpu @elastic/obs-infraobs-integrations
334+
/packages/system/data_stream/diskio @elastic/obs-infraobs-integrations
335+
/packages/system/data_stream/filesystem @elastic/obs-infraobs-integrations
336+
/packages/system/data_stream/fsstat @elastic/obs-infraobs-integrations
337+
/packages/system/data_stream/load @elastic/obs-infraobs-integrations
338+
/packages/system/data_stream/memory @elastic/obs-infraobs-integrations
339+
/packages/system/data_stream/network @elastic/obs-infraobs-integrations
340+
/packages/system/data_stream/process @elastic/obs-infraobs-integrations
341+
/packages/system/data_stream/process_summary @elastic/obs-infraobs-integrations
342+
/packages/system/data_stream/socket_summary @elastic/obs-infraobs-integrations
343+
/packages/system/data_stream/syslog @elastic/obs-infraobs-integrations
344+
/packages/system/data_stream/system @elastic/obs-infraobs-integrations
345+
/packages/system/data_stream/uptime @elastic/obs-infraobs-integrations
319346
/packages/system/manifest.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
320347
/packages/system_audit @elastic/sec-linux-platform
321348
/packages/tanium @elastic/security-service-integrations
@@ -354,6 +381,7 @@
354381
/packages/universal_profiling_symbolizer @elastic/obs-ds-intake-services
355382
/packages/vectra_detect @elastic/security-service-integrations
356383
/packages/vsphere @elastic/obs-infraobs-integrations
384+
/packages/watchguard_firebox @elastic/sec-deployment-and-devices
357385
/packages/websphere_application_server @elastic/obs-infraobs-integrations
358386
/packages/windows @elastic/elastic-agent-data-plane @elastic/sec-windows-platform
359387
/packages/windows/data_stream/applocker_exe_and_dll @elastic/sec-windows-platform
@@ -366,6 +394,7 @@
366394
/packages/windows/data_stream/powershell_operational @elastic/sec-windows-platform
367395
/packages/windows/data_stream/service @elastic/elastic-agent-data-plane
368396
/packages/windows/data_stream/sysmon_operational @elastic/sec-windows-platform
397+
/packages/windows/data_stream/windows_defender @elastic/sec-windows-platform
369398
/packages/windows_etw @elastic/sec-windows-platform
370399
/packages/winlog @elastic/sec-windows-platform
371400
/packages/wiz @elastic/security-service-integrations

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ While this repository contains _sources_ for Elastic Integrations, _built_ Elast
1515
Please review the [Contributing Guide](CONTRIBUTING.md) to learn how to build and develop packages, understand the release procedure and
1616
explore the builder tools.
1717

18+
More information about the CI pipelines that are available in this repostiory [here](./docs/ci_pipelines.md).
19+
1820
## External links
1921

2022
### Package Specification

0 commit comments

Comments
 (0)