diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile deleted file mode 100644 index 166c030..0000000 --- a/.ci/Jenkinsfile +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -pipeline { - agent { label 'linux && immutable' } - environment { - REPO = 'ecs-logging-python' - BASE_DIR = "src/go.elastic.co/apm/${env.REPO}" - NOTIFY_TO = credentials('notify-to') - } - options { - timeout(time: 1, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) - quietPeriod(10) - } - triggers { - issueCommentTrigger("${obltGitHubComments()}") - // Only main branch will run on a timer basis - cron(env.BRANCH_NAME == 'main' ? '@daily' : '') - } - stages { - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) - deleteDir() - gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) - stash allowEmpty: true, name: 'source', useDefaultExcludes: false - } - } - stage('Lint') { - options { skipDefaultCheckout() } - steps { - withGithubNotify(context: 'Lint') { - deleteDir() - unstash 'source' - dir("${BASE_DIR}"){ - preCommit(commit: "${GIT_BASE_COMMIT}", junit: true) - } - python(version: '3.7') { - sh('.ci/scripts/lint.sh') - } - } - } - } - stage('Test') { - matrix { - agent { label 'linux && immutable' } - axes { - axis { - name 'VERSION' - values '3.6', '3.7', '3.8', '3.9', '3.10' - } - } - stages { - stage('Test') { - options { skipDefaultCheckout() } - steps { - withGithubNotify(context: "Test ${VERSION}") { - deleteDir() - unstash 'source' - python(version: VERSION) { - sh(".ci/scripts/test.sh ${VERSION}") - } - } - } - post { - always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/junit*.xml") - } - } - } - } - } - } - } - post { - cleanup { - notifyBuildResult() - } - } -} - -def python(Map v = [:], body) { - def dockerImage = "python:${v.version}" - // If dockerhub got issues then let's retry twice - retry(2) { - sleep 5 - sh "docker pull ${dockerImage}" - } - docker.image(dockerImage).inside("-e HOME=${env.WORKSPACE}"){ - dir("${BASE_DIR}"){ - body() - } - } -} diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml deleted file mode 100644 index 97dba5c..0000000 --- a/.ci/jobs/defaults.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- - -##### GLOBAL METADATA - -- meta: - cluster: apm-ci - -##### JOB DEFAULTS - -- job: - view: APM-CI - project-type: multibranch - logrotate: - daysToKeep: 30 - numToKeep: 100 - number-to-keep: '5' - days-to-keep: '1' - concurrent: true - node: linux - script-path: .ci/Jenkinsfile - scm: - - github: - branch-discovery: no-pr - discover-pr-forks-strategy: merge-current - discover-pr-forks-trust: permission - discover-pr-origin: merge-current - discover-tags: true - repo: ecs-logging-python - repo-owner: elastic - credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - build-strategies: - - tags: - ignore-tags-older-than: -1 - ignore-tags-newer-than: -1 - - regular-branches: true - - change-request: - ignore-target-only-changes: false - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 3 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - timeout: '15' - use-author: true - wipe-workspace: 'True' - periodic-folder-trigger: 1w - prune-dead-branches: true - publishers: - - email: - recipients: infra-root+build@elastic.co diff --git a/.ci/jobs/ecs-logging-python-mbp.yml b/.ci/jobs/ecs-logging-python-mbp.yml deleted file mode 100644 index 7165351..0000000 --- a/.ci/jobs/ecs-logging-python-mbp.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- job: - name: apm-agent-python/ecs-logging-python-mbp - display-name: ecs-logging-python - description: Logging formatters for the Elastic Common Schema (ECS) in Python diff --git a/.ci/update-specs.yml b/.ci/update-specs.yml new file mode 100644 index 0000000..0db7c87 --- /dev/null +++ b/.ci/update-specs.yml @@ -0,0 +1,57 @@ +--- +name: update specs + +scms: + githubConfig: + kind: github + spec: + user: '{{ requiredEnv "GIT_USER" }}' + email: '{{ requiredEnv "GIT_EMAIL" }}' + owner: elastic + repository: ecs-logging-python + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + username: '{{ requiredEnv "GIT_USER" }}' + branch: main + +actions: + ecs-logging-python: + kind: github/pullrequest + scmid: githubConfig + sourceid: sha + spec: + automerge: false + labels: + - dependencies + title: 'synchronize ecs-logging spec' + description: |- + ### What + + ECS logging specs automatic sync + + ### Why + + *Changeset* + * https://github.com/elastic/ecs-logging/commit/{{ source "sha" }} + +sources: + spec.json: + name: Get specs from json + kind: file + spec: + file: https://raw.githubusercontent.com/elastic/ecs-logging/main/spec/spec.json + + sha: + name: Get commit + kind: json + spec: + file: 'https://api.github.com/repos/elastic/ecs-logging/commits?path=spec%2Fspec.json&page=1&per_page=1' + key: ".[0].sha" + +targets: + spec.json-update: + name: 'synchronize ecs-logging spec' + kind: file + sourceid: spec.json + scmid: githubConfig + spec: + file: tests/resources/spec.json diff --git a/.github/workflows/opentelemetry.yml b/.github/workflows/opentelemetry.yml new file mode 100644 index 0000000..e91fc5b --- /dev/null +++ b/.github/workflows/opentelemetry.yml @@ -0,0 +1,19 @@ +--- +name: OpenTelemetry Export Trace + +on: + workflow_run: + workflows: + - test + - update-specs + types: [completed] + +jobs: + otel-export-trace: + runs-on: ubuntu-latest + steps: + - uses: elastic/apm-pipeline-library/.github/actions/opentelemetry@current + with: + vaultUrl: ${{ secrets.VAULT_ADDR }} + vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} + vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..8244824 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,49 @@ +name: test + +on: + # Run daily at midnight + schedule: + - cron: "0 0 * * *" + push: + branches: [ "main" ] + paths-ignore: [ '*.md', '*.asciidoc' ] + pull_request: + branches: [ "main" ] + paths-ignore: [ '*.md', '*.asciidoc' ] + +permissions: + contents: read + +jobs: + + pre-commit: + if: github.event_name != 'schedule' + name: Run pre-commit + runs-on: ubuntu-latest + steps: + - uses: elastic/apm-pipeline-library/.github/actions/pre-commit@current + + lint: + if: github.event_name != 'schedule' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: '3.7' + - run: .ci/scripts/lint.sh + + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + matrix: + python: [ '3.7', '3.8', '3.9', '3.10' ] + fail-fast: false + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - run: .ci/scripts/test.sh ${{ matrix.python }} diff --git a/.github/workflows/update-specs.yml b/.github/workflows/update-specs.yml new file mode 100644 index 0000000..ab101b6 --- /dev/null +++ b/.github/workflows/update-specs.yml @@ -0,0 +1,25 @@ +--- +# Send PRs to the subscribed ECS Agents if the spec files (JSON) are modified +name: update-specs + +on: + workflow_dispatch: + schedule: + - cron: '0 6 * * *' + +permissions: + contents: read + +jobs: + bump: + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v3 + + - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current + with: + vaultUrl: ${{ secrets.VAULT_ADDR }} + vaultRoleId: ${{ secrets.VAULT_ROLE_ID }} + vaultSecretId: ${{ secrets.VAULT_SECRET_ID }} + pipeline: ./.ci/update-specs.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 927c895..8a4ff5b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,28 +1,32 @@ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 hooks: - - id: check-case-conflict - - id: check-executables-have-shebangs - - id: check-merge-conflict -- repo: git@github.com:elastic/apm-pipeline-library + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-merge-conflict + - repo: https://github.com/elastic/apm-pipeline-library rev: current hooks: - - id: check-bash-syntax - - id: check-jenkins-pipelines - - id: check-jjbb -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.950 + - id: check-bash-syntax + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910 hooks: - - id: mypy - args: [--strict, --show-error-codes, --no-warn-unused-ignores, --implicit-reexport] -- repo: https://github.com/ambv/black - rev: 22.3.0 + - id: mypy + args: + [ + --strict, + --show-error-codes, + --no-warn-unused-ignores, + --implicit-reexport, + ] + - repo: https://github.com/psf/black + rev: 22.12.0 hooks: - - id: black + - id: black language_version: python3 -- repo: https://gitlab.com/pycqa/flake8 + - repo: https://github.com/pycqa/flake8 rev: 3.9.2 hooks: - - id: flake8 - exclude: tests|conftest.py|setup.py \ No newline at end of file + - id: flake8 + exclude: tests|conftest.py|setup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 983c659..03ef660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 2.0.2 (2023-05-17) + +- Allow flit-core 3+ ([#94](https://github.com/elastic/ecs-logging-python/pull/94)) +- Remove python2 leftovers ([#94](https://github.com/elastic/ecs-logging-python/pull/94)) + ## 2.0.0 (2022-05-18) - Remove python 2 support ([#78](https://github.com/elastic/ecs-logging-python/pull/78)) diff --git a/README.md b/README.md index a53e02c..e7217e4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ecs-logging-python -[![Build Status](https://apm-ci.elastic.co/job/apm-agent-python/job/ecs-logging-python-mbp/job/main/badge/icon)](https://apm-ci.elastic.co/blue/organizations/jenkins/apm-agent-python%2Fecs-logging-python-mbp/branches) +[![Build Status](https://github.com/elastic/ecs-logging-python/actions/workflows/test.yml/badge.svg)](https://github.com/elastic/ecs-logging-pythonactions/workflows/test.yml) [![PyPI](https://img.shields.io/pypi/v/ecs-logging)](https://pypi.org/project/ecs-logging) [![Versions Supported](https://img.shields.io/pypi/pyversions/ecs-logging)](https://pypi.org/project/ecs-logging) diff --git a/docs/setup.asciidoc b/docs/setup.asciidoc index 520dd17..a4d26f1 100644 --- a/docs/setup.asciidoc +++ b/docs/setup.asciidoc @@ -112,6 +112,9 @@ formatter = StdlibFormatter( [[structlog]] === Structlog Example +Note that the structlog processor should be the last processor in the list, +as it handles the conversion to JSON as well as the ECS field enrichment. + [source,python] ---- import structlog diff --git a/ecs_logging/__init__.py b/ecs_logging/__init__.py index 2f22c90..bb274a9 100644 --- a/ecs_logging/__init__.py +++ b/ecs_logging/__init__.py @@ -20,7 +20,7 @@ from ._stdlib import StdlibFormatter from ._structlog import StructlogFormatter -__version__ = "2.0.0" +__version__ = "2.0.2" __all__ = [ "ECS_VERSION", "StdlibFormatter", diff --git a/pyproject.toml b/pyproject.toml index 5c97f24..90f1b50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["flit_core >=2,<3"] +requires = ["flit_core >=2,<4"] build-backend = "flit_core.buildapi" [tool.flit.metadata] @@ -12,7 +12,6 @@ home-page = "https://github.com/elastic/ecs-logging-python" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", diff --git a/tests/resources/spec.json b/tests/resources/spec.json index a082d75..68f32d7 100644 --- a/tests/resources/spec.json +++ b/tests/resources/spec.json @@ -42,7 +42,12 @@ "ecs.version": { "type": "string", "required": true, - "url": "https://www.elastic.co/guide/en/ecs/current/ecs-ecs.html" + "top_level_field": true, + "url": "https://www.elastic.co/guide/en/ecs/current/ecs-ecs.html", + "comment": [ + "This field SHOULD NOT be a nested object field but at the top level with a dot in the property name.", + "This is to make the JSON logs more human-readable." + ] }, "labels": { "type": "object",