From 17630445977daa870fce8467e545c55717da0fcc Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Mon, 17 Mar 2025 12:12:22 -0500 Subject: [PATCH 1/4] Modify context path of OTEL transformers As of 0.119.0, the parser collector modifies certain paths of transforms. While this is not an error, the resulting log can look a little alarming, so we rewrote some of our transforms to match. Issues: [PGO-2268] --- .../generated/pgbackrest_logs_transforms.json | 2 +- .../generated/postgres_logs_transforms.json | 2 +- internal/collector/patroni.go | 20 +- internal/collector/patroni_test.go | 50 +-- internal/collector/pgadmin.go | 24 +- internal/collector/pgadmin_test.go | 58 ++-- .../collector/pgbackrest_logs_transforms.yaml | 30 +- internal/collector/pgbackrest_test.go | 66 ++-- internal/collector/pgbouncer.go | 22 +- internal/collector/pgbouncer_test.go | 56 ++-- .../collector/postgres_logs_transforms.yaml | 132 ++++---- internal/collector/postgres_test.go | 312 ++++++++++-------- 12 files changed, 430 insertions(+), 344 deletions(-) diff --git a/internal/collector/generated/pgbackrest_logs_transforms.json b/internal/collector/generated/pgbackrest_logs_transforms.json index adf3b09af..32b3809c0 100644 --- a/internal/collector/generated/pgbackrest_logs_transforms.json +++ b/internal/collector/generated/pgbackrest_logs_transforms.json @@ -1 +1 @@ -[{"context":"log","statements":["set(instrumentation_scope.name, \"pgbackrest\")","set(instrumentation_scope.schema_url, \"https://opentelemetry.io/schemas/1.29.0\")","merge_maps(cache, ExtractPatterns(body, \"^(?\u003ctimestamp\u003e\\\\d{4}-\\\\d{2}-\\\\d{2} \\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}) (?\u003cprocess_id\u003eP\\\\d{2,3})\\\\s*(?\u003cerror_severity\u003e\\\\S*): (?\u003cmessage\u003e(?s).*)$\"), \"insert\") where Len(body) \u003e 0","set(severity_text, cache[\"error_severity\"]) where IsString(cache[\"error_severity\"])","set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == \"TRACE\"","set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == \"DEBUG\"","set(severity_number, SEVERITY_NUMBER_DEBUG2) where severity_text == \"DETAIL\"","set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == \"INFO\"","set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == \"WARN\"","set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == \"ERROR\"","set(time, Time(cache[\"timestamp\"], \"%Y-%m-%d %H:%M:%S.%L\")) where IsString(cache[\"timestamp\"])","set(attributes[\"process.pid\"], cache[\"process_id\"])","set(attributes[\"log.record.original\"], body)","set(body, cache[\"message\"])"]}] +[{"context":"log","statements":["set(instrumentation_scope.name, \"pgbackrest\")","set(instrumentation_scope.schema_url, \"https://opentelemetry.io/schemas/1.29.0\")","merge_maps(log.cache, ExtractPatterns(log.body, \"^(?\u003ctimestamp\u003e\\\\d{4}-\\\\d{2}-\\\\d{2} \\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}) (?\u003cprocess_id\u003eP\\\\d{2,3})\\\\s*(?\u003cerror_severity\u003e\\\\S*): (?\u003cmessage\u003e(?s).*)$\"), \"insert\") where Len(log.body) \u003e 0","set(log.severity_text, log.cache[\"error_severity\"]) where IsString(log.cache[\"error_severity\"])","set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == \"TRACE\"","set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == \"DEBUG\"","set(log.severity_number, SEVERITY_NUMBER_DEBUG2) where log.severity_text == \"DETAIL\"","set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == \"INFO\"","set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == \"WARN\"","set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == \"ERROR\"","set(log.time, Time(log.cache[\"timestamp\"], \"%Y-%m-%d %H:%M:%S.%L\")) where IsString(log.cache[\"timestamp\"])","set(log.attributes[\"process.pid\"], log.cache[\"process_id\"])","set(log.attributes[\"log.record.original\"], log.body)","set(log.body, log.cache[\"message\"])"]}] diff --git a/internal/collector/generated/postgres_logs_transforms.json b/internal/collector/generated/postgres_logs_transforms.json index d3a2dbe47..5071e4399 100644 --- a/internal/collector/generated/postgres_logs_transforms.json +++ b/internal/collector/generated/postgres_logs_transforms.json @@ -1 +1 @@ -[{"conditions":["body[\"format\"] == \"csv\""],"context":"log","statements":["set(cache, ParseCSV(body[\"original\"], body[\"headers\"], delimiter=\",\", mode=\"strict\"))","merge_maps(cache, ExtractPatterns(cache[\"connection_from\"], \"(?:^[[]local[]]:(?\u003cremote_port\u003e.+)|:(?\u003cremote_port\u003e[^:]+))$\"), \"insert\") where Len(cache[\"connection_from\"]) \u003e 0","set(cache[\"remote_host\"], Substring(cache[\"connection_from\"], 0, Len(cache[\"connection_from\"]) - Len(cache[\"remote_port\"]) - 1)) where Len(cache[\"connection_from\"]) \u003e 0 and IsString(cache[\"remote_port\"])","set(cache[\"remote_host\"], cache[\"connection_from\"]) where Len(cache[\"connection_from\"]) \u003e 0 and not IsString(cache[\"remote_host\"])","merge_maps(cache, ExtractPatterns(cache[\"location\"], \"^(?:(?\u003cfunc_name\u003e[^,]+), )?(?\u003cfile_name\u003e[^:]+):(?\u003cfile_line_num\u003e\\\\d+)$\"), \"insert\") where Len(cache[\"location\"]) \u003e 0","set(cache[\"cursor_position\"], Double(cache[\"cursor_position\"])) where IsMatch(cache[\"cursor_position\"], \"^[0-9.]+$\")","set(cache[\"file_line_num\"], Double(cache[\"file_line_num\"])) where IsMatch(cache[\"file_line_num\"], \"^[0-9.]+$\")","set(cache[\"internal_position\"], Double(cache[\"internal_position\"])) where IsMatch(cache[\"internal_position\"], \"^[0-9.]+$\")","set(cache[\"leader_pid\"], Double(cache[\"leader_pid\"])) where IsMatch(cache[\"leader_pid\"], \"^[0-9.]+$\")","set(cache[\"line_num\"], Double(cache[\"line_num\"])) where IsMatch(cache[\"line_num\"], \"^[0-9.]+$\")","set(cache[\"pid\"], Double(cache[\"pid\"])) where IsMatch(cache[\"pid\"], \"^[0-9.]+$\")","set(cache[\"query_id\"], Double(cache[\"query_id\"])) where IsMatch(cache[\"query_id\"], \"^[0-9.]+$\")","set(cache[\"remote_port\"], Double(cache[\"remote_port\"])) where IsMatch(cache[\"remote_port\"], \"^[0-9.]+$\")","set(body[\"parsed\"], cache)"]},{"context":"log","statements":["set(instrumentation_scope.name, \"postgres\")","set(instrumentation_scope.version, resource.attributes[\"db.version\"])","set(cache, body[\"parsed\"]) where body[\"format\"] == \"csv\"","set(cache, ParseJSON(body[\"original\"])) where body[\"format\"] == \"json\"","set(severity_text, cache[\"error_severity\"])","set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == \"DEBUG5\"","set(severity_number, SEVERITY_NUMBER_TRACE2) where severity_text == \"DEBUG4\"","set(severity_number, SEVERITY_NUMBER_TRACE3) where severity_text == \"DEBUG3\"","set(severity_number, SEVERITY_NUMBER_TRACE4) where severity_text == \"DEBUG2\"","set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == \"DEBUG1\"","set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == \"INFO\" or severity_text == \"LOG\"","set(severity_number, SEVERITY_NUMBER_INFO2) where severity_text == \"NOTICE\"","set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == \"WARNING\"","set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == \"ERROR\"","set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == \"FATAL\"","set(severity_number, SEVERITY_NUMBER_FATAL2) where severity_text == \"PANIC\"","set(time, Time(cache[\"timestamp\"], \"%F %T.%L %Z\"))","set(instrumentation_scope.schema_url, \"https://opentelemetry.io/schemas/1.29.0\")","set(resource.attributes[\"db.system\"], \"postgresql\")","set(attributes[\"log.record.original\"], body[\"original\"])","set(body, cache)","set(attributes[\"client.address\"], body[\"remote_host\"]) where IsString(body[\"remote_host\"])","set(attributes[\"client.port\"], Int(body[\"remote_port\"])) where IsDouble(body[\"remote_port\"])","set(attributes[\"code.filepath\"], body[\"file_name\"]) where IsString(body[\"file_name\"])","set(attributes[\"code.function\"], body[\"func_name\"]) where IsString(body[\"func_name\"])","set(attributes[\"code.lineno\"], Int(body[\"file_line_num\"])) where IsDouble(body[\"file_line_num\"])","set(attributes[\"db.namespace\"], body[\"dbname\"]) where IsString(body[\"dbname\"])","set(attributes[\"db.response.status_code\"], body[\"state_code\"]) where IsString(body[\"state_code\"])","set(attributes[\"process.creation.time\"], Concat([ Substring(body[\"session_start\"], 0, 10), \"T\", Substring(body[\"session_start\"], 11, 8), \"Z\"], \"\")) where IsMatch(body[\"session_start\"], \"^[^ ]{10} [^ ]{8} UTC$\")","set(attributes[\"process.pid\"], Int(body[\"pid\"])) where IsDouble(body[\"pid\"])","set(attributes[\"process.title\"], body[\"ps\"]) where IsString(body[\"ps\"])","set(attributes[\"user.name\"], body[\"user\"]) where IsString(body[\"user\"])"]},{"conditions":["Len(body[\"message\"]) \u003e 7 and Substring(body[\"message\"], 0, 7) == \"AUDIT: \""],"context":"log","statements":["set(body[\"pgaudit\"], ParseCSV(Substring(body[\"message\"], 7, Len(body[\"message\"]) - 7), \"audit_type,statement_id,substatement_id,class,command,object_type,object_name,statement,parameter\", delimiter=\",\", mode=\"strict\"))","set(instrumentation_scope.name, \"pgaudit\") where Len(body[\"pgaudit\"]) \u003e 0"]}] +[{"conditions":["body[\"format\"] == \"csv\""],"context":"log","statements":["set(log.cache, ParseCSV(log.body[\"original\"], log.body[\"headers\"], delimiter=\",\", mode=\"strict\"))","merge_maps(log.cache, ExtractPatterns(log.cache[\"connection_from\"], \"(?:^[[]local[]]:(?\u003cremote_port\u003e.+)|:(?\u003cremote_port\u003e[^:]+))$\"), \"insert\") where Len(log.cache[\"connection_from\"]) \u003e 0","set(log.cache[\"remote_host\"], Substring(log.cache[\"connection_from\"], 0, Len(log.cache[\"connection_from\"]) - Len(log.cache[\"remote_port\"]) - 1)) where Len(log.cache[\"connection_from\"]) \u003e 0 and IsString(log.cache[\"remote_port\"])","set(log.cache[\"remote_host\"], log.cache[\"connection_from\"]) where Len(log.cache[\"connection_from\"]) \u003e 0 and not IsString(log.cache[\"remote_host\"])","merge_maps(log.cache, ExtractPatterns(log.cache[\"location\"], \"^(?:(?\u003cfunc_name\u003e[^,]+), )?(?\u003cfile_name\u003e[^:]+):(?\u003cfile_line_num\u003e\\\\d+)$\"), \"insert\") where Len(log.cache[\"location\"]) \u003e 0","set(log.cache[\"cursor_position\"], Double(log.cache[\"cursor_position\"])) where IsMatch(log.cache[\"cursor_position\"], \"^[0-9.]+$\")","set(log.cache[\"file_line_num\"], Double(log.cache[\"file_line_num\"])) where IsMatch(log.cache[\"file_line_num\"], \"^[0-9.]+$\")","set(log.cache[\"internal_position\"], Double(log.cache[\"internal_position\"])) where IsMatch(log.cache[\"internal_position\"], \"^[0-9.]+$\")","set(log.cache[\"leader_pid\"], Double(log.cache[\"leader_pid\"])) where IsMatch(log.cache[\"leader_pid\"], \"^[0-9.]+$\")","set(log.cache[\"line_num\"], Double(log.cache[\"line_num\"])) where IsMatch(log.cache[\"line_num\"], \"^[0-9.]+$\")","set(log.cache[\"pid\"], Double(log.cache[\"pid\"])) where IsMatch(log.cache[\"pid\"], \"^[0-9.]+$\")","set(log.cache[\"query_id\"], Double(log.cache[\"query_id\"])) where IsMatch(log.cache[\"query_id\"], \"^[0-9.]+$\")","set(log.cache[\"remote_port\"], Double(log.cache[\"remote_port\"])) where IsMatch(log.cache[\"remote_port\"], \"^[0-9.]+$\")","set(log.body[\"parsed\"], log.cache)"]},{"context":"log","statements":["set(instrumentation_scope.name, \"postgres\")","set(instrumentation_scope.version, resource.attributes[\"db.version\"])","set(log.cache, log.body[\"parsed\"]) where log.body[\"format\"] == \"csv\"","set(log.cache, ParseJSON(log.body[\"original\"])) where log.body[\"format\"] == \"json\"","set(log.severity_text, log.cache[\"error_severity\"])","set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == \"DEBUG5\"","set(log.severity_number, SEVERITY_NUMBER_TRACE2) where log.severity_text == \"DEBUG4\"","set(log.severity_number, SEVERITY_NUMBER_TRACE3) where log.severity_text == \"DEBUG3\"","set(log.severity_number, SEVERITY_NUMBER_TRACE4) where log.severity_text == \"DEBUG2\"","set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == \"DEBUG1\"","set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == \"INFO\" or log.severity_text == \"LOG\"","set(log.severity_number, SEVERITY_NUMBER_INFO2) where log.severity_text == \"NOTICE\"","set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == \"WARNING\"","set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == \"ERROR\"","set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == \"FATAL\"","set(log.severity_number, SEVERITY_NUMBER_FATAL2) where log.severity_text == \"PANIC\"","set(log.time, Time(log.cache[\"timestamp\"], \"%F %T.%L %Z\")) where IsString(log.cache[\"timestamp\"])","set(instrumentation_scope.schema_url, \"https://opentelemetry.io/schemas/1.29.0\")","set(resource.attributes[\"db.system\"], \"postgresql\")","set(log.attributes[\"log.record.original\"], log.body[\"original\"])","set(log.body, log.cache)","set(log.attributes[\"client.address\"], log.body[\"remote_host\"]) where IsString(log.body[\"remote_host\"])","set(log.attributes[\"client.port\"], Int(log.body[\"remote_port\"])) where IsDouble(log.body[\"remote_port\"])","set(log.attributes[\"code.filepath\"], log.body[\"file_name\"]) where IsString(log.body[\"file_name\"])","set(log.attributes[\"code.function\"], log.body[\"func_name\"]) where IsString(log.body[\"func_name\"])","set(log.attributes[\"code.lineno\"], Int(log.body[\"file_line_num\"])) where IsDouble(log.body[\"file_line_num\"])","set(log.attributes[\"db.namespace\"], log.body[\"dbname\"]) where IsString(log.body[\"dbname\"])","set(log.attributes[\"db.response.status_code\"], log.body[\"state_code\"]) where IsString(log.body[\"state_code\"])","set(log.attributes[\"process.creation.time\"], Concat([ Substring(log.body[\"session_start\"], 0, 10), \"T\", Substring(log.body[\"session_start\"], 11, 8), \"Z\"], \"\")) where IsMatch(log.body[\"session_start\"], \"^[^ ]{10} [^ ]{8} UTC$\")","set(log.attributes[\"process.pid\"], Int(log.body[\"pid\"])) where IsDouble(log.body[\"pid\"])","set(log.attributes[\"process.title\"], log.body[\"ps\"]) where IsString(log.body[\"ps\"])","set(log.attributes[\"user.name\"], log.body[\"user\"]) where IsString(log.body[\"user\"])"]},{"conditions":["Len(body[\"message\"]) \u003e 7 and Substring(body[\"message\"], 0, 7) == \"AUDIT: \""],"context":"log","statements":["set(log.body[\"pgaudit\"], ParseCSV(Substring(log.body[\"message\"], 7, Len(log.body[\"message\"]) - 7), \"audit_type,statement_id,substatement_id,class,command,object_type,object_name,statement,parameter\", delimiter=\",\", mode=\"strict\"))","set(instrumentation_scope.name, \"pgaudit\") where Len(log.body[\"pgaudit\"]) \u003e 0"]}] diff --git a/internal/collector/patroni.go b/internal/collector/patroni.go index 6b22df6a0..991bb9882 100644 --- a/internal/collector/patroni.go +++ b/internal/collector/patroni.go @@ -69,11 +69,11 @@ func EnablePatroniLogging(ctx context.Context, `set(instrumentation_scope.name, "patroni")`, // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/ottlfuncs#parsejson - `set(cache, ParseJSON(body["original"]))`, + `set(log.cache, ParseJSON(log.body["original"]))`, // The log severity is in the "levelname" field. // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext - `set(severity_text, cache["levelname"])`, + `set(log.severity_text, log.cache["levelname"])`, // Map Patroni (python) "logging levels" to OpenTelemetry severity levels. // @@ -81,11 +81,11 @@ func EnablePatroniLogging(ctx context.Context, // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber // https://github.com/open-telemetry/opentelemetry-python/blob/v1.29.0/opentelemetry-api/src/opentelemetry/_logs/severity/__init__.py // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#enums - `set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG"`, - `set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO"`, - `set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING"`, - `set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR"`, - `set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL"`, + `set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == "DEBUG"`, + `set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == "INFO"`, + `set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == "WARNING"`, + `set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == "ERROR"`, + `set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == "CRITICAL"`, // Parse the "asctime" field into the record timestamp. // The format is neither RFC 3339 nor ISO 8601: @@ -95,14 +95,14 @@ func EnablePatroniLogging(ctx context.Context, // // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/stanza/docs/types/timestamp.md // https://docs.python.org/3.6/library/logging.html#logging.LogRecord - `set(time, Time(cache["asctime"], "%F %T,%L"))`, + `set(log.time, Time(log.cache["asctime"], "%F %T,%L")) where IsString(log.cache["asctime"])`, // Keep the unparsed log record in a standard attribute, and replace // the log record body with the message field. // // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md - `set(attributes["log.record.original"], body["original"])`, - `set(body, cache["message"])`, + `set(log.attributes["log.record.original"], log.body["original"])`, + `set(log.body, log.cache["message"])`, }, }}, } diff --git a/internal/collector/patroni_test.go b/internal/collector/patroni_test.go index 2f7337410..fc9823f8d 100644 --- a/internal/collector/patroni_test.go +++ b/internal/collector/patroni_test.go @@ -74,16 +74,21 @@ processors: - context: log statements: - set(instrumentation_scope.name, "patroni") - - set(cache, ParseJSON(body["original"])) - - set(severity_text, cache["levelname"]) - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL" - - set(time, Time(cache["asctime"], "%F %T,%L")) - - set(attributes["log.record.original"], body["original"]) - - set(body, cache["message"]) + - set(log.cache, ParseJSON(log.body["original"])) + - set(log.severity_text, log.cache["levelname"]) + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARNING" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == + "CRITICAL" + - set(log.time, Time(log.cache["asctime"], "%F %T,%L")) where IsString(log.cache["asctime"]) + - set(log.attributes["log.record.original"], log.body["original"]) + - set(log.body, log.cache["message"]) receivers: filelog/patroni_jsonlog: include: @@ -169,16 +174,21 @@ processors: - context: log statements: - set(instrumentation_scope.name, "patroni") - - set(cache, ParseJSON(body["original"])) - - set(severity_text, cache["levelname"]) - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL" - - set(time, Time(cache["asctime"], "%F %T,%L")) - - set(attributes["log.record.original"], body["original"]) - - set(body, cache["message"]) + - set(log.cache, ParseJSON(log.body["original"])) + - set(log.severity_text, log.cache["levelname"]) + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARNING" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == + "CRITICAL" + - set(log.time, Time(log.cache["asctime"], "%F %T,%L")) where IsString(log.cache["asctime"]) + - set(log.attributes["log.record.original"], log.body["original"]) + - set(log.body, log.cache["message"]) receivers: filelog/patroni_jsonlog: include: diff --git a/internal/collector/pgadmin.go b/internal/collector/pgadmin.go index 1f8211570..9637893c5 100644 --- a/internal/collector/pgadmin.go +++ b/internal/collector/pgadmin.go @@ -61,28 +61,28 @@ func EnablePgAdminLogging(ctx context.Context, spec *v1beta1.InstrumentationSpec // the log record body with the message field. // // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md - `set(attributes["log.record.original"], body)`, - `set(cache, ParseJSON(body))`, - `merge_maps(attributes, ExtractPatterns(cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), "insert")`, - `set(body, cache["message"])`, + `set(log.attributes["log.record.original"], log.body)`, + `set(log.cache, ParseJSON(log.body))`, + `merge_maps(log.attributes, ExtractPatterns(log.cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), "insert")`, + `set(log.body, log.cache["message"])`, // Set instrumentation scope to the "name" from each log record. - `set(instrumentation_scope.name, cache["name"])`, + `set(instrumentation_scope.name, log.cache["name"])`, // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext - `set(severity_text, cache["level"])`, - `set(time_unix_nano, Int(cache["time"]*1000000000))`, + `set(log.severity_text, log.cache["level"])`, + `set(log.time_unix_nano, Int(log.cache["time"]*1000000000))`, // Map pgAdmin "logging levels" to OpenTelemetry severity levels. // // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber // https://opentelemetry.io/docs/specs/otel/logs/data-model-appendix/#appendix-b-severitynumber-example-mappings // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#enums - `set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG"`, - `set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO"`, - `set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING"`, - `set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR"`, - `set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL"`, + `set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == "DEBUG"`, + `set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == "INFO"`, + `set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == "WARNING"`, + `set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == "ERROR"`, + `set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == "CRITICAL"`, }, }, }, diff --git a/internal/collector/pgadmin_test.go b/internal/collector/pgadmin_test.go index e5db11f58..8dc2097da 100644 --- a/internal/collector/pgadmin_test.go +++ b/internal/collector/pgadmin_test.go @@ -77,19 +77,24 @@ collector.yaml: | log_statements: - context: log statements: - - set(attributes["log.record.original"], body) - - set(cache, ParseJSON(body)) - - merge_maps(attributes, ExtractPatterns(cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), + - set(log.attributes["log.record.original"], log.body) + - set(log.cache, ParseJSON(log.body)) + - merge_maps(log.attributes, ExtractPatterns(log.cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), "insert") - - set(body, cache["message"]) - - set(instrumentation_scope.name, cache["name"]) - - set(severity_text, cache["level"]) - - set(time_unix_nano, Int(cache["time"]*1000000000)) - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL" + - set(log.body, log.cache["message"]) + - set(instrumentation_scope.name, log.cache["name"]) + - set(log.severity_text, log.cache["level"]) + - set(log.time_unix_nano, Int(log.cache["time"]*1000000000)) + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARNING" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == + "CRITICAL" receivers: filelog/gunicorn: include: @@ -198,19 +203,24 @@ collector.yaml: | log_statements: - context: log statements: - - set(attributes["log.record.original"], body) - - set(cache, ParseJSON(body)) - - merge_maps(attributes, ExtractPatterns(cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), + - set(log.attributes["log.record.original"], log.body) + - set(log.cache, ParseJSON(log.body)) + - merge_maps(log.attributes, ExtractPatterns(log.cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), "insert") - - set(body, cache["message"]) - - set(instrumentation_scope.name, cache["name"]) - - set(severity_text, cache["level"]) - - set(time_unix_nano, Int(cache["time"]*1000000000)) - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL" + - set(log.body, log.cache["message"]) + - set(instrumentation_scope.name, log.cache["name"]) + - set(log.severity_text, log.cache["level"]) + - set(log.time_unix_nano, Int(log.cache["time"]*1000000000)) + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARNING" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == + "CRITICAL" receivers: filelog/gunicorn: include: diff --git a/internal/collector/pgbackrest_logs_transforms.yaml b/internal/collector/pgbackrest_logs_transforms.yaml index 31f4a48f9..9aca9c212 100644 --- a/internal/collector/pgbackrest_logs_transforms.yaml +++ b/internal/collector/pgbackrest_logs_transforms.yaml @@ -14,30 +14,30 @@ # 3) the log level (form INFO, WARN, etc.) # 4) the message (anything else, including newline -- we can do this because we have a multiline block on the receiver) - >- - merge_maps(cache, - ExtractPatterns(body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P\\d{2,3})\\s*(?\\S*): (?(?s).*)$"), + merge_maps(log.cache, + ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P\\d{2,3})\\s*(?\\S*): (?(?s).*)$"), "insert") - where Len(body) > 0 + where Len(log.body) > 0 # The log severity is the "error_severity" field. # https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext # https://pgbackrest.org/configuration.html#section-log/option-log-level-file - - set(severity_text, cache["error_severity"]) where IsString(cache["error_severity"]) - - set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == "TRACE" - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_DEBUG2) where severity_text == "DETAIL" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARN" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" + - set(log.severity_text, log.cache["error_severity"]) where IsString(log.cache["error_severity"]) + - set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == "TRACE" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG2) where log.severity_text == "DETAIL" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == "INFO" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == "WARN" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == "ERROR" # https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-timestamp - - set(time, Time(cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where IsString(cache["timestamp"]) + - set(log.time, Time(log.cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where IsString(log.cache["timestamp"]) # https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/attributes-registry/process.md - - set(attributes["process.pid"], cache["process_id"]) + - set(log.attributes["process.pid"], log.cache["process_id"]) # Keep the unparsed log record in a standard attribute, - # and replace the log record body with the message field. + # and replace the log record log.body with the message field. # https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md - - set(attributes["log.record.original"], body) - - set(body, cache["message"]) + - set(log.attributes["log.record.original"], log.body) + - set(log.body, log.cache["message"]) diff --git a/internal/collector/pgbackrest_test.go b/internal/collector/pgbackrest_test.go index e8a5a4d2d..9f69d7fe3 100644 --- a/internal/collector/pgbackrest_test.go +++ b/internal/collector/pgbackrest_test.go @@ -77,20 +77,27 @@ processors: statements: - set(instrumentation_scope.name, "pgbackrest") - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - - 'merge_maps(cache, ExtractPatterns(body, "^(?\\d{4}-\\d{2}-\\d{2} + - 'merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P\\d{2,3})\\s*(?\\S*): - (?(?s).*)$"), "insert") where Len(body) > 0' - - set(severity_text, cache["error_severity"]) where IsString(cache["error_severity"]) - - set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == "TRACE" - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_DEBUG2) where severity_text == "DETAIL" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARN" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(time, Time(cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where IsString(cache["timestamp"]) - - set(attributes["process.pid"], cache["process_id"]) - - set(attributes["log.record.original"], body) - - set(body, cache["message"]) + (?(?s).*)$"), "insert") where Len(log.body) > 0' + - set(log.severity_text, log.cache["error_severity"]) where IsString(log.cache["error_severity"]) + - set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == + "TRACE" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG2) where log.severity_text == + "DETAIL" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARN" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.time, Time(log.cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where + IsString(log.cache["timestamp"]) + - set(log.attributes["process.pid"], log.cache["process_id"]) + - set(log.attributes["log.record.original"], log.body) + - set(log.body, log.cache["message"]) receivers: filelog/pgbackrest_log: include: @@ -177,20 +184,27 @@ processors: statements: - set(instrumentation_scope.name, "pgbackrest") - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - - 'merge_maps(cache, ExtractPatterns(body, "^(?\\d{4}-\\d{2}-\\d{2} + - 'merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P\\d{2,3})\\s*(?\\S*): - (?(?s).*)$"), "insert") where Len(body) > 0' - - set(severity_text, cache["error_severity"]) where IsString(cache["error_severity"]) - - set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == "TRACE" - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_DEBUG2) where severity_text == "DETAIL" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARN" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(time, Time(cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where IsString(cache["timestamp"]) - - set(attributes["process.pid"], cache["process_id"]) - - set(attributes["log.record.original"], body) - - set(body, cache["message"]) + (?(?s).*)$"), "insert") where Len(log.body) > 0' + - set(log.severity_text, log.cache["error_severity"]) where IsString(log.cache["error_severity"]) + - set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == + "TRACE" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG2) where log.severity_text == + "DETAIL" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARN" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.time, Time(log.cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where + IsString(log.cache["timestamp"]) + - set(log.attributes["process.pid"], log.cache["process_id"]) + - set(log.attributes["log.record.original"], log.body) + - set(log.body, log.cache["message"]) receivers: filelog/pgbackrest_log: include: diff --git a/internal/collector/pgbouncer.go b/internal/collector/pgbouncer.go index 375d2b9ba..bde500f50 100644 --- a/internal/collector/pgbouncer.go +++ b/internal/collector/pgbouncer.go @@ -102,12 +102,12 @@ func EnablePgBouncerLogging(ctx context.Context, `set(instrumentation_scope.name, "pgbouncer")`, // Extract timestamp, pid, log level, and message and store in cache. - `merge_maps(cache, ExtractPatterns(body, ` + + `merge_maps(log.cache, ExtractPatterns(log.body, ` + `"^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3} [A-Z]{3}) ` + `\\[(?\\d+)\\] (?[A-Z]+) (?.*$)"), "insert")`, // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext - `set(severity_text, cache["log_level"])`, + `set(log.severity_text, log.cache["log_level"])`, // Map pgBouncer (libusual) "logging levels" to OpenTelemetry severity levels. // @@ -115,11 +115,11 @@ func EnablePgBouncerLogging(ctx context.Context, // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber // https://opentelemetry.io/docs/specs/otel/logs/data-model-appendix/#appendix-b-severitynumber-example-mappings // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#enums - `set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "NOISE" or severity_text == "DEBUG"`, - `set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "LOG"`, - `set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING"`, - `set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR"`, - `set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "FATAL"`, + `set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == "NOISE" or log.severity_text == "DEBUG"`, + `set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == "LOG"`, + `set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == "WARNING"`, + `set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == "ERROR"`, + `set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == "FATAL"`, // Parse the timestamp. // The format is neither RFC 3339 nor ISO 8601: @@ -129,19 +129,19 @@ func EnablePgBouncerLogging(ctx context.Context, // then a timezone abbreviation. // // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/stanza/docs/types/timestamp.md - `set(time, Time(cache["timestamp"], "%F %T.%L %Z"))`, + `set(log.time, Time(log.cache["timestamp"], "%F %T.%L %Z")) where IsString(log.cache["timestamp"])`, // Keep the unparsed log record in a standard attribute, and replace // the log record body with the message field. // // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md - `set(attributes["log.record.original"], body)`, + `set(log.attributes["log.record.original"], log.body)`, // Set pid as attribute - `set(attributes["process.pid"], cache["pid"])`, + `set(log.attributes["process.pid"], log.cache["pid"])`, // Set the log message to body. - `set(body, cache["msg"])`, + `set(log.body, log.cache["msg"])`, }, }}, } diff --git a/internal/collector/pgbouncer_test.go b/internal/collector/pgbouncer_test.go index 74aed710d..333fae0dd 100644 --- a/internal/collector/pgbouncer_test.go +++ b/internal/collector/pgbouncer_test.go @@ -73,20 +73,24 @@ processors: - context: log statements: - set(instrumentation_scope.name, "pgbouncer") - - merge_maps(cache, ExtractPatterns(body, "^(?\\d{4}-\\d{2}-\\d{2} + - merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3} [A-Z]{3}) \\[(?\\d+)\\] (?[A-Z]+) (?.*$)"), "insert") - - set(severity_text, cache["log_level"]) - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "NOISE" - or severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "LOG" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "FATAL" - - set(time, Time(cache["timestamp"], "%F %T.%L %Z")) - - set(attributes["log.record.original"], body) - - set(attributes["process.pid"], cache["pid"]) - - set(body, cache["msg"]) + - set(log.severity_text, log.cache["log_level"]) + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "NOISE" or log.severity_text == "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "LOG" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARNING" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == + "FATAL" + - set(log.time, Time(log.cache["timestamp"], "%F %T.%L %Z")) where IsString(log.cache["timestamp"]) + - set(log.attributes["log.record.original"], log.body) + - set(log.attributes["process.pid"], log.cache["pid"]) + - set(log.body, log.cache["msg"]) receivers: filelog/pgbouncer_log: include: @@ -170,20 +174,24 @@ processors: - context: log statements: - set(instrumentation_scope.name, "pgbouncer") - - merge_maps(cache, ExtractPatterns(body, "^(?\\d{4}-\\d{2}-\\d{2} + - merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3} [A-Z]{3}) \\[(?\\d+)\\] (?[A-Z]+) (?.*$)"), "insert") - - set(severity_text, cache["log_level"]) - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "NOISE" - or severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "LOG" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "FATAL" - - set(time, Time(cache["timestamp"], "%F %T.%L %Z")) - - set(attributes["log.record.original"], body) - - set(attributes["process.pid"], cache["pid"]) - - set(body, cache["msg"]) + - set(log.severity_text, log.cache["log_level"]) + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "NOISE" or log.severity_text == "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "LOG" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARNING" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == + "FATAL" + - set(log.time, Time(log.cache["timestamp"], "%F %T.%L %Z")) where IsString(log.cache["timestamp"]) + - set(log.attributes["log.record.original"], log.body) + - set(log.attributes["process.pid"], log.cache["pid"]) + - set(log.body, log.cache["msg"]) receivers: filelog/pgbouncer_log: include: diff --git a/internal/collector/postgres_logs_transforms.yaml b/internal/collector/postgres_logs_transforms.yaml index f397b996e..67f847a77 100644 --- a/internal/collector/postgres_logs_transforms.yaml +++ b/internal/collector/postgres_logs_transforms.yaml @@ -12,7 +12,7 @@ - body["format"] == "csv" statements: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/ottlfuncs#parsecsv - - set(cache, ParseCSV(body["original"], body["headers"], delimiter=",", mode="strict")) + - set(log.cache, ParseCSV(log.body["original"], log.body["headers"], delimiter=",", mode="strict")) # Extract the optional "remote_port" value from the "connection_from" field. It is either: # 1. a Unix socket starting with "[local]:" or @@ -24,60 +24,60 @@ # https://git.postgresql.org/gitweb/?p=postgresql.git;hb=REL_17_0;f=src/backend/utils/error/csvlog.c#l108 # https://git.postgresql.org/gitweb/?p=postgresql.git;hb=REL_17_0;f=src/common/ip.c#l224 - >- - merge_maps(cache, - ExtractPatterns(cache["connection_from"], "(?:^[[]local[]]:(?.+)|:(?[^:]+))$"), + merge_maps(log.cache, + ExtractPatterns(log.cache["connection_from"], "(?:^[[]local[]]:(?.+)|:(?[^:]+))$"), "insert") - where Len(cache["connection_from"]) > 0 + where Len(log.cache["connection_from"]) > 0 # When there is a "remote_port" value, everything before it is the "remote_host" value. - >- - set(cache["remote_host"], - Substring(cache["connection_from"], 0, Len(cache["connection_from"]) - Len(cache["remote_port"]) - 1)) - where Len(cache["connection_from"]) > 0 and IsString(cache["remote_port"]) + set(log.cache["remote_host"], + Substring(log.cache["connection_from"], 0, Len(log.cache["connection_from"]) - Len(log.cache["remote_port"]) - 1)) + where Len(log.cache["connection_from"]) > 0 and IsString(log.cache["remote_port"]) # When there is still no "remote_host" value, copy the "connection_from" value, if any. - >- - set(cache["remote_host"], cache["connection_from"]) - where Len(cache["connection_from"]) > 0 and not IsString(cache["remote_host"]) + set(log.cache["remote_host"], log.cache["connection_from"]) + where Len(log.cache["connection_from"]) > 0 and not IsString(log.cache["remote_host"]) # Extract the values encoded in the "location" field. # # https://git.postgresql.org/gitweb/?p=postgresql.git;hb=REL_10_0;f=src/backend/utils/error/elog.c#l2805 # https://git.postgresql.org/gitweb/?p=postgresql.git;hb=REL_17_0;f=src/backend/utils/error/csvlog.c#l207 - >- - merge_maps(cache, - ExtractPatterns(cache["location"], "^(?:(?[^,]+), )?(?[^:]+):(?\\d+)$"), + merge_maps(log.cache, + ExtractPatterns(log.cache["location"], "^(?:(?[^,]+), )?(?[^:]+):(?\\d+)$"), "insert") - where Len(cache["location"]) > 0 + where Len(log.cache["location"]) > 0 # These values are numeric in JSON logs. - >- - set(cache["cursor_position"], Double(cache["cursor_position"])) - where IsMatch(cache["cursor_position"], "^[0-9.]+$") + set(log.cache["cursor_position"], Double(log.cache["cursor_position"])) + where IsMatch(log.cache["cursor_position"], "^[0-9.]+$") - >- - set(cache["file_line_num"], Double(cache["file_line_num"])) - where IsMatch(cache["file_line_num"], "^[0-9.]+$") + set(log.cache["file_line_num"], Double(log.cache["file_line_num"])) + where IsMatch(log.cache["file_line_num"], "^[0-9.]+$") - >- - set(cache["internal_position"], Double(cache["internal_position"])) - where IsMatch(cache["internal_position"], "^[0-9.]+$") + set(log.cache["internal_position"], Double(log.cache["internal_position"])) + where IsMatch(log.cache["internal_position"], "^[0-9.]+$") - >- - set(cache["leader_pid"], Double(cache["leader_pid"])) - where IsMatch(cache["leader_pid"], "^[0-9.]+$") + set(log.cache["leader_pid"], Double(log.cache["leader_pid"])) + where IsMatch(log.cache["leader_pid"], "^[0-9.]+$") - >- - set(cache["line_num"], Double(cache["line_num"])) - where IsMatch(cache["line_num"], "^[0-9.]+$") + set(log.cache["line_num"], Double(log.cache["line_num"])) + where IsMatch(log.cache["line_num"], "^[0-9.]+$") - >- - set(cache["pid"], Double(cache["pid"])) - where IsMatch(cache["pid"], "^[0-9.]+$") + set(log.cache["pid"], Double(log.cache["pid"])) + where IsMatch(log.cache["pid"], "^[0-9.]+$") - >- - set(cache["query_id"], Double(cache["query_id"])) - where IsMatch(cache["query_id"], "^[0-9.]+$") + set(log.cache["query_id"], Double(log.cache["query_id"])) + where IsMatch(log.cache["query_id"], "^[0-9.]+$") - >- - set(cache["remote_port"], Double(cache["remote_port"])) - where IsMatch(cache["remote_port"], "^[0-9.]+$") + set(log.cache["remote_port"], Double(log.cache["remote_port"])) + where IsMatch(log.cache["remote_port"], "^[0-9.]+$") # Pass the results to the next set of statements. - - set(body["parsed"], cache) + - set(log.body["parsed"], log.cache) # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#readme @@ -87,14 +87,14 @@ - set(instrumentation_scope.version, resource.attributes["db.version"]) # TODO(postgres-14): We can stop parsing CSV logs when 14 is EOL. - - set(cache, body["parsed"]) where body["format"] == "csv" + - set(log.cache, log.body["parsed"]) where log.body["format"] == "csv" # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/ottlfuncs#parsejson - - set(cache, ParseJSON(body["original"])) where body["format"] == "json" + - set(log.cache, ParseJSON(log.body["original"])) where log.body["format"] == "json" # The log severity is in the "error_severity" field. # https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext - - set(severity_text, cache["error_severity"]) + - set(log.severity_text, log.cache["error_severity"]) # Map severity text to OpenTelemetry severity levels. # Postgres has levels beyond the typical ones: @@ -106,17 +106,17 @@ # https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber # https://opentelemetry.io/docs/specs/otel/logs/data-model-appendix/#appendix-b-severitynumber-example-mappings # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#enums - - set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == "DEBUG5" - - set(severity_number, SEVERITY_NUMBER_TRACE2) where severity_text == "DEBUG4" - - set(severity_number, SEVERITY_NUMBER_TRACE3) where severity_text == "DEBUG3" - - set(severity_number, SEVERITY_NUMBER_TRACE4) where severity_text == "DEBUG2" - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG1" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" or severity_text == "LOG" - - set(severity_number, SEVERITY_NUMBER_INFO2) where severity_text == "NOTICE" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "FATAL" - - set(severity_number, SEVERITY_NUMBER_FATAL2) where severity_text == "PANIC" + - set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == "DEBUG5" + - set(log.severity_number, SEVERITY_NUMBER_TRACE2) where log.severity_text == "DEBUG4" + - set(log.severity_number, SEVERITY_NUMBER_TRACE3) where log.severity_text == "DEBUG3" + - set(log.severity_number, SEVERITY_NUMBER_TRACE4) where log.severity_text == "DEBUG2" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == "DEBUG1" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == "INFO" or log.severity_text == "LOG" + - set(log.severity_number, SEVERITY_NUMBER_INFO2) where log.severity_text == "NOTICE" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == "WARNING" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == "ERROR" + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == "FATAL" + - set(log.severity_number, SEVERITY_NUMBER_FATAL2) where log.severity_text == "PANIC" # Parse the "timestamp" field into the record timestamp. # The format is neither RFC 3339 nor ISO 8601: @@ -128,7 +128,7 @@ # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/ottlfuncs#time # https://git.postgresql.org/gitweb/?p=postgresql.git;hb=REL_10_0;f=src/backend/utils/error/elog.c#l2246 # https://git.postgresql.org/gitweb/?p=postgresql.git;hb=REL_17_0;f=src/backend/utils/error/elog.c#l2671 - - set(time, Time(cache["timestamp"], "%F %T.%L %Z")) + - set(log.time, Time(log.cache["timestamp"], "%F %T.%L %Z")) where IsString(log.cache["timestamp"]) # Rename fields emitted by Postgres to align with OpenTelemetry semantic conventions. # @@ -140,27 +140,27 @@ - set(resource.attributes["db.system"], "postgresql") # Keep the unparsed log record in a standard attribute, - # and replace the log record body with the parsed fields. + # and replace the log record log.body with the parsed fields. # # https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md - - set(attributes["log.record.original"], body["original"]) - - set(body, cache) + - set(log.attributes["log.record.original"], log.body["original"]) + - set(log.body, log.cache) # https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/attributes-registry/client.md - - set(attributes["client.address"], body["remote_host"]) where IsString(body["remote_host"]) - - set(attributes["client.port"], Int(body["remote_port"])) where IsDouble(body["remote_port"]) + - set(log.attributes["client.address"], log.body["remote_host"]) where IsString(log.body["remote_host"]) + - set(log.attributes["client.port"], Int(log.body["remote_port"])) where IsDouble(log.body["remote_port"]) # These values are populated when the "log_error_verbosity" parameter is VERBOSE. # # https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-LOG-ERROR-VERBOSITY # https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/attributes-registry/code.md - - set(attributes["code.filepath"], body["file_name"]) where IsString(body["file_name"]) - - set(attributes["code.function"], body["func_name"]) where IsString(body["func_name"]) - - set(attributes["code.lineno"], Int(body["file_line_num"])) where IsDouble(body["file_line_num"]) + - set(log.attributes["code.filepath"], log.body["file_name"]) where IsString(log.body["file_name"]) + - set(log.attributes["code.function"], log.body["func_name"]) where IsString(log.body["func_name"]) + - set(log.attributes["code.lineno"], Int(log.body["file_line_num"])) where IsDouble(log.body["file_line_num"]) # https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/attributes-registry/db.md - - set(attributes["db.namespace"], body["dbname"]) where IsString(body["dbname"]) - - set(attributes["db.response.status_code"], body["state_code"]) where IsString(body["state_code"]) + - set(log.attributes["db.namespace"], log.body["dbname"]) where IsString(log.body["dbname"]) + - set(log.attributes["db.response.status_code"], log.body["state_code"]) where IsString(log.body["state_code"]) # Postgres is multiprocess so some client/backend details align here. # @@ -170,21 +170,21 @@ # https://git.postgresql.org/gitweb/?p=postgresql.git;f=src/backend/utils/error/elog.c;hb=REL_17_0#l2697 # https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/attributes-registry/process.md - >- - set(attributes["process.creation.time"], Concat([ - Substring(body["session_start"], 0, 10), "T", - Substring(body["session_start"], 11, 8), "Z"], "")) - where IsMatch(body["session_start"], "^[^ ]{10} [^ ]{8} UTC$") + set(log.attributes["process.creation.time"], Concat([ + Substring(log.body["session_start"], 0, 10), "T", + Substring(log.body["session_start"], 11, 8), "Z"], "")) + where IsMatch(log.body["session_start"], "^[^ ]{10} [^ ]{8} UTC$") - >- - set(attributes["process.pid"], Int(body["pid"])) - where IsDouble(body["pid"]) + set(log.attributes["process.pid"], Int(log.body["pid"])) + where IsDouble(log.body["pid"]) - >- - set(attributes["process.title"], body["ps"]) - where IsString(body["ps"]) + set(log.attributes["process.title"], log.body["ps"]) + where IsString(log.body["ps"]) # https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/attributes-registry/user.md - >- - set(attributes["user.name"], body["user"]) - where IsString(body["user"]) + set(log.attributes["user.name"], log.body["user"]) + where IsString(log.body["user"]) # Look for and parse the CSV of a pgAudit message. @@ -203,9 +203,9 @@ statements: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/ottlfuncs#parsecsv - >- - set(body["pgaudit"], ParseCSV(Substring(body["message"], 7, Len(body["message"]) - 7), + set(log.body["pgaudit"], ParseCSV(Substring(log.body["message"], 7, Len(log.body["message"]) - 7), "audit_type,statement_id,substatement_id,class,command,object_type,object_name,statement,parameter", delimiter=",", mode="strict")) - >- set(instrumentation_scope.name, "pgaudit") - where Len(body["pgaudit"]) > 0 + where Len(log.body["pgaudit"]) > 0 diff --git a/internal/collector/postgres_test.go b/internal/collector/postgres_test.go index 3bdf33c61..d57e47dda 100644 --- a/internal/collector/postgres_test.go +++ b/internal/collector/postgres_test.go @@ -100,99 +100,121 @@ processors: statements: - set(instrumentation_scope.name, "pgbackrest") - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - - 'merge_maps(cache, ExtractPatterns(body, "^(?\\d{4}-\\d{2}-\\d{2} + - 'merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P\\d{2,3})\\s*(?\\S*): - (?(?s).*)$"), "insert") where Len(body) > 0' - - set(severity_text, cache["error_severity"]) where IsString(cache["error_severity"]) - - set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == "TRACE" - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_DEBUG2) where severity_text == "DETAIL" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARN" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(time, Time(cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where IsString(cache["timestamp"]) - - set(attributes["process.pid"], cache["process_id"]) - - set(attributes["log.record.original"], body) - - set(body, cache["message"]) + (?(?s).*)$"), "insert") where Len(log.body) > 0' + - set(log.severity_text, log.cache["error_severity"]) where IsString(log.cache["error_severity"]) + - set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == + "TRACE" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG2) where log.severity_text == + "DETAIL" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARN" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.time, Time(log.cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where + IsString(log.cache["timestamp"]) + - set(log.attributes["process.pid"], log.cache["process_id"]) + - set(log.attributes["log.record.original"], log.body) + - set(log.body, log.cache["message"]) transform/postgres_logs: log_statements: - conditions: - body["format"] == "csv" context: log statements: - - set(cache, ParseCSV(body["original"], body["headers"], delimiter=",", mode="strict")) - - merge_maps(cache, ExtractPatterns(cache["connection_from"], "(?:^[[]local[]]:(?.+)|:(?[^:]+))$"), - "insert") where Len(cache["connection_from"]) > 0 - - set(cache["remote_host"], Substring(cache["connection_from"], 0, Len(cache["connection_from"]) - - Len(cache["remote_port"]) - 1)) where Len(cache["connection_from"]) > 0 - and IsString(cache["remote_port"]) - - set(cache["remote_host"], cache["connection_from"]) where Len(cache["connection_from"]) - > 0 and not IsString(cache["remote_host"]) - - merge_maps(cache, ExtractPatterns(cache["location"], "^(?:(?[^,]+), - )?(?[^:]+):(?\\d+)$"), "insert") where Len(cache["location"]) + - set(log.cache, ParseCSV(log.body["original"], log.body["headers"], delimiter=",", + mode="strict")) + - merge_maps(log.cache, ExtractPatterns(log.cache["connection_from"], "(?:^[[]local[]]:(?.+)|:(?[^:]+))$"), + "insert") where Len(log.cache["connection_from"]) > 0 + - set(log.cache["remote_host"], Substring(log.cache["connection_from"], 0, Len(log.cache["connection_from"]) + - Len(log.cache["remote_port"]) - 1)) where Len(log.cache["connection_from"]) + > 0 and IsString(log.cache["remote_port"]) + - set(log.cache["remote_host"], log.cache["connection_from"]) where Len(log.cache["connection_from"]) + > 0 and not IsString(log.cache["remote_host"]) + - merge_maps(log.cache, ExtractPatterns(log.cache["location"], "^(?:(?[^,]+), + )?(?[^:]+):(?\\d+)$"), "insert") where Len(log.cache["location"]) > 0 - - set(cache["cursor_position"], Double(cache["cursor_position"])) where IsMatch(cache["cursor_position"], + - set(log.cache["cursor_position"], Double(log.cache["cursor_position"])) where + IsMatch(log.cache["cursor_position"], "^[0-9.]+$") + - set(log.cache["file_line_num"], Double(log.cache["file_line_num"])) where + IsMatch(log.cache["file_line_num"], "^[0-9.]+$") + - set(log.cache["internal_position"], Double(log.cache["internal_position"])) + where IsMatch(log.cache["internal_position"], "^[0-9.]+$") + - set(log.cache["leader_pid"], Double(log.cache["leader_pid"])) where IsMatch(log.cache["leader_pid"], "^[0-9.]+$") - - set(cache["file_line_num"], Double(cache["file_line_num"])) where IsMatch(cache["file_line_num"], + - set(log.cache["line_num"], Double(log.cache["line_num"])) where IsMatch(log.cache["line_num"], "^[0-9.]+$") - - set(cache["internal_position"], Double(cache["internal_position"])) where - IsMatch(cache["internal_position"], "^[0-9.]+$") - - set(cache["leader_pid"], Double(cache["leader_pid"])) where IsMatch(cache["leader_pid"], + - set(log.cache["pid"], Double(log.cache["pid"])) where IsMatch(log.cache["pid"], "^[0-9.]+$") - - set(cache["line_num"], Double(cache["line_num"])) where IsMatch(cache["line_num"], + - set(log.cache["query_id"], Double(log.cache["query_id"])) where IsMatch(log.cache["query_id"], "^[0-9.]+$") - - set(cache["pid"], Double(cache["pid"])) where IsMatch(cache["pid"], "^[0-9.]+$") - - set(cache["query_id"], Double(cache["query_id"])) where IsMatch(cache["query_id"], + - set(log.cache["remote_port"], Double(log.cache["remote_port"])) where IsMatch(log.cache["remote_port"], "^[0-9.]+$") - - set(cache["remote_port"], Double(cache["remote_port"])) where IsMatch(cache["remote_port"], - "^[0-9.]+$") - - set(body["parsed"], cache) + - set(log.body["parsed"], log.cache) - context: log statements: - set(instrumentation_scope.name, "postgres") - set(instrumentation_scope.version, resource.attributes["db.version"]) - - set(cache, body["parsed"]) where body["format"] == "csv" - - set(cache, ParseJSON(body["original"])) where body["format"] == "json" - - set(severity_text, cache["error_severity"]) - - set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == "DEBUG5" - - set(severity_number, SEVERITY_NUMBER_TRACE2) where severity_text == "DEBUG4" - - set(severity_number, SEVERITY_NUMBER_TRACE3) where severity_text == "DEBUG3" - - set(severity_number, SEVERITY_NUMBER_TRACE4) where severity_text == "DEBUG2" - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG1" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - or severity_text == "LOG" - - set(severity_number, SEVERITY_NUMBER_INFO2) where severity_text == "NOTICE" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "FATAL" - - set(severity_number, SEVERITY_NUMBER_FATAL2) where severity_text == "PANIC" - - set(time, Time(cache["timestamp"], "%F %T.%L %Z")) + - set(log.cache, log.body["parsed"]) where log.body["format"] == "csv" + - set(log.cache, ParseJSON(log.body["original"])) where log.body["format"] == + "json" + - set(log.severity_text, log.cache["error_severity"]) + - set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == + "DEBUG5" + - set(log.severity_number, SEVERITY_NUMBER_TRACE2) where log.severity_text == + "DEBUG4" + - set(log.severity_number, SEVERITY_NUMBER_TRACE3) where log.severity_text == + "DEBUG3" + - set(log.severity_number, SEVERITY_NUMBER_TRACE4) where log.severity_text == + "DEBUG2" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG1" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" or log.severity_text == "LOG" + - set(log.severity_number, SEVERITY_NUMBER_INFO2) where log.severity_text == + "NOTICE" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARNING" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == + "FATAL" + - set(log.severity_number, SEVERITY_NUMBER_FATAL2) where log.severity_text == + "PANIC" + - set(log.time, Time(log.cache["timestamp"], "%F %T.%L %Z")) where IsString(log.cache["timestamp"]) - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - set(resource.attributes["db.system"], "postgresql") - - set(attributes["log.record.original"], body["original"]) - - set(body, cache) - - set(attributes["client.address"], body["remote_host"]) where IsString(body["remote_host"]) - - set(attributes["client.port"], Int(body["remote_port"])) where IsDouble(body["remote_port"]) - - set(attributes["code.filepath"], body["file_name"]) where IsString(body["file_name"]) - - set(attributes["code.function"], body["func_name"]) where IsString(body["func_name"]) - - set(attributes["code.lineno"], Int(body["file_line_num"])) where IsDouble(body["file_line_num"]) - - set(attributes["db.namespace"], body["dbname"]) where IsString(body["dbname"]) - - set(attributes["db.response.status_code"], body["state_code"]) where IsString(body["state_code"]) - - set(attributes["process.creation.time"], Concat([ Substring(body["session_start"], - 0, 10), "T", Substring(body["session_start"], 11, 8), "Z"], "")) where IsMatch(body["session_start"], - "^[^ ]{10} [^ ]{8} UTC$") - - set(attributes["process.pid"], Int(body["pid"])) where IsDouble(body["pid"]) - - set(attributes["process.title"], body["ps"]) where IsString(body["ps"]) - - set(attributes["user.name"], body["user"]) where IsString(body["user"]) + - set(log.attributes["log.record.original"], log.body["original"]) + - set(log.body, log.cache) + - set(log.attributes["client.address"], log.body["remote_host"]) where IsString(log.body["remote_host"]) + - set(log.attributes["client.port"], Int(log.body["remote_port"])) where IsDouble(log.body["remote_port"]) + - set(log.attributes["code.filepath"], log.body["file_name"]) where IsString(log.body["file_name"]) + - set(log.attributes["code.function"], log.body["func_name"]) where IsString(log.body["func_name"]) + - set(log.attributes["code.lineno"], Int(log.body["file_line_num"])) where IsDouble(log.body["file_line_num"]) + - set(log.attributes["db.namespace"], log.body["dbname"]) where IsString(log.body["dbname"]) + - set(log.attributes["db.response.status_code"], log.body["state_code"]) where + IsString(log.body["state_code"]) + - set(log.attributes["process.creation.time"], Concat([ Substring(log.body["session_start"], + 0, 10), "T", Substring(log.body["session_start"], 11, 8), "Z"], "")) where + IsMatch(log.body["session_start"], "^[^ ]{10} [^ ]{8} UTC$") + - set(log.attributes["process.pid"], Int(log.body["pid"])) where IsDouble(log.body["pid"]) + - set(log.attributes["process.title"], log.body["ps"]) where IsString(log.body["ps"]) + - set(log.attributes["user.name"], log.body["user"]) where IsString(log.body["user"]) - conditions: - 'Len(body["message"]) > 7 and Substring(body["message"], 0, 7) == "AUDIT: "' context: log statements: - - set(body["pgaudit"], ParseCSV(Substring(body["message"], 7, Len(body["message"]) + - set(log.body["pgaudit"], ParseCSV(Substring(log.body["message"], 7, Len(log.body["message"]) - 7), "audit_type,statement_id,substatement_id,class,command,object_type,object_name,statement,parameter", delimiter=",", mode="strict")) - - set(instrumentation_scope.name, "pgaudit") where Len(body["pgaudit"]) > 0 + - set(instrumentation_scope.name, "pgaudit") where Len(log.body["pgaudit"]) + > 0 receivers: filelog/pgbackrest_log: include: @@ -341,99 +363,121 @@ processors: statements: - set(instrumentation_scope.name, "pgbackrest") - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - - 'merge_maps(cache, ExtractPatterns(body, "^(?\\d{4}-\\d{2}-\\d{2} + - 'merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?P\\d{2,3})\\s*(?\\S*): - (?(?s).*)$"), "insert") where Len(body) > 0' - - set(severity_text, cache["error_severity"]) where IsString(cache["error_severity"]) - - set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == "TRACE" - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG" - - set(severity_number, SEVERITY_NUMBER_DEBUG2) where severity_text == "DETAIL" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARN" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(time, Time(cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where IsString(cache["timestamp"]) - - set(attributes["process.pid"], cache["process_id"]) - - set(attributes["log.record.original"], body) - - set(body, cache["message"]) + (?(?s).*)$"), "insert") where Len(log.body) > 0' + - set(log.severity_text, log.cache["error_severity"]) where IsString(log.cache["error_severity"]) + - set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == + "TRACE" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG2) where log.severity_text == + "DETAIL" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARN" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.time, Time(log.cache["timestamp"], "%Y-%m-%d %H:%M:%S.%L")) where + IsString(log.cache["timestamp"]) + - set(log.attributes["process.pid"], log.cache["process_id"]) + - set(log.attributes["log.record.original"], log.body) + - set(log.body, log.cache["message"]) transform/postgres_logs: log_statements: - conditions: - body["format"] == "csv" context: log statements: - - set(cache, ParseCSV(body["original"], body["headers"], delimiter=",", mode="strict")) - - merge_maps(cache, ExtractPatterns(cache["connection_from"], "(?:^[[]local[]]:(?.+)|:(?[^:]+))$"), - "insert") where Len(cache["connection_from"]) > 0 - - set(cache["remote_host"], Substring(cache["connection_from"], 0, Len(cache["connection_from"]) - - Len(cache["remote_port"]) - 1)) where Len(cache["connection_from"]) > 0 - and IsString(cache["remote_port"]) - - set(cache["remote_host"], cache["connection_from"]) where Len(cache["connection_from"]) - > 0 and not IsString(cache["remote_host"]) - - merge_maps(cache, ExtractPatterns(cache["location"], "^(?:(?[^,]+), - )?(?[^:]+):(?\\d+)$"), "insert") where Len(cache["location"]) + - set(log.cache, ParseCSV(log.body["original"], log.body["headers"], delimiter=",", + mode="strict")) + - merge_maps(log.cache, ExtractPatterns(log.cache["connection_from"], "(?:^[[]local[]]:(?.+)|:(?[^:]+))$"), + "insert") where Len(log.cache["connection_from"]) > 0 + - set(log.cache["remote_host"], Substring(log.cache["connection_from"], 0, Len(log.cache["connection_from"]) + - Len(log.cache["remote_port"]) - 1)) where Len(log.cache["connection_from"]) + > 0 and IsString(log.cache["remote_port"]) + - set(log.cache["remote_host"], log.cache["connection_from"]) where Len(log.cache["connection_from"]) + > 0 and not IsString(log.cache["remote_host"]) + - merge_maps(log.cache, ExtractPatterns(log.cache["location"], "^(?:(?[^,]+), + )?(?[^:]+):(?\\d+)$"), "insert") where Len(log.cache["location"]) > 0 - - set(cache["cursor_position"], Double(cache["cursor_position"])) where IsMatch(cache["cursor_position"], + - set(log.cache["cursor_position"], Double(log.cache["cursor_position"])) where + IsMatch(log.cache["cursor_position"], "^[0-9.]+$") + - set(log.cache["file_line_num"], Double(log.cache["file_line_num"])) where + IsMatch(log.cache["file_line_num"], "^[0-9.]+$") + - set(log.cache["internal_position"], Double(log.cache["internal_position"])) + where IsMatch(log.cache["internal_position"], "^[0-9.]+$") + - set(log.cache["leader_pid"], Double(log.cache["leader_pid"])) where IsMatch(log.cache["leader_pid"], "^[0-9.]+$") - - set(cache["file_line_num"], Double(cache["file_line_num"])) where IsMatch(cache["file_line_num"], + - set(log.cache["line_num"], Double(log.cache["line_num"])) where IsMatch(log.cache["line_num"], "^[0-9.]+$") - - set(cache["internal_position"], Double(cache["internal_position"])) where - IsMatch(cache["internal_position"], "^[0-9.]+$") - - set(cache["leader_pid"], Double(cache["leader_pid"])) where IsMatch(cache["leader_pid"], + - set(log.cache["pid"], Double(log.cache["pid"])) where IsMatch(log.cache["pid"], "^[0-9.]+$") - - set(cache["line_num"], Double(cache["line_num"])) where IsMatch(cache["line_num"], + - set(log.cache["query_id"], Double(log.cache["query_id"])) where IsMatch(log.cache["query_id"], "^[0-9.]+$") - - set(cache["pid"], Double(cache["pid"])) where IsMatch(cache["pid"], "^[0-9.]+$") - - set(cache["query_id"], Double(cache["query_id"])) where IsMatch(cache["query_id"], + - set(log.cache["remote_port"], Double(log.cache["remote_port"])) where IsMatch(log.cache["remote_port"], "^[0-9.]+$") - - set(cache["remote_port"], Double(cache["remote_port"])) where IsMatch(cache["remote_port"], - "^[0-9.]+$") - - set(body["parsed"], cache) + - set(log.body["parsed"], log.cache) - context: log statements: - set(instrumentation_scope.name, "postgres") - set(instrumentation_scope.version, resource.attributes["db.version"]) - - set(cache, body["parsed"]) where body["format"] == "csv" - - set(cache, ParseJSON(body["original"])) where body["format"] == "json" - - set(severity_text, cache["error_severity"]) - - set(severity_number, SEVERITY_NUMBER_TRACE) where severity_text == "DEBUG5" - - set(severity_number, SEVERITY_NUMBER_TRACE2) where severity_text == "DEBUG4" - - set(severity_number, SEVERITY_NUMBER_TRACE3) where severity_text == "DEBUG3" - - set(severity_number, SEVERITY_NUMBER_TRACE4) where severity_text == "DEBUG2" - - set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG1" - - set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO" - or severity_text == "LOG" - - set(severity_number, SEVERITY_NUMBER_INFO2) where severity_text == "NOTICE" - - set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING" - - set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR" - - set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "FATAL" - - set(severity_number, SEVERITY_NUMBER_FATAL2) where severity_text == "PANIC" - - set(time, Time(cache["timestamp"], "%F %T.%L %Z")) + - set(log.cache, log.body["parsed"]) where log.body["format"] == "csv" + - set(log.cache, ParseJSON(log.body["original"])) where log.body["format"] == + "json" + - set(log.severity_text, log.cache["error_severity"]) + - set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == + "DEBUG5" + - set(log.severity_number, SEVERITY_NUMBER_TRACE2) where log.severity_text == + "DEBUG4" + - set(log.severity_number, SEVERITY_NUMBER_TRACE3) where log.severity_text == + "DEBUG3" + - set(log.severity_number, SEVERITY_NUMBER_TRACE4) where log.severity_text == + "DEBUG2" + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == + "DEBUG1" + - set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == + "INFO" or log.severity_text == "LOG" + - set(log.severity_number, SEVERITY_NUMBER_INFO2) where log.severity_text == + "NOTICE" + - set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == + "WARNING" + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == + "ERROR" + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == + "FATAL" + - set(log.severity_number, SEVERITY_NUMBER_FATAL2) where log.severity_text == + "PANIC" + - set(log.time, Time(log.cache["timestamp"], "%F %T.%L %Z")) where IsString(log.cache["timestamp"]) - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - set(resource.attributes["db.system"], "postgresql") - - set(attributes["log.record.original"], body["original"]) - - set(body, cache) - - set(attributes["client.address"], body["remote_host"]) where IsString(body["remote_host"]) - - set(attributes["client.port"], Int(body["remote_port"])) where IsDouble(body["remote_port"]) - - set(attributes["code.filepath"], body["file_name"]) where IsString(body["file_name"]) - - set(attributes["code.function"], body["func_name"]) where IsString(body["func_name"]) - - set(attributes["code.lineno"], Int(body["file_line_num"])) where IsDouble(body["file_line_num"]) - - set(attributes["db.namespace"], body["dbname"]) where IsString(body["dbname"]) - - set(attributes["db.response.status_code"], body["state_code"]) where IsString(body["state_code"]) - - set(attributes["process.creation.time"], Concat([ Substring(body["session_start"], - 0, 10), "T", Substring(body["session_start"], 11, 8), "Z"], "")) where IsMatch(body["session_start"], - "^[^ ]{10} [^ ]{8} UTC$") - - set(attributes["process.pid"], Int(body["pid"])) where IsDouble(body["pid"]) - - set(attributes["process.title"], body["ps"]) where IsString(body["ps"]) - - set(attributes["user.name"], body["user"]) where IsString(body["user"]) + - set(log.attributes["log.record.original"], log.body["original"]) + - set(log.body, log.cache) + - set(log.attributes["client.address"], log.body["remote_host"]) where IsString(log.body["remote_host"]) + - set(log.attributes["client.port"], Int(log.body["remote_port"])) where IsDouble(log.body["remote_port"]) + - set(log.attributes["code.filepath"], log.body["file_name"]) where IsString(log.body["file_name"]) + - set(log.attributes["code.function"], log.body["func_name"]) where IsString(log.body["func_name"]) + - set(log.attributes["code.lineno"], Int(log.body["file_line_num"])) where IsDouble(log.body["file_line_num"]) + - set(log.attributes["db.namespace"], log.body["dbname"]) where IsString(log.body["dbname"]) + - set(log.attributes["db.response.status_code"], log.body["state_code"]) where + IsString(log.body["state_code"]) + - set(log.attributes["process.creation.time"], Concat([ Substring(log.body["session_start"], + 0, 10), "T", Substring(log.body["session_start"], 11, 8), "Z"], "")) where + IsMatch(log.body["session_start"], "^[^ ]{10} [^ ]{8} UTC$") + - set(log.attributes["process.pid"], Int(log.body["pid"])) where IsDouble(log.body["pid"]) + - set(log.attributes["process.title"], log.body["ps"]) where IsString(log.body["ps"]) + - set(log.attributes["user.name"], log.body["user"]) where IsString(log.body["user"]) - conditions: - 'Len(body["message"]) > 7 and Substring(body["message"], 0, 7) == "AUDIT: "' context: log statements: - - set(body["pgaudit"], ParseCSV(Substring(body["message"], 7, Len(body["message"]) + - set(log.body["pgaudit"], ParseCSV(Substring(log.body["message"], 7, Len(log.body["message"]) - 7), "audit_type,statement_id,substatement_id,class,command,object_type,object_name,statement,parameter", delimiter=",", mode="strict")) - - set(instrumentation_scope.name, "pgaudit") where Len(body["pgaudit"]) > 0 + - set(instrumentation_scope.name, "pgaudit") where Len(log.body["pgaudit"]) + > 0 receivers: filelog/pgbackrest_log: include: From 3fab41d4888ac9621c4a6d08160041f25cc01a51 Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Thu, 20 Mar 2025 11:13:23 -0500 Subject: [PATCH 2/4] Update for pgbouncer --- .../generated/pgbouncer_metrics_queries.json | 2 +- .../collector/pgbouncer_metrics_queries.yaml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/collector/generated/pgbouncer_metrics_queries.json b/internal/collector/generated/pgbouncer_metrics_queries.json index 0248051d9..78260bcf4 100644 --- a/internal/collector/generated/pgbouncer_metrics_queries.json +++ b/internal/collector/generated/pgbouncer_metrics_queries.json @@ -1 +1 @@ -[{"metrics":[{"attribute_columns":["database","user","state","application_name","link"],"description":"Current waiting time in seconds","metric_name":"ccp_pgbouncer_clients_wait_seconds","value_column":"wait"}],"sql":"SHOW CLIENTS"},{"metrics":[{"attribute_columns":["name","port","database","force_user","pool_mode"],"description":"Maximum number of server connections","metric_name":"ccp_pgbouncer_databases_pool_size","value_column":"pool_size"},{"attribute_columns":["name","port","database","force_user","pool_mode"],"description":"Minimum number of server connections","metric_name":"ccp_pgbouncer_databases_min_pool_size","value_column":"min_pool_size"},{"attribute_columns":["name","port","database","force_user","pool_mode"],"description":"Maximum number of additional connections for this database","metric_name":"ccp_pgbouncer_databases_reserve_pool","value_column":"reserve_pool"},{"attribute_columns":["name","port","database","force_user","pool_mode"],"description":"Maximum number of allowed connections for this database, as set by max_db_connections, either globally or per database","metric_name":"ccp_pgbouncer_databases_max_connections","value_column":"max_connections"},{"attribute_columns":["name","port","database","force_user","pool_mode"],"description":"Current number of connections for this database","metric_name":"ccp_pgbouncer_databases_current_connections","value_column":"current_connections"},{"attribute_columns":["name","port","database","force_user","pool_mode"],"description":"1 if this database is currently paused, else 0","metric_name":"ccp_pgbouncer_databases_paused","value_column":"paused"},{"attribute_columns":["name","port","database","force_user","pool_mode"],"description":"1 if this database is currently disabled, else 0","metric_name":"ccp_pgbouncer_databases_disabled","value_column":"disabled"}],"sql":"SHOW DATABASES"},{"metrics":[{"attribute_columns":["list"],"description":"Count of items registered with pgBouncer","metric_name":"ccp_pgbouncer_lists_item_count","value_column":"items"}],"sql":"SHOW LISTS"},{"metrics":[{"attribute_columns":["database","user"],"description":"Client connections that are either linked to server connections or are idle with no queries waiting to be processed","metric_name":"ccp_pgbouncer_pools_client_active","value_column":"cl_active"},{"attribute_columns":["database","user"],"description":"Client connections that have sent queries but have not yet got a server connection","metric_name":"ccp_pgbouncer_pools_client_waiting","value_column":"cl_waiting"},{"attribute_columns":["database","user"],"description":"Server connections that are linked to a client","metric_name":"ccp_pgbouncer_pools_server_active","value_column":"sv_active"},{"attribute_columns":["database","user"],"description":"Server connections that are unused and immediately usable for client queries","metric_name":"ccp_pgbouncer_pools_server_idle","value_column":"sv_idle"},{"attribute_columns":["database","user"],"description":"Server connections that have been idle for more than server_check_delay, so they need server_check_query to run on them before they can be used again","metric_name":"ccp_pgbouncer_pools_server_used","value_column":"sv_used"}],"sql":"SHOW POOLS"},{"metrics":[{"attribute_columns":["database","user","state","application_name","link"],"description":"1 if the connection will be closed as soon as possible, because a configuration file reload or DNS update changed the connection information or RECONNECT was issued","metric_name":"ccp_pgbouncer_servers_close_needed","value_column":"close_needed"}],"sql":"SHOW SERVERS"}] +[{"metrics":[{"attribute_columns":["database","user","state","application_name","link"],"description":"Current waiting time in seconds","metric_name":"ccp_pgbouncer_clients_wait_seconds","value_column":"wait"}],"sql":"SHOW CLIENTS"},{"metrics":[{"attribute_columns":["name","port","database"],"description":"Maximum number of server connections","metric_name":"ccp_pgbouncer_databases_pool_size","value_column":"pool_size"},{"attribute_columns":["name","port","database"],"description":"Minimum number of server connections","metric_name":"ccp_pgbouncer_databases_min_pool_size","value_column":"min_pool_size"},{"attribute_columns":["name","port","database"],"description":"Maximum number of additional connections for this database","metric_name":"ccp_pgbouncer_databases_reserve_pool","value_column":"reserve_pool_size"},{"attribute_columns":["name","port","database"],"description":"Maximum number of allowed connections for this database, as set by max_db_connections, either globally or per database","metric_name":"ccp_pgbouncer_databases_max_connections","value_column":"max_connections"},{"attribute_columns":["name","port","database"],"description":"Current number of connections for this database","metric_name":"ccp_pgbouncer_databases_current_connections","value_column":"current_connections"},{"attribute_columns":["name","port","database"],"description":"1 if this database is currently paused, else 0","metric_name":"ccp_pgbouncer_databases_paused","value_column":"paused"},{"attribute_columns":["name","port","database"],"description":"1 if this database is currently disabled, else 0","metric_name":"ccp_pgbouncer_databases_disabled","value_column":"disabled"}],"sql":"SHOW DATABASES"},{"metrics":[{"attribute_columns":["list"],"description":"Count of items registered with pgBouncer","metric_name":"ccp_pgbouncer_lists_item_count","value_column":"items"}],"sql":"SHOW LISTS"},{"metrics":[{"attribute_columns":["database","user"],"description":"Client connections that are either linked to server connections or are idle with no queries waiting to be processed","metric_name":"ccp_pgbouncer_pools_client_active","value_column":"cl_active"},{"attribute_columns":["database","user"],"description":"Client connections that have sent queries but have not yet got a server connection","metric_name":"ccp_pgbouncer_pools_client_waiting","value_column":"cl_waiting"},{"attribute_columns":["database","user"],"description":"Server connections that are linked to a client","metric_name":"ccp_pgbouncer_pools_server_active","value_column":"sv_active"},{"attribute_columns":["database","user"],"description":"Server connections that are unused and immediately usable for client queries","metric_name":"ccp_pgbouncer_pools_server_idle","value_column":"sv_idle"},{"attribute_columns":["database","user"],"description":"Server connections that have been idle for more than server_check_delay, so they need server_check_query to run on them before they can be used again","metric_name":"ccp_pgbouncer_pools_server_used","value_column":"sv_used"}],"sql":"SHOW POOLS"},{"metrics":[{"attribute_columns":["database","user","state","application_name","link"],"description":"1 if the connection will be closed as soon as possible, because a configuration file reload or DNS update changed the connection information or RECONNECT was issued","metric_name":"ccp_pgbouncer_servers_close_needed","value_column":"close_needed"}],"sql":"SHOW SERVERS"}] diff --git a/internal/collector/pgbouncer_metrics_queries.yaml b/internal/collector/pgbouncer_metrics_queries.yaml index 228fef1cc..ef09e2d02 100644 --- a/internal/collector/pgbouncer_metrics_queries.yaml +++ b/internal/collector/pgbouncer_metrics_queries.yaml @@ -17,39 +17,39 @@ metrics: - metric_name: ccp_pgbouncer_databases_pool_size value_column: pool_size - attribute_columns: ["name", "port", "database", "force_user", "pool_mode"] + attribute_columns: ["name", "port", "database"] description: "Maximum number of server connections" - metric_name: ccp_pgbouncer_databases_min_pool_size value_column: min_pool_size - attribute_columns: ["name", "port", "database", "force_user", "pool_mode"] + attribute_columns: ["name", "port", "database"] description: "Minimum number of server connections" - metric_name: ccp_pgbouncer_databases_reserve_pool - value_column: reserve_pool - attribute_columns: ["name", "port", "database", "force_user", "pool_mode"] + value_column: reserve_pool_size + attribute_columns: ["name", "port", "database"] description: "Maximum number of additional connections for this database" - metric_name: ccp_pgbouncer_databases_max_connections value_column: max_connections - attribute_columns: ["name", "port", "database", "force_user", "pool_mode"] + attribute_columns: ["name", "port", "database"] description: >- Maximum number of allowed connections for this database, as set by max_db_connections, either globally or per database - metric_name: ccp_pgbouncer_databases_current_connections value_column: current_connections - attribute_columns: ["name", "port", "database", "force_user", "pool_mode"] + attribute_columns: ["name", "port", "database"] description: "Current number of connections for this database" - metric_name: ccp_pgbouncer_databases_paused value_column: paused - attribute_columns: ["name", "port", "database", "force_user", "pool_mode"] + attribute_columns: ["name", "port", "database"] description: "1 if this database is currently paused, else 0" - metric_name: ccp_pgbouncer_databases_disabled value_column: disabled - attribute_columns: ["name", "port", "database", "force_user", "pool_mode"] + attribute_columns: ["name", "port", "database"] description: "1 if this database is currently disabled, else 0" - sql: "SHOW LISTS" From 4f80c64f356921d019ff65f8cc19ee76718e9c02 Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Thu, 20 Mar 2025 11:14:08 -0500 Subject: [PATCH 3/4] remove context: log --- .../generated/pgbackrest_logs_transforms.json | 2 +- .../generated/postgres_logs_transforms.json | 2 +- internal/collector/patroni.go | 1 - internal/collector/patroni_test.go | 6 ++---- internal/collector/pgadmin.go | 1 - internal/collector/pgadmin_test.go | 6 ++---- .../collector/pgbackrest_logs_transforms.yaml | 3 +-- internal/collector/pgbackrest_test.go | 6 ++---- internal/collector/pgbouncer.go | 1 - internal/collector/pgbouncer_test.go | 6 ++---- internal/collector/postgres_logs_transforms.yaml | 9 +++------ internal/collector/postgres_test.go | 16 ++++------------ 12 files changed, 18 insertions(+), 41 deletions(-) diff --git a/internal/collector/generated/pgbackrest_logs_transforms.json b/internal/collector/generated/pgbackrest_logs_transforms.json index 32b3809c0..3f8cf5137 100644 --- a/internal/collector/generated/pgbackrest_logs_transforms.json +++ b/internal/collector/generated/pgbackrest_logs_transforms.json @@ -1 +1 @@ -[{"context":"log","statements":["set(instrumentation_scope.name, \"pgbackrest\")","set(instrumentation_scope.schema_url, \"https://opentelemetry.io/schemas/1.29.0\")","merge_maps(log.cache, ExtractPatterns(log.body, \"^(?\u003ctimestamp\u003e\\\\d{4}-\\\\d{2}-\\\\d{2} \\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}) (?\u003cprocess_id\u003eP\\\\d{2,3})\\\\s*(?\u003cerror_severity\u003e\\\\S*): (?\u003cmessage\u003e(?s).*)$\"), \"insert\") where Len(log.body) \u003e 0","set(log.severity_text, log.cache[\"error_severity\"]) where IsString(log.cache[\"error_severity\"])","set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == \"TRACE\"","set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == \"DEBUG\"","set(log.severity_number, SEVERITY_NUMBER_DEBUG2) where log.severity_text == \"DETAIL\"","set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == \"INFO\"","set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == \"WARN\"","set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == \"ERROR\"","set(log.time, Time(log.cache[\"timestamp\"], \"%Y-%m-%d %H:%M:%S.%L\")) where IsString(log.cache[\"timestamp\"])","set(log.attributes[\"process.pid\"], log.cache[\"process_id\"])","set(log.attributes[\"log.record.original\"], log.body)","set(log.body, log.cache[\"message\"])"]}] +[{"statements":["set(instrumentation_scope.name, \"pgbackrest\")","set(instrumentation_scope.schema_url, \"https://opentelemetry.io/schemas/1.29.0\")","merge_maps(log.cache, ExtractPatterns(log.body, \"^(?\u003ctimestamp\u003e\\\\d{4}-\\\\d{2}-\\\\d{2} \\\\d{2}:\\\\d{2}:\\\\d{2}\\\\.\\\\d{3}) (?\u003cprocess_id\u003eP\\\\d{2,3})\\\\s*(?\u003cerror_severity\u003e\\\\S*): (?\u003cmessage\u003e(?s).*)$\"), \"insert\") where Len(log.body) \u003e 0","set(log.severity_text, log.cache[\"error_severity\"]) where IsString(log.cache[\"error_severity\"])","set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == \"TRACE\"","set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == \"DEBUG\"","set(log.severity_number, SEVERITY_NUMBER_DEBUG2) where log.severity_text == \"DETAIL\"","set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == \"INFO\"","set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == \"WARN\"","set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == \"ERROR\"","set(log.time, Time(log.cache[\"timestamp\"], \"%Y-%m-%d %H:%M:%S.%L\")) where IsString(log.cache[\"timestamp\"])","set(log.attributes[\"process.pid\"], log.cache[\"process_id\"])","set(log.attributes[\"log.record.original\"], log.body)","set(log.body, log.cache[\"message\"])"]}] diff --git a/internal/collector/generated/postgres_logs_transforms.json b/internal/collector/generated/postgres_logs_transforms.json index 5071e4399..f7409174e 100644 --- a/internal/collector/generated/postgres_logs_transforms.json +++ b/internal/collector/generated/postgres_logs_transforms.json @@ -1 +1 @@ -[{"conditions":["body[\"format\"] == \"csv\""],"context":"log","statements":["set(log.cache, ParseCSV(log.body[\"original\"], log.body[\"headers\"], delimiter=\",\", mode=\"strict\"))","merge_maps(log.cache, ExtractPatterns(log.cache[\"connection_from\"], \"(?:^[[]local[]]:(?\u003cremote_port\u003e.+)|:(?\u003cremote_port\u003e[^:]+))$\"), \"insert\") where Len(log.cache[\"connection_from\"]) \u003e 0","set(log.cache[\"remote_host\"], Substring(log.cache[\"connection_from\"], 0, Len(log.cache[\"connection_from\"]) - Len(log.cache[\"remote_port\"]) - 1)) where Len(log.cache[\"connection_from\"]) \u003e 0 and IsString(log.cache[\"remote_port\"])","set(log.cache[\"remote_host\"], log.cache[\"connection_from\"]) where Len(log.cache[\"connection_from\"]) \u003e 0 and not IsString(log.cache[\"remote_host\"])","merge_maps(log.cache, ExtractPatterns(log.cache[\"location\"], \"^(?:(?\u003cfunc_name\u003e[^,]+), )?(?\u003cfile_name\u003e[^:]+):(?\u003cfile_line_num\u003e\\\\d+)$\"), \"insert\") where Len(log.cache[\"location\"]) \u003e 0","set(log.cache[\"cursor_position\"], Double(log.cache[\"cursor_position\"])) where IsMatch(log.cache[\"cursor_position\"], \"^[0-9.]+$\")","set(log.cache[\"file_line_num\"], Double(log.cache[\"file_line_num\"])) where IsMatch(log.cache[\"file_line_num\"], \"^[0-9.]+$\")","set(log.cache[\"internal_position\"], Double(log.cache[\"internal_position\"])) where IsMatch(log.cache[\"internal_position\"], \"^[0-9.]+$\")","set(log.cache[\"leader_pid\"], Double(log.cache[\"leader_pid\"])) where IsMatch(log.cache[\"leader_pid\"], \"^[0-9.]+$\")","set(log.cache[\"line_num\"], Double(log.cache[\"line_num\"])) where IsMatch(log.cache[\"line_num\"], \"^[0-9.]+$\")","set(log.cache[\"pid\"], Double(log.cache[\"pid\"])) where IsMatch(log.cache[\"pid\"], \"^[0-9.]+$\")","set(log.cache[\"query_id\"], Double(log.cache[\"query_id\"])) where IsMatch(log.cache[\"query_id\"], \"^[0-9.]+$\")","set(log.cache[\"remote_port\"], Double(log.cache[\"remote_port\"])) where IsMatch(log.cache[\"remote_port\"], \"^[0-9.]+$\")","set(log.body[\"parsed\"], log.cache)"]},{"context":"log","statements":["set(instrumentation_scope.name, \"postgres\")","set(instrumentation_scope.version, resource.attributes[\"db.version\"])","set(log.cache, log.body[\"parsed\"]) where log.body[\"format\"] == \"csv\"","set(log.cache, ParseJSON(log.body[\"original\"])) where log.body[\"format\"] == \"json\"","set(log.severity_text, log.cache[\"error_severity\"])","set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == \"DEBUG5\"","set(log.severity_number, SEVERITY_NUMBER_TRACE2) where log.severity_text == \"DEBUG4\"","set(log.severity_number, SEVERITY_NUMBER_TRACE3) where log.severity_text == \"DEBUG3\"","set(log.severity_number, SEVERITY_NUMBER_TRACE4) where log.severity_text == \"DEBUG2\"","set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == \"DEBUG1\"","set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == \"INFO\" or log.severity_text == \"LOG\"","set(log.severity_number, SEVERITY_NUMBER_INFO2) where log.severity_text == \"NOTICE\"","set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == \"WARNING\"","set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == \"ERROR\"","set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == \"FATAL\"","set(log.severity_number, SEVERITY_NUMBER_FATAL2) where log.severity_text == \"PANIC\"","set(log.time, Time(log.cache[\"timestamp\"], \"%F %T.%L %Z\")) where IsString(log.cache[\"timestamp\"])","set(instrumentation_scope.schema_url, \"https://opentelemetry.io/schemas/1.29.0\")","set(resource.attributes[\"db.system\"], \"postgresql\")","set(log.attributes[\"log.record.original\"], log.body[\"original\"])","set(log.body, log.cache)","set(log.attributes[\"client.address\"], log.body[\"remote_host\"]) where IsString(log.body[\"remote_host\"])","set(log.attributes[\"client.port\"], Int(log.body[\"remote_port\"])) where IsDouble(log.body[\"remote_port\"])","set(log.attributes[\"code.filepath\"], log.body[\"file_name\"]) where IsString(log.body[\"file_name\"])","set(log.attributes[\"code.function\"], log.body[\"func_name\"]) where IsString(log.body[\"func_name\"])","set(log.attributes[\"code.lineno\"], Int(log.body[\"file_line_num\"])) where IsDouble(log.body[\"file_line_num\"])","set(log.attributes[\"db.namespace\"], log.body[\"dbname\"]) where IsString(log.body[\"dbname\"])","set(log.attributes[\"db.response.status_code\"], log.body[\"state_code\"]) where IsString(log.body[\"state_code\"])","set(log.attributes[\"process.creation.time\"], Concat([ Substring(log.body[\"session_start\"], 0, 10), \"T\", Substring(log.body[\"session_start\"], 11, 8), \"Z\"], \"\")) where IsMatch(log.body[\"session_start\"], \"^[^ ]{10} [^ ]{8} UTC$\")","set(log.attributes[\"process.pid\"], Int(log.body[\"pid\"])) where IsDouble(log.body[\"pid\"])","set(log.attributes[\"process.title\"], log.body[\"ps\"]) where IsString(log.body[\"ps\"])","set(log.attributes[\"user.name\"], log.body[\"user\"]) where IsString(log.body[\"user\"])"]},{"conditions":["Len(body[\"message\"]) \u003e 7 and Substring(body[\"message\"], 0, 7) == \"AUDIT: \""],"context":"log","statements":["set(log.body[\"pgaudit\"], ParseCSV(Substring(log.body[\"message\"], 7, Len(log.body[\"message\"]) - 7), \"audit_type,statement_id,substatement_id,class,command,object_type,object_name,statement,parameter\", delimiter=\",\", mode=\"strict\"))","set(instrumentation_scope.name, \"pgaudit\") where Len(log.body[\"pgaudit\"]) \u003e 0"]}] +[{"conditions":["body[\"format\"] == \"csv\""],"statements":["set(log.cache, ParseCSV(log.body[\"original\"], log.body[\"headers\"], delimiter=\",\", mode=\"strict\"))","merge_maps(log.cache, ExtractPatterns(log.cache[\"connection_from\"], \"(?:^[[]local[]]:(?\u003cremote_port\u003e.+)|:(?\u003cremote_port\u003e[^:]+))$\"), \"insert\") where Len(log.cache[\"connection_from\"]) \u003e 0","set(log.cache[\"remote_host\"], Substring(log.cache[\"connection_from\"], 0, Len(log.cache[\"connection_from\"]) - Len(log.cache[\"remote_port\"]) - 1)) where Len(log.cache[\"connection_from\"]) \u003e 0 and IsString(log.cache[\"remote_port\"])","set(log.cache[\"remote_host\"], log.cache[\"connection_from\"]) where Len(log.cache[\"connection_from\"]) \u003e 0 and not IsString(log.cache[\"remote_host\"])","merge_maps(log.cache, ExtractPatterns(log.cache[\"location\"], \"^(?:(?\u003cfunc_name\u003e[^,]+), )?(?\u003cfile_name\u003e[^:]+):(?\u003cfile_line_num\u003e\\\\d+)$\"), \"insert\") where Len(log.cache[\"location\"]) \u003e 0","set(log.cache[\"cursor_position\"], Double(log.cache[\"cursor_position\"])) where IsMatch(log.cache[\"cursor_position\"], \"^[0-9.]+$\")","set(log.cache[\"file_line_num\"], Double(log.cache[\"file_line_num\"])) where IsMatch(log.cache[\"file_line_num\"], \"^[0-9.]+$\")","set(log.cache[\"internal_position\"], Double(log.cache[\"internal_position\"])) where IsMatch(log.cache[\"internal_position\"], \"^[0-9.]+$\")","set(log.cache[\"leader_pid\"], Double(log.cache[\"leader_pid\"])) where IsMatch(log.cache[\"leader_pid\"], \"^[0-9.]+$\")","set(log.cache[\"line_num\"], Double(log.cache[\"line_num\"])) where IsMatch(log.cache[\"line_num\"], \"^[0-9.]+$\")","set(log.cache[\"pid\"], Double(log.cache[\"pid\"])) where IsMatch(log.cache[\"pid\"], \"^[0-9.]+$\")","set(log.cache[\"query_id\"], Double(log.cache[\"query_id\"])) where IsMatch(log.cache[\"query_id\"], \"^[0-9.]+$\")","set(log.cache[\"remote_port\"], Double(log.cache[\"remote_port\"])) where IsMatch(log.cache[\"remote_port\"], \"^[0-9.]+$\")","set(log.body[\"parsed\"], log.cache)"]},{"statements":["set(instrumentation_scope.name, \"postgres\")","set(instrumentation_scope.version, resource.attributes[\"db.version\"])","set(log.cache, log.body[\"parsed\"]) where log.body[\"format\"] == \"csv\"","set(log.cache, ParseJSON(log.body[\"original\"])) where log.body[\"format\"] == \"json\"","set(log.severity_text, log.cache[\"error_severity\"])","set(log.severity_number, SEVERITY_NUMBER_TRACE) where log.severity_text == \"DEBUG5\"","set(log.severity_number, SEVERITY_NUMBER_TRACE2) where log.severity_text == \"DEBUG4\"","set(log.severity_number, SEVERITY_NUMBER_TRACE3) where log.severity_text == \"DEBUG3\"","set(log.severity_number, SEVERITY_NUMBER_TRACE4) where log.severity_text == \"DEBUG2\"","set(log.severity_number, SEVERITY_NUMBER_DEBUG) where log.severity_text == \"DEBUG1\"","set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_text == \"INFO\" or log.severity_text == \"LOG\"","set(log.severity_number, SEVERITY_NUMBER_INFO2) where log.severity_text == \"NOTICE\"","set(log.severity_number, SEVERITY_NUMBER_WARN) where log.severity_text == \"WARNING\"","set(log.severity_number, SEVERITY_NUMBER_ERROR) where log.severity_text == \"ERROR\"","set(log.severity_number, SEVERITY_NUMBER_FATAL) where log.severity_text == \"FATAL\"","set(log.severity_number, SEVERITY_NUMBER_FATAL2) where log.severity_text == \"PANIC\"","set(log.time, Time(log.cache[\"timestamp\"], \"%F %T.%L %Z\")) where IsString(log.cache[\"timestamp\"])","set(instrumentation_scope.schema_url, \"https://opentelemetry.io/schemas/1.29.0\")","set(resource.attributes[\"db.system\"], \"postgresql\")","set(log.attributes[\"log.record.original\"], log.body[\"original\"])","set(log.body, log.cache)","set(log.attributes[\"client.address\"], log.body[\"remote_host\"]) where IsString(log.body[\"remote_host\"])","set(log.attributes[\"client.port\"], Int(log.body[\"remote_port\"])) where IsDouble(log.body[\"remote_port\"])","set(log.attributes[\"code.filepath\"], log.body[\"file_name\"]) where IsString(log.body[\"file_name\"])","set(log.attributes[\"code.function\"], log.body[\"func_name\"]) where IsString(log.body[\"func_name\"])","set(log.attributes[\"code.lineno\"], Int(log.body[\"file_line_num\"])) where IsDouble(log.body[\"file_line_num\"])","set(log.attributes[\"db.namespace\"], log.body[\"dbname\"]) where IsString(log.body[\"dbname\"])","set(log.attributes[\"db.response.status_code\"], log.body[\"state_code\"]) where IsString(log.body[\"state_code\"])","set(log.attributes[\"process.creation.time\"], Concat([ Substring(log.body[\"session_start\"], 0, 10), \"T\", Substring(log.body[\"session_start\"], 11, 8), \"Z\"], \"\")) where IsMatch(log.body[\"session_start\"], \"^[^ ]{10} [^ ]{8} UTC$\")","set(log.attributes[\"process.pid\"], Int(log.body[\"pid\"])) where IsDouble(log.body[\"pid\"])","set(log.attributes[\"process.title\"], log.body[\"ps\"]) where IsString(log.body[\"ps\"])","set(log.attributes[\"user.name\"], log.body[\"user\"]) where IsString(log.body[\"user\"])"]},{"conditions":["Len(body[\"message\"]) \u003e 7 and Substring(body[\"message\"], 0, 7) == \"AUDIT: \""],"statements":["set(log.body[\"pgaudit\"], ParseCSV(Substring(log.body[\"message\"], 7, Len(log.body[\"message\"]) - 7), \"audit_type,statement_id,substatement_id,class,command,object_type,object_name,statement,parameter\", delimiter=\",\", mode=\"strict\"))","set(instrumentation_scope.name, \"pgaudit\") where Len(log.body[\"pgaudit\"]) \u003e 0"]}] diff --git a/internal/collector/patroni.go b/internal/collector/patroni.go index 991bb9882..017ed4867 100644 --- a/internal/collector/patroni.go +++ b/internal/collector/patroni.go @@ -64,7 +64,6 @@ func EnablePatroniLogging(ctx context.Context, // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/processor/transformprocessor#readme outConfig.Processors["transform/patroni_logs"] = map[string]any{ "log_statements": []map[string]any{{ - "context": "log", "statements": []string{ `set(instrumentation_scope.name, "patroni")`, diff --git a/internal/collector/patroni_test.go b/internal/collector/patroni_test.go index fc9823f8d..f55ff2124 100644 --- a/internal/collector/patroni_test.go +++ b/internal/collector/patroni_test.go @@ -71,8 +71,7 @@ processors: timeout: 30s transform/patroni_logs: log_statements: - - context: log - statements: + - statements: - set(instrumentation_scope.name, "patroni") - set(log.cache, ParseJSON(log.body["original"])) - set(log.severity_text, log.cache["levelname"]) @@ -171,8 +170,7 @@ processors: timeout: 30s transform/patroni_logs: log_statements: - - context: log - statements: + - statements: - set(instrumentation_scope.name, "patroni") - set(log.cache, ParseJSON(log.body["original"])) - set(log.severity_text, log.cache["levelname"]) diff --git a/internal/collector/pgadmin.go b/internal/collector/pgadmin.go index 9637893c5..5c4840cd6 100644 --- a/internal/collector/pgadmin.go +++ b/internal/collector/pgadmin.go @@ -55,7 +55,6 @@ func EnablePgAdminLogging(ctx context.Context, spec *v1beta1.InstrumentationSpec otelConfig.Processors["transform/pgadmin_log"] = map[string]any{ "log_statements": []map[string]any{ { - "context": "log", "statements": []string{ // Keep the unparsed log record in a standard attribute, and replace // the log record body with the message field. diff --git a/internal/collector/pgadmin_test.go b/internal/collector/pgadmin_test.go index 8dc2097da..b856baab0 100644 --- a/internal/collector/pgadmin_test.go +++ b/internal/collector/pgadmin_test.go @@ -75,8 +75,7 @@ collector.yaml: | timeout: 30s transform/pgadmin_log: log_statements: - - context: log - statements: + - statements: - set(log.attributes["log.record.original"], log.body) - set(log.cache, ParseJSON(log.body)) - merge_maps(log.attributes, ExtractPatterns(log.cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), @@ -201,8 +200,7 @@ collector.yaml: | timeout: 30s transform/pgadmin_log: log_statements: - - context: log - statements: + - statements: - set(log.attributes["log.record.original"], log.body) - set(log.cache, ParseJSON(log.body)) - merge_maps(log.attributes, ExtractPatterns(log.cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), diff --git a/internal/collector/pgbackrest_logs_transforms.yaml b/internal/collector/pgbackrest_logs_transforms.yaml index 9aca9c212..389f9d0a2 100644 --- a/internal/collector/pgbackrest_logs_transforms.yaml +++ b/internal/collector/pgbackrest_logs_transforms.yaml @@ -3,8 +3,7 @@ # # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/processor/transformprocessor#readme -- context: log - statements: +- statements: - set(instrumentation_scope.name, "pgbackrest") - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") diff --git a/internal/collector/pgbackrest_test.go b/internal/collector/pgbackrest_test.go index 9f69d7fe3..a916786df 100644 --- a/internal/collector/pgbackrest_test.go +++ b/internal/collector/pgbackrest_test.go @@ -73,8 +73,7 @@ processors: timeout: 30s transform/pgbackrest_logs: log_statements: - - context: log - statements: + - statements: - set(instrumentation_scope.name, "pgbackrest") - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - 'merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} @@ -180,8 +179,7 @@ processors: timeout: 30s transform/pgbackrest_logs: log_statements: - - context: log - statements: + - statements: - set(instrumentation_scope.name, "pgbackrest") - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - 'merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} diff --git a/internal/collector/pgbouncer.go b/internal/collector/pgbouncer.go index bde500f50..700b9a372 100644 --- a/internal/collector/pgbouncer.go +++ b/internal/collector/pgbouncer.go @@ -96,7 +96,6 @@ func EnablePgBouncerLogging(ctx context.Context, // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/processor/transformprocessor#readme outConfig.Processors["transform/pgbouncer_logs"] = map[string]any{ "log_statements": []map[string]any{{ - "context": "log", "statements": []string{ // Set instrumentation scope `set(instrumentation_scope.name, "pgbouncer")`, diff --git a/internal/collector/pgbouncer_test.go b/internal/collector/pgbouncer_test.go index 333fae0dd..cbd69cbd0 100644 --- a/internal/collector/pgbouncer_test.go +++ b/internal/collector/pgbouncer_test.go @@ -70,8 +70,7 @@ processors: timeout: 30s transform/pgbouncer_logs: log_statements: - - context: log - statements: + - statements: - set(instrumentation_scope.name, "pgbouncer") - merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3} [A-Z]{3}) \\[(?\\d+)\\] (?[A-Z]+) @@ -171,8 +170,7 @@ processors: timeout: 30s transform/pgbouncer_logs: log_statements: - - context: log - statements: + - statements: - set(instrumentation_scope.name, "pgbouncer") - merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3} [A-Z]{3}) \\[(?\\d+)\\] (?[A-Z]+) diff --git a/internal/collector/postgres_logs_transforms.yaml b/internal/collector/postgres_logs_transforms.yaml index 67f847a77..c8178f2d6 100644 --- a/internal/collector/postgres_logs_transforms.yaml +++ b/internal/collector/postgres_logs_transforms.yaml @@ -7,8 +7,7 @@ # TODO(postgres-14): We can stop parsing CSV logs when 14 is EOL. # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#readme -- context: log - conditions: +- conditions: - body["format"] == "csv" statements: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/ottlfuncs#parsecsv @@ -81,8 +80,7 @@ # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#readme -- context: log - statements: +- statements: - set(instrumentation_scope.name, "postgres") - set(instrumentation_scope.version, resource.attributes["db.version"]) @@ -191,8 +189,7 @@ # # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#readme # https://github.com/pgaudit/pgaudit#format -- context: log - conditions: +- conditions: # Messages from pgAudit have always been prefixed with "AUDIT:", but that # could change in the future. # diff --git a/internal/collector/postgres_test.go b/internal/collector/postgres_test.go index d57e47dda..ba188a129 100644 --- a/internal/collector/postgres_test.go +++ b/internal/collector/postgres_test.go @@ -96,8 +96,7 @@ processors: timeout: 30s transform/pgbackrest_logs: log_statements: - - context: log - statements: + - statements: - set(instrumentation_scope.name, "pgbackrest") - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - 'merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} @@ -125,7 +124,6 @@ processors: log_statements: - conditions: - body["format"] == "csv" - context: log statements: - set(log.cache, ParseCSV(log.body["original"], log.body["headers"], delimiter=",", mode="strict")) @@ -156,8 +154,7 @@ processors: - set(log.cache["remote_port"], Double(log.cache["remote_port"])) where IsMatch(log.cache["remote_port"], "^[0-9.]+$") - set(log.body["parsed"], log.cache) - - context: log - statements: + - statements: - set(instrumentation_scope.name, "postgres") - set(instrumentation_scope.version, resource.attributes["db.version"]) - set(log.cache, log.body["parsed"]) where log.body["format"] == "csv" @@ -208,7 +205,6 @@ processors: - conditions: - 'Len(body["message"]) > 7 and Substring(body["message"], 0, 7) == "AUDIT: "' - context: log statements: - set(log.body["pgaudit"], ParseCSV(Substring(log.body["message"], 7, Len(log.body["message"]) - 7), "audit_type,statement_id,substatement_id,class,command,object_type,object_name,statement,parameter", @@ -359,8 +355,7 @@ processors: timeout: 30s transform/pgbackrest_logs: log_statements: - - context: log - statements: + - statements: - set(instrumentation_scope.name, "pgbackrest") - set(instrumentation_scope.schema_url, "https://opentelemetry.io/schemas/1.29.0") - 'merge_maps(log.cache, ExtractPatterns(log.body, "^(?\\d{4}-\\d{2}-\\d{2} @@ -388,7 +383,6 @@ processors: log_statements: - conditions: - body["format"] == "csv" - context: log statements: - set(log.cache, ParseCSV(log.body["original"], log.body["headers"], delimiter=",", mode="strict")) @@ -419,8 +413,7 @@ processors: - set(log.cache["remote_port"], Double(log.cache["remote_port"])) where IsMatch(log.cache["remote_port"], "^[0-9.]+$") - set(log.body["parsed"], log.cache) - - context: log - statements: + - statements: - set(instrumentation_scope.name, "postgres") - set(instrumentation_scope.version, resource.attributes["db.version"]) - set(log.cache, log.body["parsed"]) where log.body["format"] == "csv" @@ -471,7 +464,6 @@ processors: - conditions: - 'Len(body["message"]) > 7 and Substring(body["message"], 0, 7) == "AUDIT: "' - context: log statements: - set(log.body["pgaudit"], ParseCSV(Substring(log.body["message"], 7, Len(log.body["message"]) - 7), "audit_type,statement_id,substatement_id,class,command,object_type,object_name,statement,parameter", From af7bb9c60ee1fd29105e3eb213f6e571c55f6cc4 Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Thu, 20 Mar 2025 11:19:00 -0500 Subject: [PATCH 4/4] add info around pgbouncer columns avoided --- internal/collector/pgbouncer_metrics_queries.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/collector/pgbouncer_metrics_queries.yaml b/internal/collector/pgbouncer_metrics_queries.yaml index ef09e2d02..a4e3a918f 100644 --- a/internal/collector/pgbouncer_metrics_queries.yaml +++ b/internal/collector/pgbouncer_metrics_queries.yaml @@ -11,7 +11,9 @@ attribute_columns: ["database", "user", "state", "application_name", "link"] description: "Current waiting time in seconds" - # NOTE: Avoid collecting "host" column because it can be null; the collector will warn against null. + # NOTE: Avoid collecting/using "host", "force_user", and "pool_mode" columns because they + # can be NULL; the collector will warn against NULL even when not used. But it will emit + # an error log if those columns are used. # The host column should always point either to pgBouncer's virtual database (the null case) or to the primary. - sql: "SHOW DATABASES" metrics: