From 51b76a3e4dac029d6a69448c9207cfabb8d382c5 Mon Sep 17 00:00:00 2001 From: Eguzki Astiz Lezaun Date: Fri, 30 Sep 2022 18:38:26 +0200 Subject: [PATCH 1/4] add opentelemetry instrumentation --- Dockerfile | 66 +++++++++++++++++++ Makefile | 6 ++ docker-compose.yml | 39 +++++++++-- examples/opentelemetry/apicast-config.json | 12 ++++ examples/opentelemetry/otel.toml | 20 ++++++ examples/opentracing/apicast-config.json | 12 ++++ examples/opentracing/jaeger-config.json | 26 ++++++++ gateway/conf.d/opentelemetry/otel.conf.liquid | 7 ++ .../conf.d/opentelemetry/otel.example.toml | 25 +++++++ gateway/conf/nginx.conf.liquid | 15 +++++ gateway/http.d/apicast.conf.liquid | 20 +++++- gateway/src/apicast/cli/environment.lua | 4 ++ 12 files changed, 244 insertions(+), 8 deletions(-) create mode 100644 examples/opentelemetry/apicast-config.json create mode 100644 examples/opentelemetry/otel.toml create mode 100644 examples/opentracing/apicast-config.json create mode 100644 examples/opentracing/jaeger-config.json create mode 100644 gateway/conf.d/opentelemetry/otel.conf.liquid create mode 100644 gateway/conf.d/opentelemetry/otel.example.toml diff --git a/Dockerfile b/Dockerfile index d529a2412..317236028 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,67 @@ +FROM registry.access.redhat.com/ubi8:8.5 AS opentelemetry-builder + +ARG OPENRESTY_RPM_VERSION="1.19.3" + +RUN sed -i s/enabled=./enabled=0/g /etc/yum/pluginconf.d/subscription-manager.conf + +RUN yum install -y pcre-devel zlib-devel libcurl-devel cmake gcc gcc-c++ git + +WORKDIR /tmp + +RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.36.4 \ + https://github.com/grpc/grpc \ + && cd grpc \ + && mkdir -p cmake/build \ + && cd cmake/build \ + && cmake \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DCMAKE_INSTALL_PREFIX=/install \ + -DCMAKE_BUILD_TYPE=Release \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ + ../.. \ + && make -j2 \ + && make install + +RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.6.1 \ + https://github.com/open-telemetry/opentelemetry-cpp.git \ + && cd opentelemetry-cpp \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/install \ + -DCMAKE_PREFIX_PATH=/install \ + -DWITH_OTLP=ON \ + -DWITH_OTLP_GRPC=ON \ + -DWITH_OTLP_HTTP=OFF \ + -DBUILD_TESTING=OFF \ + -DWITH_EXAMPLES=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + .. \ + && make -j2 \ + && make install + +ADD https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/refs/tags/webserver/v1.0.1.tar.gz ./ + +RUN tar -xzvf v1.0.1.tar.gz && rm v1.0.1.tar.gz + +RUN mkdir -p otel-nginx/build && mkdir -p otel-nginx/src +RUN cp -r /tmp/opentelemetry-cpp-contrib-webserver-v1.0.1/instrumentation/nginx/src otel-nginx/ +RUN cp /tmp/opentelemetry-cpp-contrib-webserver-v1.0.1/instrumentation/nginx/CMakeLists.txt /tmp/opentelemetry-cpp-contrib-webserver-v1.0.1/instrumentation/nginx/nginx.cmake otel-nginx/ +RUN cd otel-nginx/build \ + && cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=/install \ + -DCMAKE_INSTALL_PREFIX=/usr/share/nginx/modules \ + -DNGINX_VERSION=1.19.3 \ + .. \ + && make -j2 \ + && make install + FROM registry.access.redhat.com/ubi8:8.5 ARG OPENRESTY_RPM_VERSION="1.19.3" @@ -38,6 +102,8 @@ ENV PATH="./lua_modules/bin:/usr/local/openresty/luajit/bin/:${PATH}" \ LUA_CPATH="./lua_modules/lib/lua/5.1/?.so;;" \ LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/app-root/lib" +COPY --from=opentelemetry-builder /usr/share/nginx/modules/otel_ngx_module.so /usr/local/openresty/nginx/modules + RUN luarocks install --deps-mode=none --tree /usr/local https://luarocks.org/manifests/pintsized/lua-resty-http-0.15-0.src.rock RUN luarocks install --deps-mode=none --tree /usr/local https://luarocks.org/manifests/kikito/router-2.1-0.src.rock RUN luarocks install --deps-mode=none --tree /usr/local https://luarocks.org/manifests/kikito/inspect-3.1.1-0.src.rock diff --git a/Makefile b/Makefile index dc8a667d6..a6ff0bd96 100644 --- a/Makefile +++ b/Makefile @@ -164,6 +164,12 @@ gateway-logs: export IMAGE_NAME = does-not-matter gateway-logs: $(DOCKER_COMPOSE) logs gateway +opentelemetry-gateway: ## run gateway instrumented with opentelemetry + $(DOCKER_COMPOSE) run opentelemetry-instrumented-gateway + +opentracing-gateway: ## run gateway instrumented with opentracing + $(DOCKER_COMPOSE) run opentracing-instrumented-gateway + test-runtime-image: export IMAGE_NAME ?= $(RUNTIME_IMAGE) test-runtime-image: clean-containers ## Smoke test the runtime image. Pass any docker image in IMAGE_NAME parameter. $(DOCKER_COMPOSE) --version diff --git a/docker-compose.yml b/docker-compose.yml index 8ff521c41..a92da573e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,15 +53,40 @@ services: KEYCLOAK_LOGLEVEL: INFO ports: - "8080" + opentracing-instrumented-gateway: + image: ${IMAGE_NAME:-apicast-test} + depends_on: + - jaeger + environment: + THREESCALE_CONFIG_FILE: /tmp/config.json + THREESCALE_DEPLOYMENT_ENV: staging + APICAST_CONFIGURATION_LOADER: lazy + APICAST_LOG_LEVEL: debug + APICAST_CONFIGURATION_CACHE: "0" + OPENTRACING_TRACER: jaeger + OPENTRACING_CONFIG: /opt/app-root/src/tracing-configs/tracing-config-jaeger-jaeger-config.json + volumes: + - ./examples/opentracing/apicast-config.json:/tmp/config.json + - ./examples/opentracing/jaeger-config.json:/opt/app-root/src/tracing-configs/tracing-config-jaeger-jaeger-config.json + opentelemetry-instrumented-gateway: + image: ${IMAGE_NAME:-apicast-test} + depends_on: + - jaeger + environment: + THREESCALE_CONFIG_FILE: /tmp/config.json + THREESCALE_DEPLOYMENT_ENV: staging + APICAST_CONFIGURATION_LOADER: lazy + APICAST_LOG_LEVEL: debug + APICAST_CONFIGURATION_CACHE: "0" + OPENTELEMETRY: "1" + OPENTELEMETRY_CONFIG: /opt/app-root/src/tracing-configs/otel.toml + volumes: + - ./examples/opentelemetry/apicast-config.json:/tmp/config.json + - ./examples/opentelemetry/otel.toml:/opt/app-root/src/tracing-configs/otel.toml jaeger: image: jaegertracing/all-in-one:latest environment: - COLLECTOR_ZIPKIN_HTTP_PORT: 9411 + JAEGER_DISABLED: "false" + COLLECTOR_OTLP_ENABLED: "true" ports: - - 5775:5775/udp - - 6831:6831/udp - - 6832:6832/udp - - 5778:5778 - 16686:16686 - - 14268:14268 - - 9411:9411 diff --git a/examples/opentelemetry/apicast-config.json b/examples/opentelemetry/apicast-config.json new file mode 100644 index 000000000..621e88573 --- /dev/null +++ b/examples/opentelemetry/apicast-config.json @@ -0,0 +1,12 @@ +{ + "services": [ + { + "proxy": { + "hosts": ["one"], + "proxy_rules": [], + "api_backend": "https://echo-api.3scale.net", + "policy_chain": [] + } + } + ] +} diff --git a/examples/opentelemetry/otel.toml b/examples/opentelemetry/otel.toml new file mode 100644 index 000000000..8af04a1e6 --- /dev/null +++ b/examples/opentelemetry/otel.toml @@ -0,0 +1,20 @@ +exporter = "otlp" +processor = "simple" + +[exporters.otlp] +# Alternatively the OTEL_EXPORTER_OTLP_ENDPOINT environment variable can also be used. +host = "jaeger" +port = 4317 +# Optional: enable SSL, for endpoints that support it +# use_ssl = true +# Optional: set a filesystem path to a pem file to be used for SSL encryption +# (when use_ssl = true) +# ssl_cert_path = "/path/to/cert.pem" + +[processors.batch] +max_queue_size = 2048 +schedule_delay_millis = 5000 +max_export_batch_size = 512 + +[service] +name = "apicast" # Opentelemetry resource name diff --git a/examples/opentracing/apicast-config.json b/examples/opentracing/apicast-config.json new file mode 100644 index 000000000..621e88573 --- /dev/null +++ b/examples/opentracing/apicast-config.json @@ -0,0 +1,12 @@ +{ + "services": [ + { + "proxy": { + "hosts": ["one"], + "proxy_rules": [], + "api_backend": "https://echo-api.3scale.net", + "policy_chain": [] + } + } + ] +} diff --git a/examples/opentracing/jaeger-config.json b/examples/opentracing/jaeger-config.json new file mode 100644 index 000000000..aba954417 --- /dev/null +++ b/examples/opentracing/jaeger-config.json @@ -0,0 +1,26 @@ +{ + "service_name": "apicast", + "disabled": false, + "sampler": { + "type": "const", + "param": 1 + }, + "reporter": { + "queueSize": 100, + "bufferFlushInterval": 10, + "logSpans": false, + "localAgentHostPort": "jaeger:6831" + }, + "headers": { + "jaegerDebugHeader": "debug-id", + "jaegerBaggageHeader": "baggage", + "TraceContextHeaderName": "uber-trace-id", + "traceBaggageHeaderPrefix": "testctx-" + }, + "baggage_restrictions": { + "denyBaggageOnInitializationFailure": false, + "hostPort": "127.0.0.1:5778", + "refreshInterval": 60 + } +} + diff --git a/gateway/conf.d/opentelemetry/otel.conf.liquid b/gateway/conf.d/opentelemetry/otel.conf.liquid new file mode 100644 index 000000000..784247115 --- /dev/null +++ b/gateway/conf.d/opentelemetry/otel.conf.liquid @@ -0,0 +1,7 @@ +opentelemetry on; + +{% if opentelemetry_config_file == nil or opentelemetry_config_file == empty %} + {% assign opentelemetry_config_file = "conf.d/opentelemetry/jaeger.example.toml" | filesystem | first%} +{% endif %} + +opentelemetry_config {{ opentelemetry_config_file }}; diff --git a/gateway/conf.d/opentelemetry/otel.example.toml b/gateway/conf.d/opentelemetry/otel.example.toml new file mode 100644 index 000000000..023b2f100 --- /dev/null +++ b/gateway/conf.d/opentelemetry/otel.example.toml @@ -0,0 +1,25 @@ +exporter = "otlp" +processor = "batch" + +[exporters.otlp] +# Alternatively the OTEL_EXPORTER_OTLP_ENDPOINT environment variable can also be used. +host = "localhost" +port = 4317 +# Optional: enable SSL, for endpoints that support it +# use_ssl = true +# Optional: set a filesystem path to a pem file to be used for SSL encryption +# (when use_ssl = true) +# ssl_cert_path = "/path/to/cert.pem" + +[processors.batch] +max_queue_size = 2048 +schedule_delay_millis = 5000 +max_export_batch_size = 512 + +[service] +name = "apicast" # Opentelemetry resource name + +[sampler] +name = "AlwaysOn" # Also: AlwaysOff, TraceIdRatioBased +ratio = 0.1 +parent_based = false diff --git a/gateway/conf/nginx.conf.liquid b/gateway/conf/nginx.conf.liquid index c9c1054b1..806da984c 100644 --- a/gateway/conf/nginx.conf.liquid +++ b/gateway/conf/nginx.conf.liquid @@ -15,6 +15,16 @@ load_module {{file}}; {% endif %} {% endif %} +{% if opentelemetry != empty %} + {% for file in "modules/otel_ngx_module.so" | filesystem %} +load_module {{file}}; + {% endfor %} +{% else %} + {% if timer_resolution %} + timer_resolution {{ timer_resolution }}; + {% endif %} +{% endif %} + daemon {{ daemon | default: 'off' }}; master_process {{ master_process | default: 'on' }}; worker_processes {{ worker_processes | default: 'auto' }}; @@ -81,6 +91,11 @@ http { {% include tracer_conf %} {% endif %} + {% if opentelemetry != empty %} + {%- capture otel_conf %}conf.d/opentelemetry/otel.conf.liquid{%- endcapture -%} + {% include otel_conf %} + {% endif %} + {% for file in template | default: 'http.d/apicast.conf.liquid' | filesystem %} {% include file %} {% endfor %} diff --git a/gateway/http.d/apicast.conf.liquid b/gateway/http.d/apicast.conf.liquid index 724bb5bc4..17ee1ad1a 100644 --- a/gateway/http.d/apicast.conf.liquid +++ b/gateway/http.d/apicast.conf.liquid @@ -1,6 +1,6 @@ log_format time '[$time_local] $target_host:$server_port $remote_addr:$remote_port "$request" $status $body_bytes_sent ($request_time) $post_action_impact'; -# Use maps as variables because some logs can be raised out of the server context +# Use maps as variables because some logs can be raised out of the server context # where variables cannot be set, this allow us to avoid a warning map "" $extended_access_log { default ''; @@ -33,6 +33,10 @@ server { opentracing_trace_locations off; {% endif %} + {% if opentelemetry != empty %} + opentelemetry_operation_name apicast_management; + {% endif %} + {% include "conf.d/management.conf" %} } @@ -45,6 +49,10 @@ server { opentracing_trace_locations off; {% endif %} + {% if opentelemetry != empty %} + opentelemetry_operation_name apicast_mockbackend; + {% endif %} + {% include "conf.d/backend.conf" %} } @@ -62,6 +70,10 @@ server { opentracing_trace_locations off; {% endif %} + {% if opentelemetry != empty %} + opentelemetry_operation_name apicast_echo; + {% endif %} + {% include "conf.d/echo.conf" %} } @@ -115,6 +127,12 @@ server { opentracing_tag original_request_uri $original_request_uri; {% endif %} + {% if opentelemetry != empty %} + opentelemetry_operation_name apicast; + opentelemetry_propagate; + opentelemetry_attribute original_request_uri $original_request_uri; + {% endif %} + {% include "http.d/ssl.conf" %} {% for file in "apicast.d/*.conf" | filesystem %} diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua index f23df6cbb..eb1985605 100644 --- a/gateway/src/apicast/cli/environment.lua +++ b/gateway/src/apicast/cli/environment.lua @@ -98,6 +98,8 @@ _M.default_environment = 'production' -- @tfield ?string opentracing_tracer loads an opentracing tracer library, for example: jaeger -- @tfield ?string opentracing_config opentracing config file to load -- @tfield ?string opentracing_forward_header opentracing http header to forward upstream +-- @tfield ?string opentelemetry enables server instrumentation using opentelemetry SDKs +-- @tfield ?string opentelemetry_config_file opentelemetry config file to load -- @tfield ?string upstream_retry_cases error cases where the call to the upstream should be retried -- follows the same format as https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream -- @tfield ?policy_chain policy_chain @{policy_chain} instance @@ -116,6 +118,8 @@ _M.default_config = { opentracing_tracer = env_value_ref('OPENTRACING_TRACER'), opentracing_config = env_value_ref('OPENTRACING_CONFIG'), opentracing_forward_header = env_value_ref('OPENTRACING_FORWARD_HEADER'), + opentelemetry = env_value_ref('OPENTELEMETRY'), + opentelemetry_config_file = env_value_ref('OPENTELEMETRY_CONFIG'), upstream_retry_cases = env_value_ref('APICAST_UPSTREAM_RETRY_CASES'), http_keepalive_timeout = env_value_ref('HTTP_KEEPALIVE_TIMEOUT'), policy_chain = require('apicast.policy_chain').default(), From 83daaa4e8ff2c8830a4ca0645a3ee2e46e5dd233 Mon Sep 17 00:00:00 2001 From: Eguzki Astiz Lezaun Date: Wed, 19 Oct 2022 17:17:53 +0200 Subject: [PATCH 2/4] verbose --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 317236028..baa0f8967 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM registry.access.redhat.com/ubi8:8.5 AS opentelemetry-builder ARG OPENRESTY_RPM_VERSION="1.19.3" +ARG OTHER="1.19.3" RUN sed -i s/enabled=./enabled=0/g /etc/yum/pluginconf.d/subscription-manager.conf @@ -25,7 +26,7 @@ RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.36.4 \ -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF \ -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ ../.. \ - && make -j2 \ + && make -j8 \ && make install RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.6.1 \ @@ -43,7 +44,7 @@ RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.6.1 \ -DWITH_EXAMPLES=OFF \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ .. \ - && make -j2 \ + && make -j8 \ && make install ADD https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/refs/tags/webserver/v1.0.1.tar.gz ./ @@ -59,7 +60,7 @@ RUN cd otel-nginx/build \ -DCMAKE_INSTALL_PREFIX=/usr/share/nginx/modules \ -DNGINX_VERSION=1.19.3 \ .. \ - && make -j2 \ + && make -j8 VERBOSE=1 \ && make install FROM registry.access.redhat.com/ubi8:8.5 From e9631ef76f8b70f1d8e90be5285c7a99975a3232 Mon Sep 17 00:00:00 2001 From: Eguzki Astiz Lezaun Date: Wed, 19 Oct 2022 17:18:18 +0200 Subject: [PATCH 3/4] remove unused --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index baa0f8967..43f3d15b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ FROM registry.access.redhat.com/ubi8:8.5 AS opentelemetry-builder ARG OPENRESTY_RPM_VERSION="1.19.3" -ARG OTHER="1.19.3" RUN sed -i s/enabled=./enabled=0/g /etc/yum/pluginconf.d/subscription-manager.conf From 60d793baead6e7aa9f40f2b3bed12b23195530ff Mon Sep 17 00:00:00 2001 From: Eguzki Astiz Lezaun Date: Thu, 20 Oct 2022 01:34:21 +0200 Subject: [PATCH 4/4] wip --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 43f3d15b2..ddc0d883b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.36.4 \ -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF \ -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ ../.. \ - && make -j8 \ + && make -j8 VERBOSE=1 \ && make install RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.6.1 \ @@ -43,7 +43,7 @@ RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.6.1 \ -DWITH_EXAMPLES=OFF \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ .. \ - && make -j8 \ + && make -j8 VERBOSE=1 \ && make install ADD https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/refs/tags/webserver/v1.0.1.tar.gz ./