3scale · eguzki · Sep 30, 2022 · Oct 19, 2022 · Oct 19, 2022 · Oct 19, 2022
@@ -1,3 +1,67 @@
+FROM registry.access.redhat.com/ubi8:8.5 AS opentelemetry-builder
+
+ARG OPENRESTY_RPM_VERSION="1.19.3"
+
+RUN sed -i s/enabled=./enabled=0/g /etc/yum/pluginconf.d/subscription-manager.conf
+
+RUN yum install -y pcre-devel zlib-devel libcurl-devel cmake gcc gcc-c++ git
+
+WORKDIR /tmp
+
+RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.36.4 \
+  https://github.com/grpc/grpc \
+  && cd grpc \
+  && mkdir -p cmake/build \
+  && cd cmake/build \
+  && cmake \
+    -DgRPC_INSTALL=ON \
+    -DgRPC_BUILD_TESTS=OFF \
+    -DCMAKE_INSTALL_PREFIX=/install \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \
+    -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \
+    -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \
+    -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \
+    -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF \
+    -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \
+    ../.. \
+  && make -j8 VERBOSE=1 \
+  && make install
+
+RUN git clone --shallow-submodules --depth 1 --recurse-submodules -b v1.6.1 \
+  https://github.com/open-telemetry/opentelemetry-cpp.git \
+  && cd opentelemetry-cpp \
+  && mkdir build \
+  && cd build \
+  && cmake -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=/install \
+    -DCMAKE_PREFIX_PATH=/install \
+    -DWITH_OTLP=ON \
+    -DWITH_OTLP_GRPC=ON \
+    -DWITH_OTLP_HTTP=OFF \
+    -DBUILD_TESTING=OFF \
+    -DWITH_EXAMPLES=OFF \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+    .. \
+  && make -j8 VERBOSE=1 \
+  && make install
+
+ADD https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/refs/tags/webserver/v1.0.1.tar.gz ./
+
+RUN tar -xzvf v1.0.1.tar.gz && rm v1.0.1.tar.gz
+
+RUN mkdir -p otel-nginx/build && mkdir -p otel-nginx/src
+RUN cp -r /tmp/opentelemetry-cpp-contrib-webserver-v1.0.1/instrumentation/nginx/src otel-nginx/
+RUN cp /tmp/opentelemetry-cpp-contrib-webserver-v1.0.1/instrumentation/nginx/CMakeLists.txt /tmp/opentelemetry-cpp-contrib-webserver-v1.0.1/instrumentation/nginx/nginx.cmake otel-nginx/
+RUN cd otel-nginx/build \
+  && cmake -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_PREFIX_PATH=/install \
+    -DCMAKE_INSTALL_PREFIX=/usr/share/nginx/modules \
+    -DNGINX_VERSION=1.19.3 \
+    .. \
+  && make -j8 VERBOSE=1 \
+  && make install
+
 FROM registry.access.redhat.com/ubi8:8.5
 
 ARG OPENRESTY_RPM_VERSION="1.19.3"
@@ -38,6 +102,8 @@ ENV PATH="./lua_modules/bin:/usr/local/openresty/luajit/bin/:${PATH}" \
     LUA_CPATH="./lua_modules/lib/lua/5.1/?.so;;" \
     LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/app-root/lib"
 
+COPY --from=opentelemetry-builder /usr/share/nginx/modules/otel_ngx_module.so /usr/local/openresty/nginx/modules
+
 RUN luarocks install --deps-mode=none --tree /usr/local https://luarocks.org/manifests/pintsized/lua-resty-http-0.15-0.src.rock
 RUN luarocks install --deps-mode=none --tree /usr/local https://luarocks.org/manifests/kikito/router-2.1-0.src.rock
 RUN luarocks install --deps-mode=none --tree /usr/local https://luarocks.org/manifests/kikito/inspect-3.1.1-0.src.rock

@@ -164,6 +164,12 @@ gateway-logs: export IMAGE_NAME = does-not-matter
 gateway-logs:
 	$(DOCKER_COMPOSE) logs gateway
 
+opentelemetry-gateway: ## run gateway instrumented with opentelemetry
+	$(DOCKER_COMPOSE) run opentelemetry-instrumented-gateway
+
+opentracing-gateway: ## run gateway instrumented with opentracing
+	$(DOCKER_COMPOSE) run opentracing-instrumented-gateway
+
 test-runtime-image: export IMAGE_NAME ?= $(RUNTIME_IMAGE)
 test-runtime-image: clean-containers ## Smoke test the runtime image. Pass any docker image in IMAGE_NAME parameter.
 	$(DOCKER_COMPOSE) --version

@@ -53,15 +53,40 @@ services:
       KEYCLOAK_LOGLEVEL: INFO
     ports:
       - "8080"
+  opentracing-instrumented-gateway:
+    image: ${IMAGE_NAME:-apicast-test}
+    depends_on:
+    - jaeger
+    environment:
+      THREESCALE_CONFIG_FILE: /tmp/config.json
+      THREESCALE_DEPLOYMENT_ENV: staging
+      APICAST_CONFIGURATION_LOADER: lazy
+      APICAST_LOG_LEVEL: debug
+      APICAST_CONFIGURATION_CACHE: "0"
+      OPENTRACING_TRACER: jaeger
+      OPENTRACING_CONFIG: /opt/app-root/src/tracing-configs/tracing-config-jaeger-jaeger-config.json
+    volumes:
+      - ./examples/opentracing/apicast-config.json:/tmp/config.json
+      - ./examples/opentracing/jaeger-config.json:/opt/app-root/src/tracing-configs/tracing-config-jaeger-jaeger-config.json
+  opentelemetry-instrumented-gateway:
+    image: ${IMAGE_NAME:-apicast-test}
+    depends_on:
+    - jaeger
+    environment:
+      THREESCALE_CONFIG_FILE: /tmp/config.json
+      THREESCALE_DEPLOYMENT_ENV: staging
+      APICAST_CONFIGURATION_LOADER: lazy
+      APICAST_LOG_LEVEL: debug
+      APICAST_CONFIGURATION_CACHE: "0"
+      OPENTELEMETRY: "1"
+      OPENTELEMETRY_CONFIG: /opt/app-root/src/tracing-configs/otel.toml
+    volumes:
+      - ./examples/opentelemetry/apicast-config.json:/tmp/config.json
+      - ./examples/opentelemetry/otel.toml:/opt/app-root/src/tracing-configs/otel.toml
   jaeger:
     image: jaegertracing/all-in-one:latest
     environment:
-      COLLECTOR_ZIPKIN_HTTP_PORT: 9411
+      JAEGER_DISABLED: "false"
+      COLLECTOR_OTLP_ENABLED: "true"
     ports:
-      - 5775:5775/udp
-      - 6831:6831/udp
-      - 6832:6832/udp
-      - 5778:5778
       - 16686:16686
-      - 14268:14268
-      - 9411:9411
@@ -0,0 +1,12 @@
+{
+    "services": [
+        {
+            "proxy": {
+                "hosts": ["one"],
+                "proxy_rules": [],
+                "api_backend": "https://echo-api.3scale.net",
+                "policy_chain": []
+            }
+        }
+    ]
+}
@@ -0,0 +1,20 @@
+exporter = "otlp"
+processor = "simple"
+
+[exporters.otlp]
+# Alternatively the OTEL_EXPORTER_OTLP_ENDPOINT environment variable can also be used.
+host = "jaeger"
+port = 4317
+# Optional: enable SSL, for endpoints that support it
+# use_ssl = true
+# Optional: set a filesystem path to a pem file to be used for SSL encryption
+# (when use_ssl = true)
+# ssl_cert_path = "/path/to/cert.pem"
+
+[processors.batch]
+max_queue_size = 2048
+schedule_delay_millis = 5000
+max_export_batch_size = 512
+
+[service]
+name = "apicast" # Opentelemetry resource name
@@ -0,0 +1,12 @@
+{
+    "services": [
+        {
+            "proxy": {
+                "hosts": ["one"],
+                "proxy_rules": [],
+                "api_backend": "https://echo-api.3scale.net",
+                "policy_chain": []
+            }
+        }
+    ]
+}
@@ -0,0 +1,26 @@
+{
+    "service_name": "apicast",
+    "disabled": false,
+    "sampler": {
+        "type": "const",
+        "param": 1
+    },
+    "reporter": {
+        "queueSize": 100,
+        "bufferFlushInterval": 10,
+        "logSpans": false,
+        "localAgentHostPort": "jaeger:6831"
+    },
+    "headers": {
+        "jaegerDebugHeader": "debug-id",
+        "jaegerBaggageHeader": "baggage",
+        "TraceContextHeaderName": "uber-trace-id",
+        "traceBaggageHeaderPrefix": "testctx-"
+    },
+    "baggage_restrictions": {
+        "denyBaggageOnInitializationFailure": false,
+        "hostPort": "127.0.0.1:5778",
+        "refreshInterval": 60
+    }
+}
+
@@ -0,0 +1,7 @@
+opentelemetry on;
+
+{% if opentelemetry_config_file == nil or opentelemetry_config_file == empty %}
+    {% assign opentelemetry_config_file = "conf.d/opentelemetry/jaeger.example.toml" | filesystem | first%}
+{% endif %}
+
+opentelemetry_config {{ opentelemetry_config_file }};
@@ -0,0 +1,25 @@
+exporter = "otlp"
+processor = "batch"
+
+[exporters.otlp]
+# Alternatively the OTEL_EXPORTER_OTLP_ENDPOINT environment variable can also be used.
+host = "localhost"
+port = 4317
+# Optional: enable SSL, for endpoints that support it
+# use_ssl = true
+# Optional: set a filesystem path to a pem file to be used for SSL encryption
+# (when use_ssl = true)
+# ssl_cert_path = "/path/to/cert.pem"
+
+[processors.batch]
+max_queue_size = 2048
+schedule_delay_millis = 5000
+max_export_batch_size = 512
+
+[service]
+name = "apicast" # Opentelemetry resource name
+
+[sampler]
+name = "AlwaysOn" # Also: AlwaysOff, TraceIdRatioBased
+ratio = 0.1
+parent_based = false
@@ -15,6 +15,16 @@ load_module {{file}};
   {% endif %}
 {% endif %}
 
+{% if opentelemetry != empty %}
+  {% for file in "modules/otel_ngx_module.so" | filesystem %}
+load_module {{file}};
+  {% endfor %}
+{% else %}
+  {% if timer_resolution %}
+    timer_resolution {{ timer_resolution }};
+  {% endif %}
+{% endif %}
+
 daemon {{ daemon | default: 'off' }};
 master_process {{ master_process | default: 'on' }};
 worker_processes {{ worker_processes | default: 'auto' }};
@@ -81,6 +91,11 @@ http {
     {% include tracer_conf %}
   {% endif %}
 
+  {% if opentelemetry != empty %}
+    {%- capture otel_conf %}conf.d/opentelemetry/otel.conf.liquid{%- endcapture -%}
+    {% include otel_conf %}
+  {% endif %}
+
   {% for file in template | default: 'http.d/apicast.conf.liquid' | filesystem %}
     {% include file %}
   {% endfor %}

@@ -1,6 +1,6 @@
 log_format time '[$time_local] $target_host:$server_port $remote_addr:$remote_port "$request" $status $body_bytes_sent ($request_time) $post_action_impact';
 
-# Use maps as variables because some logs can be raised out of the server context 
+# Use maps as variables because some logs can be raised out of the server context
 # where variables cannot be set, this allow us to avoid a warning
 map "" $extended_access_log {
     default '';
@@ -33,6 +33,10 @@ server {
   opentracing_trace_locations off;
   {% endif %}
 
+  {% if opentelemetry != empty %}
+  opentelemetry_operation_name apicast_management;
+  {% endif %}
+
   {% include "conf.d/management.conf" %}
 }
 
@@ -45,6 +49,10 @@ server {
   opentracing_trace_locations off;
   {% endif %}
 
+  {% if opentelemetry != empty %}
+  opentelemetry_operation_name apicast_mockbackend;
+  {% endif %}
+
   {% include "conf.d/backend.conf" %}
 }
 
@@ -62,6 +70,10 @@ server {
   opentracing_trace_locations off;
   {% endif %}
 
+  {% if opentelemetry != empty %}
+  opentelemetry_operation_name apicast_echo;
+  {% endif %}
+
   {% include "conf.d/echo.conf" %}
 }
 
@@ -115,6 +127,12 @@ server {
   opentracing_tag original_request_uri $original_request_uri;
   {% endif %}
 
+  {% if opentelemetry != empty %}
+  opentelemetry_operation_name apicast;
+  opentelemetry_propagate;
+  opentelemetry_attribute original_request_uri $original_request_uri;
+  {% endif %}
+
   {% include "http.d/ssl.conf" %}
 
   {% for file in "apicast.d/*.conf" | filesystem %}

@@ -98,6 +98,8 @@ _M.default_environment = 'production'
 -- @tfield ?string opentracing_tracer loads an opentracing tracer library, for example: jaeger
 -- @tfield ?string opentracing_config opentracing config file to load
 -- @tfield ?string opentracing_forward_header opentracing http header to forward upstream
+-- @tfield ?string opentelemetry enables server instrumentation using opentelemetry SDKs
+-- @tfield ?string opentelemetry_config_file opentelemetry config file to load
 -- @tfield ?string upstream_retry_cases error cases where the call to the upstream should be retried
 --         follows the same format as https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream
 -- @tfield ?policy_chain policy_chain @{policy_chain} instance
@@ -116,6 +118,8 @@ _M.default_config = {
     opentracing_tracer = env_value_ref('OPENTRACING_TRACER'),
     opentracing_config = env_value_ref('OPENTRACING_CONFIG'),
     opentracing_forward_header = env_value_ref('OPENTRACING_FORWARD_HEADER'),
+    opentelemetry = env_value_ref('OPENTELEMETRY'),
+    opentelemetry_config_file = env_value_ref('OPENTELEMETRY_CONFIG'),
     upstream_retry_cases = env_value_ref('APICAST_UPSTREAM_RETRY_CASES'),
     http_keepalive_timeout = env_value_ref('HTTP_KEEPALIVE_TIMEOUT'),
     policy_chain = require('apicast.policy_chain').default(),