Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Configure OpenTelemetry Tracing #4883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 3 commits into from
Oct 25, 2021
Merged

Configure OpenTelemetry Tracing #4883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c16429e
opentelemetry: add gRPC tracing
sallyom May 6, 2021
37418e1
pin go.opentelemetry grpc/otelgrpc v0.25.0
sallyom Oct 25, 2021
ec63057
bump(deps-opentelemetry)
sallyom Oct 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 31 additions & 6 deletions cmd/crio/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/containers/storage/pkg/reexec"
"github.com/cri-o/cri-o/internal/criocli"
"github.com/cri-o/cri-o/internal/log"
"github.com/cri-o/cri-o/internal/opentelemetry"
"github.com/cri-o/cri-o/internal/signals"
"github.com/cri-o/cri-o/internal/version"
libconfig "github.com/cri-o/cri-o/pkg/config"
Expand All @@ -31,6 +32,8 @@ import (
"github.com/sirupsen/logrus"
"github.com/soheilhy/cmux"
"github.com/urfave/cli/v2"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"golang.org/x/sys/unix"
"google.golang.org/grpc"
)
Expand All @@ -45,7 +48,7 @@ func writeCrioGoroutineStacks() {
}
}

func catchShutdown(ctx context.Context, cancel context.CancelFunc, gserver *grpc.Server, sserver *server.Server, hserver *http.Server, signalled *bool) {
func catchShutdown(ctx context.Context, cancel context.CancelFunc, gserver *grpc.Server, tp *sdktrace.TracerProvider, sserver *server.Server, hserver *http.Server, signalled *bool) {
sig := make(chan os.Signal, 2048)
signal.Notify(sig, signals.Interrupt, signals.Term, unix.SIGUSR1, unix.SIGUSR2, unix.SIGPIPE, signals.Hup)
go func() {
Expand All @@ -70,6 +73,11 @@ func catchShutdown(ctx context.Context, cancel context.CancelFunc, gserver *grpc
continue
}
*signalled = true
if tp != nil {
if err := tp.Shutdown(ctx); err != nil {
logrus.Warnf("Error shutting down opentelemetry tracer provider: %v", err)
}
}
gserver.GracefulStop()
hserver.Shutdown(ctx) // nolint: errcheck
if err := sserver.StopStreamServer(); err != nil {
Expand Down Expand Up @@ -236,12 +244,29 @@ func main() {
logrus.Fatalf("Failed to chmod listen socket %s: %v", config.Listen, err)
}

grpcServer := grpc.NewServer(
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
var tracerProvider *sdktrace.TracerProvider
chainUnaryServer := grpc_middleware.ChainUnaryServer(metrics.UnaryInterceptor(), log.UnaryInterceptor())
chainStreamServer := grpc_middleware.ChainStreamServer(log.StreamInterceptor())
if config.EnableTracing {
var opts []otelgrpc.Option
tracerProvider, opts, err = opentelemetry.InitTracing(
ctx,
config.TracingEndpoint,
config.TracingSamplingRatePerMillion,
)
if err != nil {
logrus.Fatalf("Failed to initialize tracer provider: %v", err)
}
chainUnaryServer = grpc_middleware.ChainUnaryServer(
metrics.UnaryInterceptor(),
log.UnaryInterceptor(),
)),
grpc.StreamInterceptor(log.StreamInterceptor()),
otelgrpc.UnaryServerInterceptor(opts...),
)
chainStreamServer = grpc_middleware.ChainStreamServer(log.StreamInterceptor(), otelgrpc.StreamServerInterceptor(opts...))
}
grpcServer := grpc.NewServer(
grpc.UnaryInterceptor(chainUnaryServer),
grpc.StreamInterceptor(chainStreamServer),
grpc.MaxSendMsgSize(config.GRPCMaxSendMsgSize),
grpc.MaxRecvMsgSize(config.GRPCMaxRecvMsgSize),
)
Expand Down Expand Up @@ -317,7 +342,7 @@ func main() {
}

graceful := false
catchShutdown(ctx, cancel, grpcServer, crioServer, httpServer, &graceful)
catchShutdown(ctx, cancel, grpcServer, tracerProvider, crioServer, httpServer, &graceful)

go func() {
if err := grpcServer.Serve(grpcL); err != nil {
Expand Down
3 changes: 3 additions & 0 deletions completions/bash/crio
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ h
--drop-infra-ctr
--enable-metrics
--enable-profile-unix-socket
--enable-tracing
--gid-mappings
--global-auth-file
--grpc-max-recv-msg-size
Expand Down Expand Up @@ -100,6 +101,8 @@ h
--stream-tls-ca
--stream-tls-cert
--stream-tls-key
--tracing-endpoint
--tracing-sampling-rate-per-million
--uid-mappings
--version-file
--version-file-persist
Expand Down
3 changes: 3 additions & 0 deletions completions/fish/crio.fish
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l device-ownership-from-secu
complete -c crio -n '__fish_crio_no_subcommand' -f -l drop-infra-ctr -d 'Determines whether pods are created without an infra container, when the pod is not using a pod level PID namespace (default: true)'
complete -c crio -n '__fish_crio_no_subcommand' -f -l enable-metrics -d 'Enable metrics endpoint for the server on localhost:9090'
complete -c crio -n '__fish_crio_no_subcommand' -f -l enable-profile-unix-socket -d 'Enable pprof profiler on crio unix domain socket'
complete -c crio -n '__fish_crio_no_subcommand' -f -l enable-tracing -d 'Enable OpenTelemetry trace data exporting'
complete -c crio -n '__fish_crio_no_subcommand' -f -l gid-mappings -r -d 'Specify the GID mappings to use for the user namespace (default: "")'
complete -c crio -n '__fish_crio_no_subcommand' -l global-auth-file -r -d 'Path to a file like /var/lib/kubelet/config.json holding credentials necessary for pulling images from secure registries (default: "")'
complete -c crio -n '__fish_crio_no_subcommand' -f -l grpc-max-recv-msg-size -r -d 'Maximum grpc receive message size in bytes'
Expand Down Expand Up @@ -135,6 +136,8 @@ complete -c crio -n '__fish_crio_no_subcommand' -f -l stream-port -r -d 'Bind po
complete -c crio -n '__fish_crio_no_subcommand' -l stream-tls-ca -r -d 'Path to the x509 CA(s) file used to verify and authenticate client communication with the encrypted stream. This file can change and CRI-O will automatically pick up the changes within 5 minutes (default: "")'
complete -c crio -n '__fish_crio_no_subcommand' -l stream-tls-cert -r -d 'Path to the x509 certificate file used to serve the encrypted stream. This file can change and CRI-O will automatically pick up the changes within 5 minutes (default: "")'
complete -c crio -n '__fish_crio_no_subcommand' -l stream-tls-key -r -d 'Path to the key file used to serve the encrypted stream. This file can change and CRI-O will automatically pick up the changes within 5 minutes (default: "")'
complete -c crio -n '__fish_crio_no_subcommand' -f -l tracing-endpoint -r -d 'Address on which the gRPC tracing collector will listen'
complete -c crio -n '__fish_crio_no_subcommand' -f -l tracing-sampling-rate-per-million -r -d 'Number of samples to collect per million OpenTelemetry spans'
complete -c crio -n '__fish_crio_no_subcommand' -f -l uid-mappings -r -d 'Specify the UID mappings to use for the user namespace (default: "")'
complete -c crio -n '__fish_crio_no_subcommand' -l version-file -r -d 'Location for CRI-O to lay down the temporary version file. It is used to check if crio wipe should wipe containers, which should always happen on a node reboot'
complete -c crio -n '__fish_crio_no_subcommand' -l version-file-persist -r -d 'Location for CRI-O to lay down the persistent version file. It is used to check if crio wipe should wipe images, which should only happen when CRI-O has been upgraded'
Expand Down
2 changes: 1 addition & 1 deletion completions/zsh/_crio
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ it later with **--config**. Global options will modify the output.' 'version:dis
_describe 'commands' cmds

local -a opts
opts=('--absent-mount-sources-to-reject' '--additional-devices' '--address' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--blockio-config-file' '--cgroup-manager' '--clean-shutdown-file' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--device-ownership-from-security-context' '--drop-infra-ctr' '--enable-metrics' '--enable-profile-unix-socket' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--infra-ctr-cpuset' '--insecure-registry' '--internal-wipe' '--irqbalance-config-file' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--metrics-cert' '--metrics-collectors' '--metrics-key' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-cpu' '--profile-mem' '--profile-port' '--rdt-config-file' '--read-only' '--registries-conf' '--registries-conf-dir' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-idle-timeout' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
opts=('--absent-mount-sources-to-reject' '--additional-devices' '--address' '--apparmor-profile' '--big-files-temporary-dir' '--bind-mount-prefix' '--blockio-config-file' '--cgroup-manager' '--clean-shutdown-file' '--cni-config-dir' '--cni-default-network' '--cni-plugin-dir' '--config' '--config-dir' '--conmon' '--conmon-cgroup' '--conmon-env' '--container-attach-socket-dir' '--container-exits-dir' '--ctr-stop-timeout' '--decryption-keys-path' '--default-capabilities' '--default-env' '--default-mounts-file' '--default-runtime' '--default-sysctls' '--default-transport' '--default-ulimits' '--device-ownership-from-security-context' '--drop-infra-ctr' '--enable-metrics' '--enable-profile-unix-socket' '--enable-tracing' '--gid-mappings' '--global-auth-file' '--grpc-max-recv-msg-size' '--grpc-max-send-msg-size' '--hooks-dir' '--image-volumes' '--infra-ctr-cpuset' '--insecure-registry' '--internal-wipe' '--irqbalance-config-file' '--listen' '--log' '--log-dir' '--log-filter' '--log-format' '--log-journald' '--log-level' '--log-size-max' '--metrics-cert' '--metrics-collectors' '--metrics-key' '--metrics-port' '--metrics-socket' '--namespaces-dir' '--no-pivot' '--pause-command' '--pause-image' '--pause-image-auth-file' '--pids-limit' '--pinns-path' '--profile' '--profile-cpu' '--profile-mem' '--profile-port' '--rdt-config-file' '--read-only' '--registries-conf' '--registries-conf-dir' '--registry' '--root' '--runroot' '--runtimes' '--seccomp-profile' '--seccomp-use-default-when-empty' '--selinux' '--separate-pull-cgroup' '--signature-policy' '--storage-driver' '--storage-opt' '--stream-address' '--stream-enable-tls' '--stream-idle-timeout' '--stream-port' '--stream-tls-ca' '--stream-tls-cert' '--stream-tls-key' '--tracing-endpoint' '--tracing-sampling-rate-per-million' '--uid-mappings' '--version-file' '--version-file-persist' '--help' '--version')
_describe 'global options' opts

return
Expand Down
13 changes: 11 additions & 2 deletions docs/crio.8.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ crio
[--drop-infra-ctr]
[--enable-metrics]
[--enable-profile-unix-socket]
[--enable-tracing]
[--gid-mappings]=[value]
[--global-auth-file]=[value]
[--grpc-max-recv-msg-size]=[value]
Expand Down Expand Up @@ -97,6 +98,8 @@ crio
[--stream-tls-ca]=[value]
[--stream-tls-cert]=[value]
[--stream-tls-key]=[value]
[--tracing-endpoint]=[value]
[--tracing-sampling-rate-per-million]=[value]
[--uid-mappings]=[value]
[--version-file-persist]=[value]
[--version-file]=[value]
Expand Down Expand Up @@ -198,13 +201,15 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]

**--enable-profile-unix-socket**: Enable pprof profiler on crio unix domain socket

**--enable-tracing**: Enable OpenTelemetry trace data exporting

**--gid-mappings**="": Specify the GID mappings to use for the user namespace (default: "")

**--global-auth-file**="": Path to a file like /var/lib/kubelet/config.json holding credentials necessary for pulling images from secure registries (default: "")

**--grpc-max-recv-msg-size**="": Maximum grpc receive message size in bytes (default: 16777216)
**--grpc-max-recv-msg-size**="": Maximum grpc receive message size in bytes (default: 83886080)

**--grpc-max-send-msg-size**="": Maximum grpc receive message size (default: 16777216)
**--grpc-max-send-msg-size**="": Maximum grpc receive message size (default: 83886080)

**--help, -h**: show help

Expand Down Expand Up @@ -340,6 +345,10 @@ crio [GLOBAL OPTIONS] command [COMMAND OPTIONS] [ARGUMENTS...]

**--stream-tls-key**="": Path to the key file used to serve the encrypted stream. This file can change and CRI-O will automatically pick up the changes within 5 minutes (default: "")

**--tracing-endpoint**="": Address on which the gRPC tracing collector will listen (default: 0.0.0.0:4317)

**--tracing-sampling-rate-per-million**="": Number of samples to collect per million OpenTelemetry spans (default: 0)

**--uid-mappings**="": Specify the UID mappings to use for the user namespace (default: "")

**--version, -v**: print the version
Expand Down
20 changes: 16 additions & 4 deletions docs/crio.conf.5.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,11 @@ The `crio.api` table contains settings for the kubelet/gRPC interface.
**stream_tls_ca**=""
Path to the x509 CA(s) file used to verify and authenticate client communication with the encrypted stream. This file can change and CRI-O will automatically pick up the changes within 5 minutes.

**grpc_max_send_msg_size**=16777216
Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
**grpc_max_send_msg_size**=83886080
Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 80 * 1024 * 1024.

**grpc_max_recv_msg_size**=16777216
Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
**grpc_max_recv_msg_size**=83886080
Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 80 * 1024 * 1024.

## CRIO.RUNTIME TABLE
The `crio.runtime` table contains settings pertaining to the OCI runtime used and options for how to set up and manage the OCI runtime.
Expand Down Expand Up @@ -390,6 +390,18 @@ If set to true, enable users to set a custom shm size instead of using the defau
**metrics_key**=""
The certificate key for the secure metrics server.

## CRIO.TRACING TABLE
[EXPERIMENTAL] The `crio.tracing` table containers settings pertaining to the export of OpenTelemetry trace data.

**enable_tracing**=false
Globally enable or disable OpenTelemetry trace data exporting.

**tracing_endpoint**="0.0.0.0:4317"
Address on which the gRPC trace collector will listen.

**tracing_sampling_rate_per_million**=""
Number of samples to collect per million OpenTelemetry spans.

# SEE ALSO
crio.conf.d(5), containers-storage.conf(5), containers-policy.json(5), containers-registries.conf(5), crio(8)

Expand Down
4 changes: 4 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ require (
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
github.com/urfave/cli/v2 v2.3.0
github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0
go.opentelemetry.io/otel v1.0.1
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1
go.opentelemetry.io/otel/sdk v1.0.1
golang.org/x/net v0.0.0-20210525063256-abc453219eb5
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0
Expand Down
Loading