A parquet Handler for slog Go library.
Sponsored by:
Simple to use, built on open standards, and designed for full cost control
See also:
- slog-multi: 
slog.Handlerchaining, fanout, routing, failover, load balancing... - slog-formatter: 
slogattribute formatting - slog-sampling: 
slogsampling policy - slog-mock: 
slog.Handlerfor test purposes 
HTTP middlewares:
- slog-gin: Gin middleware for 
sloglogger - slog-echo: Echo middleware for 
sloglogger - slog-fiber: Fiber middleware for 
sloglogger - slog-chi: Chi middleware for 
sloglogger - slog-http: 
net/httpmiddleware forsloglogger 
Loggers:
- slog-zap: A 
sloghandler forZap - slog-zerolog: A 
sloghandler forZerolog - slog-logrus: A 
sloghandler forLogrus 
Log sinks:
- slog-datadog: A 
sloghandler forDatadog - slog-betterstack: A 
sloghandler forBetterstack - slog-rollbar: A 
sloghandler forRollbar - slog-loki: A 
sloghandler forLoki - slog-sentry: A 
sloghandler forSentry - slog-syslog: A 
sloghandler forSyslog - slog-logstash: A 
sloghandler forLogstash - slog-fluentd: A 
sloghandler forFluentd - slog-graylog: A 
sloghandler forGraylog - slog-quickwit: A 
sloghandler forQuickwit - slog-slack: A 
sloghandler forSlack - slog-telegram: A 
sloghandler forTelegram - slog-mattermost: A 
sloghandler forMattermost - slog-microsoft-teams: A 
sloghandler forMicrosoft Teams - slog-webhook: A 
sloghandler forWebhook - slog-kafka: A 
sloghandler forKafka - slog-nats: A 
sloghandler forNATS - slog-parquet: A 
sloghandler forParquet+Object Storage - slog-channel: A 
sloghandler for Go channels 
go get github.com/samber/slog-parquet/v2Compatibility: go >= 1.21
No breaking changes will be made to exported APIs before v3.0.0.
GoDoc: https://pkg.go.dev/github.com/samber/slog-parquet/v2
type Option struct {
    // log level (default: debug)
    Level slog.Leveler
    // parquet rows buffer
    Buffer slogparquet.ParquetBuffer
    // optional: customize json payload builder
    Converter Converter
    // optional: fetch attributes from context
    AttrFromContext []func(ctx context.Context) []slog.Attr
    // optional: see slog.HandlerOptions
    AddSource   bool
    ReplaceAttr func(groups []string, a slog.Attr) slog.Attr
}Other global parameters:
slogparquet.SourceKey = "source"
slogparquet.ErrorKeys = []string{"error", "err"}func NewParquetBuffer(bucket objstore.Bucket, prefix string, maxRecords int, maxInterval time.Duration) slogparquet.ParquetBufferAttributes will be injected in log payload.
See github.com/thanos-io/objstore.
import (
    "log/slog"
    slogparquet "github.com/samber/slog-parquet/v2"
    "github.com/thanos-io/objstore/providers/s3"
)
func main() {
    bucket, _ := s3.NewBucketWithConfig(
        slogparquet.NewLogger(),
        s3.Config{
            Endpoint:  os.Getenv("AWS_S3_ENDPOINT"),
            Region:    os.Getenv("AWS_S3_REGION"),
            Bucket:    os.Getenv("AWS_S3_BUCKET"),
            AccessKey: os.Getenv("AWS_ACCESS_KEY"),
            SecretKey: os.Getenv("AWS_SECRET_KEY"),
            PartSize:  16 * 1024 * 1024, // 16MB
        },
        "logger",
    )
    buffer := slogparquet.NewParquetBuffer(bucket, "api/logs", 10*1024*1024)
    logger := slog.New(slogparquet.Option{Level: slog.LevelDebug, Buffer: buffer}.NewParquetHandler())
    logger = logger.
        With("environment", "dev").
        With("release", "v1.0.0")
    // log error
    logger.
        With("category", "sql").
        With("query.statement", "SELECT COUNT(*) FROM users;").
        With("query.duration", 1*time.Second).
        With("error", fmt.Errorf("could not count users")).
        Error("caramba!")
    // log user signup
    logger.
        With(
            slog.Group("user",
                slog.String("id", "user-123"),
                slog.Time("created_at", time.Now()),
            ),
        ).
        Info("user registration")
    buffer.Flush(true)
    bucket.Close()
}Output:
$ parquet meta ~/Downloads/00_17_08.d4d9f.parquet
File path:  /Users/samber/Downloads/00_17_08.d4d9f.parquet
Created by: github.com/samber/slog-parquet version (devel)(build )
Properties: (none)
Schema:
message log {
  required int64 time (TIMESTAMP(NANOS,true));
  required binary log_level (STRING);
  required binary message (STRING);
  required binary attributes;
  required binary source (STRING);
}
Row group 0:  count: 2  279.00 B records  start: 51  total(compressed): 558 B total(uncompressed):644 B 
--------------------------------------------------------------------------------
            type      encodings count     avg size   nulls   min / max
time        INT64     F   _     2         22.50 B            "2023-08-19T00:17:08.14408..." / "2023-08-19T00:17:08.14420..."
log_level   BINARY    F         2         26.50 B            "ERROR" / "INFO"
message     BINARY    F         2         35.00 B            "caramba!" / "user registration"
attributes  BINARY    F         2         155.50 B           "0x7B2263617465676F7279223..." / "0x7B22656E7669726F6E6D656..."
source      BINARY    F         2         39.50 B            "samber/slog-parquet" / "samber/slog-parquet"Import the samber/slog-otel library.
import (
	slogparquet "github.com/samber/slog-parquet"
	slogotel "github.com/samber/slog-otel"
	"go.opentelemetry.io/otel/sdk/trace"
)
func main() {
	tp := trace.NewTracerProvider(
		trace.WithSampler(trace.AlwaysSample()),
	)
	tracer := tp.Tracer("hello/world")
	ctx, span := tracer.Start(context.Background(), "foo")
	defer span.End()
	span.AddEvent("bar")
	logger := slog.New(
		slogparquet.Option{
			// ...
			AttrFromContext: []func(ctx context.Context) []slog.Attr{
				slogotel.ExtractOtelAttrFromContext([]string{"tracing"}, "trace_id", "span_id"),
			},
		}.NewParquetHandler(),
	)
	logger.ErrorContext(ctx, "a message")
}- Ping me on twitter @samuelberthe (DMs, mentions, whatever :))
 - Fork the project
 - Fix open issues or request new features
 
Don't hesitate ;)
# Install some dev dependencies
make tools
# Run tests
make test
# or
make watch-testGive a βοΈ if this project helped you!
Copyright Β© 2023 Samuel Berthe.
This project is MIT licensed.