Skip to content

[3.5] Backport add sampling rate to fix distributed tracing #16951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
serathius merged 3 commits into from
Nov 26, 2023
Merged

[3.5] Backport add sampling rate to fix distributed tracing #16951

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3a54851
Backport server: Add sampling rate to distributed tracing.
jmhbnz Nov 16, 2023
5a564d5
Backport server: Have tracingExporter own resources it initialises.
jmhbnz Nov 25, 2023
f952197
Backport embed: Add tracing integration test.
jmhbnz Nov 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Backport server: Have tracingExporter own resources it initialises. 
Signed-off-by: James Blair <[email protected]>
  • Loading branch information
@jmhbnz
jmhbnz committed Nov 25, 2023
commit 5a564d56d74fb25bcb52aa6e3f0d9b39497bc5f5
50 changes: 37 additions & 13 deletions server/embed/config_tracing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,19 @@ func validateTracingConfig(samplingRate int) error {
return nil
}

func setupTracingExporter(ctx context.Context, cfg *Config) (exporter tracesdk.SpanExporter, options []otelgrpc.Option, err error) {
exporter, err = otlptracegrpc.New(ctx,
type tracingExporter struct {
exporter tracesdk.SpanExporter
opts []otelgrpc.Option
provider *tracesdk.TracerProvider
}

func newTracingExporter(ctx context.Context, cfg *Config) (*tracingExporter, error) {
exporter, err := otlptracegrpc.New(ctx,
otlptracegrpc.WithInsecure(),
otlptracegrpc.WithEndpoint(cfg.ExperimentalDistributedTracingAddress),
)
if err != nil {
return nil, nil, err
return nil, err
}

res, err := resource.New(ctx,
Expand All @@ -55,33 +61,37 @@ func setupTracingExporter(ctx context.Context, cfg *Config) (exporter tracesdk.S
),
)
if err != nil {
return nil, nil, err
return nil, err
}

if resWithIDKey := determineResourceWithIDKey(cfg.ExperimentalDistributedTracingServiceInstanceID); resWithIDKey != nil {
// Merge resources into a new
// resource in case of duplicates.
res, err = resource.Merge(res, resWithIDKey)
if err != nil {
return nil, nil, err
return nil, err
}
}

options = append(options,
traceProvider := tracesdk.NewTracerProvider(
tracesdk.WithBatcher(exporter),
tracesdk.WithResource(res),
tracesdk.WithSampler(
tracesdk.ParentBased(determineSampler(cfg.ExperimentalDistributedTracingSamplingRatePerMillion)),
),
)

options := []otelgrpc.Option{
otelgrpc.WithPropagators(
propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
),
),
otelgrpc.WithTracerProvider(
tracesdk.NewTracerProvider(
tracesdk.WithBatcher(exporter),
tracesdk.WithResource(res),
tracesdk.WithSampler(tracesdk.ParentBased(tracesdk.NeverSample())),
),
traceProvider,
),
)
}

cfg.logger.Debug(
"distributed tracing enabled",
Expand All @@ -90,7 +100,21 @@ func setupTracingExporter(ctx context.Context, cfg *Config) (exporter tracesdk.S
zap.String("service-instance-id", cfg.ExperimentalDistributedTracingServiceInstanceID),
)

return exporter, options, err
return &tracingExporter{
exporter: exporter,
opts: options,
provider: traceProvider,
}, nil
}

func (te *tracingExporter) Close(ctx context.Context) {
if te.provider != nil {
te.provider.Shutdown(ctx)
}

if te.exporter != nil {
te.exporter.Shutdown(ctx)
}
}

func determineSampler(samplingRate int) tracesdk.Sampler {
Expand Down
9 changes: 4 additions & 5 deletions server/embed/etcd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,15 +228,14 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {

if srvcfg.ExperimentalEnableDistributedTracing {
tctx := context.Background()
tracingExporter, opts, err := setupTracingExporter(tctx, cfg)
tracingExporter, err := newTracingExporter(tctx, cfg)
if err != nil {
return e, err
}
if tracingExporter == nil || len(opts) == 0 {
return e, fmt.Errorf("error setting up distributed tracing")
e.tracingExporterShutdown = func() {
tracingExporter.Close(tctx)
}
e.tracingExporterShutdown = func() { tracingExporter.Shutdown(tctx) }
srvcfg.ExperimentalTracerOptions = opts
srvcfg.ExperimentalTracerOptions = tracingExporter.opts

e.cfg.logger.Info("distributed tracing setup enabled")
}
Expand Down