Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[POC] migrate to OTEL using one OTEL instance #1895

Closed
wants to merge 9 commits into from
Closed

[POC] migrate to OTEL using one OTEL instance #1895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
08cb1b8
feat: migrate exporter to OTEL
mutianf May 26, 2023
26a648a
address comments
mutianf Jun 16, 2023
91fcd80
filter out only bigtable metrics
mutianf Jun 28, 2023
32ac2be
fix test
mutianf Jun 28, 2023
af6cbb2
use the bom
mutianf Jun 28, 2023
e87cf91
update
mutianf Jun 30, 2023
e2b7e58
update
mutianf Jul 5, 2023
6d127e3
feat: migrate builtin metrics to OTEl
mutianf Jun 15, 2023
2fd01a5
feat: migrate to OTEL [poc]
mutianf Aug 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions google-cloud-bigtable-deps-bom/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-bom</artifactId>
<version>1.27.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down
29 changes: 29 additions & 0 deletions google-cloud-bigtable/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,35 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-api</artifactId>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-sdk</artifactId>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-sdk-testing</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-sdk-metrics</artifactId>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
<artifactId>opentelemetry-sdk-common</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-monitoring</artifactId>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-cloud-monitoring-v3</artifactId>
</dependency>
</dependencies>

<profiles>
Expand Down
40 changes: 28 additions & 12 deletions ...-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,10 @@
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
import com.google.api.gax.rpc.UnaryCallSettings;
import com.google.auth.Credentials;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.bigtable.data.v2.models.Query;
import com.google.cloud.bigtable.data.v2.models.Row;
import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings;
import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings;
import com.google.cloud.bigtable.stats.BigtableStackdriverStatsExporter;
import com.google.cloud.bigtable.stats.BuiltinViews;
import com.google.common.base.MoreObjects;
import com.google.common.base.Strings;
import io.grpc.ManagedChannelBuilder;
Expand Down Expand Up @@ -197,23 +194,26 @@ public static void enableGfeOpenCensusStats() {
com.google.cloud.bigtable.data.v2.stub.metrics.RpcViews.registerBigtableClientGfeViews();
}

/** Register built in metrics. */
/**
* Register built in metrics.
*
* @deprecated Please use {@link BigtableDataSettings.Builder#setBuiltinMetricsEnabled(boolean)}
* to enable or disable built-in metrics.
*/
@Deprecated
public static void enableBuiltinMetrics() throws IOException {
if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
BuiltinViews.registerBigtableBuiltinViews();
BigtableStackdriverStatsExporter.register(GoogleCredentials.getApplicationDefault());
}
BUILTIN_METRICS_REGISTERED.compareAndSet(false, true);
}

/**
* Register built in metrics with credentials. The credentials need to have metric write access
* for all the projects you're publishing to.
*
* @deprecated Please use {@link BigtableDataSettings.Builder#setBuiltinMetricsEnabled(boolean)}
* to enable or disable built-in metrics.
*/
public static void enableBuiltinMetrics(Credentials credentials) throws IOException {
if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) {
BuiltinViews.registerBigtableBuiltinViews();
BigtableStackdriverStatsExporter.register(credentials);
}
BUILTIN_METRICS_REGISTERED.compareAndSet(false, true);
}

/** Returns the target project id. */
Expand Down Expand Up @@ -278,6 +278,11 @@ public boolean isBulkMutationFlowControlEnabled() {
return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
}

/** Gets if built-in metrics is enabled. */
public boolean isBuiltinMetricsEnabled() {
return stubSettings.isBuiltinMetricsEnabled();
}

/** Returns the underlying RPC settings. */
public EnhancedBigtableStubSettings getStubSettings() {
return stubSettings;
Expand Down Expand Up @@ -527,6 +532,17 @@ public boolean isBulkMutationFlowControlEnabled() {
return stubSettings.bulkMutateRowsSettings().isServerInitiatedFlowControlEnabled();
}

/** Set if built-in metrics will be enabled. */
public Builder setBuiltinMetricsEnabled(boolean enable) {
stubSettings.setBuiltinMetricsEnabled(enable);
return this;
}

/** Gets if built-in metrics is enabled. */
public boolean isBuiltinMetricsEnabled() {
return stubSettings.isBuiltinMetricsEnabled();
}

/**
* Returns the underlying settings for making RPC calls. The settings should be changed with
* care.
Expand Down
131 changes: 87 additions & 44 deletions ...d-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,26 @@
*/
package com.google.cloud.bigtable.data.v2.stub;

import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.APPLICATION_BLOCKING_LATENCIES_SELECTOR;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.APPLICATION_BLOCKING_LATENCIES_VIEW;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.APP_PROFILE;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.ATTEMPT_LATENCIES_SELECTOR;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.ATTEMPT_LATENCIES_VIEW;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.CLIENT_BLOCKING_LATENCIES_SELECTOR;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.CLIENT_BLOCKING_LATENCIES_VIEW;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.CONNECTIVITY_ERROR_COUNT_SELECTOR;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.CONNECTIVITY_ERROR_COUNT_VIEW;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.FIRST_RESPONSE_LATENCIES_SELECTOR;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.FIRST_RESPONSE_LATENCIES_VIEW;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.INSTANCE_ID;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.OPERATION_LATENCIES_SELECTOR;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.OPERATION_LATENCIES_VIEW;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.PROJECT_ID;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.RETRY_COUNT_SELECTOR;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.RETRY_COUNT_VIEW;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.SERVER_LATENCIES_SELECTOR;
import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsAttributes.SERVER_LATENCIES_VIEW;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.api.gax.batching.Batcher;
Expand All @@ -38,6 +58,7 @@
import com.google.api.gax.rpc.ServerStreamingCallSettings;
import com.google.api.gax.rpc.ServerStreamingCallable;
import com.google.api.gax.rpc.UnaryCallable;
import com.google.api.gax.tracing.ApiTracerFactory;
import com.google.api.gax.tracing.OpencensusTracerFactory;
import com.google.api.gax.tracing.SpanName;
import com.google.api.gax.tracing.TracedServerStreamingCallable;
Expand Down Expand Up @@ -87,6 +108,7 @@
import com.google.cloud.bigtable.data.v2.stub.changestream.GenerateInitialChangeStreamPartitionsUserCallable;
import com.google.cloud.bigtable.data.v2.stub.changestream.ReadChangeStreamResumptionStrategy;
import com.google.cloud.bigtable.data.v2.stub.changestream.ReadChangeStreamUserCallable;
import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableCloudMonitoringExporter;
import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerBatchedUnaryCallable;
import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerStreamingCallable;
import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerUnaryCallable;
Expand Down Expand Up @@ -115,10 +137,14 @@
import com.google.protobuf.ByteString;
import io.opencensus.stats.Stats;
import io.opencensus.stats.StatsRecorder;
import io.opencensus.tags.TagKey;
import io.opencensus.tags.TagValue;
import io.opencensus.tags.Tagger;
import io.opencensus.tags.Tags;
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
import io.opentelemetry.sdk.metrics.export.MetricExporter;
import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
Expand Down Expand Up @@ -211,48 +237,8 @@ public static EnhancedBigtableStubSettings finalizeSettings(
.build());
}

ImmutableMap<TagKey, TagValue> attributes =
ImmutableMap.<TagKey, TagValue>builder()
.put(RpcMeasureConstants.BIGTABLE_PROJECT_ID, TagValue.create(settings.getProjectId()))
.put(
RpcMeasureConstants.BIGTABLE_INSTANCE_ID, TagValue.create(settings.getInstanceId()))
.put(
RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID,
TagValue.create(settings.getAppProfileId()))
.build();
ImmutableMap<String, String> builtinAttributes =
ImmutableMap.<String, String>builder()
.put("project_id", settings.getProjectId())
.put("instance", settings.getInstanceId())
.put("app_profile", settings.getAppProfileId())
.build();
// Inject Opencensus instrumentation
builder.setTracerFactory(
new CompositeTracerFactory(
ImmutableList.of(
// Add OpenCensus Tracing
new OpencensusTracerFactory(
ImmutableMap.<String, String>builder()
// Annotate traces with the same tags as metrics
.put(
RpcMeasureConstants.BIGTABLE_PROJECT_ID.getName(),
settings.getProjectId())
.put(
RpcMeasureConstants.BIGTABLE_INSTANCE_ID.getName(),
settings.getInstanceId())
.put(
RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID.getName(),
settings.getAppProfileId())
// Also annotate traces with library versions
.put("gax", GaxGrpcProperties.getGaxGrpcVersion())
.put("grpc", GaxGrpcProperties.getGrpcVersion())
.put("gapic", Version.VERSION)
.build()),
// Add OpenCensus Metrics
MetricsTracerFactory.create(tagger, stats, attributes),
BuiltinMetricsTracerFactory.create(builtinAttributes),
// Add user configured tracer
settings.getTracerFactory())));
builder = setupTracerFactory(builder);

return builder.build();
}

Expand Down Expand Up @@ -291,6 +277,63 @@ private static void patchCredentials(EnhancedBigtableStubSettings.Builder settin
settings.setCredentialsProvider(FixedCredentialsProvider.create(patchedCreds));
}

private static EnhancedBigtableStubSettings.Builder setupTracerFactory(
EnhancedBigtableStubSettings.Builder settings) throws IOException {
ImmutableList.Builder<ApiTracerFactory> tracerFactories = ImmutableList.builder();
// Set up OpenCensusTracerFactory and user provided tracer factory
tracerFactories
.add(
new OpencensusTracerFactory(
ImmutableMap.<String, String>builder()
// Annotate traces with the same tags as metrics
.put(RpcMeasureConstants.BIGTABLE_PROJECT_ID.getKey(), settings.getProjectId())
.put(
RpcMeasureConstants.BIGTABLE_INSTANCE_ID.getKey(), settings.getInstanceId())
.put(
RpcMeasureConstants.BIGTABLE_APP_PROFILE_ID.getKey(),
settings.getAppProfileId())
// Also annotate traces with library versions
.put("gax", GaxGrpcProperties.getGaxGrpcVersion())
.put("grpc", GaxGrpcProperties.getGrpcVersion())
.put("gapic", Version.VERSION)
.build()))
.add(settings.getTracerFactory());

// By default, provide an otel instance and register builtin metrics
// customer can pass in their own OTEL instance and register builtin metrics with the meter
// provider
Attributes attributes =
Attributes.of(
PROJECT_ID, settings.getProjectId(),
INSTANCE_ID, settings.getInstanceId(),
APP_PROFILE, settings.getAppProfileId());
if (settings.getOpenTelemetry() != null) {
tracerFactories.add(MetricsTracerFactory.create(settings.getOpenTelemetry(), attributes));
} else if (settings.isBuiltinMetricsEnabled()) {
MetricExporter metricExporter =
BigtableCloudMonitoringExporter.create(
settings.getProjectId(), settings.getCredentialsProvider().getCredentials());
SdkMeterProvider sdkMeterProvider =
SdkMeterProvider.builder()
.registerMetricReader(PeriodicMetricReader.create(metricExporter))
.registerView(OPERATION_LATENCIES_SELECTOR, OPERATION_LATENCIES_VIEW)
.registerView(ATTEMPT_LATENCIES_SELECTOR, ATTEMPT_LATENCIES_VIEW)
.registerView(SERVER_LATENCIES_SELECTOR, SERVER_LATENCIES_VIEW)
.registerView(FIRST_RESPONSE_LATENCIES_SELECTOR, FIRST_RESPONSE_LATENCIES_VIEW)
.registerView(
APPLICATION_BLOCKING_LATENCIES_SELECTOR, APPLICATION_BLOCKING_LATENCIES_VIEW)
.registerView(CLIENT_BLOCKING_LATENCIES_SELECTOR, CLIENT_BLOCKING_LATENCIES_VIEW)
.registerView(RETRY_COUNT_SELECTOR, RETRY_COUNT_VIEW)
.registerView(CONNECTIVITY_ERROR_COUNT_SELECTOR, CONNECTIVITY_ERROR_COUNT_VIEW)
.build();
OpenTelemetry openTelemetry =
OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
tracerFactories.add(BuiltinMetricsTracerFactory.create(openTelemetry, attributes));
}
settings.setTracerFactory(new CompositeTracerFactory(tracerFactories.build()));
return settings;
}

public EnhancedBigtableStub(EnhancedBigtableStubSettings settings, ClientContext clientContext) {
this.settings = settings;
this.clientContext = clientContext;
Expand Down
Loading
Loading