Merge pull request #373 from alltilla/google-pubsub-grpc

Add `google-pubsub-grpc()` destination
axoflow · Nov 27, 2024 · a2fbad4 · a2fbad4
2 parents fd273a4 + 2c49846
commit a2fbad4
Show file tree

Hide file tree

Showing 24 changed files with 3,408 additions and 18 deletions.
diff --git a/configure.ac b/configure.ac
@@ -313,7 +313,7 @@ AC_ARG_WITH(libpaho-mqtt,
                             [use libpaho-mqtt library from (prefix) directory DIR]),,)
 
 AC_ARG_ENABLE(grpc,
-              [  --enable-grpc           Enable GRPC based modules support (OpenTelemetry, Loki, BigQuery, ClickHouse) (default: auto)]
+              [  --enable-grpc           Enable GRPC based modules support (OpenTelemetry, Loki, BigQuery, Pub/Sub, ClickHouse) (default: auto)]
               ,,enable_grpc="auto")
 
 AC_ARG_WITH(protoc,

diff --git a/docker/apkbuild/axoflow/axosyslog/APKBUILD b/docker/apkbuild/axoflow/axosyslog/APKBUILD
@@ -155,6 +155,7 @@ _grpc() {
 	_submv usr/lib/syslog-ng/libotel.so \
 		usr/lib/syslog-ng/libloki.so \
 		usr/lib/syslog-ng/libbigquery.so \
+		usr/lib/syslog-ng/libpubsub.so \
 		usr/lib/syslog-ng/libclickhouse.so
 }
 

diff --git a/modules/grpc/CMakeLists.txt b/modules/grpc/CMakeLists.txt
@@ -80,4 +80,5 @@ endif()
 add_subdirectory(loki)
 add_subdirectory(otel)
 add_subdirectory(bigquery)
+add_subdirectory(pubsub)
 add_subdirectory(clickhouse)
diff --git a/modules/grpc/Makefile.am b/modules/grpc/Makefile.am
@@ -5,6 +5,7 @@ include modules/grpc/common/Makefile.am
 include modules/grpc/otel/Makefile.am
 include modules/grpc/loki/Makefile.am
 include modules/grpc/bigquery/Makefile.am
+include modules/grpc/pubsub/Makefile.am
 include modules/grpc/clickhouse/Makefile.am
 
 if ENABLE_GRPC

diff --git a/modules/grpc/protos/CMakeLists.txt b/modules/grpc/protos/CMakeLists.txt
@@ -6,9 +6,9 @@ set(LOKI_PROTO_SRCDIR "${CMAKE_CURRENT_SOURCE_DIR}/grafana-loki")
 set(LOKI_PROTO_BUILDDIR "${CMAKE_CURRENT_BINARY_DIR}/grafana-loki")
 set(LOKI_PROTO_BUILDDIR ${LOKI_PROTO_BUILDDIR} PARENT_SCOPE)
 
-set(BIGQUERY_PROTO_SRCDIR "${CMAKE_CURRENT_SOURCE_DIR}/googleapis-proto")
-set(BIGQUERY_PROTO_BUILDDIR "${CMAKE_CURRENT_BINARY_DIR}/googleapis-proto")
-set(BIGQUERY_PROTO_BUILDDIR ${BIGQUERY_PROTO_BUILDDIR} PARENT_SCOPE)
+set(GOOGLEAPIS_PROTO_SRCDIR "${CMAKE_CURRENT_SOURCE_DIR}/googleapis-proto")
+set(GOOGLEAPIS_PROTO_BUILDDIR "${CMAKE_CURRENT_BINARY_DIR}/googleapis-proto")
+set(GOOGLEAPIS_PROTO_BUILDDIR ${GOOGLEAPIS_PROTO_BUILDDIR} PARENT_SCOPE)
 
 set(CLICKHOUSE_PROTO_SRCDIR "${CMAKE_CURRENT_SOURCE_DIR}/clickhouse-proto")
 set(CLICKHOUSE_PROTO_BUILDDIR "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-proto")
@@ -26,7 +26,7 @@ set(OTEL_PROTO_GRPC_SOURCES
   opentelemetry/proto/collector/metrics/v1/metrics_service.proto
   opentelemetry/proto/collector/trace/v1/trace_service.proto)
 
-set(BIGQUERY_PROTO_SOURCES
+set(GOOGLEAPIS_PROTO_SOURCES
   google/api/annotations.proto
   google/api/client.proto
   google/api/field_behavior.proto
@@ -42,8 +42,10 @@ set(BIGQUERY_PROTO_SOURCES
   google/rpc/status.proto
 )
 
-set(BIGQUERY_PROTO_GRPC_SOURCES
+set(GOOGLEAPIS_PROTO_GRPC_SOURCES
   google/cloud/bigquery/storage/v1/storage.proto
+  google/pubsub/v1/pubsub.proto
+  google/pubsub/v1/schema.proto
 )
 
 set(LOKI_PROTO_GRPC_SOURCES
@@ -72,16 +74,16 @@ protobuf_generate_cpp_grpc(
   PROTOS ${LOKI_PROTO_GRPC_SOURCES})
 
 protobuf_generate_cpp(
-  PROTO_PATH ${BIGQUERY_PROTO_SRCDIR}
-  CPP_OUT ${BIGQUERY_PROTO_BUILDDIR}
-  OUT_SRCS BIGQUERY_PROTO_GENERATED_SOURCES
-  PROTOS ${BIGQUERY_PROTO_SOURCES})
+  PROTO_PATH ${GOOGLEAPIS_PROTO_SRCDIR}
+  CPP_OUT ${GOOGLEAPIS_PROTO_BUILDDIR}
+  OUT_SRCS GOOGLEAPIS_PROTO_GENERATED_SOURCES
+  PROTOS ${GOOGLEAPIS_PROTO_SOURCES})
 
 protobuf_generate_cpp_grpc(
-  PROTO_PATH ${BIGQUERY_PROTO_SRCDIR}
-  CPP_OUT ${BIGQUERY_PROTO_BUILDDIR}
-  OUT_SRCS BIGQUERY_PROTO_GENERATED_GRPC_SOURCES
-  PROTOS ${BIGQUERY_PROTO_GRPC_SOURCES})
+  PROTO_PATH ${GOOGLEAPIS_PROTO_SRCDIR}
+  CPP_OUT ${GOOGLEAPIS_PROTO_BUILDDIR}
+  OUT_SRCS GOOGLEAPIS_PROTO_GENERATED_GRPC_SOURCES
+  PROTOS ${GOOGLEAPIS_PROTO_GRPC_SOURCES})
 
 protobuf_generate_cpp(
   PROTO_PATH ${CLICKHOUSE_PROTO_SRCDIR}
@@ -99,16 +101,16 @@ add_library(grpc-protos SHARED
   ${OTEL_PROTO_GENERATED_SOURCES}
   ${OTEL_PROTO_GENERATED_GRPC_SOURCES}
   ${LOKI_PROTO_GENERATED_GRPC_SOURCES}
-  ${BIGQUERY_PROTO_GENERATED_SOURCES}
-  ${BIGQUERY_PROTO_GENERATED_GRPC_SOURCES}
+  ${GOOGLEAPIS_PROTO_GENERATED_SOURCES}
+  ${GOOGLEAPIS_PROTO_GENERATED_GRPC_SOURCES}
   ${CLICKHOUSE_PROTO_GENERATED_SOURCES}
   ${CLICKHOUSE_PROTO_GENERATED_GRPC_SOURCES}
   apphook.cpp
 )
 
 target_link_libraries(grpc-protos PRIVATE ${MODULE_GRPC_LIBS} syslog-ng)
 target_include_directories(grpc-protos
-  PUBLIC ${OTEL_PROTO_BUILDDIR} ${LOKI_PROTO_BUILDDIR} ${BIGQUERY_PROTO_BUILDDIR} ${CLICKHOUSE_PROTO_BUILDDIR}
+  PUBLIC ${OTEL_PROTO_BUILDDIR} ${LOKI_PROTO_BUILDDIR} ${GOOGLEAPIS_PROTO_BUILDDIR} ${CLICKHOUSE_PROTO_BUILDDIR}
   PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/modules/grpc
   PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}
 )

diff --git a/modules/grpc/protos/Makefile.am b/modules/grpc/protos/Makefile.am
@@ -115,6 +115,14 @@ GRPC_PROTOS_BUILT_SOURCES = \
   $(GOOGLEAPIS_PROTO_BUILDDIR)/google/cloud/bigquery/storage/v1/stream.pb.h \
   $(GOOGLEAPIS_PROTO_BUILDDIR)/google/cloud/bigquery/storage/v1/table.pb.cc \
   $(GOOGLEAPIS_PROTO_BUILDDIR)/google/cloud/bigquery/storage/v1/table.pb.h \
+  $(GOOGLEAPIS_PROTO_BUILDDIR)/google/pubsub/v1/pubsub.grpc.pb.cc \
+  $(GOOGLEAPIS_PROTO_BUILDDIR)/google/pubsub/v1/pubsub.grpc.pb.h \
+  $(GOOGLEAPIS_PROTO_BUILDDIR)/google/pubsub/v1/pubsub.pb.cc \
+  $(GOOGLEAPIS_PROTO_BUILDDIR)/google/pubsub/v1/pubsub.pb.h \
+  $(GOOGLEAPIS_PROTO_BUILDDIR)/google/pubsub/v1/schema.grpc.pb.cc \
+  $(GOOGLEAPIS_PROTO_BUILDDIR)/google/pubsub/v1/schema.grpc.pb.h \
+  $(GOOGLEAPIS_PROTO_BUILDDIR)/google/pubsub/v1/schema.pb.cc \
+  $(GOOGLEAPIS_PROTO_BUILDDIR)/google/pubsub/v1/schema.pb.h \
   $(GOOGLEAPIS_PROTO_BUILDDIR)/google/rpc/status.pb.cc \
   $(GOOGLEAPIS_PROTO_BUILDDIR)/google/rpc/status.pb.h \
   \