Skip to content

Commit

Permalink
feat: SSAPI Checkpointing (BPS-277) (#1969)
Browse files Browse the repository at this point in the history
  • Loading branch information
Caleb-Hurshman authored Nov 19, 2024
2 parents 421795d + 34907e1 commit c24eb17
Show file tree
Hide file tree
Showing 10 changed files with 398 additions and 41 deletions.
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ require (
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.9.0
go.opentelemetry.io/collector/component v0.113.0
go.opentelemetry.io/collector/confmap v1.19.0
go.opentelemetry.io/collector/confmap v1.20.0
go.opentelemetry.io/collector/confmap/provider/envprovider v1.19.0
go.opentelemetry.io/collector/confmap/provider/fileprovider v1.19.0
go.opentelemetry.io/collector/confmap/provider/httpsprovider v1.19.0
Expand Down Expand Up @@ -347,7 +347,7 @@ require (
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/kelseyhightower/envconfig v1.4.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/knadh/koanf/v2 v2.1.1 // indirect
github.com/knadh/koanf/v2 v2.1.2 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/leodido/go-syslog/v4 v4.2.0 // indirect
github.com/lestrrat-go/strftime v1.1.0 // indirect
Expand Down Expand Up @@ -424,7 +424,7 @@ require (
go.opentelemetry.io/collector/extension/auth v0.113.0 // indirect
go.opentelemetry.io/collector/extension/experimental/storage v0.113.0 // indirect
go.opentelemetry.io/collector/extension/extensioncapabilities v0.113.0 // indirect
go.opentelemetry.io/collector/filter v0.113.0 // indirect
go.opentelemetry.io/collector/filter v0.114.0 // indirect
go.opentelemetry.io/collector/internal/fanoutconsumer v0.113.0 // indirect
go.opentelemetry.io/collector/internal/memorylimiter v0.113.0 // indirect
go.opentelemetry.io/collector/internal/sharedcomponent v0.113.0 // indirect
Expand Down
12 changes: 6 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1729,8 +1729,8 @@ github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/q
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/knadh/koanf v1.5.0 h1:q2TSd/3Pyc/5yP9ldIrSdIz26MCcyNQzW0pEAugLPNs=
github.com/knadh/koanf v1.5.0/go.mod h1:Hgyjp4y8v44hpZtPzs7JZfRAW5AhN7KfZcwv1RYggDs=
github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM=
github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es=
github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ=
github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo=
github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00=
github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
Expand Down Expand Up @@ -2657,8 +2657,8 @@ go.opentelemetry.io/collector/config/configtls v1.19.0 h1:GQ/cF1hgNqHVBq2oSSrOFX
go.opentelemetry.io/collector/config/configtls v1.19.0/go.mod h1:1hyqnYB3JqEUlk1ME/s9HYz4oCRcxQCRxsJitFFT/cA=
go.opentelemetry.io/collector/config/internal v0.113.0 h1:9RAzH8v7ItFT1npHpvP0SvUzBHcZDliCGRo9Spp6v7c=
go.opentelemetry.io/collector/config/internal v0.113.0/go.mod h1:yC7E4h1Uj0SubxcFImh6OvBHFTjMh99+A5PuyIgDWqc=
go.opentelemetry.io/collector/confmap v1.19.0 h1:TQ0lZpAKqgsE0EKk+u4JA+uBbPYeFRmWP3GH43w40CY=
go.opentelemetry.io/collector/confmap v1.19.0/go.mod h1:GgNu1ElPGmLn9govqIfjaopvdspw4PJ9KeDtWC4E2Q4=
go.opentelemetry.io/collector/confmap v1.20.0 h1:ARfOwmkKxFOud1njl03yAHQ30+uenlzqCO6LBYamDTE=
go.opentelemetry.io/collector/confmap v1.20.0/go.mod h1:DMpd9Ay/ffls3JoQBQ73vWeRsz1rNuLbwjo6WtjSQus=
go.opentelemetry.io/collector/confmap/provider/envprovider v1.19.0 h1:f8O/I5pVRN86Gx5mHekNx92S6fGdOS4VcooRJKWe6Bs=
go.opentelemetry.io/collector/confmap/provider/envprovider v1.19.0/go.mod h1:AiaW5YW1LD0/WlZuc8eZuZPBH6PA9QqsiAYRX1iC6T0=
go.opentelemetry.io/collector/confmap/provider/fileprovider v1.19.0 h1:TYwyk4ea3U+5MYcEjrzZAaonBcLlabQu8CZeB7ekAYY=
Expand Down Expand Up @@ -2715,8 +2715,8 @@ go.opentelemetry.io/collector/extension/zpagesextension v0.113.0 h1:b/Clxso9uVwL
go.opentelemetry.io/collector/extension/zpagesextension v0.113.0/go.mod h1:5csGYy9Ydfy6Hpw3Tod864P6HUEZpA6UiuPJPG3TjSU=
go.opentelemetry.io/collector/featuregate v1.19.0 h1:ASea2sU+tdpKI3RxIJC/pufDAfwAmrvcQ4EmTHVu0B0=
go.opentelemetry.io/collector/featuregate v1.19.0/go.mod h1:47xrISO71vJ83LSMm8+yIDsUbKktUp48Ovt7RR6VbRs=
go.opentelemetry.io/collector/filter v0.113.0 h1:5ODwM8QEOzZq08H8DJilBa4PHieXpBreJVKZ0D2YshA=
go.opentelemetry.io/collector/filter v0.113.0/go.mod h1:Mh3N6cpVijdamUJj1tAgSU1RG/Ek4FuY2ODKYxKZDtk=
go.opentelemetry.io/collector/filter v0.114.0 h1:5I97yblUxc6rXCYRn542aSrsNQLo/dE+87XROW2b5oU=
go.opentelemetry.io/collector/filter v0.114.0/go.mod h1:Nxwc+RD9AH4y/qYtkTP+Ac19CxgW5GAB+sJU4ACLr6g=
go.opentelemetry.io/collector/internal/fanoutconsumer v0.113.0 h1:Beu2zAN6/EDXQ6hMFU6FT1BsnU5FXmWNOlfTAhrgbGc=
go.opentelemetry.io/collector/internal/fanoutconsumer v0.113.0/go.mod h1:WUXbc4L6KJ3SpmsxBgId0OYzRDuS7n274kNpqrgnSmY=
go.opentelemetry.io/collector/internal/memorylimiter v0.113.0 h1:qe3xZYB4BgSuPDgFMQbcJ5gDy8t+S1vt6pL+OKrdx9E=
Expand Down
6 changes: 6 additions & 0 deletions receiver/splunksearchapireceiver/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"strings"
"time"

"go.opentelemetry.io/collector/component"
"go.opentelemetry.io/collector/config/confighttp"
)

Expand All @@ -33,6 +34,7 @@ type Config struct {
Password string `mapstructure:"splunk_password"`
Searches []Search `mapstructure:"searches"`
JobPollInterval time.Duration `mapstructure:"job_poll_interval"`
StorageID *component.ID `mapstructure:"storage"`
}

// Search struct to represent a Splunk search
Expand All @@ -59,6 +61,10 @@ func (cfg *Config) Validate() error {
return errors.New("at least one search must be provided")
}

if cfg.StorageID == nil {
return errors.New("storage configuration is required for this receiver")
}

for _, search := range cfg.Searches {
if search.Query == "" {
return errors.New("missing query in search")
Expand Down
36 changes: 35 additions & 1 deletion receiver/splunksearchapireceiver/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"testing"

"github.com/stretchr/testify/require"
"go.opentelemetry.io/collector/component"
)

func TestValidate(t *testing.T) {
Expand All @@ -26,6 +27,7 @@ func TestValidate(t *testing.T) {
endpoint string
username string
password string
storage string
searches []Search
errExpected bool
errText string
Expand All @@ -34,6 +36,7 @@ func TestValidate(t *testing.T) {
desc: "Missing endpoint",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -48,6 +51,7 @@ func TestValidate(t *testing.T) {
desc: "Missing username",
endpoint: "http://localhost:8089",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -62,6 +66,7 @@ func TestValidate(t *testing.T) {
desc: "Missing password",
endpoint: "http://localhost:8089",
username: "user",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -72,11 +77,27 @@ func TestValidate(t *testing.T) {
errExpected: true,
errText: "missing Splunk password",
},
{
desc: "Missing storage",
endpoint: "http://localhost:8089",
username: "user",
password: "password",
searches: []Search{
{
Query: "search index=_internal",
EarliestTime: "2024-10-30T04:00:00.000Z",
LatestTime: "2024-10-30T14:00:00.000Z",
},
},
errExpected: true,
errText: "storage configuration is required for this receiver",
},
{
desc: "Missing searches",
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
errExpected: true,
errText: "at least one search must be provided",
},
Expand All @@ -85,6 +106,7 @@ func TestValidate(t *testing.T) {
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
EarliestTime: "2024-10-30T04:00:00.000Z",
Expand All @@ -99,6 +121,7 @@ func TestValidate(t *testing.T) {
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -113,6 +136,7 @@ func TestValidate(t *testing.T) {
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -128,6 +152,7 @@ func TestValidate(t *testing.T) {
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -142,6 +167,7 @@ func TestValidate(t *testing.T) {
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -157,6 +183,7 @@ func TestValidate(t *testing.T) {
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal | stats count by sourcetype",
Expand All @@ -172,6 +199,7 @@ func TestValidate(t *testing.T) {
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -186,6 +214,7 @@ func TestValidate(t *testing.T) {
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -205,6 +234,7 @@ func TestValidate(t *testing.T) {
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal",
Expand All @@ -216,10 +246,11 @@ func TestValidate(t *testing.T) {
errExpected: false,
},
{
desc: "Query with ealiest and latest time",
desc: "Query with earliest and latest time",
endpoint: "http://localhost:8089",
username: "user",
password: "password",
storage: "file_storage",
searches: []Search{
{
Query: "search index=_internal earliest=2024-10-30T04:00:00.000Z latest=2024-10-30T14:00:00.000Z",
Expand All @@ -238,6 +269,9 @@ func TestValidate(t *testing.T) {
cfg.Username = tc.username
cfg.Password = tc.password
cfg.Searches = tc.searches
if tc.storage != "" {
cfg.StorageID = &component.ID{}
}
err := cfg.Validate()
if tc.errExpected {
require.EqualError(t, err, tc.errText)
Expand Down
10 changes: 6 additions & 4 deletions receiver/splunksearchapireceiver/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,12 @@ func createLogsReceiver(_ context.Context,
) (receiver.Logs, error) {
ssapirConfig := cfg.(*Config)
ssapir := &splunksearchapireceiver{
logger: params.Logger,
logsConsumer: consumer,
config: ssapirConfig,
settings: params.TelemetrySettings,
logger: params.Logger,
logsConsumer: consumer,
config: ssapirConfig,
id: params.ID,
settings: params.TelemetrySettings,
checkpointRecord: &EventRecord{},
}
return ssapir, nil
}
Expand Down
34 changes: 32 additions & 2 deletions receiver/splunksearchapireceiver/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,34 +3,64 @@ module github.com/open-telemetry/opentelemtry-collector-contrib/receiver/splunks
go 1.22.5

require (
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.113.0
github.com/stretchr/testify v1.9.0
go.opentelemetry.io/collector/component v0.113.0
go.opentelemetry.io/collector/consumer v0.113.0
go.opentelemetry.io/collector/consumer/consumertest v0.113.0
go.opentelemetry.io/collector/extension/experimental/storage v0.113.0
go.opentelemetry.io/collector/pdata v1.19.0
go.opentelemetry.io/collector/receiver v0.112.0
go.opentelemetry.io/collector/receiver v0.113.0
go.uber.org/zap v1.27.0
)

require (
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/elastic/lunes v0.1.0 // indirect
github.com/expr-lang/expr v1.16.9 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/go-version v1.7.0 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/knadh/koanf/maps v0.1.1 // indirect
github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
github.com/knadh/koanf/v2 v2.1.2 // indirect
github.com/leodido/go-syslog/v4 v4.2.0 // indirect
github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect
github.com/magefile/mage v1.15.0 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.113.0 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/cors v1.11.1 // indirect
github.com/valyala/fastjson v1.6.4 // indirect
go.opentelemetry.io/collector/client v1.19.0 // indirect
go.opentelemetry.io/collector/config/configauth v0.113.0 // indirect
go.opentelemetry.io/collector/config/configcompression v1.19.0 // indirect
go.opentelemetry.io/collector/config/configopaque v1.19.0 // indirect
go.opentelemetry.io/collector/config/configtls v1.19.0 // indirect
go.opentelemetry.io/collector/config/internal v0.113.0 // indirect
go.opentelemetry.io/collector/confmap v1.20.0 // indirect
go.opentelemetry.io/collector/consumer/consumererror v0.113.0 // indirect
go.opentelemetry.io/collector/consumer/consumerprofiles v0.113.0 // indirect
go.opentelemetry.io/collector/extension v0.113.0 // indirect
go.opentelemetry.io/collector/extension/auth v0.113.0 // indirect
go.opentelemetry.io/collector/featuregate v1.19.0 // indirect
go.opentelemetry.io/collector/pdata/pprofile v0.113.0 // indirect
go.opentelemetry.io/collector/semconv v0.113.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
go.opentelemetry.io/otel/sdk v1.31.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect
gonum.org/v1/gonum v0.15.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

Expand All @@ -41,7 +71,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
go.opentelemetry.io/collector/config/confighttp v0.113.0
go.opentelemetry.io/collector/config/configtelemetry v0.113.0 // indirect
go.opentelemetry.io/collector/pipeline v0.112.0 // indirect
go.opentelemetry.io/collector/pipeline v0.113.0 // indirect
go.opentelemetry.io/otel v1.31.0 // indirect
go.opentelemetry.io/otel/metric v1.31.0 // indirect
go.opentelemetry.io/otel/trace v1.31.0 // indirect
Expand Down
Loading

0 comments on commit c24eb17

Please sign in to comment.