feat: SSAPI Checkpointing (BPS-277)

go.mod

@@ -166,7 +166,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/component v0.113.0
-	go.opentelemetry.io/collector/confmap v1.19.0
+	go.opentelemetry.io/collector/confmap v1.20.0
 	go.opentelemetry.io/collector/confmap/provider/envprovider v1.19.0
 	go.opentelemetry.io/collector/confmap/provider/fileprovider v1.19.0
 	go.opentelemetry.io/collector/confmap/provider/httpsprovider v1.19.0
@@ -347,7 +347,7 @@ require (
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
-	github.com/knadh/koanf/v2 v2.1.1 // indirect
+	github.com/knadh/koanf/v2 v2.1.2 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/leodido/go-syslog/v4 v4.2.0 // indirect
 	github.com/lestrrat-go/strftime v1.1.0 // indirect
@@ -424,7 +424,7 @@ require (
 	go.opentelemetry.io/collector/extension/auth v0.113.0 // indirect
 	go.opentelemetry.io/collector/extension/experimental/storage v0.113.0 // indirect
 	go.opentelemetry.io/collector/extension/extensioncapabilities v0.113.0 // indirect
-	go.opentelemetry.io/collector/filter v0.113.0 // indirect
+	go.opentelemetry.io/collector/filter v0.114.0 // indirect
 	go.opentelemetry.io/collector/internal/fanoutconsumer v0.113.0 // indirect
 	go.opentelemetry.io/collector/internal/memorylimiter v0.113.0 // indirect
 	go.opentelemetry.io/collector/internal/sharedcomponent v0.113.0 // indirect

go.sum

@@ -1729,8 +1729,8 @@ github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/q
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knadh/koanf v1.5.0 h1:q2TSd/3Pyc/5yP9ldIrSdIz26MCcyNQzW0pEAugLPNs=
 github.com/knadh/koanf v1.5.0/go.mod h1:Hgyjp4y8v44hpZtPzs7JZfRAW5AhN7KfZcwv1RYggDs=
-github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM=
-github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es=
+github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ=
+github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo=
 github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00=
 github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -2657,8 +2657,8 @@ go.opentelemetry.io/collector/config/configtls v1.19.0 h1:GQ/cF1hgNqHVBq2oSSrOFX
 go.opentelemetry.io/collector/config/configtls v1.19.0/go.mod h1:1hyqnYB3JqEUlk1ME/s9HYz4oCRcxQCRxsJitFFT/cA=
 go.opentelemetry.io/collector/config/internal v0.113.0 h1:9RAzH8v7ItFT1npHpvP0SvUzBHcZDliCGRo9Spp6v7c=
 go.opentelemetry.io/collector/config/internal v0.113.0/go.mod h1:yC7E4h1Uj0SubxcFImh6OvBHFTjMh99+A5PuyIgDWqc=
-go.opentelemetry.io/collector/confmap v1.19.0 h1:TQ0lZpAKqgsE0EKk+u4JA+uBbPYeFRmWP3GH43w40CY=
-go.opentelemetry.io/collector/confmap v1.19.0/go.mod h1:GgNu1ElPGmLn9govqIfjaopvdspw4PJ9KeDtWC4E2Q4=
+go.opentelemetry.io/collector/confmap v1.20.0 h1:ARfOwmkKxFOud1njl03yAHQ30+uenlzqCO6LBYamDTE=
+go.opentelemetry.io/collector/confmap v1.20.0/go.mod h1:DMpd9Ay/ffls3JoQBQ73vWeRsz1rNuLbwjo6WtjSQus=
 go.opentelemetry.io/collector/confmap/provider/envprovider v1.19.0 h1:f8O/I5pVRN86Gx5mHekNx92S6fGdOS4VcooRJKWe6Bs=
 go.opentelemetry.io/collector/confmap/provider/envprovider v1.19.0/go.mod h1:AiaW5YW1LD0/WlZuc8eZuZPBH6PA9QqsiAYRX1iC6T0=
 go.opentelemetry.io/collector/confmap/provider/fileprovider v1.19.0 h1:TYwyk4ea3U+5MYcEjrzZAaonBcLlabQu8CZeB7ekAYY=
@@ -2715,8 +2715,8 @@ go.opentelemetry.io/collector/extension/zpagesextension v0.113.0 h1:b/Clxso9uVwL
 go.opentelemetry.io/collector/extension/zpagesextension v0.113.0/go.mod h1:5csGYy9Ydfy6Hpw3Tod864P6HUEZpA6UiuPJPG3TjSU=
 go.opentelemetry.io/collector/featuregate v1.19.0 h1:ASea2sU+tdpKI3RxIJC/pufDAfwAmrvcQ4EmTHVu0B0=
 go.opentelemetry.io/collector/featuregate v1.19.0/go.mod h1:47xrISO71vJ83LSMm8+yIDsUbKktUp48Ovt7RR6VbRs=
-go.opentelemetry.io/collector/filter v0.113.0 h1:5ODwM8QEOzZq08H8DJilBa4PHieXpBreJVKZ0D2YshA=
-go.opentelemetry.io/collector/filter v0.113.0/go.mod h1:Mh3N6cpVijdamUJj1tAgSU1RG/Ek4FuY2ODKYxKZDtk=
+go.opentelemetry.io/collector/filter v0.114.0 h1:5I97yblUxc6rXCYRn542aSrsNQLo/dE+87XROW2b5oU=
+go.opentelemetry.io/collector/filter v0.114.0/go.mod h1:Nxwc+RD9AH4y/qYtkTP+Ac19CxgW5GAB+sJU4ACLr6g=
 go.opentelemetry.io/collector/internal/fanoutconsumer v0.113.0 h1:Beu2zAN6/EDXQ6hMFU6FT1BsnU5FXmWNOlfTAhrgbGc=
 go.opentelemetry.io/collector/internal/fanoutconsumer v0.113.0/go.mod h1:WUXbc4L6KJ3SpmsxBgId0OYzRDuS7n274kNpqrgnSmY=
 go.opentelemetry.io/collector/internal/memorylimiter v0.113.0 h1:qe3xZYB4BgSuPDgFMQbcJ5gDy8t+S1vt6pL+OKrdx9E=

receiver/splunksearchapireceiver/config.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config/confighttp"
 )
 
@@ -33,6 +34,7 @@ type Config struct {
 	Password                string        `mapstructure:"splunk_password"`
 	Searches                []Search      `mapstructure:"searches"`
 	JobPollInterval         time.Duration `mapstructure:"job_poll_interval"`
+	StorageID               *component.ID `mapstructure:"storage"`
 }
 
 // Search struct to represent a Splunk search
@@ -59,6 +61,10 @@ func (cfg *Config) Validate() error {
 		return errors.New("at least one search must be provided")
 	}
 
+	if cfg.StorageID == nil {
+		return errors.New("storage configuration is required for this receiver")
+	}
+
 	for _, search := range cfg.Searches {
 		if search.Query == "" {
 			return errors.New("missing query in search")

receiver/splunksearchapireceiver/config_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component"
 )
 
 func TestValidate(t *testing.T) {
@@ -26,6 +27,7 @@ func TestValidate(t *testing.T) {
 		endpoint    string
 		username    string
 		password    string
+		storage     string
 		searches    []Search
 		errExpected bool
 		errText     string
@@ -34,6 +36,7 @@ func TestValidate(t *testing.T) {
 			desc:     "Missing endpoint",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal",
@@ -48,6 +51,7 @@ func TestValidate(t *testing.T) {
 			desc:     "Missing username",
 			endpoint: "http://localhost:8089",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal",
@@ -62,6 +66,7 @@ func TestValidate(t *testing.T) {
 			desc:     "Missing password",
 			endpoint: "http://localhost:8089",
 			username: "user",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal",
@@ -72,11 +77,27 @@ func TestValidate(t *testing.T) {
 			errExpected: true,
 			errText:     "missing Splunk password",
 		},
+		{
+			desc:     "Missing storage",
+			endpoint: "http://localhost:8089",
+			username: "user",
+			password: "password",
+			searches: []Search{
+				{
+					Query:        "search index=_internal",
+					EarliestTime: "2024-10-30T04:00:00.000Z",
+					LatestTime:   "2024-10-30T14:00:00.000Z",
+				},
+			},
+			errExpected: true,
+			errText:     "storage configuration is required for this receiver",
+		},
 		{
 			desc:        "Missing searches",
 			endpoint:    "http://localhost:8089",
 			username:    "user",
 			password:    "password",
+			storage:     "file_storage",
 			errExpected: true,
 			errText:     "at least one search must be provided",
 		},
@@ -85,6 +106,7 @@ func TestValidate(t *testing.T) {
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					EarliestTime: "2024-10-30T04:00:00.000Z",
@@ -99,6 +121,7 @@ func TestValidate(t *testing.T) {
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:      "search index=_internal",
@@ -113,6 +136,7 @@ func TestValidate(t *testing.T) {
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal",
@@ -128,6 +152,7 @@ func TestValidate(t *testing.T) {
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal",
@@ -142,6 +167,7 @@ func TestValidate(t *testing.T) {
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal",
@@ -157,6 +183,7 @@ func TestValidate(t *testing.T) {
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal | stats count by sourcetype",
@@ -172,6 +199,7 @@ func TestValidate(t *testing.T) {
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal",
@@ -186,6 +214,7 @@ func TestValidate(t *testing.T) {
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal",
@@ -205,6 +234,7 @@ func TestValidate(t *testing.T) {
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal",
@@ -216,10 +246,11 @@ func TestValidate(t *testing.T) {
 			errExpected: false,
 		},
 		{
-			desc:     "Query with ealiest and latest time",
+			desc:     "Query with earliest and latest time",
 			endpoint: "http://localhost:8089",
 			username: "user",
 			password: "password",
+			storage:  "file_storage",
 			searches: []Search{
 				{
 					Query:        "search index=_internal earliest=2024-10-30T04:00:00.000Z latest=2024-10-30T14:00:00.000Z",
@@ -238,6 +269,9 @@ func TestValidate(t *testing.T) {
 			cfg.Username = tc.username
 			cfg.Password = tc.password
 			cfg.Searches = tc.searches
+			if tc.storage != "" {
+				cfg.StorageID = &component.ID{}
+			}
 			err := cfg.Validate()
 			if tc.errExpected {
 				require.EqualError(t, err, tc.errText)

receiver/splunksearchapireceiver/factory.go

@@ -42,10 +42,12 @@ func createLogsReceiver(_ context.Context,
 ) (receiver.Logs, error) {
 	ssapirConfig := cfg.(*Config)
 	ssapir := &splunksearchapireceiver{
-		logger:       params.Logger,
-		logsConsumer: consumer,
-		config:       ssapirConfig,
-		settings:     params.TelemetrySettings,
+		logger:           params.Logger,
+		logsConsumer:     consumer,
+		config:           ssapirConfig,
+		id:               params.ID,
+		settings:         params.TelemetrySettings,
+		checkpointRecord: &EventRecord{},
 	}
 	return ssapir, nil
 }

receiver/splunksearchapireceiver/go.mod

@@ -3,34 +3,64 @@ module github.com/open-telemetry/opentelemtry-collector-contrib/receiver/splunks
 go 1.22.5
 
 require (
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.113.0
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/component v0.113.0
 	go.opentelemetry.io/collector/consumer v0.113.0
+	go.opentelemetry.io/collector/consumer/consumertest v0.113.0
+	go.opentelemetry.io/collector/extension/experimental/storage v0.113.0
 	go.opentelemetry.io/collector/pdata v1.19.0
-	go.opentelemetry.io/collector/receiver v0.112.0
+	go.opentelemetry.io/collector/receiver v0.113.0
 	go.uber.org/zap v1.27.0
 )
 
 require (
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/elastic/lunes v0.1.0 // indirect
+	github.com/expr-lang/expr v1.16.9 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/knadh/koanf/providers/confmap v0.1.0 // indirect
+	github.com/knadh/koanf/v2 v2.1.2 // indirect
+	github.com/leodido/go-syslog/v4 v4.2.0 // indirect
+	github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect
+	github.com/magefile/mage v1.15.0 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.113.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rs/cors v1.11.1 // indirect
+	github.com/valyala/fastjson v1.6.4 // indirect
 	go.opentelemetry.io/collector/client v1.19.0 // indirect
 	go.opentelemetry.io/collector/config/configauth v0.113.0 // indirect
 	go.opentelemetry.io/collector/config/configcompression v1.19.0 // indirect
 	go.opentelemetry.io/collector/config/configopaque v1.19.0 // indirect
 	go.opentelemetry.io/collector/config/configtls v1.19.0 // indirect
 	go.opentelemetry.io/collector/config/internal v0.113.0 // indirect
+	go.opentelemetry.io/collector/confmap v1.20.0 // indirect
+	go.opentelemetry.io/collector/consumer/consumererror v0.113.0 // indirect
+	go.opentelemetry.io/collector/consumer/consumerprofiles v0.113.0 // indirect
 	go.opentelemetry.io/collector/extension v0.113.0 // indirect
 	go.opentelemetry.io/collector/extension/auth v0.113.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.19.0 // indirect
+	go.opentelemetry.io/collector/pdata/pprofile v0.113.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.113.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.31.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect
+	gonum.org/v1/gonum v0.15.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
@@ -41,7 +71,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	go.opentelemetry.io/collector/config/confighttp v0.113.0
 	go.opentelemetry.io/collector/config/configtelemetry v0.113.0 // indirect
-	go.opentelemetry.io/collector/pipeline v0.112.0 // indirect
+	go.opentelemetry.io/collector/pipeline v0.113.0 // indirect
 	go.opentelemetry.io/otel v1.31.0 // indirect
 	go.opentelemetry.io/otel/metric v1.31.0 // indirect
 	go.opentelemetry.io/otel/trace v1.31.0 // indirect