From eb8eea9afb3a6292ad617086d9aae0660113959f Mon Sep 17 00:00:00 2001 From: neil-xie <104041627+neil-xie@users.noreply.github.com> Date: Thu, 19 Oct 2023 13:37:10 -0700 Subject: [PATCH] Add Pinot as new advanced visibility store option (#5201) * Updated server start up to choose visibility manager based on the advanced visibility option * Added new triple visibility manager to provide options to write to both ES and Pinot for future migration * Added pinotVisibilityStore and pinotClient to support storing visibility messages in Pinot * Added util methods to flatten the search attributes into columns * Added util methods to validate pinot response and query * Added new kafka indexer message type for Pinot since Pinot doesn't need indexer to process * Added integration test to set up Pinot test cluster and test Pinot functionality --------- Co-authored-by: Bowen Xiao Co-authored-by: David Porter Co-authored-by: Shijie Sheng --- cmd/server/cadence/server.go | 26 +- common/config/config.go | 2 + common/config/pinot.go | 31 + common/constants.go | 12 +- common/dynamicconfig/constants.go | 11 + common/log/tag/values.go | 1 + common/messaging/kafka/producerImpl.go | 7 + common/metrics/defs.go | 73 + common/persistence/client/bean.go | 4 + common/persistence/client/factory.go | 59 +- .../pinot/pinotVisibilityMetricClients.go | 385 ++++++ .../persistence/pinot/pinotVisibilityStore.go | 1086 +++++++++++++++ .../pinot/pinotVisibilityStore_test.go | 674 ++++++++++ common/persistence/pinotResponseComparator.go | 524 ++++++++ .../pinotResponseComparator_test.go | 541 ++++++++ .../pinotVisibilityTripleManager.go | 407 ++++++ .../persistence/pinotiVsibilityDualManager.go | 338 +++++ common/pinot/interfaces.go | 55 + common/pinot/page_token.go | 76 ++ common/pinot/pinotClient.go | 156 +++ common/pinot/pinotClient_test.go | 305 +++++ common/pinot/pinotQueryValidator.go | 271 ++++ common/pinot/pinotQueryValidator_test.go | 130 ++ common/pinot/responseUtility.go | 128 ++ common/resource/params.go | 3 + common/resource/resourceImpl.go | 2 + common/service/config.go | 2 + config/development_pinot.yaml | 45 + config/dynamicconfig/development.yaml | 1 + config/dynamicconfig/development_pinot.yaml | 49 + docker/dev/cassandra-pinot-kafka.yml | 12 +- docker/docker-compose-pinot.yml | 2 +- go.mod | 4 +- go.sum | 8 +- host/dynamicconfig.go | 5 + host/integrationbase.go | 40 + host/onebox.go | 27 +- host/pinot_test.go | 1183 +++++++++++++++++ host/pinotutils/pinotClient.go | 38 + host/testcluster.go | 67 + host/testdata/integration_pinot_cluster.yaml | 42 + schema/{Pinot => pinot}/README.md | 0 .../cadence-visibility-config.json | 16 +- .../cadence-visibility-schema.json | 26 +- service/frontend/service.go | 7 +- service/history/resource/resource.go | 7 +- 46 files changed, 6854 insertions(+), 34 deletions(-) create mode 100644 common/config/pinot.go create mode 100644 common/persistence/pinot/pinotVisibilityMetricClients.go create mode 100644 common/persistence/pinot/pinotVisibilityStore.go create mode 100644 common/persistence/pinot/pinotVisibilityStore_test.go create mode 100644 common/persistence/pinotResponseComparator.go create mode 100644 common/persistence/pinotResponseComparator_test.go create mode 100644 common/persistence/pinotVisibilityTripleManager.go create mode 100644 common/persistence/pinotiVsibilityDualManager.go create mode 100644 common/pinot/interfaces.go create mode 100644 common/pinot/page_token.go create mode 100644 common/pinot/pinotClient.go create mode 100644 common/pinot/pinotClient_test.go create mode 100644 common/pinot/pinotQueryValidator.go create mode 100644 common/pinot/pinotQueryValidator_test.go create mode 100644 common/pinot/responseUtility.go create mode 100644 config/development_pinot.yaml create mode 100644 config/dynamicconfig/development_pinot.yaml create mode 100644 host/pinot_test.go create mode 100644 host/pinotutils/pinotClient.go create mode 100644 host/testdata/integration_pinot_cluster.yaml rename schema/{Pinot => pinot}/README.md (100%) rename schema/{Pinot => pinot}/cadence-visibility-config.json (78%) rename schema/{Pinot => pinot}/cadence-visibility-schema.json (79%) diff --git a/cmd/server/cadence/server.go b/cmd/server/cadence/server.go index 361dae93d5f..a3001d81331 100644 --- a/cmd/server/cadence/server.go +++ b/cmd/server/cadence/server.go @@ -24,12 +24,12 @@ import ( "log" "time" - "github.com/uber/cadence/common/persistence" - "go.uber.org/cadence/.gen/go/cadence/workflowserviceclient" "go.uber.org/cadence/compatibility" apiv1 "github.com/uber/cadence-idl/go/proto/api/v1" + "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/service/worker" "github.com/uber/cadence/common" "github.com/uber/cadence/common/archiver" @@ -52,7 +52,10 @@ import ( "github.com/uber/cadence/service/frontend" "github.com/uber/cadence/service/history" "github.com/uber/cadence/service/matching" - "github.com/uber/cadence/service/worker" + + "github.com/startreedata/pinot-client-go/pinot" + + pnt "github.com/uber/cadence/common/pinot" ) type ( @@ -221,6 +224,23 @@ func (s *server) startService() common.Daemon { } params.ESConfig = advancedVisStore.ElasticSearch + if params.PersistenceConfig.AdvancedVisibilityStore == common.PinotVisibilityStoreName { + // components like ESAnalyzer is still using ElasticSearch + // The plan is to clean those after we switch to operate on Pinot + esVisibilityStore, ok := s.cfg.Persistence.DataStores[common.ESVisibilityStoreName] + if !ok { + log.Fatalf("Missing Elasticsearch config") + } + params.ESConfig = esVisibilityStore.ElasticSearch + params.PinotConfig = advancedVisStore.Pinot + pinotBroker := params.PinotConfig.Broker + pinotRawClient, err := pinot.NewFromBrokerList([]string{pinotBroker}) + if err != nil || pinotRawClient == nil { + log.Fatalf("Creating Pinot visibility client failed: %v", err) + } + pinotClient := pnt.NewPinotClient(pinotRawClient, params.Logger, params.PinotConfig) + params.PinotClient = pinotClient + } params.ESConfig.SetUsernamePassword() esClient, err := elasticsearch.NewGenericClient(params.ESConfig, params.Logger) if err != nil { diff --git a/common/config/config.go b/common/config/config.go index 7d80854bb3e..35f1b0eb081 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -219,6 +219,8 @@ type ( ShardedNoSQL *ShardedNoSQL `yaml:"shardedNosql"` // ElasticSearch contains the config for a ElasticSearch datastore ElasticSearch *ElasticSearchConfig `yaml:"elasticsearch"` + // Pinot contains the config for a Pinot datastore + Pinot *PinotVisibilityConfig `yaml:"pinot"` } // Cassandra contains configuration to connect to Cassandra cluster diff --git a/common/config/pinot.go b/common/config/pinot.go new file mode 100644 index 00000000000..9c2ec698fe7 --- /dev/null +++ b/common/config/pinot.go @@ -0,0 +1,31 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package config + +// PinotVisibilityConfig for connecting to Pinot +type ( + PinotVisibilityConfig struct { + Cluster string `yaml:"cluster"` //nolint:govet + Broker string `yaml:"broker"` //nolint:govet + Table string `yaml:"table"` //nolint:govet + ServiceName string `yaml:"serviceName"` //nolint:govet + } +) diff --git a/common/constants.go b/common/constants.go index 40d9a313735..128b2a82193 100644 --- a/common/constants.go +++ b/common/constants.go @@ -85,7 +85,15 @@ const ( const ( // VisibilityAppName is used to find kafka topics and ES indexName for visibility - VisibilityAppName = "visibility" + VisibilityAppName = "visibility" + PinotVisibilityAppName = "pinot-visibility" +) + +const ( + // ESVisibilityStoreName is used to find es advanced visibility store + ESVisibilityStoreName = "es-visibility" + // PinotVisibilityStoreName is used to find pinot advanced visibility store + PinotVisibilityStoreName = "pinot-visibility" ) // This was flagged by salus as potentially hardcoded credentials. This is a false positive by the scanner and should be @@ -149,6 +157,8 @@ const ( AdvancedVisibilityWritingModeOn = "on" // AdvancedVisibilityWritingModeDual means write to both normal visibility and advanced visibility store AdvancedVisibilityWritingModeDual = "dual" + // AdvacnedVisibilityWritingModeTriple means write to both normal visibility and advanced visibility store, includes ES and Pinot + AdvacnedVisibilityWritingModeTriple = "triple" ) const ( diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index a41b41914a8..903a80488df 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -1438,6 +1438,12 @@ const ( // Default value: true // Allowed filters: DomainName EnableReadVisibilityFromES + // EnableReadVisibilityFromPinot is key for enable read from pinot or db visibility, usually using with AdvancedVisibilityWritingMode for seamless migration from db visibility to advanced visibility + // KeyName: system.enableReadVisibilityFromPinot + // Value type: Bool + // Default value: true + // Allowed filters: DomainName + EnableReadVisibilityFromPinot // EmitShardDiffLog is whether emit the shard diff log // KeyName: history.emitShardDiffLog // Value type: Bool @@ -3675,6 +3681,11 @@ var BoolKeys = map[BoolKey]DynamicBool{ Description: "EnableReadVisibilityFromES is key for enable read from elastic search or db visibility, usually using with AdvancedVisibilityWritingMode for seamless migration from db visibility to advanced visibility", DefaultValue: true, }, + EnableReadVisibilityFromPinot: DynamicBool{ + KeyName: "system.enableReadVisibilityFromPinot", + Description: "EnableReadVisibilityFromPinot is key for enable read from pinot or db visibility, usually using with AdvancedVisibilityWritingMode for seamless migration from db visibility to advanced visibility", + DefaultValue: true, + }, EmitShardDiffLog: DynamicBool{ KeyName: "history.emitShardDiffLog", Description: "EmitShardDiffLog is whether emit the shard diff log", diff --git a/common/log/tag/values.go b/common/log/tag/values.go index a0c4a65de15..93acc4cff81 100644 --- a/common/log/tag/values.go +++ b/common/log/tag/values.go @@ -132,6 +132,7 @@ var ( ComponentCrossClusterTaskFetcher = component("cross-cluster-task-fetcher") ComponentShardScanner = component("shardscanner-scanner") ComponentShardFixer = component("shardscanner-fixer") + ComponentPinotVisibilityManager = component("pinot-visibility-manager") ) // Pre-defined values for TagSysLifecycle diff --git a/common/messaging/kafka/producerImpl.go b/common/messaging/kafka/producerImpl.go index 4515375ee51..8e50db3a7b3 100644 --- a/common/messaging/kafka/producerImpl.go +++ b/common/messaging/kafka/producerImpl.go @@ -111,6 +111,13 @@ func (p *producerImpl) getProducerMessage(message interface{}) (*sarama.Producer Value: sarama.ByteEncoder(message.Value), } return msg, nil + case *indexer.PinotMessage: + msg := &sarama.ProducerMessage{ + Topic: p.topic, + Key: sarama.StringEncoder(message.GetWorkflowID()), + Value: sarama.ByteEncoder(message.GetPayload()), + } + return msg, nil default: return nil, errors.New("unknown producer message type") } diff --git a/common/metrics/defs.go b/common/metrics/defs.go index b87402ff43d..ad123c2da94 100644 --- a/common/metrics/defs.go +++ b/common/metrics/defs.go @@ -709,6 +709,41 @@ const ( // ElasticsearchDeleteUninitializedWorkflowExecutionsScope tracks DeleteUninitializedWorkflowExecution calls made by service to persistence layer ElasticsearchDeleteUninitializedWorkflowExecutionsScope + // PinotRecordWorkflowExecutionStartedScope tracks RecordWorkflowExecutionStarted calls made by service to persistence layer + PinotRecordWorkflowExecutionStartedScope + // PinotRecordWorkflowExecutionClosedScope tracks RecordWorkflowExecutionClosed calls made by service to persistence layer + PinotRecordWorkflowExecutionClosedScope + // PinotRecordWorkflowExecutionUninitializedScope tracks RecordWorkflowExecutionUninitialized calls made by service to persistence layer + PinotRecordWorkflowExecutionUninitializedScope + // PinotUpsertWorkflowExecutionScope tracks UpsertWorkflowExecution calls made by service to persistence layer + PinotUpsertWorkflowExecutionScope + // PinotListOpenWorkflowExecutionsScope tracks ListOpenWorkflowExecutions calls made by service to persistence layer + PinotListOpenWorkflowExecutionsScope + // PinotListClosedWorkflowExecutionsScope tracks ListClosedWorkflowExecutions calls made by service to persistence layer + PinotListClosedWorkflowExecutionsScope + // PinotListOpenWorkflowExecutionsByTypeScope tracks ListOpenWorkflowExecutionsByType calls made by service to persistence layer + PinotListOpenWorkflowExecutionsByTypeScope + // PinotListClosedWorkflowExecutionsByTypeScope tracks ListClosedWorkflowExecutionsByType calls made by service to persistence layer + PinotListClosedWorkflowExecutionsByTypeScope + // PinotListOpenWorkflowExecutionsByWorkflowIDScope tracks ListOpenWorkflowExecutionsByWorkflowID calls made by service to persistence layer + PinotListOpenWorkflowExecutionsByWorkflowIDScope + // PinotListClosedWorkflowExecutionsByWorkflowIDScope tracks ListClosedWorkflowExecutionsByWorkflowID calls made by service to persistence layer + PinotListClosedWorkflowExecutionsByWorkflowIDScope + // PinotListClosedWorkflowExecutionsByStatusScope tracks ListClosedWorkflowExecutionsByStatus calls made by service to persistence layer + PinotListClosedWorkflowExecutionsByStatusScope + // PinotGetClosedWorkflowExecutionScope tracks GetClosedWorkflowExecution calls made by service to persistence layer + PinotGetClosedWorkflowExecutionScope + // PinotListWorkflowExecutionsScope tracks ListWorkflowExecutions calls made by service to persistence layer + PinotListWorkflowExecutionsScope + // PinotScanWorkflowExecutionsScope tracks ScanWorkflowExecutions calls made by service to persistence layer + PinotScanWorkflowExecutionsScope + // PinotCountWorkflowExecutionsScope tracks CountWorkflowExecutions calls made by service to persistence layer + PinotCountWorkflowExecutionsScope + // PinotDeleteWorkflowExecutionsScope tracks DeleteWorkflowExecution calls made by service to persistence layer + PinotDeleteWorkflowExecutionsScope + // PinotDeleteUninitializedWorkflowExecutionsScope tracks DeleteUninitializedWorkflowExecution calls made by service to persistence layer + PinotDeleteUninitializedWorkflowExecutionsScope + // SequentialTaskProcessingScope is used by sequential task processing logic SequentialTaskProcessingScope // ParallelTaskProcessingScope is used by parallel task processing logic @@ -1551,6 +1586,23 @@ var ScopeDefs = map[ServiceIdx]map[int]scopeDefinition{ ElasticsearchCountWorkflowExecutionsScope: {operation: "CountWorkflowExecutions"}, ElasticsearchDeleteWorkflowExecutionsScope: {operation: "DeleteWorkflowExecution"}, ElasticsearchDeleteUninitializedWorkflowExecutionsScope: {operation: "DeleteUninitializedWorkflowExecution"}, + PinotRecordWorkflowExecutionStartedScope: {operation: "RecordWorkflowExecutionStarted"}, + PinotRecordWorkflowExecutionClosedScope: {operation: "RecordWorkflowExecutionClosed"}, + PinotRecordWorkflowExecutionUninitializedScope: {operation: "RecordWorkflowExecutionUninitialized"}, + PinotUpsertWorkflowExecutionScope: {operation: "UpsertWorkflowExecution"}, + PinotListOpenWorkflowExecutionsScope: {operation: "ListOpenWorkflowExecutions"}, + PinotListClosedWorkflowExecutionsScope: {operation: "ListClosedWorkflowExecutions"}, + PinotListOpenWorkflowExecutionsByTypeScope: {operation: "ListOpenWorkflowExecutionsByType"}, + PinotListClosedWorkflowExecutionsByTypeScope: {operation: "ListClosedWorkflowExecutionsByType"}, + PinotListOpenWorkflowExecutionsByWorkflowIDScope: {operation: "ListOpenWorkflowExecutionsByWorkflowID"}, + PinotListClosedWorkflowExecutionsByWorkflowIDScope: {operation: "ListClosedWorkflowExecutionsByWorkflowID"}, + PinotListClosedWorkflowExecutionsByStatusScope: {operation: "ListClosedWorkflowExecutionsByStatus"}, + PinotGetClosedWorkflowExecutionScope: {operation: "GetClosedWorkflowExecution"}, + PinotListWorkflowExecutionsScope: {operation: "ListWorkflowExecutions"}, + PinotScanWorkflowExecutionsScope: {operation: "ScanWorkflowExecutions"}, + PinotCountWorkflowExecutionsScope: {operation: "CountWorkflowExecutions"}, + PinotDeleteWorkflowExecutionsScope: {operation: "DeleteWorkflowExecution"}, + PinotDeleteUninitializedWorkflowExecutionsScope: {operation: "DeleteUninitializedWorkflowExecution"}, SequentialTaskProcessingScope: {operation: "SequentialTaskProcessing"}, ParallelTaskProcessingScope: {operation: "ParallelTaskProcessing"}, TaskSchedulerScope: {operation: "TaskScheduler"}, @@ -1935,6 +1987,17 @@ const ( ElasticsearchErrBadRequestCounterPerDomain ElasticsearchErrBusyCounterPerDomain + PinotRequests + PinotFailures + PinotLatency + PinotErrBadRequestCounter + PinotErrBusyCounter + PinotRequestsPerDomain + PinotFailuresPerDomain + PinotLatencyPerDomain + PinotErrBadRequestCounterPerDomain + PinotErrBusyCounterPerDomain + SequentialTaskSubmitRequest SequentialTaskSubmitRequestTaskQueueExist SequentialTaskSubmitRequestTaskQueueMissing @@ -2522,6 +2585,16 @@ var MetricDefs = map[ServiceIdx]map[int]metricDefinition{ ElasticsearchLatencyPerDomain: {metricName: "elasticsearch_latency_per_domain", metricRollupName: "elasticsearch_latency", metricType: Timer}, ElasticsearchErrBadRequestCounterPerDomain: {metricName: "elasticsearch_errors_bad_request_per_domain", metricRollupName: "elasticsearch_errors_bad_request", metricType: Counter}, ElasticsearchErrBusyCounterPerDomain: {metricName: "elasticsearch_errors_busy_per_domain", metricRollupName: "elasticsearch_errors_busy", metricType: Counter}, + PinotRequests: {metricName: "pinot_requests", metricType: Counter}, + PinotFailures: {metricName: "pinot_errors", metricType: Counter}, + PinotLatency: {metricName: "pinot_latency", metricType: Timer}, + PinotErrBadRequestCounter: {metricName: "pinot_errors_bad_request", metricType: Counter}, + PinotErrBusyCounter: {metricName: "pinot_errors_busy", metricType: Counter}, + PinotRequestsPerDomain: {metricName: "pinot_requests_per_domain", metricRollupName: "pinot_requests", metricType: Counter}, + PinotFailuresPerDomain: {metricName: "pinot_errors_per_domain", metricRollupName: "pinot_errors", metricType: Counter}, + PinotLatencyPerDomain: {metricName: "pinot_latency_per_domain", metricRollupName: "pinot_latency", metricType: Timer}, + PinotErrBadRequestCounterPerDomain: {metricName: "pinot_errors_bad_request_per_domain", metricRollupName: "pinot_errors_bad_request", metricType: Counter}, + PinotErrBusyCounterPerDomain: {metricName: "pinot_errors_busy_per_domain", metricRollupName: "pinot_errors_busy", metricType: Counter}, SequentialTaskSubmitRequest: {metricName: "sequentialtask_submit_request", metricType: Counter}, SequentialTaskSubmitRequestTaskQueueExist: {metricName: "sequentialtask_submit_request_taskqueue_exist", metricType: Counter}, SequentialTaskSubmitRequestTaskQueueMissing: {metricName: "sequentialtask_submit_request_taskqueue_missing", metricType: Counter}, diff --git a/common/persistence/client/bean.go b/common/persistence/client/bean.go index 07a59d50d7c..252925eb75a 100644 --- a/common/persistence/client/bean.go +++ b/common/persistence/client/bean.go @@ -25,6 +25,8 @@ package client import ( "sync" + "github.com/uber/cadence/common/pinot" + "github.com/uber/cadence/common/config" es "github.com/uber/cadence/common/elasticsearch" "github.com/uber/cadence/common/messaging" @@ -85,6 +87,8 @@ type ( MessagingClient messaging.Client ESClient es.GenericClient ESConfig *config.ElasticSearchConfig + PinotConfig *config.PinotVisibilityConfig + PinotClient pinot.GenericClient } ) diff --git a/common/persistence/client/factory.go b/common/persistence/client/factory.go index 64d595e2bc5..fae3d9c1844 100644 --- a/common/persistence/client/factory.go +++ b/common/persistence/client/factory.go @@ -23,6 +23,10 @@ package client import ( "sync" + pnt "github.com/uber/cadence/common/pinot" + + pinotVisibility "github.com/uber/cadence/common/persistence/pinot" + "github.com/uber/cadence/common" "github.com/uber/cadence/common/config" es "github.com/uber/cadence/common/elasticsearch" @@ -265,7 +269,7 @@ func (f *factoryImpl) NewVisibilityManager( // No need to create visibility manager as no read/write needed return nil, nil } - var visibilityFromDB, visibilityFromES p.VisibilityManager + var visibilityFromDB, visibilityFromES, visibilityFromPinot p.VisibilityManager var err error if params.PersistenceConfig.VisibilityStore != "" { visibilityFromDB, err = f.newDBVisibilityManager(resourceConfig) @@ -273,7 +277,31 @@ func (f *factoryImpl) NewVisibilityManager( return nil, err } } - if params.PersistenceConfig.AdvancedVisibilityStore != "" { + if params.PersistenceConfig.AdvancedVisibilityStore == common.PinotVisibilityStoreName { + visibilityProducer, err := params.MessagingClient.NewProducer(common.PinotVisibilityAppName) + if err != nil { + f.logger.Fatal("Creating visibility producer failed", tag.Error(err)) + } + + visibilityFromPinot = newPinotVisibilityManager( + params.PinotClient, resourceConfig, visibilityProducer, params.MetricsClient, f.logger) + + esVisibilityProducer, err := params.MessagingClient.NewProducer(common.VisibilityAppName) + visibilityIndexName := params.ESConfig.Indices[common.VisibilityAppName] + visibilityFromES = newESVisibilityManager( + visibilityIndexName, params.ESClient, resourceConfig, esVisibilityProducer, params.MetricsClient, f.logger, + ) + + return p.NewPinotVisibilityTripleManager( + visibilityFromDB, + visibilityFromPinot, + visibilityFromES, + resourceConfig.EnableReadVisibilityFromPinot, + resourceConfig.EnableReadVisibilityFromES, + resourceConfig.AdvancedVisibilityWritingMode, + f.logger, + ), nil + } else if params.PersistenceConfig.AdvancedVisibilityStore != "" { visibilityIndexName := params.ESConfig.Indices[common.VisibilityAppName] visibilityProducer, err := params.MessagingClient.NewProducer(common.VisibilityAppName) if err != nil { @@ -292,6 +320,33 @@ func (f *factoryImpl) NewVisibilityManager( ), nil } +// NewESVisibilityManager create a visibility manager for ElasticSearch +// In history, it only needs kafka producer for writing data; +// In frontend, it only needs ES client and related config for reading data +func newPinotVisibilityManager( + pinotClient pnt.GenericClient, + visibilityConfig *service.Config, + producer messaging.Producer, + metricsClient metrics.Client, + log log.Logger, +) p.VisibilityManager { + visibilityFromPinotStore := pinotVisibility.NewPinotVisibilityStore(pinotClient, visibilityConfig, producer, log) + visibilityFromPinot := p.NewVisibilityManagerImpl(visibilityFromPinotStore, log) + + // wrap with rate limiter + if visibilityConfig.PersistenceMaxQPS != nil && visibilityConfig.PersistenceMaxQPS() != 0 { + pinotRateLimiter := quotas.NewDynamicRateLimiter(visibilityConfig.PersistenceMaxQPS.AsFloat64()) + visibilityFromPinot = p.NewVisibilityPersistenceRateLimitedClient(visibilityFromPinot, pinotRateLimiter, log) + } + + if metricsClient != nil { + // wrap with metrics + visibilityFromPinot = pinotVisibility.NewPinotVisibilityMetricsClient(visibilityFromPinot, metricsClient, log) + } + + return visibilityFromPinot +} + // NewESVisibilityManager create a visibility manager for ElasticSearch // In history, it only needs kafka producer for writing data; // In frontend, it only needs ES client and related config for reading data diff --git a/common/persistence/pinot/pinotVisibilityMetricClients.go b/common/persistence/pinot/pinotVisibilityMetricClients.go new file mode 100644 index 00000000000..44f15d469d7 --- /dev/null +++ b/common/persistence/pinot/pinotVisibilityMetricClients.go @@ -0,0 +1,385 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package pinotvisibility + +import ( + "context" + + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/metrics" + p "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/types" +) + +type pinotVisibilityMetricsClient struct { + metricClient metrics.Client + persistence p.VisibilityManager + logger log.Logger +} + +var _ p.VisibilityManager = (*pinotVisibilityMetricsClient)(nil) + +// NewPinotVisibilityMetricsClient wrap visibility client with metrics client +func NewPinotVisibilityMetricsClient(persistence p.VisibilityManager, metricClient metrics.Client, logger log.Logger) p.VisibilityManager { + return &pinotVisibilityMetricsClient{ + persistence: persistence, + metricClient: metricClient, + logger: logger, + } +} + +func (p *pinotVisibilityMetricsClient) GetName() string { + return p.persistence.GetName() +} + +func (p *pinotVisibilityMetricsClient) RecordWorkflowExecutionStarted( + ctx context.Context, + request *p.RecordWorkflowExecutionStartedRequest, +) error { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotRecordWorkflowExecutionStartedScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + err := p.persistence.RecordWorkflowExecutionStarted(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotRecordWorkflowExecutionStartedScope, err) + } + + return err +} + +func (p *pinotVisibilityMetricsClient) RecordWorkflowExecutionClosed( + ctx context.Context, + request *p.RecordWorkflowExecutionClosedRequest, +) error { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotRecordWorkflowExecutionClosedScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + err := p.persistence.RecordWorkflowExecutionClosed(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotRecordWorkflowExecutionClosedScope, err) + } + + return err +} + +func (p *pinotVisibilityMetricsClient) RecordWorkflowExecutionUninitialized( + ctx context.Context, + request *p.RecordWorkflowExecutionUninitializedRequest, +) error { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotRecordWorkflowExecutionUninitializedScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + err := p.persistence.RecordWorkflowExecutionUninitialized(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotRecordWorkflowExecutionUninitializedScope, err) + } + + return err +} + +func (p *pinotVisibilityMetricsClient) UpsertWorkflowExecution( + ctx context.Context, + request *p.UpsertWorkflowExecutionRequest, +) error { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotUpsertWorkflowExecutionScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + err := p.persistence.UpsertWorkflowExecution(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotUpsertWorkflowExecutionScope, err) + } + + return err +} + +func (p *pinotVisibilityMetricsClient) ListOpenWorkflowExecutions( + ctx context.Context, + request *p.ListWorkflowExecutionsRequest, +) (*p.ListWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotListOpenWorkflowExecutionsScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.ListOpenWorkflowExecutions(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotListOpenWorkflowExecutionsScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) ListClosedWorkflowExecutions( + ctx context.Context, + request *p.ListWorkflowExecutionsRequest, +) (*p.ListWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotListClosedWorkflowExecutionsScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.ListClosedWorkflowExecutions(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotListClosedWorkflowExecutionsScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) ListOpenWorkflowExecutionsByType( + ctx context.Context, + request *p.ListWorkflowExecutionsByTypeRequest, +) (*p.ListWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotListOpenWorkflowExecutionsByTypeScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.ListOpenWorkflowExecutionsByType(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotListOpenWorkflowExecutionsByTypeScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) ListClosedWorkflowExecutionsByType( + ctx context.Context, + request *p.ListWorkflowExecutionsByTypeRequest, +) (*p.ListWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotListClosedWorkflowExecutionsByTypeScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.ListClosedWorkflowExecutionsByType(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotListClosedWorkflowExecutionsByTypeScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) ListOpenWorkflowExecutionsByWorkflowID( + ctx context.Context, + request *p.ListWorkflowExecutionsByWorkflowIDRequest, +) (*p.ListWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotListOpenWorkflowExecutionsByWorkflowIDScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.ListOpenWorkflowExecutionsByWorkflowID(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotListOpenWorkflowExecutionsByWorkflowIDScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) ListClosedWorkflowExecutionsByWorkflowID( + ctx context.Context, + request *p.ListWorkflowExecutionsByWorkflowIDRequest, +) (*p.ListWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotListClosedWorkflowExecutionsByWorkflowIDScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.ListClosedWorkflowExecutionsByWorkflowID(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotListClosedWorkflowExecutionsByWorkflowIDScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) ListClosedWorkflowExecutionsByStatus( + ctx context.Context, + request *p.ListClosedWorkflowExecutionsByStatusRequest, +) (*p.ListWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotListClosedWorkflowExecutionsByStatusScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.ListClosedWorkflowExecutionsByStatus(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotListClosedWorkflowExecutionsByStatusScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) GetClosedWorkflowExecution( + ctx context.Context, + request *p.GetClosedWorkflowExecutionRequest, +) (*p.GetClosedWorkflowExecutionResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotGetClosedWorkflowExecutionScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.GetClosedWorkflowExecution(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotGetClosedWorkflowExecutionScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) ListWorkflowExecutions( + ctx context.Context, + request *p.ListWorkflowExecutionsByQueryRequest, +) (*p.ListWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotListWorkflowExecutionsScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.ListWorkflowExecutions(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotListWorkflowExecutionsScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) ScanWorkflowExecutions( + ctx context.Context, + request *p.ListWorkflowExecutionsByQueryRequest, +) (*p.ListWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotScanWorkflowExecutionsScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.ScanWorkflowExecutions(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotScanWorkflowExecutionsScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) CountWorkflowExecutions( + ctx context.Context, + request *p.CountWorkflowExecutionsRequest, +) (*p.CountWorkflowExecutionsResponse, error) { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotCountWorkflowExecutionsScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + response, err := p.persistence.CountWorkflowExecutions(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotCountWorkflowExecutionsScope, err) + } + + return response, err +} + +func (p *pinotVisibilityMetricsClient) DeleteWorkflowExecution( + ctx context.Context, + request *p.VisibilityDeleteWorkflowExecutionRequest, +) error { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotDeleteWorkflowExecutionsScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + err := p.persistence.DeleteWorkflowExecution(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotDeleteWorkflowExecutionsScope, err) + } + + return err +} + +func (p *pinotVisibilityMetricsClient) DeleteUninitializedWorkflowExecution( + ctx context.Context, + request *p.VisibilityDeleteWorkflowExecutionRequest, +) error { + + scopeWithDomainTag := p.metricClient.Scope(metrics.PinotDeleteWorkflowExecutionsScope, metrics.DomainTag(request.Domain)) + scopeWithDomainTag.IncCounter(metrics.PinotRequestsPerDomain) + sw := scopeWithDomainTag.StartTimer(metrics.PinotLatencyPerDomain) + defer sw.Stop() + err := p.persistence.DeleteWorkflowExecution(ctx, request) + + if err != nil { + p.updateErrorMetric(scopeWithDomainTag, metrics.PinotDeleteWorkflowExecutionsScope, err) + } + + return err +} + +func (p *pinotVisibilityMetricsClient) updateErrorMetric(scopeWithDomainTag metrics.Scope, scope int, err error) { + + switch err.(type) { + case *types.BadRequestError: + scopeWithDomainTag.IncCounter(metrics.PinotErrBadRequestCounterPerDomain) + scopeWithDomainTag.IncCounter(metrics.PinotFailuresPerDomain) + + case *types.ServiceBusyError: + scopeWithDomainTag.IncCounter(metrics.PinotErrBusyCounterPerDomain) + scopeWithDomainTag.IncCounter(metrics.PinotFailuresPerDomain) + default: + p.logger.Error("Operation failed with internal error.", tag.MetricScope(scope), tag.Error(err)) + scopeWithDomainTag.IncCounter(metrics.PinotFailuresPerDomain) + } +} + +func (p *pinotVisibilityMetricsClient) Close() { + p.persistence.Close() +} diff --git a/common/persistence/pinot/pinotVisibilityStore.go b/common/persistence/pinot/pinotVisibilityStore.go new file mode 100644 index 00000000000..be15ff5e744 --- /dev/null +++ b/common/persistence/pinot/pinotVisibilityStore.go @@ -0,0 +1,1086 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package pinotvisibility + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/uber/cadence/.gen/go/indexer" + + workflow "github.com/uber/cadence/.gen/go/shared" + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/messaging" + p "github.com/uber/cadence/common/persistence" + pnt "github.com/uber/cadence/common/pinot" + "github.com/uber/cadence/common/service" + "github.com/uber/cadence/common/types" + "github.com/uber/cadence/common/types/mapper/thrift" +) + +const ( + pinotPersistenceName = "pinot" + DescendingOrder = "DESC" + AcendingOrder = "ASC" + DomainID = "DomainID" + WorkflowID = "WorkflowID" + RunID = "RunID" + WorkflowType = "WorkflowType" + CloseStatus = "CloseStatus" + HistoryLength = "HistoryLength" + TaskList = "TaskList" + IsCron = "IsCron" + NumClusters = "NumClusters" + ShardID = "ShardID" + Attr = "Attr" + StartTime = "StartTime" + CloseTime = "CloseTime" + UpdateTime = "UpdateTime" + ExecutionTime = "ExecutionTime" + Encoding = "Encoding" + LikeStatement = "%s LIKE '%%%s%%'" + IsDeleted = "IsDeleted" // used for Pinot deletion/rolling upsert only, not visible to user + SecondsSinceEpoch = "SecondsSinceEpoch" // used for Pinot deletion/rolling upsert only, not visible to user + + // used to be micro second + oneMicroSecondInNano = int64(time.Microsecond / time.Nanosecond) +) + +type ( + pinotVisibilityStore struct { + pinotClient pnt.GenericClient + producer messaging.Producer + logger log.Logger + config *service.Config + pinotQueryValidator *pnt.VisibilityQueryValidator + } +) + +var _ p.VisibilityStore = (*pinotVisibilityStore)(nil) + +func NewPinotVisibilityStore( + pinotClient pnt.GenericClient, + config *service.Config, + producer messaging.Producer, + logger log.Logger, +) p.VisibilityStore { + if producer == nil { + // must be bug, check history setup + logger.Fatal("message producer is nil") + } + return &pinotVisibilityStore{ + pinotClient: pinotClient, + producer: producer, + logger: logger.WithTags(tag.ComponentPinotVisibilityManager), + config: config, + pinotQueryValidator: pnt.NewPinotQueryValidator(config.ValidSearchAttributes()), + } +} + +func (v *pinotVisibilityStore) Close() { + // Not needed for pinot, just keep for visibility store interface +} + +func (v *pinotVisibilityStore) GetName() string { + return pinotPersistenceName +} + +func (v *pinotVisibilityStore) RecordWorkflowExecutionStarted( + ctx context.Context, + request *p.InternalRecordWorkflowExecutionStartedRequest, +) error { + + msg, err := createVisibilityMessage( + request.DomainUUID, + request.WorkflowID, + request.RunID, + request.WorkflowTypeName, + request.TaskList, + request.StartTimestamp.UnixMilli(), + request.ExecutionTimestamp.UnixMilli(), + request.TaskID, + request.Memo.Data, + request.Memo.GetEncoding(), + request.IsCron, + request.NumClusters, + -1, // represent invalid close time, means open workflow execution + -1, // represent invalid close status, means open workflow execution + 0, // will be updated when workflow execution updates + request.UpdateTimestamp.UnixMilli(), + int64(request.ShardID), + request.SearchAttributes, + false, + ) + + if err != nil { + return err + } + + return v.producer.Publish(ctx, msg) +} + +func (v *pinotVisibilityStore) RecordWorkflowExecutionClosed(ctx context.Context, request *p.InternalRecordWorkflowExecutionClosedRequest) error { + + msg, err := createVisibilityMessage( + request.DomainUUID, + request.WorkflowID, + request.RunID, + request.WorkflowTypeName, + request.TaskList, + request.StartTimestamp.UnixMilli(), + request.ExecutionTimestamp.UnixMilli(), + request.TaskID, + request.Memo.Data, + request.Memo.GetEncoding(), + request.IsCron, + request.NumClusters, + request.CloseTimestamp.UnixMilli(), + *thrift.FromWorkflowExecutionCloseStatus(&request.Status), + request.HistoryLength, + request.UpdateTimestamp.UnixMilli(), + int64(request.ShardID), + request.SearchAttributes, + false, + ) + + if err != nil { + return err + } + + return v.producer.Publish(ctx, msg) +} + +func (v *pinotVisibilityStore) RecordWorkflowExecutionUninitialized(ctx context.Context, request *p.InternalRecordWorkflowExecutionUninitializedRequest) error { + + msg, err := createVisibilityMessage( + request.DomainUUID, + request.WorkflowID, + request.RunID, + request.WorkflowTypeName, + "", + -1, + -1, + 0, + nil, + "", + false, + 0, + -1, // represent invalid close time, means open workflow execution + -1, // represent invalid close status, means open workflow execution + 0, //will be updated when workflow execution updates + request.UpdateTimestamp.UnixMilli(), + request.ShardID, + nil, + false, + ) + + if err != nil { + return err + } + + return v.producer.Publish(ctx, msg) +} + +func (v *pinotVisibilityStore) UpsertWorkflowExecution(ctx context.Context, request *p.InternalUpsertWorkflowExecutionRequest) error { + + msg, err := createVisibilityMessage( + request.DomainUUID, + request.WorkflowID, + request.RunID, + request.WorkflowTypeName, + request.TaskList, + request.StartTimestamp.UnixMilli(), + request.ExecutionTimestamp.UnixMilli(), + request.TaskID, + request.Memo.Data, + request.Memo.GetEncoding(), + request.IsCron, + request.NumClusters, + -1, // represent invalid close time, means open workflow execution + -1, // represent invalid close status, means open workflow execution + 0, // will not be used + request.UpdateTimestamp.UnixMilli(), + request.ShardID, + request.SearchAttributes, + false, + ) + + if err != nil { + return err + } + + return v.producer.Publish(ctx, msg) +} + +func (v *pinotVisibilityStore) DeleteWorkflowExecution( + ctx context.Context, + request *p.VisibilityDeleteWorkflowExecutionRequest, +) error { + + msg, err := createDeleteVisibilityMessage( + request.DomainID, + request.WorkflowID, + request.RunID, + true, + ) + + if err != nil { + return err + } + + return v.producer.Publish(ctx, msg) +} + +func (v *pinotVisibilityStore) DeleteUninitializedWorkflowExecution( + ctx context.Context, + request *p.VisibilityDeleteWorkflowExecutionRequest, +) error { + // verify if it is uninitialized workflow execution record + // if it is, then call the existing delete method to delete + query := fmt.Sprintf("StartTime = missing and DomainID = %s and RunID = %s", request.DomainID, request.RunID) + queryRequest := &p.CountWorkflowExecutionsRequest{ + Domain: request.Domain, + Query: query, + } + resp, err := v.CountWorkflowExecutions(ctx, queryRequest) + if err != nil { + return err + } + if resp.Count > 0 { + if err = v.DeleteWorkflowExecution(ctx, request); err != nil { + return err + } + } + return nil +} + +func (v *pinotVisibilityStore) ListOpenWorkflowExecutions( + ctx context.Context, + request *p.InternalListWorkflowExecutionsRequest, +) (*p.InternalListWorkflowExecutionsResponse, error) { + isRecordValid := func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool { + return !request.EarliestTime.After(rec.StartTime) && !rec.StartTime.After(request.LatestTime) + } + query := getListWorkflowExecutionsQuery(v.pinotClient.GetTableName(), request, false) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: isRecordValid, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: request, + } + return v.pinotClient.Search(req) +} + +func (v *pinotVisibilityStore) ListClosedWorkflowExecutions( + ctx context.Context, + request *p.InternalListWorkflowExecutionsRequest, +) (*p.InternalListWorkflowExecutionsResponse, error) { + isRecordValid := func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool { + return !request.EarliestTime.After(rec.CloseTime) && !rec.CloseTime.After(request.LatestTime) + } + + query := getListWorkflowExecutionsQuery(v.pinotClient.GetTableName(), request, true) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: isRecordValid, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: request, + } + + return v.pinotClient.Search(req) +} + +func (v *pinotVisibilityStore) ListOpenWorkflowExecutionsByType(ctx context.Context, request *p.InternalListWorkflowExecutionsByTypeRequest) (*p.InternalListWorkflowExecutionsResponse, error) { + isRecordValid := func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool { + return !request.EarliestTime.After(rec.StartTime) && !rec.StartTime.After(request.LatestTime) + } + + query := getListWorkflowExecutionsByTypeQuery(v.pinotClient.GetTableName(), request, false) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: isRecordValid, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: &request.InternalListWorkflowExecutionsRequest, + } + + return v.pinotClient.Search(req) +} + +func (v *pinotVisibilityStore) ListClosedWorkflowExecutionsByType(ctx context.Context, request *p.InternalListWorkflowExecutionsByTypeRequest) (*p.InternalListWorkflowExecutionsResponse, error) { + isRecordValid := func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool { + return !request.EarliestTime.After(rec.CloseTime) && !rec.CloseTime.After(request.LatestTime) + } + + query := getListWorkflowExecutionsByTypeQuery(v.pinotClient.GetTableName(), request, true) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: isRecordValid, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: &request.InternalListWorkflowExecutionsRequest, + } + + return v.pinotClient.Search(req) +} + +func (v *pinotVisibilityStore) ListOpenWorkflowExecutionsByWorkflowID(ctx context.Context, request *p.InternalListWorkflowExecutionsByWorkflowIDRequest) (*p.InternalListWorkflowExecutionsResponse, error) { + isRecordValid := func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool { + return !request.EarliestTime.After(rec.StartTime) && !rec.StartTime.After(request.LatestTime) + } + + query := getListWorkflowExecutionsByWorkflowIDQuery(v.pinotClient.GetTableName(), request, false) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: isRecordValid, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: &request.InternalListWorkflowExecutionsRequest, + } + + return v.pinotClient.Search(req) +} + +func (v *pinotVisibilityStore) ListClosedWorkflowExecutionsByWorkflowID(ctx context.Context, request *p.InternalListWorkflowExecutionsByWorkflowIDRequest) (*p.InternalListWorkflowExecutionsResponse, error) { + isRecordValid := func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool { + return !request.EarliestTime.After(rec.CloseTime) && !rec.CloseTime.After(request.LatestTime) + } + + query := getListWorkflowExecutionsByWorkflowIDQuery(v.pinotClient.GetTableName(), request, true) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: isRecordValid, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: &request.InternalListWorkflowExecutionsRequest, + } + + return v.pinotClient.Search(req) +} + +func (v *pinotVisibilityStore) ListClosedWorkflowExecutionsByStatus(ctx context.Context, request *p.InternalListClosedWorkflowExecutionsByStatusRequest) (*p.InternalListWorkflowExecutionsResponse, error) { + isRecordValid := func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool { + return !request.EarliestTime.After(rec.CloseTime) && !rec.CloseTime.After(request.LatestTime) + } + + query := getListWorkflowExecutionsByStatusQuery(v.pinotClient.GetTableName(), request) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: isRecordValid, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: &request.InternalListWorkflowExecutionsRequest, + } + + return v.pinotClient.Search(req) +} + +func (v *pinotVisibilityStore) GetClosedWorkflowExecution(ctx context.Context, request *p.InternalGetClosedWorkflowExecutionRequest) (*p.InternalGetClosedWorkflowExecutionResponse, error) { + query := getGetClosedWorkflowExecutionQuery(v.pinotClient.GetTableName(), request) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: nil, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: &p.InternalListWorkflowExecutionsRequest{ // create a new request to avoid nil pointer exceptions + DomainUUID: request.DomainUUID, + Domain: request.Domain, + EarliestTime: time.Time{}, + LatestTime: time.Time{}, + PageSize: 1, + NextPageToken: nil, + }, + } + + resp, err := v.pinotClient.Search(req) + + if err != nil { + return nil, &types.InternalServiceError{ + Message: fmt.Sprintf("Pinot GetClosedWorkflowExecution failed, %v", err), + } + } + + return &p.InternalGetClosedWorkflowExecutionResponse{ + Execution: resp.Executions[0], + }, nil +} + +func (v *pinotVisibilityStore) ListWorkflowExecutions(ctx context.Context, request *p.ListWorkflowExecutionsByQueryRequest) (*p.InternalListWorkflowExecutionsResponse, error) { + checkPageSize(request) + + query := v.getListWorkflowExecutionsByQueryQuery(v.pinotClient.GetTableName(), request) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: nil, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: &p.InternalListWorkflowExecutionsRequest{ + DomainUUID: request.DomainUUID, + Domain: request.Domain, + EarliestTime: time.Time{}, + LatestTime: time.Time{}, + NextPageToken: request.NextPageToken, + PageSize: request.PageSize, + }, + } + + return v.pinotClient.Search(req) +} + +func (v *pinotVisibilityStore) ScanWorkflowExecutions(ctx context.Context, request *p.ListWorkflowExecutionsByQueryRequest) (*p.InternalListWorkflowExecutionsResponse, error) { + checkPageSize(request) + + query := v.getListWorkflowExecutionsByQueryQuery(v.pinotClient.GetTableName(), request) + + req := &pnt.SearchRequest{ + Query: query, + IsOpen: true, + Filter: nil, + MaxResultWindow: v.config.ESIndexMaxResultWindow(), + ListRequest: &p.InternalListWorkflowExecutionsRequest{ + DomainUUID: request.DomainUUID, + Domain: request.Domain, + EarliestTime: time.Time{}, + LatestTime: time.Time{}, + NextPageToken: request.NextPageToken, + PageSize: request.PageSize, + }, + } + + return v.pinotClient.Search(req) +} + +func (v *pinotVisibilityStore) CountWorkflowExecutions(ctx context.Context, request *p.CountWorkflowExecutionsRequest) (*p.CountWorkflowExecutionsResponse, error) { + query := v.getCountWorkflowExecutionsQuery(v.pinotClient.GetTableName(), request) + + resp, err := v.pinotClient.CountByQuery(query) + if err != nil { + return nil, &types.InternalServiceError{ + Message: fmt.Sprintf("CountClosedWorkflowExecutions failed, %v", err), + } + } + + return &p.CountWorkflowExecutionsResponse{ + Count: resp, + }, nil +} + +// a new function to create visibility message for deletion +// don't use the other function and provide some nil values because it may cause nil pointer exceptions +func createDeleteVisibilityMessage(domainID string, + wid, + rid string, + isDeleted bool, +) (*indexer.PinotMessage, error) { + m := make(map[string]interface{}) + m[DomainID] = domainID + m[WorkflowID] = wid + m[RunID] = rid + m[IsDeleted] = isDeleted + m[SecondsSinceEpoch] = time.Now().UnixNano() + serializedMsg, err := json.Marshal(m) + if err != nil { + return nil, err + } + + msg := &indexer.PinotMessage{ + WorkflowID: common.StringPtr(wid), + Payload: serializedMsg, + } + return msg, nil +} + +func createVisibilityMessage( + // common parameters + domainID string, + wid, + rid string, + workflowTypeName string, + taskList string, + startTimeUnixNano int64, + executionTimeUnixNano int64, + taskID int64, + memo []byte, + encoding common.EncodingType, + isCron bool, + numClusters int16, + // specific to certain status + closeTimeUnixNano int64, // close execution + closeStatus workflow.WorkflowExecutionCloseStatus, // close execution + historyLength int64, // close execution + updateTimeUnixNano int64, // update execution, + shardID int64, + rawSearchAttributes map[string][]byte, + isDeleted bool, +) (*indexer.PinotMessage, error) { + m := make(map[string]interface{}) + //loop through all input parameters + m[DomainID] = domainID + m[WorkflowID] = wid + m[RunID] = rid + m[WorkflowType] = workflowTypeName + m[TaskList] = taskList + m[StartTime] = startTimeUnixNano + m[ExecutionTime] = executionTimeUnixNano + m[IsCron] = isCron + m[NumClusters] = numClusters + m[CloseTime] = closeTimeUnixNano + m[CloseStatus] = int(closeStatus) + m[HistoryLength] = historyLength + m[UpdateTime] = updateTimeUnixNano + m[ShardID] = shardID + m[IsDeleted] = isDeleted + m[SecondsSinceEpoch] = updateTimeUnixNano // same as update time when record is upserted, could not use updateTime directly since this will be modified by Pinot + + SearchAttributes := make(map[string]interface{}) + var err error + for key, value := range rawSearchAttributes { + value, err = isTimeStruct(value) + if err != nil { + return nil, err + } + + var val interface{} + err = json.Unmarshal(value, &val) + if err != nil { + return nil, err + } + SearchAttributes[key] = val + } + m[Attr] = SearchAttributes + serializedMsg, err := json.Marshal(m) + if err != nil { + return nil, err + } + + msg := &indexer.PinotMessage{ + WorkflowID: common.StringPtr(wid), + Payload: serializedMsg, + } + return msg, nil + +} + +// check if value is time.Time type +// if it is, convert it to unixMilli +// if it isn't time, return the original value +func isTimeStruct(value []byte) ([]byte, error) { + var time time.Time + err := json.Unmarshal(value, &time) + if err == nil { + unixTime := time.UnixMilli() + value, err = json.Marshal(unixTime) + if err != nil { + return nil, err + } + } + return value, nil +} + +/****************************** Request Translator ******************************/ + +type PinotQuery struct { + query string + filters PinotQueryFilter + sorters string + limits string +} + +type PinotQueryFilter struct { + string +} + +func NewPinotQuery(tableName string) PinotQuery { + return PinotQuery{ + query: fmt.Sprintf("SELECT *\nFROM %s\n", tableName), + filters: PinotQueryFilter{}, + sorters: "", + limits: "", + } +} + +func NewPinotCountQuery(tableName string) PinotQuery { + return PinotQuery{ + query: fmt.Sprintf("SELECT COUNT(*)\nFROM %s\n", tableName), + filters: PinotQueryFilter{}, + sorters: "", + limits: "", + } +} + +func (q *PinotQuery) String() string { + return fmt.Sprintf("%s%s%s%s", q.query, q.filters.string, q.sorters, q.limits) +} + +func (q *PinotQuery) concatSorter(sorter string) { + q.sorters += sorter + "\n" +} + +func (q *PinotQuery) addPinotSorter(orderBy string, order string) { + if q.sorters == "" { + q.sorters = "Order BY " + } else { + q.sorters += ", " + } + q.sorters += fmt.Sprintf("%s %s\n", orderBy, order) +} + +func (q *PinotQuery) addLimits(limit int) { + q.limits += fmt.Sprintf("LIMIT %d\n", limit) +} + +func (q *PinotQuery) addOffsetAndLimits(offset int, limit int) { + q.limits += fmt.Sprintf("LIMIT %d, %d\n", offset, limit) +} + +func (f *PinotQueryFilter) checkFirstFilter() { + if f.string == "" { + f.string = "WHERE " + } else { + f.string += "AND " + } +} + +func (f *PinotQueryFilter) addEqual(obj string, val interface{}) { + f.checkFirstFilter() + if _, ok := val.(string); ok { + val = fmt.Sprintf("'%s'", val) + } else { + val = fmt.Sprintf("%v", val) + } + quotedVal := fmt.Sprintf("%s", val) + f.string += fmt.Sprintf("%s = %s\n", obj, quotedVal) +} + +// addQuery adds a complete query into the filter +func (f *PinotQueryFilter) addQuery(query string) { + f.checkFirstFilter() + f.string += fmt.Sprintf("%s\n", query) +} + +// addGte check object is greater than or equals to val +func (f *PinotQueryFilter) addGte(obj string, val int) { + f.checkFirstFilter() + f.string += fmt.Sprintf("%s >= %s\n", obj, fmt.Sprintf("%v", val)) +} + +// addLte check object is less than val +func (f *PinotQueryFilter) addLt(obj string, val interface{}) { + f.checkFirstFilter() + f.string += fmt.Sprintf("%s < %s\n", obj, fmt.Sprintf("%v", val)) +} + +func (f *PinotQueryFilter) addTimeRange(obj string, earliest interface{}, latest interface{}) { + f.checkFirstFilter() + f.string += fmt.Sprintf("%s BETWEEN %v AND %v\n", obj, earliest, latest) +} + +func (f *PinotQueryFilter) addPartialMatch(key string, val string) { + f.checkFirstFilter() + f.string += fmt.Sprintf("%s\n", getPartialFormatString(key, val)) +} + +func getPartialFormatString(key string, val string) string { + return fmt.Sprintf(LikeStatement, key, val) +} + +func (v *pinotVisibilityStore) getCountWorkflowExecutionsQuery(tableName string, request *p.CountWorkflowExecutionsRequest) string { + if request == nil { + return "" + } + + query := NewPinotCountQuery(tableName) + + // need to add Domain ID + query.filters.addEqual(DomainID, request.DomainUUID) + query.filters.addEqual(IsDeleted, false) + + requestQuery := strings.TrimSpace(request.Query) + + // if customized query is empty, directly return + if requestQuery == "" { + return query.String() + } + + requestQuery = filterPrefix(requestQuery) + comparExpr, _ := parseOrderBy(requestQuery) + comparExpr, err := v.pinotQueryValidator.ValidateQuery(comparExpr) + if err != nil { + v.logger.Error(fmt.Sprintf("pinot query validator error: %s", err)) + } + + comparExpr = filterPrefix(comparExpr) + if comparExpr != "" { + query.filters.addQuery(comparExpr) + } + + return query.String() +} + +func (v *pinotVisibilityStore) getListWorkflowExecutionsByQueryQuery(tableName string, request *p.ListWorkflowExecutionsByQueryRequest) string { + if request == nil { + return "" + } + + token, err := pnt.GetNextPageToken(request.NextPageToken) + if err != nil { + panic(fmt.Sprintf("deserialize next page token error: %s", err)) + } + + query := NewPinotQuery(tableName) + + // need to add Domain ID + query.filters.addEqual(DomainID, request.DomainUUID) + query.filters.addEqual(IsDeleted, false) + + requestQuery := strings.TrimSpace(request.Query) + + // if customized query is empty, directly return + if requestQuery == "" { + query.addOffsetAndLimits(token.From, request.PageSize) + return query.String() + } + + requestQuery = filterPrefix(requestQuery) + if common.IsJustOrderByClause(requestQuery) { + query.concatSorter(requestQuery) + query.addOffsetAndLimits(token.From, request.PageSize) + return query.String() + } + + comparExpr, orderBy := parseOrderBy(requestQuery) + comparExpr, err = v.pinotQueryValidator.ValidateQuery(comparExpr) + if err != nil { + v.logger.Error(fmt.Sprintf("pinot query validator error: %s", err)) + } + + comparExpr = filterPrefix(comparExpr) + if comparExpr != "" { + query.filters.addQuery(comparExpr) + } + if orderBy != "" { + query.concatSorter(orderBy) + } + + // MUST HAVE! because pagination wouldn't work without order by clause! + if query.sorters == "" { + query.addPinotSorter(StartTime, "DESC") + } + + query.addOffsetAndLimits(token.From, request.PageSize) + return query.String() +} + +func filterPrefix(query string) string { + prefix := fmt.Sprintf("`%s.", Attr) + postfix := "`" + + query = strings.ReplaceAll(query, prefix, "") + return strings.ReplaceAll(query, postfix, "") +} + +/* +Can have cases: +1. A = B +2. A < B +3. A > B +4. A <= B +5. A >= B +*/ +func splitElement(element string) (string, string, string) { + if element == "" { + return "", "", "" + } + + listLE := strings.Split(element, "<=") + listGE := strings.Split(element, ">=") + listE := strings.Split(element, "=") + listL := strings.Split(element, "<") + listG := strings.Split(element, ">") + + if len(listLE) > 1 { + return strings.TrimSpace(listLE[0]), strings.TrimSpace(listLE[1]), "<=" + } + + if len(listGE) > 1 { + return strings.TrimSpace(listGE[0]), strings.TrimSpace(listGE[1]), ">=" + } + + if len(listE) > 1 { + return strings.TrimSpace(listE[0]), strings.TrimSpace(listE[1]), "=" + } + + if len(listL) > 1 { + return strings.TrimSpace(listL[0]), strings.TrimSpace(listL[1]), "<" + } + + if len(listG) > 1 { + return strings.TrimSpace(listG[0]), strings.TrimSpace(listG[1]), ">" + } + + return "", "", "" +} + +/* +Order by XXX DESC +-> if startWith("Order by") -> return "", element + +CustomizedString = 'cannot be used in order by' +-> if last character is ‘ or " -> return element, "" + +CustomizedInt = 1 (without order by clause) +-> if !contains("Order by") -> return element, "" + +CustomizedString = 'cannot be used in order by' Order by XXX DESC +-> Find the index x of last appearance of "order by" -> return element[0, x], element[x, len] + +CustomizedInt = 1 Order by XXX DESC +-> Find the index x of last appearance of "order by" -> return element[0, x], element[x, len] +*/ +func parseOrderBy(element string) (string, string) { + // case 1: when order by query also passed in + if common.IsJustOrderByClause(element) { + return "", element + } + + // case 2: when last element is a string + if element[len(element)-1] == '\'' || element[len(element)-1] == '"' { + return element, "" + } + + // case 3: when last element doesn't contain "order by" + if !strings.Contains(strings.ToLower(element), "order by") { + return element, "" + } + + // case 4: general case + elementArray := strings.Split(element, " ") + orderByIndex := findLastOrderBy(elementArray) // find the last appearance of "order by" is the answer + if orderByIndex == 0 { + return element, "" + } + return strings.Join(elementArray[:orderByIndex], " "), strings.Join(elementArray[orderByIndex:], " ") +} + +func findLastOrderBy(list []string) int { + for i := len(list) - 2; i >= 0; i-- { + if strings.Contains(list[i], "\"") || strings.Contains(list[i], "'") { + return 0 // means order by is inside a string + } + + if strings.ToLower(list[i]) == "order" && strings.ToLower(list[i+1]) == "by" { + return i + } + } + return 0 +} + +func getListWorkflowExecutionsQuery(tableName string, request *p.InternalListWorkflowExecutionsRequest, isClosed bool) string { + if request == nil { + return "" + } + + token, err := pnt.GetNextPageToken(request.NextPageToken) + if err != nil { + panic(fmt.Sprintf("deserialize next page token error: %s", err)) + } + + from := token.From + pageSize := request.PageSize + + query := NewPinotQuery(tableName) + query.filters.addEqual(DomainID, request.DomainUUID) + query.filters.addEqual(IsDeleted, false) + + earliest := request.EarliestTime.UnixMilli() - oneMicroSecondInNano + latest := request.LatestTime.UnixMilli() + oneMicroSecondInNano + + if isClosed { + query.filters.addTimeRange(CloseTime, earliest, latest) //convert Unix Time to miliseconds + query.filters.addGte(CloseStatus, 0) + } else { + query.filters.addTimeRange(StartTime, earliest, latest) //convert Unix Time to miliseconds + query.filters.addLt(CloseStatus, 0) + query.filters.addEqual(CloseTime, -1) + } + + query.addPinotSorter(StartTime, DescendingOrder) + query.addOffsetAndLimits(from, pageSize) + + return query.String() +} + +func getListWorkflowExecutionsByTypeQuery(tableName string, request *p.InternalListWorkflowExecutionsByTypeRequest, isClosed bool) string { + if request == nil { + return "" + } + + query := NewPinotQuery(tableName) + + query.filters.addEqual(DomainID, request.DomainUUID) + query.filters.addEqual(IsDeleted, false) + query.filters.addEqual(WorkflowType, request.WorkflowTypeName) + earliest := request.EarliestTime.UnixMilli() - oneMicroSecondInNano + latest := request.LatestTime.UnixMilli() + oneMicroSecondInNano + + if isClosed { + query.filters.addTimeRange(CloseTime, earliest, latest) //convert Unix Time to miliseconds + query.filters.addGte(CloseStatus, 0) + } else { + query.filters.addTimeRange(StartTime, earliest, latest) //convert Unix Time to miliseconds + query.filters.addLt(CloseStatus, 0) + query.filters.addEqual(CloseTime, -1) + } + + query.addPinotSorter(StartTime, DescendingOrder) + + token, err := pnt.GetNextPageToken(request.NextPageToken) + if err != nil { + panic(fmt.Sprintf("deserialize next page token error: %s", err)) + } + + from := token.From + pageSize := request.PageSize + query.addOffsetAndLimits(from, pageSize) + + return query.String() +} + +func getListWorkflowExecutionsByWorkflowIDQuery(tableName string, request *p.InternalListWorkflowExecutionsByWorkflowIDRequest, isClosed bool) string { + if request == nil { + return "" + } + + query := NewPinotQuery(tableName) + + query.filters.addEqual(DomainID, request.DomainUUID) + query.filters.addEqual(IsDeleted, false) + query.filters.addEqual(WorkflowID, request.WorkflowID) + earliest := request.EarliestTime.UnixMilli() - oneMicroSecondInNano + latest := request.LatestTime.UnixMilli() + oneMicroSecondInNano + + if isClosed { + query.filters.addTimeRange(CloseTime, earliest, latest) //convert Unix Time to miliseconds + query.filters.addGte(CloseStatus, 0) + } else { + query.filters.addTimeRange(StartTime, earliest, latest) //convert Unix Time to miliseconds + query.filters.addLt(CloseStatus, 0) + query.filters.addEqual(CloseTime, -1) + } + + query.addPinotSorter(StartTime, DescendingOrder) + + token, err := pnt.GetNextPageToken(request.NextPageToken) + if err != nil { + panic(fmt.Sprintf("deserialize next page token error: %s", err)) + } + + from := token.From + pageSize := request.PageSize + query.addOffsetAndLimits(from, pageSize) + + return query.String() +} + +func getListWorkflowExecutionsByStatusQuery(tableName string, request *p.InternalListClosedWorkflowExecutionsByStatusRequest) string { + if request == nil { + return "" + } + + query := NewPinotQuery(tableName) + + query.filters.addEqual(DomainID, request.DomainUUID) + query.filters.addEqual(IsDeleted, false) + + status := "0" + switch request.Status.String() { + case "COMPLETED": + status = "0" + case "FAILED": + status = "1" + case "CANCELED": + status = "2" + case "TERMINATED": + status = "3" + case "CONTINUED_AS_NEW": + status = "4" + case "TIMED_OUT": + status = "5" + } + + query.filters.addEqual(CloseStatus, status) + query.filters.addTimeRange(CloseTime, request.EarliestTime.UnixMilli(), request.LatestTime.UnixMilli()) //convert Unix Time to miliseconds + + query.addPinotSorter(StartTime, DescendingOrder) + + token, err := pnt.GetNextPageToken(request.NextPageToken) + if err != nil { + panic(fmt.Sprintf("deserialize next page token error: %s", err)) + } + + from := token.From + pageSize := request.PageSize + query.addOffsetAndLimits(from, pageSize) + + return query.String() +} + +func getGetClosedWorkflowExecutionQuery(tableName string, request *p.InternalGetClosedWorkflowExecutionRequest) string { + if request == nil { + return "" + } + + query := NewPinotQuery(tableName) + + query.filters.addEqual(DomainID, request.DomainUUID) + query.filters.addEqual(IsDeleted, false) + query.filters.addGte(CloseStatus, 0) + query.filters.addEqual(WorkflowID, request.Execution.GetWorkflowID()) + + rid := request.Execution.GetRunID() + if rid != "" { + query.filters.addEqual(RunID, rid) + } + + return query.String() +} + +func checkPageSize(request *p.ListWorkflowExecutionsByQueryRequest) { + if request.PageSize == 0 { + request.PageSize = 1000 + } +} diff --git a/common/persistence/pinot/pinotVisibilityStore_test.go b/common/persistence/pinot/pinotVisibilityStore_test.go new file mode 100644 index 00000000000..af164bc0b85 --- /dev/null +++ b/common/persistence/pinot/pinotVisibilityStore_test.go @@ -0,0 +1,674 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package pinotvisibility + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/uber/cadence/common/definition" + + pnt "github.com/uber/cadence/common/pinot" + + "github.com/stretchr/testify/assert" + + p "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/types" + + "github.com/uber/cadence/common/log" +) + +var ( + testIndex = "test-index" + testDomain = "test-domain" + testDomainID = "bfd5c907-f899-4baf-a7b2-2ab85e623ebd" + testPageSize = 10 + testEarliestTime = int64(1547596872371000000) + testLatestTime = int64(2547596872371000000) + testWorkflowType = "test-wf-type" + testWorkflowID = "test-wid" + testCloseStatus = int32(1) + testTableName = "test-table-name" + + validSearchAttr = definition.GetDefaultIndexedKeys() + + visibilityStore = pinotVisibilityStore{ + pinotClient: nil, + producer: nil, + logger: log.NewNoop(), + config: nil, + pinotQueryValidator: pnt.NewPinotQueryValidator(validSearchAttr), + } +) + +func TestGetCountWorkflowExecutionsQuery(t *testing.T) { + request := &p.CountWorkflowExecutionsRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + Query: "WorkflowID = 'wfid'", + } + + result := visibilityStore.getCountWorkflowExecutionsQuery(testTableName, request) + expectResult := fmt.Sprintf(`SELECT COUNT(*) +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND WorkflowID = 'wfid' +`, testTableName) + + assert.Equal(t, result, expectResult) + + nilResult := visibilityStore.getCountWorkflowExecutionsQuery(testTableName, nil) + assert.Equal(t, nilResult, "") +} + +func TestGetListWorkflowExecutionQuery(t *testing.T) { + + token := pnt.PinotVisibilityPageToken{ + From: 11, + } + + serializedToken, err := json.Marshal(token) + if err != nil { + panic(fmt.Sprintf("Serialized error in PinotVisibilityStoreTest!!!, %s", err)) + } + + tests := map[string]struct { + input *p.ListWorkflowExecutionsByQueryRequest + expectedOutput string + }{ + "complete request with keyword query only": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "`Attr.CustomKeywordField` = 'keywordCustomized'", + }, + expectedOutput: fmt.Sprintf( + `SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName), + }, + + "complete request from search attribute worker": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "CustomIntField=2 and CustomKeywordField='Update2' order by `Attr.CustomDatetimeField` DESC", + }, + expectedOutput: fmt.Sprintf( + `SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND JSON_MATCH(Attr, '"$.CustomIntField"=''2''') and (JSON_MATCH(Attr, '"$.CustomKeywordField"=''Update2''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''Update2''')) +order by CustomDatetimeField DESC +LIMIT 0, 10 +`, testTableName), + }, + + "complete request with keyword query and other customized query": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "CustomKeywordField = 'keywordCustomized' and CustomStringField = 'String and or order by'", + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'String and or order by*')) +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName), + }, + + "complete request with or query & customized attributes": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "CustomStringField = 'Or' or CustomStringField = 'and' Order by StartTime DESC", + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND ((JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'Or*')) or (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'and*'))) +Order by StartTime DESC +LIMIT 0, 10 +`, testTableName), + }, + + "complex query": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "WorkflowID = 'wid' and ((CustomStringField = 'custom and custom2 or custom3 order by') or CustomIntField between 1 and 10)", + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND WorkflowID = 'wid' and ((JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom and custom2 or custom3 order by*')) or CustomIntField between 1 and 10) +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName), + }, + + "or clause with custom attributes": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "CustomIntField = 1 or CustomIntField = 2", + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND (JSON_MATCH(Attr, '"$.CustomIntField"=''1''') or JSON_MATCH(Attr, '"$.CustomIntField"=''2''')) +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName), + }, + + "complete request with customized query with missing": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "CloseTime = missing anD WorkflowType = 'some-test-workflow'", + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND CloseTime = -1 and WorkflowType = 'some-test-workflow' +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName), + }, + + "complete request with customized query with NextPageToken": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: serializedToken, + Query: "CloseStatus < 0 and CustomKeywordField = 'keywordCustomized' AND CustomIntField<=10 and CustomStringField = 'String field is for text' Order by DomainID Desc", + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND CloseStatus < 0 and (JSON_MATCH(Attr, '"$.CustomKeywordField"=''keywordCustomized''') or JSON_MATCH(Attr, '"$.CustomKeywordField[*]"=''keywordCustomized''')) and JSON_MATCH(Attr, '"$.CustomIntField"=''10''') and (JSON_MATCH(Attr, '"$.CustomStringField" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'String field is for text*')) +Order by DomainID Desc +LIMIT 11, 10 +`, testTableName), + }, + + "complete request with order by query": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "Order by DomainId Desc", + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +Order by DomainId Desc +LIMIT 0, 10 +`, testTableName), + }, + + "complete request with filter query": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "CloseStatus < 0", + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND CloseStatus < 0 +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName), + }, + + "complete request with empty query": { + input: &p.ListWorkflowExecutionsByQueryRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + PageSize: testPageSize, + NextPageToken: nil, + Query: "", + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +LIMIT 0, 10 +`, testTableName), + }, + + "empty request": { + input: &p.ListWorkflowExecutionsByQueryRequest{}, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = '' +AND IsDeleted = false +LIMIT 0, 0 +`, testTableName), + }, + + "nil request": { + input: nil, + expectedOutput: "", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + output := visibilityStore.getListWorkflowExecutionsByQueryQuery(testTableName, test.input) + assert.Equal(t, test.expectedOutput, output) + }) + }) + } +} + +func TestGetListWorkflowExecutionsQuery(t *testing.T) { + request := &p.InternalListWorkflowExecutionsRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + EarliestTime: time.Unix(0, testEarliestTime), + LatestTime: time.Unix(0, testLatestTime), + PageSize: testPageSize, + NextPageToken: nil, + } + + closeResult := getListWorkflowExecutionsQuery(testTableName, request, true) + openResult := getListWorkflowExecutionsQuery(testTableName, request, false) + nilResult := getListWorkflowExecutionsQuery(testTableName, nil, true) + expectCloseResult := fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND CloseTime BETWEEN 1547596871371 AND 2547596873371 +AND CloseStatus >= 0 +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName) + expectOpenResult := fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND StartTime BETWEEN 1547596871371 AND 2547596873371 +AND CloseStatus < 0 +AND CloseTime = -1 +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName) + expectNilResult := "" + + assert.Equal(t, closeResult, expectCloseResult) + assert.Equal(t, openResult, expectOpenResult) + assert.Equal(t, nilResult, expectNilResult) +} + +func TestGetListWorkflowExecutionsByTypeQuery(t *testing.T) { + request := &p.InternalListWorkflowExecutionsByTypeRequest{ + InternalListWorkflowExecutionsRequest: p.InternalListWorkflowExecutionsRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + EarliestTime: time.Unix(0, testEarliestTime), + LatestTime: time.Unix(0, testLatestTime), + PageSize: testPageSize, + NextPageToken: nil, + }, + WorkflowTypeName: testWorkflowType, + } + + closeResult := getListWorkflowExecutionsByTypeQuery(testTableName, request, true) + openResult := getListWorkflowExecutionsByTypeQuery(testTableName, request, false) + nilResult := getListWorkflowExecutionsByTypeQuery(testTableName, nil, true) + expectCloseResult := fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND WorkflowType = 'test-wf-type' +AND CloseTime BETWEEN 1547596871371 AND 2547596873371 +AND CloseStatus >= 0 +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName) + expectOpenResult := fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND WorkflowType = 'test-wf-type' +AND StartTime BETWEEN 1547596871371 AND 2547596873371 +AND CloseStatus < 0 +AND CloseTime = -1 +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName) + expectNilResult := "" + + assert.Equal(t, closeResult, expectCloseResult) + assert.Equal(t, openResult, expectOpenResult) + assert.Equal(t, nilResult, expectNilResult) +} + +func TestGetListWorkflowExecutionsByWorkflowIDQuery(t *testing.T) { + request := &p.InternalListWorkflowExecutionsByWorkflowIDRequest{ + InternalListWorkflowExecutionsRequest: p.InternalListWorkflowExecutionsRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + EarliestTime: time.Unix(0, testEarliestTime), + LatestTime: time.Unix(0, testLatestTime), + PageSize: testPageSize, + NextPageToken: nil, + }, + WorkflowID: testWorkflowID, + } + + closeResult := getListWorkflowExecutionsByWorkflowIDQuery(testTableName, request, true) + openResult := getListWorkflowExecutionsByWorkflowIDQuery(testTableName, request, false) + nilResult := getListWorkflowExecutionsByWorkflowIDQuery(testTableName, nil, true) + expectCloseResult := fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND WorkflowID = 'test-wid' +AND CloseTime BETWEEN 1547596871371 AND 2547596873371 +AND CloseStatus >= 0 +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName) + expectOpenResult := fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND WorkflowID = 'test-wid' +AND StartTime BETWEEN 1547596871371 AND 2547596873371 +AND CloseStatus < 0 +AND CloseTime = -1 +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName) + expectNilResult := "" + + assert.Equal(t, closeResult, expectCloseResult) + assert.Equal(t, openResult, expectOpenResult) + assert.Equal(t, nilResult, expectNilResult) +} + +func TestGetListWorkflowExecutionsByStatusQuery(t *testing.T) { + request := &p.InternalListClosedWorkflowExecutionsByStatusRequest{ + InternalListWorkflowExecutionsRequest: p.InternalListWorkflowExecutionsRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + EarliestTime: time.Unix(0, testEarliestTime), + LatestTime: time.Unix(0, testLatestTime), + PageSize: testPageSize, + NextPageToken: nil, + }, + Status: types.WorkflowExecutionCloseStatus(0), + } + + closeResult := getListWorkflowExecutionsByStatusQuery(testTableName, request) + nilResult := getListWorkflowExecutionsByStatusQuery(testTableName, nil) + expectCloseResult := fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND CloseStatus = '0' +AND CloseTime BETWEEN 1547596872371 AND 2547596872371 +Order BY StartTime DESC +LIMIT 0, 10 +`, testTableName) + expectNilResult := "" + + assert.Equal(t, expectCloseResult, closeResult) + assert.Equal(t, expectNilResult, nilResult) +} + +func TestGetGetClosedWorkflowExecutionQuery(t *testing.T) { + tests := map[string]struct { + input *p.InternalGetClosedWorkflowExecutionRequest + expectedOutput string + }{ + "complete request with empty RunId": { + input: &p.InternalGetClosedWorkflowExecutionRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + Execution: types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: "", + }, + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND CloseStatus >= 0 +AND WorkflowID = 'test-wid' +`, testTableName), + }, + + "complete request with runId": { + input: &p.InternalGetClosedWorkflowExecutionRequest{ + DomainUUID: testDomainID, + Domain: testDomain, + Execution: types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: "runid", + }, + }, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = 'bfd5c907-f899-4baf-a7b2-2ab85e623ebd' +AND IsDeleted = false +AND CloseStatus >= 0 +AND WorkflowID = 'test-wid' +AND RunID = 'runid' +`, testTableName), + }, + + "empty request": { + input: &p.InternalGetClosedWorkflowExecutionRequest{}, + expectedOutput: fmt.Sprintf(`SELECT * +FROM %s +WHERE DomainID = '' +AND IsDeleted = false +AND CloseStatus >= 0 +AND WorkflowID = '' +`, testTableName), + }, + + "nil request": { + input: nil, + expectedOutput: "", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + output := getGetClosedWorkflowExecutionQuery(testTableName, test.input) + assert.Equal(t, test.expectedOutput, output) + }) + }) + } +} + +func TestStringFormatting(t *testing.T) { + key := "CustomizedStringField" + val := "When query; select * from users_secret_table;" + + assert.Equal(t, `CustomizedStringField LIKE '%When query; select * from users_secret_table;%'`, getPartialFormatString(key, val)) +} + +func TestParseLastElement(t *testing.T) { + tests := map[string]struct { + input string + expectedElement string + expectedOrderBy string + }{ + "Case1: only contains order by": { + input: "Order by TestInt DESC", + expectedElement: "", + expectedOrderBy: "Order by TestInt DESC", + }, + "Case2: only contains order by": { + input: "TestString = 'cannot be used in order by'", + expectedElement: "TestString = 'cannot be used in order by'", + expectedOrderBy: "", + }, + "Case3: not contains any order by": { + input: "TestInt = 1", + expectedElement: "TestInt = 1", + expectedOrderBy: "", + }, + "Case4-1: with order by in string & real order by": { + input: "TestString = 'cannot be used in order by' Order by TestInt DESC", + expectedElement: "TestString = 'cannot be used in order by'", + expectedOrderBy: "Order by TestInt DESC", + }, + "Case4-2: with non-string attribute & real order by": { + input: "TestDouble = 1.0 Order by TestInt DESC", + expectedElement: "TestDouble = 1.0", + expectedOrderBy: "Order by TestInt DESC", + }, + "Case5: with random case order by": { + input: "TestString = 'cannot be used in OrDer by' ORdeR by TestInt DESC", + expectedElement: "TestString = 'cannot be used in OrDer by'", + expectedOrderBy: "ORdeR by TestInt DESC", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + element, orderBy := parseOrderBy(test.input) + assert.Equal(t, test.expectedElement, element) + assert.Equal(t, test.expectedOrderBy, orderBy) + }) + }) + } +} + +func TestSplitElement(t *testing.T) { + tests := map[string]struct { + input string + expectedKey string + expectedVal string + expectedOp string + }{ + "Case1-1: A=B": { + input: "CustomizedTestField=Test", + expectedKey: "CustomizedTestField", + expectedVal: "Test", + expectedOp: "=", + }, + "Case1-2: A=\"B\"": { + input: "CustomizedTestField=\"Test\"", + expectedKey: "CustomizedTestField", + expectedVal: "\"Test\"", + expectedOp: "=", + }, + "Case1-3: A='B'": { + input: "CustomizedTestField='Test'", + expectedKey: "CustomizedTestField", + expectedVal: "'Test'", + expectedOp: "=", + }, + "Case2: A<=B": { + input: "CustomizedTestField<=Test", + expectedKey: "CustomizedTestField", + expectedVal: "Test", + expectedOp: "<=", + }, + "Case3: A>=B": { + input: "CustomizedTestField>=Test", + expectedKey: "CustomizedTestField", + expectedVal: "Test", + expectedOp: ">=", + }, + "Case4: A = B": { + input: "CustomizedTestField = Test", + expectedKey: "CustomizedTestField", + expectedVal: "Test", + expectedOp: "=", + }, + "Case5: A <= B": { + input: "CustomizedTestField <= Test", + expectedKey: "CustomizedTestField", + expectedVal: "Test", + expectedOp: "<=", + }, + "Case6: A >= B": { + input: "CustomizedTestField >= Test", + expectedKey: "CustomizedTestField", + expectedVal: "Test", + expectedOp: ">=", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + key, val, op := splitElement(test.input) + assert.Equal(t, test.expectedKey, key) + assert.Equal(t, test.expectedVal, val) + assert.Equal(t, test.expectedOp, op) + }) + }) + } +} diff --git a/common/persistence/pinotResponseComparator.go b/common/persistence/pinotResponseComparator.go new file mode 100644 index 00000000000..6b075858c57 --- /dev/null +++ b/common/persistence/pinotResponseComparator.go @@ -0,0 +1,524 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package persistence + +import ( + "bytes" + "context" + "fmt" + "reflect" + "strings" + + "github.com/uber/cadence/common/log/tag" + + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/types" +) + +func interfaceToMap(in interface{}) (map[string][]byte, error) { + if in == nil || in == "" { + return map[string][]byte{}, nil + } + + v, ok := in.(map[string][]byte) + if !ok { + return map[string][]byte{}, fmt.Errorf(fmt.Sprintf("interface to map error in ES/Pinot comparator: %#v", in)) + } + + return v, nil +} + +func compareSearchAttributes(esSearchAttribute interface{}, pinotSearchAttribute interface{}) error { + esAttr, ok := esSearchAttribute.(*types.SearchAttributes) + if !ok { + return fmt.Errorf("interface is not an ES SearchAttributes! ") + } + + pinotAttr, ok := pinotSearchAttribute.(*types.SearchAttributes) + if !ok { + return fmt.Errorf("interface is not a pinot SearchAttributes! ") + } + + esSearchAttributeList, err := interfaceToMap(esAttr.GetIndexedFields()) + if err != nil { + return err + } + pinotSearchAttributeList, err := interfaceToMap(pinotAttr.GetIndexedFields()) + if err != nil { + return err + } + + for key, esValue := range esSearchAttributeList { // length(esAttribute) <= length(pinotAttribute) + pinotValue := pinotSearchAttributeList[key] + if !bytes.Equal(esValue, pinotValue) { + return fmt.Errorf(fmt.Sprintf("Comparison Failed: response.%s are not equal. ES value = %s, Pinot value = %s", key, esValue, pinotValue)) + } + } + + return nil +} + +func compareExecutions(esInput interface{}, pinotInput interface{}) error { + esExecution, ok := esInput.(*types.WorkflowExecution) + if !ok { + return fmt.Errorf("interface is not an ES WorkflowExecution! ") + } + + pinotExecution, ok := pinotInput.(*types.WorkflowExecution) + if !ok { + return fmt.Errorf("interface is not a pinot WorkflowExecution! ") + } + + if esExecution.GetWorkflowID() != pinotExecution.GetWorkflowID() { + return fmt.Errorf(fmt.Sprintf("Comparison Failed: Execution.WorkflowID are not equal. ES value = %s, Pinot value = %s", esExecution.GetWorkflowID(), pinotExecution.GetWorkflowID())) + } + + if esExecution.GetRunID() != pinotExecution.GetRunID() { + return fmt.Errorf(fmt.Sprintf("Comparison Failed: Execution.RunID are not equal. ES value = %s, Pinot value = %s", esExecution.GetRunID(), pinotExecution.GetRunID())) + } + + return nil +} + +func compareType(esInput interface{}, pinotInput interface{}) error { + esType, ok := esInput.(*types.WorkflowType) + if !ok { + return fmt.Errorf("interface is not an ES WorkflowType! ") + } + + pinotType, ok := pinotInput.(*types.WorkflowType) + if !ok { + return fmt.Errorf("interface is not a pinot WorkflowType! ") + } + + if esType.GetName() != pinotType.GetName() { + return fmt.Errorf(fmt.Sprintf("Comparison Failed: WorkflowTypes are not equal. ES value = %s, Pinot value = %s", esType.GetName(), pinotType.GetName())) + } + + return nil +} + +func compareCloseStatus(esInput interface{}, pinotInput interface{}) error { + esStatus, ok := esInput.(*types.WorkflowExecutionCloseStatus) + if !ok { + return fmt.Errorf("interface is not an ES WorkflowExecutionCloseStatus! ") + } + + pinotStatus, ok := pinotInput.(*types.WorkflowExecutionCloseStatus) + if !ok { + return fmt.Errorf("interface is not a pinot WorkflowExecutionCloseStatus! ") + } + + if esStatus != pinotStatus { + return fmt.Errorf(fmt.Sprintf("Comparison Failed: WorkflowExecutionCloseStatus are not equal. ES value = %s, Pinot value = %s", esStatus, pinotStatus)) + } + + return nil +} + +func compareListWorkflowExecutionInfo( + esExecutionInfo *types.WorkflowExecutionInfo, + pinotExecutionInfo *types.WorkflowExecutionInfo, +) error { + vOfES := reflect.ValueOf(*esExecutionInfo) + typeOfesExecutionInfo := vOfES.Type() + vOfPinot := reflect.ValueOf(*pinotExecutionInfo) + + for i := 0; i < vOfES.NumField(); i++ { + esFieldName := typeOfesExecutionInfo.Field(i).Name + esValue := vOfES.Field(i).Interface() + pinotValue := vOfPinot.Field(i).Interface() + + // if the value in ES is nil, then we don't need to compare + if esValue == nil { + continue + } + + // if the value in ES is not nil but in pinot is nil, then there's an error + if pinotValue == nil { + return fmt.Errorf("Pinot result is nil while ES result is not. ") + } + + switch strings.ToLower(esFieldName) { + case "memo", "autoresetpoints", "partitionconfig": + + case "searchattributes": + err := compareSearchAttributes(esValue, pinotValue) + if err != nil { + return err + } + case "execution", "parentexecution": + err := compareExecutions(esValue, pinotValue) + if err != nil { + return err + } + case "type": + err := compareType(esValue, pinotValue) + if err != nil { + return err + } + case "closestatus": + err := compareCloseStatus(esValue, pinotValue) + if err != nil { + return err + } + default: + if esValue != pinotValue { + return fmt.Errorf(fmt.Sprintf("Comparison Failed: response.%s are not equal. ES value = %s, Pinot value = %s", esFieldName, esValue, pinotValue)) + } + } + } + + return nil +} + +func compareListWorkflowExecutions( + esExecutionInfos []*types.WorkflowExecutionInfo, + pinotExecutionInfos []*types.WorkflowExecutionInfo, +) error { + if esExecutionInfos == nil && pinotExecutionInfos == nil { + return nil + } + if esExecutionInfos == nil || pinotExecutionInfos == nil { + return fmt.Errorf(fmt.Sprintf("Comparison failed. One of the response is nil. ")) + } + if len(esExecutionInfos) != len(pinotExecutionInfos) { + return fmt.Errorf(fmt.Sprintf("Comparison failed. result length doesn't equal. ")) + } + + for i := 0; i < len(esExecutionInfos); i++ { + err := compareListWorkflowExecutionInfo(esExecutionInfos[i], pinotExecutionInfos[i]) + if err != nil { + return err + } + } + + return nil +} + +func comparePinotESListOpenResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *ListWorkflowExecutionsRequest, + logger log.Logger, +) (*ListWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.ListOpenWorkflowExecutions(ctx, request) + pinotResponse, err2 := PinotManager.ListOpenWorkflowExecutions(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("ListOpenWorkflowExecutions in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("ListOpenWorkflowExecutions in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("ListOpenWorkflowExecutions in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + err := compareListWorkflowExecutions(esResponse.Executions, pinotResponse.Executions) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + return esResponse, nil +} + +func comparePinotESListClosedResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *ListWorkflowExecutionsRequest, + logger log.Logger, +) (*ListWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.ListClosedWorkflowExecutions(ctx, request) + pinotResponse, err2 := PinotManager.ListClosedWorkflowExecutions(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("ListClosedWorkflowExecutions in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("ListClosedWorkflowExecutions in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("ListClosedWorkflowExecutions in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + err := compareListWorkflowExecutions(esResponse.Executions, pinotResponse.Executions) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + return esResponse, nil +} + +func comparePinotESListOpenByTypeResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *ListWorkflowExecutionsByTypeRequest, + logger log.Logger, +) (*ListWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.ListOpenWorkflowExecutionsByType(ctx, request) + pinotResponse, err2 := PinotManager.ListOpenWorkflowExecutionsByType(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("ListOpenWorkflowExecutionsByType in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("ListOpenWorkflowExecutionsByType in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("ListOpenWorkflowExecutionsByType in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + err := compareListWorkflowExecutions(esResponse.Executions, pinotResponse.Executions) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + return esResponse, nil +} + +func comparePinotESListClosedByTypeResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *ListWorkflowExecutionsByTypeRequest, + logger log.Logger, +) (*ListWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.ListClosedWorkflowExecutionsByType(ctx, request) + pinotResponse, err2 := PinotManager.ListClosedWorkflowExecutionsByType(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("ListClosedWorkflowExecutionsByType in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("ListClosedWorkflowExecutionsByType in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("ListClosedWorkflowExecutionsByType in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + err := compareListWorkflowExecutions(esResponse.Executions, pinotResponse.Executions) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + return esResponse, nil +} + +func comparePinotESListOpenByWorkflowIDResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *ListWorkflowExecutionsByWorkflowIDRequest, + logger log.Logger, +) (*ListWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.ListOpenWorkflowExecutionsByWorkflowID(ctx, request) + pinotResponse, err2 := PinotManager.ListOpenWorkflowExecutionsByWorkflowID(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("ListOpenWorkflowExecutionsByWorkflowID in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("ListOpenWorkflowExecutionsByWorkflowID in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("ListOpenWorkflowExecutionsByWorkflowID in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + err := compareListWorkflowExecutions(esResponse.Executions, pinotResponse.Executions) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + return esResponse, nil +} + +func comparePinotESListClosedByWorkflowIDResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *ListWorkflowExecutionsByWorkflowIDRequest, + logger log.Logger, +) (*ListWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.ListClosedWorkflowExecutionsByWorkflowID(ctx, request) + pinotResponse, err2 := PinotManager.ListClosedWorkflowExecutionsByWorkflowID(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("ListClosedWorkflowExecutionsByWorkflowID in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("ListClosedWorkflowExecutionsByWorkflowID in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("ListClosedWorkflowExecutionsByWorkflowID in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + err := compareListWorkflowExecutions(esResponse.Executions, pinotResponse.Executions) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + } + return esResponse, nil +} + +func comparePinotESListClosedByStatusResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *ListClosedWorkflowExecutionsByStatusRequest, + logger log.Logger, +) (*ListWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.ListClosedWorkflowExecutionsByStatus(ctx, request) + pinotResponse, err2 := PinotManager.ListClosedWorkflowExecutionsByStatus(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("ListClosedWorkflowExecutionsByStatus in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("ListClosedWorkflowExecutionsByStatus in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("ListClosedWorkflowExecutionsByStatus in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + err := compareListWorkflowExecutions(esResponse.Executions, pinotResponse.Executions) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + return esResponse, nil +} + +func comparePinotESGetClosedByStatusResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *GetClosedWorkflowExecutionRequest, + logger log.Logger, +) (*GetClosedWorkflowExecutionResponse, error) { + esResponse, err1 := ESManager.GetClosedWorkflowExecution(ctx, request) + pinotResponse, err2 := PinotManager.GetClosedWorkflowExecution(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("GetClosedWorkflowExecutions in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("GetClosedWorkflowExecutions in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("GetClosedWorkflowExecutions in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + err := compareListWorkflowExecutionInfo(esResponse.Execution, pinotResponse.Execution) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + return esResponse, nil +} + +func comparePinotESListByQueryResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *ListWorkflowExecutionsByQueryRequest, + logger log.Logger, +) (*ListWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.ListWorkflowExecutions(ctx, request) + pinotResponse, err2 := PinotManager.ListWorkflowExecutions(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("ListOpenWorkflowExecutionsByQuery in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("ListOpenWorkflowExecutionsByQuery in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("ListOpenWorkflowExecutionsByQuery in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + err := compareListWorkflowExecutions(esResponse.Executions, pinotResponse.Executions) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + return esResponse, nil +} + +func comparePinotESScanResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *ListWorkflowExecutionsByQueryRequest, + logger log.Logger, +) (*ListWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.ScanWorkflowExecutions(ctx, request) + pinotResponse, err2 := PinotManager.ScanWorkflowExecutions(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("ScanWorkflowExecutions in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("ScanWorkflowExecutions in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("ScanWorkflowExecutions in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + err := compareListWorkflowExecutions(esResponse.Executions, pinotResponse.Executions) + if err != nil { + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + return esResponse, nil +} + +func comparePinotESCountResponse( + ctx context.Context, + ESManager VisibilityManager, + PinotManager VisibilityManager, + request *CountWorkflowExecutionsRequest, + logger log.Logger, +) (*CountWorkflowExecutionsResponse, error) { + esResponse, err1 := ESManager.CountWorkflowExecutions(ctx, request) + pinotResponse, err2 := PinotManager.CountWorkflowExecutions(ctx, request) + + if err1 != nil && err2 != nil { + return nil, fmt.Errorf("CountOpenWorkflowExecutions in comparator error, no available results") + } else if err1 != nil { + logger.Error(fmt.Sprintf("CountOpenWorkflowExecutions in comparator error, ES: %s", err1)) + return pinotResponse, nil + } else if err2 != nil { + logger.Error(fmt.Sprintf("CountOpenWorkflowExecutions in comparator error, Pinot: %s", err2)) + return esResponse, nil + } + + if esResponse.Count != pinotResponse.Count { + err := fmt.Errorf(fmt.Sprintf("Comparison Failed: counts are not equal. ES value = %v, Pinot value = %v", esResponse.Count, pinotResponse.Count)) + logger.Error("ES/Pinot Response comparison Error! ", tag.Error(err)) + return esResponse, nil + } + + return esResponse, nil +} diff --git a/common/persistence/pinotResponseComparator_test.go b/common/persistence/pinotResponseComparator_test.go new file mode 100644 index 00000000000..a7e5e5df021 --- /dev/null +++ b/common/persistence/pinotResponseComparator_test.go @@ -0,0 +1,541 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package persistence + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/uber/cadence/common/types" +) + +var ( + testIndex = "test-index" + testDomain = "test-domain" + testDomainID = "bfd5c907-f899-4baf-a7b2-2ab85e623ebd" + testPageSize = 10 + testEarliestTime = int64(1547596872371000000) + testLatestTime = int64(2547596872371000000) + testWorkflowType = "test-wf-type" + testWorkflowID = "test-wid" + testCloseStatus = int32(1) + testTableName = "test-table-name" + testRunID = "test-run-id" + testSearchAttributes1 = map[string]interface{}{"TestAttr1": "val1", "TestAttr2": 2, "TestAttr3": false} + testSearchAttributes2 = map[string]interface{}{"TestAttr1": "val2", "TestAttr2": 2, "TestAttr3": false} + testSearchAttributes3 = map[string]interface{}{"TestAttr2": 2, "TestAttr3": false} +) + +func TestInterfaceToMap(t *testing.T) { + tests := map[string]struct { + input interface{} + expectedResult map[string][]byte + expectedError error + }{ + "Case1: nil input case": { + input: nil, + expectedResult: map[string][]byte{}, + expectedError: nil, + }, + "Case2: empty input case": { + input: "", + expectedResult: map[string][]byte{}, + expectedError: nil, + }, + "Case3: normal input case": { + input: transferMap(testSearchAttributes1), + expectedResult: transferMap(testSearchAttributes1), + expectedError: nil, + }, + "Case4: error input case": { + input: 0, + expectedResult: map[string][]byte{}, + expectedError: fmt.Errorf("interface to map error in ES/Pinot comparator: 0"), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + actualResult, actualError := interfaceToMap(test.input) + assert.Equal(t, test.expectedResult, actualResult) + assert.Equal(t, test.expectedError, actualError) + }) + }) + } +} + +func TestCompareSearchAttributes(t *testing.T) { + tests := map[string]struct { + pinotInput interface{} + esInput interface{} + expectedResult error + }{ + "Case1: pass case": { + pinotInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes1)}, + esInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes1)}, + expectedResult: nil, + }, + "Case2: error case": { + pinotInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes1)}, + esInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes2)}, + expectedResult: fmt.Errorf(fmt.Sprintf("Comparison Failed: response.%s are not equal. ES value = \"%s\", Pinot value = \"%s\"", "TestAttr1", "val2", "val1")), + }, + "Case3: pass case with different response": { + pinotInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes1)}, + esInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes3)}, + expectedResult: nil, + }, + "Case4: error case with different response": { + pinotInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes3)}, + esInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes2)}, + expectedResult: fmt.Errorf(fmt.Sprintf("Comparison Failed: response.%s are not equal. ES value = \"%s\", Pinot value = %s", "TestAttr1", "val2", "")), + }, + "Case5: error input case1": { + pinotInput: 0, + esInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes2)}, + expectedResult: fmt.Errorf(fmt.Sprintf("interface is not a pinot SearchAttributes! ")), + }, + "Case6: error input case2": { + pinotInput: &types.SearchAttributes{IndexedFields: transferMap(testSearchAttributes2)}, + esInput: 0, + expectedResult: fmt.Errorf(fmt.Sprintf("interface is not an ES SearchAttributes! ")), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + actualResult := compareSearchAttributes(test.esInput, test.pinotInput) + assert.Equal(t, test.expectedResult, actualResult) + }) + }) + } +} + +func TestCompareExecutions(t *testing.T) { + tests := map[string]struct { + pinotInput interface{} + esInput interface{} + expectedResult error + }{ + "Case1: pass case": { + pinotInput: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + esInput: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, expectedResult: nil, + }, + "Case2: error case": { + pinotInput: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: "testRunID", + }, + esInput: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, expectedResult: fmt.Errorf(fmt.Sprintf("Comparison Failed: Execution.RunID are not equal. ES value = test-run-id, Pinot value = testRunID")), + }, + "Case3: error input case1": { + pinotInput: 0, + esInput: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: "testRunID", + }, + expectedResult: fmt.Errorf(fmt.Sprintf("interface is not a pinot WorkflowExecution! ")), + }, + "Case4: error input case2": { + pinotInput: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: "testRunID", + }, + esInput: 0, + expectedResult: fmt.Errorf(fmt.Sprintf("interface is not an ES WorkflowExecution! ")), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + actualResult := compareExecutions(test.esInput, test.pinotInput) + assert.Equal(t, test.expectedResult, actualResult) + }) + }) + } +} + +func TestCompareType(t *testing.T) { + tests := map[string]struct { + pinotInput interface{} + esInput interface{} + expectedResult error + }{ + "Case1: pass case": { + pinotInput: &types.WorkflowType{Name: testWorkflowType}, + esInput: &types.WorkflowType{Name: testWorkflowType}, + expectedResult: nil, + }, + "Case2: error case": { + pinotInput: &types.WorkflowType{Name: "testWorkflowType"}, + esInput: &types.WorkflowType{Name: testWorkflowType}, + expectedResult: fmt.Errorf("Comparison Failed: WorkflowTypes are not equal. ES value = test-wf-type, Pinot value = testWorkflowType"), + }, + "Case3: error input case1": { + pinotInput: 0, + esInput: &types.WorkflowType{Name: testWorkflowType}, + expectedResult: fmt.Errorf(fmt.Sprintf("interface is not a pinot WorkflowType! ")), + }, + "Case4: error input case2": { + pinotInput: &types.WorkflowType{Name: testWorkflowType}, + esInput: 0, + expectedResult: fmt.Errorf(fmt.Sprintf("interface is not an ES WorkflowType! ")), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + actualResult := compareType(test.esInput, test.pinotInput) + assert.Equal(t, test.expectedResult, actualResult) + }) + }) + } +} + +func TestCompareCloseStatus(t *testing.T) { + testVal1 := types.WorkflowExecutionCloseStatus(0) + testVal2 := types.WorkflowExecutionCloseStatus(1) + + tests := map[string]struct { + pinotInput interface{} + esInput interface{} + expectedResult error + }{ + "Case1: pass case": { + pinotInput: &testVal1, + esInput: &testVal1, + expectedResult: nil, + }, + "Case2: error case": { + pinotInput: &testVal1, + esInput: &testVal2, + expectedResult: fmt.Errorf("Comparison Failed: WorkflowExecutionCloseStatus are not equal. ES value = FAILED, Pinot value = COMPLETED"), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + actualResult := compareCloseStatus(test.esInput, test.pinotInput) + assert.Equal(t, test.expectedResult, actualResult) + }) + }) + } +} + +func TestCompareListWorkflowExecutionInfo(t *testing.T) { + testSearchAttributeMap1 := transferMap(testSearchAttributes1) + testSearchAttributeMap2 := transferMap(testSearchAttributes2) + + tests := map[string]struct { + esInfo *types.WorkflowExecutionInfo + pinotInfo *types.WorkflowExecutionInfo + expectedResult error + }{ + "Case1: pass case": { + esInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: nil, + }, + pinotInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: nil, + }, + expectedResult: nil, + }, + "Case2: pass case with search attributes": { + esInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + pinotInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + expectedResult: nil, + }, + "Case3: error case with wrong type": { + esInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: "testWorkflowType"}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + pinotInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + expectedResult: fmt.Errorf("Comparison Failed: WorkflowTypes are not equal. ES value = testWorkflowType, Pinot value = test-wf-type"), + }, + "Case4: error case with wrong workflowID": { + esInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: "testWorkflowID", + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + pinotInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + expectedResult: fmt.Errorf("Comparison Failed: Execution.WorkflowID are not equal. ES value = testWorkflowID, Pinot value = test-wid"), + }, + "Case5: error case with wrong runID": { + esInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: "testRunID", + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + pinotInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + expectedResult: fmt.Errorf("Comparison Failed: Execution.RunID are not equal. ES value = testRunID, Pinot value = test-run-id"), + }, + "Case6: error case with wrong SearchAttributes": { + esInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + pinotInfo: &types.WorkflowExecutionInfo{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap2}, + }, + expectedResult: fmt.Errorf("Comparison Failed: response.TestAttr1 are not equal. ES value = \"val1\", Pinot value = \"val2\""), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + actualResult := compareListWorkflowExecutionInfo(test.esInfo, test.pinotInfo) + assert.Equal(t, test.expectedResult, actualResult) + }) + }) + } +} + +func TestCompareListWorkflowExecutions(t *testing.T) { + testSearchAttributeMap1 := transferMap(testSearchAttributes1) + + tests := map[string]struct { + esInfo []*types.WorkflowExecutionInfo + pinotInfo []*types.WorkflowExecutionInfo + expectedResult error + }{ + "Case1: pass case": { + esInfo: []*types.WorkflowExecutionInfo{{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + }, + { + Execution: &types.WorkflowExecution{ + WorkflowID: "testWorkflowID", + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }}, + pinotInfo: []*types.WorkflowExecutionInfo{{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + }, + { + Execution: &types.WorkflowExecution{ + WorkflowID: "testWorkflowID", + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }}, + expectedResult: nil, + }, + "Case2: nil case": { + esInfo: nil, + pinotInfo: nil, + expectedResult: nil, + }, + "Case3: one nil case": { + esInfo: []*types.WorkflowExecutionInfo{{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }}, + pinotInfo: nil, + expectedResult: fmt.Errorf("Comparison failed. One of the response is nil. "), + }, + "Case4: length not equal case": { + esInfo: []*types.WorkflowExecutionInfo{{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }, + { + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }}, + pinotInfo: []*types.WorkflowExecutionInfo{{ + Execution: &types.WorkflowExecution{ + WorkflowID: testWorkflowID, + RunID: testRunID, + }, + Type: &types.WorkflowType{Name: testWorkflowType}, + StartTime: &testEarliestTime, + CloseTime: &testLatestTime, + SearchAttributes: &types.SearchAttributes{IndexedFields: testSearchAttributeMap1}, + }}, + expectedResult: fmt.Errorf("Comparison failed. result length doesn't equal. "), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + actualResult := compareListWorkflowExecutions(test.esInfo, test.pinotInfo) + assert.Equal(t, test.expectedResult, actualResult) + }) + }) + } +} + +func transferMap(input map[string]interface{}) map[string][]byte { + res := make(map[string][]byte) + for key := range input { + marshalVal, _ := json.Marshal(input[key]) + res[key] = marshalVal + } + return res +} diff --git a/common/persistence/pinotVisibilityTripleManager.go b/common/persistence/pinotVisibilityTripleManager.go new file mode 100644 index 00000000000..43ad5b0abc0 --- /dev/null +++ b/common/persistence/pinotVisibilityTripleManager.go @@ -0,0 +1,407 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package persistence + +import ( + "context" + "fmt" + + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/dynamicconfig" + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/types" +) + +type ( + pinotVisibilityTripleManager struct { + logger log.Logger + dbVisibilityManager VisibilityManager + pinotVisibilityManager VisibilityManager + esVisibilityManager VisibilityManager + readModeIsFromPinot dynamicconfig.BoolPropertyFnWithDomainFilter + readModeIsFromES dynamicconfig.BoolPropertyFnWithDomainFilter + writeMode dynamicconfig.StringPropertyFn + } +) + +var _ VisibilityManager = (*pinotVisibilityTripleManager)(nil) + +// NewPinotVisibilityTripleManager create a visibility manager that operate on DB or Pinot based on dynamic config. +func NewPinotVisibilityTripleManager( + dbVisibilityManager VisibilityManager, // one of the VisibilityManager can be nil + pinotVisibilityManager VisibilityManager, + esVisibilityManager VisibilityManager, + readModeIsFromPinot dynamicconfig.BoolPropertyFnWithDomainFilter, + readModeIsFromES dynamicconfig.BoolPropertyFnWithDomainFilter, + visWritingMode dynamicconfig.StringPropertyFn, + logger log.Logger, +) VisibilityManager { + if dbVisibilityManager == nil && pinotVisibilityManager == nil && esVisibilityManager == nil { + logger.Fatal("require one of dbVisibilityManager or pinotVisibilityManager or esVisibilityManager") + return nil + } + return &pinotVisibilityTripleManager{ + dbVisibilityManager: dbVisibilityManager, + pinotVisibilityManager: pinotVisibilityManager, + esVisibilityManager: esVisibilityManager, + readModeIsFromPinot: readModeIsFromPinot, + readModeIsFromES: readModeIsFromES, + writeMode: visWritingMode, + logger: logger, + } +} + +func (v *pinotVisibilityTripleManager) Close() { + if v.dbVisibilityManager != nil { + v.dbVisibilityManager.Close() + } + if v.pinotVisibilityManager != nil { + v.pinotVisibilityManager.Close() + } + if v.esVisibilityManager != nil { + v.esVisibilityManager.Close() + } +} + +func (v *pinotVisibilityTripleManager) GetName() string { + if v.pinotVisibilityManager != nil { + return v.pinotVisibilityManager.GetName() + } else if v.esVisibilityManager != nil { + return v.esVisibilityManager.GetName() + } + return v.dbVisibilityManager.GetName() +} + +func (v *pinotVisibilityTripleManager) RecordWorkflowExecutionStarted( + ctx context.Context, + request *RecordWorkflowExecutionStartedRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.RecordWorkflowExecutionStarted(ctx, request) + }, + func() error { + return v.esVisibilityManager.RecordWorkflowExecutionStarted(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.RecordWorkflowExecutionStarted(ctx, request) + }, + ) +} + +func (v *pinotVisibilityTripleManager) RecordWorkflowExecutionClosed( + ctx context.Context, + request *RecordWorkflowExecutionClosedRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.RecordWorkflowExecutionClosed(ctx, request) + }, + func() error { + return v.esVisibilityManager.RecordWorkflowExecutionClosed(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.RecordWorkflowExecutionClosed(ctx, request) + }, + ) +} + +func (v *pinotVisibilityTripleManager) RecordWorkflowExecutionUninitialized( + ctx context.Context, + request *RecordWorkflowExecutionUninitializedRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.RecordWorkflowExecutionUninitialized(ctx, request) + }, + func() error { + return v.esVisibilityManager.RecordWorkflowExecutionUninitialized(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.RecordWorkflowExecutionUninitialized(ctx, request) + }, + ) +} + +func (v *pinotVisibilityTripleManager) DeleteWorkflowExecution( + ctx context.Context, + request *VisibilityDeleteWorkflowExecutionRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.DeleteWorkflowExecution(ctx, request) + }, + func() error { + return v.esVisibilityManager.DeleteWorkflowExecution(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.DeleteWorkflowExecution(ctx, request) + }, + ) +} + +func (v *pinotVisibilityTripleManager) DeleteUninitializedWorkflowExecution( + ctx context.Context, + request *VisibilityDeleteWorkflowExecutionRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.DeleteUninitializedWorkflowExecution(ctx, request) + }, + func() error { + return v.esVisibilityManager.DeleteUninitializedWorkflowExecution(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.DeleteUninitializedWorkflowExecution(ctx, request) + }, + ) +} + +func (v *pinotVisibilityTripleManager) UpsertWorkflowExecution( + ctx context.Context, + request *UpsertWorkflowExecutionRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.UpsertWorkflowExecution(ctx, request) + }, + func() error { + return v.esVisibilityManager.UpsertWorkflowExecution(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.UpsertWorkflowExecution(ctx, request) + }, + ) +} + +func (v *pinotVisibilityTripleManager) chooseVisibilityModeForAdmin() string { + switch { + case v.dbVisibilityManager != nil && v.pinotVisibilityManager != nil: + return common.AdvancedVisibilityWritingModeDual + case v.pinotVisibilityManager != nil: + return common.AdvancedVisibilityWritingModeOn + case v.dbVisibilityManager != nil: + return common.AdvancedVisibilityWritingModeOff + default: + return "INVALID_ADMIN_MODE" + } +} + +func (v *pinotVisibilityTripleManager) chooseVisibilityManagerForWrite(ctx context.Context, dbVisFunc, esVisFunc, pinotVisFunc func() error) error { + var writeMode string + if v.writeMode != nil { + writeMode = v.writeMode() + } else { + key := VisibilityAdminDeletionKey("visibilityAdminDelete") + if value := ctx.Value(key); value != nil && value.(bool) { + writeMode = v.chooseVisibilityModeForAdmin() + } + } + + switch writeMode { + //only perform as triple manager during migration by setting write mode to triple, + //other time perform as a dual visibility manager of pinot and db + case common.AdvancedVisibilityWritingModeOff: + if v.dbVisibilityManager != nil { + return dbVisFunc() + } + v.logger.Warn("basic visibility is not available to write, fall back to advanced visibility") + return pinotVisFunc() + case common.AdvancedVisibilityWritingModeOn: + // this is the way to make it work for migration, will clean up after migration is done + // by default the AdvancedVisibilityWritingMode is set to ON for ES + // if we change this dynamic config before deployment, ES will stop working and block task processing + // we have to change it after deployment. But need to make sure double writes are working, so the only way is changing the behavior of this function + if v.pinotVisibilityManager != nil && v.esVisibilityManager != nil { + if err := esVisFunc(); err != nil { + return err + } + return pinotVisFunc() + } else if v.pinotVisibilityManager != nil { + v.logger.Warn("ES visibility is not available to write, fall back to pinot visibility") + return pinotVisFunc() + } else if v.esVisibilityManager != nil { + v.logger.Warn("Pinot visibility is not available to write, fall back to es visibility") + return esVisFunc() + } else { + v.logger.Warn("advanced visibility is not available to write, fall back to basic visibility") + return dbVisFunc() + } + case common.AdvancedVisibilityWritingModeDual: + if v.pinotVisibilityManager != nil { + if err := pinotVisFunc(); err != nil { + return err + } + if v.dbVisibilityManager != nil { + return dbVisFunc() + } + v.logger.Warn("basic visibility is not available to write") + return nil + } + v.logger.Warn("advanced visibility is not available to write") + return dbVisFunc() + case common.AdvacnedVisibilityWritingModeTriple: + if v.pinotVisibilityManager != nil && v.esVisibilityManager != nil { + if err := pinotVisFunc(); err != nil { + return err + } + if err := esVisFunc(); err != nil { + return err + } + if v.dbVisibilityManager != nil { + return dbVisFunc() + } + v.logger.Warn("basic visibility is not available to write") + return nil + } + v.logger.Warn("advanced visibility is not available to write") + return dbVisFunc() + default: + return &types.InternalServiceError{ + Message: fmt.Sprintf("Unknown visibility writing mode: %s", writeMode), + } + } +} + +func (v *pinotVisibilityTripleManager) ListOpenWorkflowExecutions( + ctx context.Context, + request *ListWorkflowExecutionsRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListOpenWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityTripleManager) ListClosedWorkflowExecutions( + ctx context.Context, + request *ListWorkflowExecutionsRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListClosedWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityTripleManager) ListOpenWorkflowExecutionsByType( + ctx context.Context, + request *ListWorkflowExecutionsByTypeRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListOpenWorkflowExecutionsByType(ctx, request) +} + +func (v *pinotVisibilityTripleManager) ListClosedWorkflowExecutionsByType( + ctx context.Context, + request *ListWorkflowExecutionsByTypeRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListClosedWorkflowExecutionsByType(ctx, request) +} + +func (v *pinotVisibilityTripleManager) ListOpenWorkflowExecutionsByWorkflowID( + ctx context.Context, + request *ListWorkflowExecutionsByWorkflowIDRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListOpenWorkflowExecutionsByWorkflowID(ctx, request) +} + +func (v *pinotVisibilityTripleManager) ListClosedWorkflowExecutionsByWorkflowID( + ctx context.Context, + request *ListWorkflowExecutionsByWorkflowIDRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListClosedWorkflowExecutionsByWorkflowID(ctx, request) +} + +func (v *pinotVisibilityTripleManager) ListClosedWorkflowExecutionsByStatus( + ctx context.Context, + request *ListClosedWorkflowExecutionsByStatusRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListClosedWorkflowExecutionsByStatus(ctx, request) +} + +func (v *pinotVisibilityTripleManager) GetClosedWorkflowExecution( + ctx context.Context, + request *GetClosedWorkflowExecutionRequest, +) (*GetClosedWorkflowExecutionResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.GetClosedWorkflowExecution(ctx, request) +} + +func (v *pinotVisibilityTripleManager) ListWorkflowExecutions( + ctx context.Context, + request *ListWorkflowExecutionsByQueryRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityTripleManager) ScanWorkflowExecutions( + ctx context.Context, + request *ListWorkflowExecutionsByQueryRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ScanWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityTripleManager) CountWorkflowExecutions( + ctx context.Context, + request *CountWorkflowExecutionsRequest, +) (*CountWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.CountWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityTripleManager) chooseVisibilityManagerForRead(domain string) VisibilityManager { + var visibilityMgr VisibilityManager + if v.readModeIsFromES(domain) { + if v.esVisibilityManager != nil { + visibilityMgr = v.esVisibilityManager + } else { + visibilityMgr = v.dbVisibilityManager + v.logger.Warn("domain is configured to read from advanced visibility(ElasticSearch based) but it's not available, fall back to basic visibility", + tag.WorkflowDomainName(domain)) + } + } else if v.readModeIsFromPinot(domain) { + if v.pinotVisibilityManager != nil { + visibilityMgr = v.pinotVisibilityManager + } else { + visibilityMgr = v.dbVisibilityManager + v.logger.Warn("domain is configured to read from advanced visibility(Pinot based) but it's not available, fall back to basic visibility", + tag.WorkflowDomainName(domain)) + } + } else { + if v.dbVisibilityManager != nil { + visibilityMgr = v.dbVisibilityManager + } else { + visibilityMgr = v.pinotVisibilityManager + v.logger.Warn("domain is configured to read from basic visibility but it's not available, fall back to advanced visibility", + tag.WorkflowDomainName(domain)) + } + } + return visibilityMgr +} diff --git a/common/persistence/pinotiVsibilityDualManager.go b/common/persistence/pinotiVsibilityDualManager.go new file mode 100644 index 00000000000..de4c80b8e50 --- /dev/null +++ b/common/persistence/pinotiVsibilityDualManager.go @@ -0,0 +1,338 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package persistence + +import ( + "context" + "fmt" + + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/dynamicconfig" + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/log/tag" + "github.com/uber/cadence/common/types" +) + +type ( + pinotVisibilityDualManager struct { + logger log.Logger + dbVisibilityManager VisibilityManager + pinotVisibilityManager VisibilityManager + readModeIsFromPinot dynamicconfig.BoolPropertyFnWithDomainFilter + writeMode dynamicconfig.StringPropertyFn + } +) + +var _ VisibilityManager = (*pinotVisibilityDualManager)(nil) + +// NewPinotVisibilityDualManager create a visibility manager that operate on DB or Pinot based on dynamic config. +func NewPinotVisibilityDualManager( + dbVisibilityManager VisibilityManager, // one of the VisibilityManager can be nil + pinotVisibilityManager VisibilityManager, // one of the VisibilityManager can be nil + readModeIsFromPinot dynamicconfig.BoolPropertyFnWithDomainFilter, + visWritingMode dynamicconfig.StringPropertyFn, + logger log.Logger, +) VisibilityManager { + if dbVisibilityManager == nil && pinotVisibilityManager == nil { + logger.Fatal("require one of dbVisibilityManager or pinotVisibilityManager") + return nil + } + return &pinotVisibilityDualManager{ + dbVisibilityManager: dbVisibilityManager, + pinotVisibilityManager: pinotVisibilityManager, + readModeIsFromPinot: readModeIsFromPinot, + writeMode: visWritingMode, + logger: logger, + } +} + +func (v *pinotVisibilityDualManager) Close() { + if v.dbVisibilityManager != nil { + v.dbVisibilityManager.Close() + } + if v.pinotVisibilityManager != nil { + v.pinotVisibilityManager.Close() + } +} + +func (v *pinotVisibilityDualManager) GetName() string { + if v.pinotVisibilityManager != nil { + return v.pinotVisibilityManager.GetName() + } + return v.dbVisibilityManager.GetName() +} + +func (v *pinotVisibilityDualManager) RecordWorkflowExecutionStarted( + ctx context.Context, + request *RecordWorkflowExecutionStartedRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.RecordWorkflowExecutionStarted(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.RecordWorkflowExecutionStarted(ctx, request) + }, + ) +} + +func (v *pinotVisibilityDualManager) RecordWorkflowExecutionClosed( + ctx context.Context, + request *RecordWorkflowExecutionClosedRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.RecordWorkflowExecutionClosed(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.RecordWorkflowExecutionClosed(ctx, request) + }, + ) +} + +func (v *pinotVisibilityDualManager) RecordWorkflowExecutionUninitialized( + ctx context.Context, + request *RecordWorkflowExecutionUninitializedRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.RecordWorkflowExecutionUninitialized(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.RecordWorkflowExecutionUninitialized(ctx, request) + }, + ) +} + +func (v *pinotVisibilityDualManager) DeleteWorkflowExecution( + ctx context.Context, + request *VisibilityDeleteWorkflowExecutionRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.DeleteWorkflowExecution(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.DeleteWorkflowExecution(ctx, request) + }, + ) +} + +func (v *pinotVisibilityDualManager) DeleteUninitializedWorkflowExecution( + ctx context.Context, + request *VisibilityDeleteWorkflowExecutionRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.DeleteUninitializedWorkflowExecution(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.DeleteUninitializedWorkflowExecution(ctx, request) + }, + ) +} + +func (v *pinotVisibilityDualManager) UpsertWorkflowExecution( + ctx context.Context, + request *UpsertWorkflowExecutionRequest, +) error { + return v.chooseVisibilityManagerForWrite( + ctx, + func() error { + return v.dbVisibilityManager.UpsertWorkflowExecution(ctx, request) + }, + func() error { + return v.pinotVisibilityManager.UpsertWorkflowExecution(ctx, request) + }, + ) +} + +func (v *pinotVisibilityDualManager) chooseVisibilityModeForAdmin() string { + switch { + case v.dbVisibilityManager != nil && v.pinotVisibilityManager != nil: + return common.AdvancedVisibilityWritingModeDual + case v.pinotVisibilityManager != nil: + return common.AdvancedVisibilityWritingModeOn + case v.dbVisibilityManager != nil: + return common.AdvancedVisibilityWritingModeOff + default: + return "INVALID_ADMIN_MODE" + } +} + +func (v *pinotVisibilityDualManager) chooseVisibilityManagerForWrite(ctx context.Context, dbVisFunc, pinotVisFunc func() error) error { + var writeMode string + if v.writeMode != nil { + writeMode = v.writeMode() + } else { + key := VisibilityAdminDeletionKey("visibilityAdminDelete") + if value := ctx.Value(key); value != nil && value.(bool) { + writeMode = v.chooseVisibilityModeForAdmin() + } + } + + switch writeMode { + case common.AdvancedVisibilityWritingModeOff: + if v.dbVisibilityManager != nil { + return dbVisFunc() + } + v.logger.Warn("basic visibility is not available to write, fall back to advanced visibility") + return pinotVisFunc() + case common.AdvancedVisibilityWritingModeOn: + if v.pinotVisibilityManager != nil { + return pinotVisFunc() + } + v.logger.Warn("advanced visibility is not available to write, fall back to basic visibility") + return dbVisFunc() + case common.AdvancedVisibilityWritingModeDual: + if v.pinotVisibilityManager != nil { + if err := pinotVisFunc(); err != nil { + return err + } + if v.dbVisibilityManager != nil { + return dbVisFunc() + } + v.logger.Warn("basic visibility is not available to write") + return nil + } + v.logger.Warn("advanced visibility is not available to write") + return dbVisFunc() + default: + return &types.InternalServiceError{ + Message: fmt.Sprintf("Unknown visibility writing mode: %s", writeMode), + } + } +} + +func (v *pinotVisibilityDualManager) ListOpenWorkflowExecutions( + ctx context.Context, + request *ListWorkflowExecutionsRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListOpenWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityDualManager) ListClosedWorkflowExecutions( + ctx context.Context, + request *ListWorkflowExecutionsRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListClosedWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityDualManager) ListOpenWorkflowExecutionsByType( + ctx context.Context, + request *ListWorkflowExecutionsByTypeRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListOpenWorkflowExecutionsByType(ctx, request) +} + +func (v *pinotVisibilityDualManager) ListClosedWorkflowExecutionsByType( + ctx context.Context, + request *ListWorkflowExecutionsByTypeRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListClosedWorkflowExecutionsByType(ctx, request) +} + +func (v *pinotVisibilityDualManager) ListOpenWorkflowExecutionsByWorkflowID( + ctx context.Context, + request *ListWorkflowExecutionsByWorkflowIDRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListOpenWorkflowExecutionsByWorkflowID(ctx, request) +} + +func (v *pinotVisibilityDualManager) ListClosedWorkflowExecutionsByWorkflowID( + ctx context.Context, + request *ListWorkflowExecutionsByWorkflowIDRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListClosedWorkflowExecutionsByWorkflowID(ctx, request) +} + +func (v *pinotVisibilityDualManager) ListClosedWorkflowExecutionsByStatus( + ctx context.Context, + request *ListClosedWorkflowExecutionsByStatusRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListClosedWorkflowExecutionsByStatus(ctx, request) +} + +func (v *pinotVisibilityDualManager) GetClosedWorkflowExecution( + ctx context.Context, + request *GetClosedWorkflowExecutionRequest, +) (*GetClosedWorkflowExecutionResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.GetClosedWorkflowExecution(ctx, request) +} + +func (v *pinotVisibilityDualManager) ListWorkflowExecutions( + ctx context.Context, + request *ListWorkflowExecutionsByQueryRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ListWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityDualManager) ScanWorkflowExecutions( + ctx context.Context, + request *ListWorkflowExecutionsByQueryRequest, +) (*ListWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.ScanWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityDualManager) CountWorkflowExecutions( + ctx context.Context, + request *CountWorkflowExecutionsRequest, +) (*CountWorkflowExecutionsResponse, error) { + manager := v.chooseVisibilityManagerForRead(request.Domain) + return manager.CountWorkflowExecutions(ctx, request) +} + +func (v *pinotVisibilityDualManager) chooseVisibilityManagerForRead(domain string) VisibilityManager { + var visibilityMgr VisibilityManager + if v.readModeIsFromPinot(domain) { + if v.pinotVisibilityManager != nil { + visibilityMgr = v.pinotVisibilityManager + } else { + visibilityMgr = v.dbVisibilityManager + v.logger.Warn("domain is configured to read from advanced visibility(Pinot based) but it's not available, fall back to basic visibility", + tag.WorkflowDomainName(domain)) + } + } else { + if v.dbVisibilityManager != nil { + visibilityMgr = v.dbVisibilityManager + } else { + visibilityMgr = v.pinotVisibilityManager + v.logger.Warn("domain is configured to read from basic visibility but it's not available, fall back to advanced visibility", + tag.WorkflowDomainName(domain)) + } + } + return visibilityMgr +} diff --git a/common/pinot/interfaces.go b/common/pinot/interfaces.go new file mode 100644 index 00000000000..3f1b3d3f059 --- /dev/null +++ b/common/pinot/interfaces.go @@ -0,0 +1,55 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +//go:generate mockgen -package $GOPACKAGE -source $GOFILE -destination GenericClient_mock.go -self_package github.com/uber/cadence/common/pinot + +package pinot + +import ( + p "github.com/uber/cadence/common/persistence" +) + +type ( + // GenericClient is a generic interface for all versions of Pinot clients + GenericClient interface { + // Search API is only for supporting various List[Open/Closed]WorkflowExecutions(ByXyz). + // Use SearchByQuery or ScanByQuery for generic purpose searching. + Search(request *SearchRequest) (*SearchResponse, error) + // CountByQuery is for returning the count of workflow executions that match the query + CountByQuery(query string) (int64, error) + GetTableName() string + } + + // IsRecordValidFilter is a function to filter visibility records + IsRecordValidFilter func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool + + // SearchRequest is request for Search + SearchRequest struct { + Query string + IsOpen bool + Filter IsRecordValidFilter + MaxResultWindow int + ListRequest *p.InternalListWorkflowExecutionsRequest + } + + // SearchResponse is a response to Search, SearchByQuery and ScanByQuery + SearchResponse = p.InternalListWorkflowExecutionsResponse +) diff --git a/common/pinot/page_token.go b/common/pinot/page_token.go new file mode 100644 index 00000000000..06eab5060c5 --- /dev/null +++ b/common/pinot/page_token.go @@ -0,0 +1,76 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package pinot + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/uber/cadence/common/types" +) + +type ( + // PinotVisibilityPageToken holds the paging token for Pinot + PinotVisibilityPageToken struct { + From int + } +) + +// DeserializePageToken return the structural token +func DeserializePageToken(data []byte) (*PinotVisibilityPageToken, error) { + var token PinotVisibilityPageToken + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + err := dec.Decode(&token) + if err != nil { + return nil, &types.BadRequestError{ + Message: fmt.Sprintf("unable to deserialize page token. err: %v", err), + } + } + return &token, nil +} + +// SerializePageToken return the token blob +func SerializePageToken(token *PinotVisibilityPageToken) ([]byte, error) { + data, err := json.Marshal(token) + if err != nil { + return nil, &types.BadRequestError{ + Message: fmt.Sprintf("unable to serialize page token. err: %v", err), + } + } + return data, nil +} + +// GetNextPageToken returns the structural token with nil handling +func GetNextPageToken(token []byte) (*PinotVisibilityPageToken, error) { + var result *PinotVisibilityPageToken + var err error + if len(token) > 0 { + result, err = DeserializePageToken(token) + if err != nil { + return nil, err + } + } else { + result = &PinotVisibilityPageToken{} + } + return result, nil +} diff --git a/common/pinot/pinotClient.go b/common/pinot/pinotClient.go new file mode 100644 index 00000000000..3a360d51abb --- /dev/null +++ b/common/pinot/pinotClient.go @@ -0,0 +1,156 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package pinot + +import ( + "encoding/json" + "fmt" + + "github.com/uber/cadence/common/config" + + "github.com/startreedata/pinot-client-go/pinot" + + "github.com/uber/cadence/common/log" + p "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/types" +) + +type PinotClient struct { + client *pinot.Connection + logger log.Logger + tableName string + serviceName string +} + +func NewPinotClient(client *pinot.Connection, logger log.Logger, pinotConfig *config.PinotVisibilityConfig) GenericClient { + return &PinotClient{ + client: client, + logger: logger, + tableName: pinotConfig.Table, + serviceName: pinotConfig.ServiceName, + } +} + +func (c *PinotClient) Search(request *SearchRequest) (*SearchResponse, error) { + resp, err := c.client.ExecuteSQL(c.tableName, request.Query) + + if err != nil { + return nil, &types.InternalServiceError{ + Message: fmt.Sprintf("Pinot Search failed, %v", err), + } + } + + token, err := GetNextPageToken(request.ListRequest.NextPageToken) + + if err != nil { + return nil, &types.InternalServiceError{ + Message: fmt.Sprintf("Get NextPage token failed, %v", err), + } + } + + return c.getInternalListWorkflowExecutionsResponse(resp, request.Filter, token, request.ListRequest.PageSize, request.MaxResultWindow) +} + +func (c *PinotClient) CountByQuery(query string) (int64, error) { + resp, err := c.client.ExecuteSQL(c.tableName, query) + if err != nil { + return 0, &types.InternalServiceError{ + Message: fmt.Sprintf("CountWorkflowExecutions ExecuteSQL failed, %v", err), + } + } + + count, err := resp.ResultTable.Rows[0][0].(json.Number).Int64() + if err == nil { + return count, nil + } + + return -1, &types.InternalServiceError{ + Message: fmt.Sprintf("can't convert result to integer!, query = %s, query result = %v, err = %v", query, resp.ResultTable.Rows[0][0], err), + } +} + +func (c *PinotClient) GetTableName() string { + return c.tableName +} + +// Pinot Response Translator +// We flattened the search attributes into columns in Pinot table +// This function converts the search result back to VisibilityRecord +func (c *PinotClient) getInternalListWorkflowExecutionsResponse( + resp *pinot.BrokerResponse, + isRecordValid func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool, + token *PinotVisibilityPageToken, + pageSize int, + maxResultWindow int, +) (*p.InternalListWorkflowExecutionsResponse, error) { + response := &p.InternalListWorkflowExecutionsResponse{} + if resp == nil || resp.ResultTable == nil || resp.ResultTable.GetRowCount() == 0 { + return response, nil + } + schema := resp.ResultTable.DataSchema // get the schema to map results + columnNames := schema.ColumnNames + actualHits := resp.ResultTable.Rows + numOfActualHits := resp.ResultTable.GetRowCount() + response.Executions = make([]*p.InternalVisibilityWorkflowExecutionInfo, 0) + for i := 0; i < numOfActualHits; i++ { + workflowExecutionInfo := ConvertSearchResultToVisibilityRecord(actualHits[i], columnNames, c.logger) + + if isRecordValid == nil || isRecordValid(workflowExecutionInfo) { + response.Executions = append(response.Executions, workflowExecutionInfo) + } + } + + if numOfActualHits == pageSize { // this means the response is not the last page + var nextPageToken []byte + var err error + + // ES Search API support pagination using From and PageSize, but has limit that From+PageSize cannot exceed a threshold + // In pinot we just skip (previous pages * page limit) items and take the next (number of page limit) items + nextPageToken, err = SerializePageToken(&PinotVisibilityPageToken{From: token.From + numOfActualHits}) + + if err != nil { + return nil, err + } + + response.NextPageToken = make([]byte, len(nextPageToken)) + copy(response.NextPageToken, nextPageToken) + } + return response, nil +} + +func (c *PinotClient) getInternalGetClosedWorkflowExecutionResponse(resp *pinot.BrokerResponse) ( + *p.InternalGetClosedWorkflowExecutionResponse, + error, +) { + if resp == nil { + return nil, nil + } + + response := &p.InternalGetClosedWorkflowExecutionResponse{} + schema := resp.ResultTable.DataSchema // get the schema to map results + columnNames := schema.ColumnNames + actualHits := resp.ResultTable.Rows + response.Execution = ConvertSearchResultToVisibilityRecord(actualHits[0], columnNames, c.logger) + + return response, nil +} diff --git a/common/pinot/pinotClient_test.go b/common/pinot/pinotClient_test.go new file mode 100644 index 00000000000..fe4c1fcfe6f --- /dev/null +++ b/common/pinot/pinotClient_test.go @@ -0,0 +1,305 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package pinot + +import ( + "fmt" + "testing" + "time" + + "github.com/startreedata/pinot-client-go/pinot" + "github.com/stretchr/testify/assert" + + p "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/types" +) + +var ( + testIndex = "test-index" + testDomain = "test-domain" + testDomainID = "bfd5c907-f899-4baf-a7b2-2ab85e623ebd" + testPageSize = 10 + testEarliestTime = int64(1547596872371000000) + testLatestTime = int64(2547596872371000000) + testWorkflowType = "test-wf-type" + testWorkflowID = "test-wid" + testCloseStatus = int32(1) + + client = PinotClient{ + client: nil, + logger: nil, + } +) + +func TestBuildMap(t *testing.T) { + columnName := []string{"WorkflowID", "RunID", "WorkflowType", "DomainID", "StartTime", "ExecutionTime", "CloseTime", "CloseStatus", "HistoryLength", "TaskList", "IsCron", "NumClusters", "UpdateTime", "CustomIntField", "CustomStringField"} + hit := []interface{}{"wfid", "rid", "wftype", "domainid", testEarliestTime, testEarliestTime, testLatestTime, 1, 1, "tsklst", true, 1, testEarliestTime, 1, "some string"} + + tests := map[string]struct { + inputColumnNames []string + inputHit []interface{} + expectedMap map[string]interface{} + }{ + "Case1: with everything": { + inputColumnNames: columnName, + inputHit: hit, + expectedMap: map[string]interface{}{"CloseStatus": 1, "CloseTime": int64(2547596872371000000), "CustomIntField": 1, "CustomStringField": "some string", "DomainID": "domainid", "ExecutionTime": int64(1547596872371000000), "HistoryLength": 1, "IsCron": true, "NumClusters": 1, "RunID": "rid", "StartTime": int64(1547596872371000000), "TaskList": "tsklst", "UpdateTime": int64(1547596872371000000), "WorkflowID": "wfid", "WorkflowType": "wftype"}, + }, + "Case2: nil result": { + inputColumnNames: nil, + inputHit: nil, + expectedMap: map[string]interface{}{}, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + resMap := buildMap(test.inputHit, test.inputColumnNames) + assert.Equal(t, test.expectedMap, resMap) + }) + }) + } +} + +func TestConvertSearchResultToVisibilityRecord(t *testing.T) { + columnName := []string{"WorkflowID", "RunID", "WorkflowType", "DomainID", "StartTime", "ExecutionTime", "CloseTime", "CloseStatus", "HistoryLength", "TaskList", "IsCron", "NumClusters", "UpdateTime", "Attr"} + hit := []interface{}{"wfid", "rid", "wftype", "domainid", testEarliestTime, testEarliestTime, testLatestTime, 1, 1, "tsklst", true, 1, testEarliestTime, "{}"} + closeStatus := types.WorkflowExecutionCloseStatusFailed + + columnNameWithAttr := []string{"WorkflowID", "RunID", "WorkflowType", "DomainID", "StartTime", "ExecutionTime", "CloseTime", "CloseStatus", "HistoryLength", "TaskList", "IsCron", "NumClusters", "UpdateTime", "Attr"} + hitWithAttr := []interface{}{"wfid", "rid", "wftype", "domainid", testEarliestTime, testEarliestTime, testLatestTime, 1, 1, "tsklst", true, 1, testEarliestTime, `{"CustomStringField": "customA and customB or customC", "CustomDoubleField": 3.14}`} + + tests := map[string]struct { + inputColumnNames []string + inputHit []interface{} + expectedVisibilityRecord *p.InternalVisibilityWorkflowExecutionInfo + }{ + "Case1: with everything except for an empty Attr": { + inputColumnNames: columnName, + inputHit: hit, + expectedVisibilityRecord: &p.InternalVisibilityWorkflowExecutionInfo{ + DomainID: "domainid", + WorkflowType: "wftype", + WorkflowID: "wfid", + RunID: "rid", + TypeName: "wftype", + StartTime: time.UnixMilli(testEarliestTime), + ExecutionTime: time.UnixMilli(testEarliestTime), + CloseTime: time.UnixMilli(testLatestTime), + Status: &closeStatus, + HistoryLength: 1, + Memo: nil, + TaskList: "tsklst", + IsCron: true, + NumClusters: 1, + UpdateTime: time.UnixMilli(testEarliestTime), + SearchAttributes: map[string]interface{}{}, + ShardID: 0, + }, + }, + "Case2: with everything": { + inputColumnNames: columnNameWithAttr, + inputHit: hitWithAttr, + expectedVisibilityRecord: &p.InternalVisibilityWorkflowExecutionInfo{ + DomainID: "domainid", + WorkflowType: "wftype", + WorkflowID: "wfid", + RunID: "rid", + TypeName: "wftype", + StartTime: time.UnixMilli(testEarliestTime), + ExecutionTime: time.UnixMilli(testEarliestTime), + CloseTime: time.UnixMilli(testLatestTime), + Status: &closeStatus, + HistoryLength: 1, + Memo: nil, + TaskList: "tsklst", + IsCron: true, + NumClusters: 1, + UpdateTime: time.UnixMilli(testEarliestTime), + SearchAttributes: map[string]interface{}{"CustomStringField": "customA and customB or customC", "CustomDoubleField": 3.14}, + ShardID: 0, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.NotPanics(t, func() { + visibilityRecord := ConvertSearchResultToVisibilityRecord(test.inputHit, test.inputColumnNames, nil) + assert.Equal(t, test.expectedVisibilityRecord, visibilityRecord) + }) + }) + } +} + +func TestGetInternalListWorkflowExecutionsResponse(t *testing.T) { + columnName := []string{"WorkflowID", "RunID", "WorkflowType", "DomainID", "StartTime", "ExecutionTime", "CloseTime", "CloseStatus", "HistoryLength", "Encoding", "TaskList", "IsCron", "NumClusters", "UpdateTime", "Attr"} + hit1 := []interface{}{"wfid1", "rid1", "wftype1", "domainid1", testEarliestTime, testEarliestTime, testLatestTime, 1, 1, "encode1", "tsklst1", true, 1, testEarliestTime, "null"} + hit2 := []interface{}{"wfid2", "rid2", "wftype2", "domainid2", testEarliestTime, testEarliestTime, testLatestTime, 1, 1, "encode2", "tsklst2", false, 1, testEarliestTime, "null"} + hit3 := []interface{}{"wfid3", "rid3", "wftype3", "domainid3", testEarliestTime, testEarliestTime, testLatestTime, 1, 1, "encode3", "tsklst3", false, 1, testEarliestTime, "null"} + hit4 := []interface{}{"wfid4", "rid4", "wftype4", "domainid4", testEarliestTime, testEarliestTime, testLatestTime, 1, 1, "encode4", "tsklst4", false, 1, testEarliestTime, "null"} + hit5 := []interface{}{"wfid5", "rid5", "wftype5", "domainid5", testEarliestTime, testEarliestTime, testLatestTime, 1, 1, "encode5", "tsklst5", false, 1, testEarliestTime, "null"} + + brokerResponse := &pinot.BrokerResponse{ + AggregationResults: nil, + SelectionResults: nil, + ResultTable: &pinot.ResultTable{ + DataSchema: pinot.RespSchema{ + ColumnDataTypes: nil, + ColumnNames: columnName, + }, + Rows: [][]interface{}{ + hit1, + hit2, + hit3, + hit4, + hit5, + }, + }, + Exceptions: nil, + TraceInfo: nil, + NumServersQueried: 1, + NumServersResponded: 1, + NumSegmentsQueried: 1, + NumSegmentsProcessed: 1, + NumSegmentsMatched: 1, + NumConsumingSegmentsQueried: 1, + NumDocsScanned: 10, + NumEntriesScannedInFilter: 1, + NumEntriesScannedPostFilter: 1, + NumGroupsLimitReached: false, + TotalDocs: 1, + TimeUsedMs: 1, + MinConsumingFreshnessTimeMs: 1, + } + + token := &PinotVisibilityPageToken{ + From: 0, + } + + // Cannot use a table test, because they are not checking the same fields + result, err := client.getInternalListWorkflowExecutionsResponse(brokerResponse, nil, token, 5, 33) + + assert.Equal(t, "wfid1", result.Executions[0].WorkflowID) + assert.Equal(t, "rid1", result.Executions[0].RunID) + assert.Equal(t, "wftype1", result.Executions[0].WorkflowType) + assert.Equal(t, "domainid1", result.Executions[0].DomainID) + assert.Equal(t, time.UnixMilli(testEarliestTime), result.Executions[0].StartTime) + assert.Equal(t, time.UnixMilli(testEarliestTime), result.Executions[0].ExecutionTime) + assert.Equal(t, time.UnixMilli(testLatestTime), result.Executions[0].CloseTime) + assert.Equal(t, types.WorkflowExecutionCloseStatus(1), *result.Executions[0].Status) + assert.Equal(t, int64(1), result.Executions[0].HistoryLength) + assert.Equal(t, "tsklst1", result.Executions[0].TaskList) + assert.Equal(t, true, result.Executions[0].IsCron) + assert.Equal(t, int16(1), result.Executions[0].NumClusters) + assert.Equal(t, time.UnixMilli(testEarliestTime), result.Executions[0].UpdateTime) + + assert.Equal(t, "wfid2", result.Executions[1].WorkflowID) + assert.Equal(t, "rid2", result.Executions[1].RunID) + assert.Equal(t, "wftype2", result.Executions[1].WorkflowType) + assert.Equal(t, "domainid2", result.Executions[1].DomainID) + assert.Equal(t, time.UnixMilli(testEarliestTime), result.Executions[1].StartTime) + assert.Equal(t, time.UnixMilli(testEarliestTime), result.Executions[1].ExecutionTime) + assert.Equal(t, time.UnixMilli(testLatestTime), result.Executions[1].CloseTime) + assert.Equal(t, types.WorkflowExecutionCloseStatus(1), *result.Executions[1].Status) + assert.Equal(t, int64(1), result.Executions[1].HistoryLength) + assert.Equal(t, "tsklst2", result.Executions[1].TaskList) + assert.Equal(t, false, result.Executions[1].IsCron) + assert.Equal(t, int16(1), result.Executions[1].NumClusters) + assert.Equal(t, time.UnixMilli(testEarliestTime), result.Executions[1].UpdateTime) + + assert.Nil(t, err) + + responseToken := result.NextPageToken + unmarshalResponseToken, err := GetNextPageToken(responseToken) + if err != nil { + panic(fmt.Sprintf("Unmarshal error in PinotClient test %s", err)) + } + assert.Equal(t, 5, unmarshalResponseToken.From) + + // check if record is not valid + isRecordValid := func(rec *p.InternalVisibilityWorkflowExecutionInfo) bool { + return false + } + emptyResult, err := client.getInternalListWorkflowExecutionsResponse(brokerResponse, isRecordValid, nil, 10, 33) + assert.Equal(t, 0, len(emptyResult.Executions)) + assert.Nil(t, err) + + // check nil input + nilResult, err := client.getInternalListWorkflowExecutionsResponse(nil, isRecordValid, nil, 10, 33) + assert.Equal(t, &p.InternalListWorkflowExecutionsResponse{}, nilResult) + assert.Nil(t, err) +} + +func TestGetInternalGetClosedWorkflowExecutionResponse(t *testing.T) { + columnName := []string{"WorkflowID", "RunID", "WorkflowType", "DomainID", "StartTime", "ExecutionTime", "CloseTime", "CloseStatus", "HistoryLength", "Encoding", "TaskList", "IsCron", "NumClusters", "UpdateTime", "Attr"} + hit1 := []interface{}{"wfid1", "rid1", "wftype1", "domainid1", testEarliestTime, testEarliestTime, testLatestTime, 1, 1, "encode1", "tsklst1", true, 1, testEarliestTime, "null"} + + brokerResponse := &pinot.BrokerResponse{ + AggregationResults: nil, + SelectionResults: nil, + ResultTable: &pinot.ResultTable{ + DataSchema: pinot.RespSchema{ + ColumnDataTypes: nil, + ColumnNames: columnName, + }, + Rows: [][]interface{}{ + hit1, + }, + }, + Exceptions: nil, + TraceInfo: nil, + NumServersQueried: 1, + NumServersResponded: 1, + NumSegmentsQueried: 1, + NumSegmentsProcessed: 1, + NumSegmentsMatched: 1, + NumConsumingSegmentsQueried: 1, + NumDocsScanned: 1, + NumEntriesScannedInFilter: 1, + NumEntriesScannedPostFilter: 1, + NumGroupsLimitReached: false, + TotalDocs: 1, + TimeUsedMs: 1, + MinConsumingFreshnessTimeMs: 1, + } + + result, err := client.getInternalGetClosedWorkflowExecutionResponse(brokerResponse) + + assert.Equal(t, "wfid1", result.Execution.WorkflowID) + assert.Equal(t, "rid1", result.Execution.RunID) + assert.Equal(t, "wftype1", result.Execution.WorkflowType) + assert.Equal(t, "domainid1", result.Execution.DomainID) + assert.Equal(t, time.UnixMilli(testEarliestTime), result.Execution.StartTime) + assert.Equal(t, time.UnixMilli(testEarliestTime), result.Execution.ExecutionTime) + assert.Equal(t, time.UnixMilli(testLatestTime), result.Execution.CloseTime) + assert.Equal(t, types.WorkflowExecutionCloseStatus(1), *result.Execution.Status) + assert.Equal(t, int64(1), result.Execution.HistoryLength) + assert.Equal(t, "tsklst1", result.Execution.TaskList) + assert.Equal(t, true, result.Execution.IsCron) + assert.Equal(t, int16(1), result.Execution.NumClusters) + assert.Equal(t, time.UnixMilli(testEarliestTime), result.Execution.UpdateTime) + + assert.Nil(t, err) +} diff --git a/common/pinot/pinotQueryValidator.go b/common/pinot/pinotQueryValidator.go new file mode 100644 index 00000000000..aa5bf9d5730 --- /dev/null +++ b/common/pinot/pinotQueryValidator.go @@ -0,0 +1,271 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package pinot + +import ( + "errors" + "fmt" + "strings" + + "github.com/uber/cadence/common/log" + + "github.com/xwb1989/sqlparser" + + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/definition" + "github.com/uber/cadence/common/types" +) + +// VisibilityQueryValidator for sql query validation +type VisibilityQueryValidator struct { + validSearchAttributes map[string]interface{} +} + +// NewPinotQueryValidator create VisibilityQueryValidator +func NewPinotQueryValidator(validSearchAttributes map[string]interface{}) *VisibilityQueryValidator { + return &VisibilityQueryValidator{ + validSearchAttributes: validSearchAttributes, + } +} + +// ValidateQuery validates that search attributes in the query and returns modified query. +func (qv *VisibilityQueryValidator) ValidateQuery(whereClause string) (string, error) { + if len(whereClause) != 0 { + // Build a placeholder query that allows us to easily parse the contents of the where clause. + // IMPORTANT: This query is never executed, it is just used to parse and validate whereClause + var placeholderQuery string + whereClause := strings.TrimSpace(whereClause) + if common.IsJustOrderByClause(whereClause) { // just order by + placeholderQuery = fmt.Sprintf("SELECT * FROM dummy %s", whereClause) + } else { + placeholderQuery = fmt.Sprintf("SELECT * FROM dummy WHERE %s", whereClause) + } + + stmt, err := sqlparser.Parse(placeholderQuery) + if err != nil { + return "", &types.BadRequestError{Message: "Invalid query."} + } + + sel, ok := stmt.(*sqlparser.Select) + if !ok { + return "", &types.BadRequestError{Message: "Invalid select query."} + } + buf := sqlparser.NewTrackedBuffer(nil) + res := "" + // validate where expr + if sel.Where != nil { + res, err = qv.validateWhereExpr(sel.Where.Expr) + if err != nil { + return "", &types.BadRequestError{Message: err.Error()} + } + } + + sel.OrderBy.Format(buf) + res += buf.String() + return res, nil + } + return whereClause, nil +} + +func (qv *VisibilityQueryValidator) validateWhereExpr(expr sqlparser.Expr) (string, error) { + if expr == nil { + return "", nil + } + buf := sqlparser.NewTrackedBuffer(nil) + + switch expr := expr.(type) { + case *sqlparser.AndExpr, *sqlparser.OrExpr: + return qv.validateAndOrExpr(expr) + case *sqlparser.ComparisonExpr: + return qv.validateComparisonExpr(expr) + case *sqlparser.RangeCond: + expr.Format(buf) + return buf.String(), nil + //return qv.validateRangeExpr(expr) + case *sqlparser.ParenExpr: + return qv.validateWhereExpr(expr.Expr) + default: + return "", errors.New("invalid where clause") + } +} + +func (qv *VisibilityQueryValidator) validateAndOrExpr(expr sqlparser.Expr) (string, error) { + var leftExpr sqlparser.Expr + var rightExpr sqlparser.Expr + isAnd := false + + switch expr := expr.(type) { + case *sqlparser.AndExpr: + leftExpr = expr.Left + rightExpr = expr.Right + isAnd = true + case *sqlparser.OrExpr: + leftExpr = expr.Left + rightExpr = expr.Right + } + + leftRes, err := qv.validateWhereExpr(leftExpr) + if err != nil { + return "", err + } + + rightRes, err := qv.validateWhereExpr(rightExpr) + if err != nil { + return "", err + } + + if isAnd { + return fmt.Sprintf("%s and %s", leftRes, rightRes), nil + } + + return fmt.Sprintf("(%s or %s)", leftRes, rightRes), nil +} + +func (qv *VisibilityQueryValidator) validateComparisonExpr(expr sqlparser.Expr) (string, error) { + comparisonExpr := expr.(*sqlparser.ComparisonExpr) + + colName, ok := comparisonExpr.Left.(*sqlparser.ColName) + if !ok { + return "", errors.New("invalid comparison expression, left") + } + + colNameStr := colName.Name.String() + + if !qv.isValidSearchAttributes(colNameStr) { + return "", fmt.Errorf("invalid search attribute %q", colNameStr) + } + + // Case1: it is system key + // this means that we don't need to change the structure of the query, + // just need to check if a value == "missing" + if definition.IsSystemIndexedKey(colNameStr) { + return qv.processSystemKey(expr) + } + // Case2: when a value is not system key + // This means, the value is from Attr so that we need to change the query to be a Json index format + return qv.processCustomKey(expr) +} + +// isValidSearchAttributes return true if key is registered +func (qv *VisibilityQueryValidator) isValidSearchAttributes(key string) bool { + validAttr := qv.validSearchAttributes + _, isValidKey := validAttr[key] + return isValidKey +} + +func (qv *VisibilityQueryValidator) processSystemKey(expr sqlparser.Expr) (string, error) { + comparisonExpr := expr.(*sqlparser.ComparisonExpr) + buf := sqlparser.NewTrackedBuffer(nil) + + colName, ok := comparisonExpr.Left.(*sqlparser.ColName) + if !ok { + return "", errors.New("invalid comparison expression, left") + } + colNameStr := colName.Name.String() + + if comparisonExpr.Operator != sqlparser.EqualStr { + expr.Format(buf) + return buf.String(), nil + } + // need to deal with missing value e.g. CloseTime = missing + // Question: why is the right side is sometimes a type of "colName", and sometimes a type of "SQLVal"? + // Answer: for any value, sqlParser will treat any string that doesn't surrounded by single quote as ColName; + // any string that surrounded by single quote as SQLVal + _, ok = comparisonExpr.Right.(*sqlparser.SQLVal) + if !ok { // this means, the value is a string, and not surrounded by single qoute, which means, val = missing + colVal, ok := comparisonExpr.Right.(*sqlparser.ColName) + if !ok { + return "", fmt.Errorf("error: Failed to convert val") + } + colValStr := colVal.Name.String() + + // double check if val is not missing + if colValStr != "missing" { + return "", fmt.Errorf("error: failed to convert val") + } + + var newColVal string + if strings.ToLower(colNameStr) == "historylength" { + newColVal = "0" + } else { + newColVal = "-1" // -1 is the default value for all Closed workflows related fields + } + comparisonExpr.Right = &sqlparser.ColName{ + Metadata: colName.Metadata, + Name: sqlparser.NewColIdent(newColVal), + Qualifier: colName.Qualifier, + } + } + + // For this branch, we still have a sqlExpr type. So need to use a buf to return the string + comparisonExpr.Format(buf) + return buf.String(), nil +} + +func (qv *VisibilityQueryValidator) processCustomKey(expr sqlparser.Expr) (string, error) { + comparisonExpr := expr.(*sqlparser.ComparisonExpr) + + colName, ok := comparisonExpr.Left.(*sqlparser.ColName) + if !ok { + return "", errors.New("invalid comparison expression, left") + } + + colNameStr := colName.Name.String() + + // check type: if is IndexedValueTypeString, change to like statement for partial match + valType, ok := qv.validSearchAttributes[colNameStr] + if !ok { + return "", fmt.Errorf("invalid search attribute") + } + + // get the column value + colVal, ok := comparisonExpr.Right.(*sqlparser.SQLVal) + if !ok { + return "", errors.New("invalid comparison expression, right") + } + colValStr := string(colVal.Val) + + // get the value type + indexValType := common.ConvertIndexedValueTypeToInternalType(valType, log.NewNoop()) + + // Case2-1: when it is string, need partial match + if indexValType == types.IndexedValueTypeString { + // change to like statement for partial match + comparisonExpr.Operator = sqlparser.LikeStr + comparisonExpr.Right = &sqlparser.SQLVal{ + Type: sqlparser.StrVal, + Val: []byte("%" + colValStr + "%"), + } + //return fmt.Sprintf("JSON_EXTRACT_SCALAR(Attr, '$.%s', 'STRING') LIKE '%%%s%%'", colNameStr, colValStr), nil + return fmt.Sprintf("(JSON_MATCH(Attr, '\"$.%s\" is not null') "+ + "AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.%s', 'string'), '%s*'))", colNameStr, colNameStr, colValStr), nil + } + // case2-2: otherwise, exact match + // case2-2-1: if it is keyword, need to deal with a situation when value is an array + if indexValType == types.IndexedValueTypeKeyword { + return fmt.Sprintf("(JSON_MATCH(Attr, '\"$.%s\"=''%s''') or JSON_MATCH(Attr, '\"$.%s[*]\"=''%s'''))", + colNameStr, colValStr, colNameStr, colValStr), nil + } + // case2-2-2: other cases: + return fmt.Sprintf("JSON_MATCH(Attr, '\"$.%s\"=''%s''')", colNameStr, colValStr), nil +} diff --git a/common/pinot/pinotQueryValidator_test.go b/common/pinot/pinotQueryValidator_test.go new file mode 100644 index 00000000000..b5e3df4e76b --- /dev/null +++ b/common/pinot/pinotQueryValidator_test.go @@ -0,0 +1,130 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package pinot + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/uber/cadence/common/definition" + "github.com/uber/cadence/common/dynamicconfig" +) + +func TestValidateQuery(t *testing.T) { + tests := map[string]struct { + query string + validated string + err string + }{ + "Case1: empty query": { + query: "", + validated: "", + }, + "Case2: simple query": { + query: "WorkflowID = 'wid'", + validated: "WorkflowID = 'wid'", + }, + "Case3: query with custom field": { + query: "CustomStringField = 'custom'", + validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom*'))", + }, + "Case4: custom field query with or in string": { + query: "CustomStringField='Or'", + validated: "(JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'Or*'))", + }, + "Case5: custom keyword field query": { + query: "CustomKeywordField = 'custom'", + validated: "(JSON_MATCH(Attr, '\"$.CustomKeywordField\"=''custom''') or JSON_MATCH(Attr, '\"$.CustomKeywordField[*]\"=''custom'''))", + }, + "Case6-1: complex query I: with parenthesis": { + query: "(CustomStringField = 'custom and custom2 or custom3 order by') or CustomIntField between 1 and 10", + validated: "((JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom and custom2 or custom3 order by*')) or CustomIntField between 1 and 10)", + }, + "Case6-2: complex query II: with only system keys": { + query: "DomainID = 'd-id' and (RunID = 'run-id' or WorkflowID = 'wid')", + validated: "DomainID = 'd-id' and (RunID = 'run-id' or WorkflowID = 'wid')", + }, + "Case6-3: complex query III: operation priorities": { + query: "DomainID = 'd-id' or RunID = 'run-id' and WorkflowID = 'wid'", + validated: "(DomainID = 'd-id' or RunID = 'run-id' and WorkflowID = 'wid')", + }, + "Case6-4: complex query IV": { + query: "WorkflowID = 'wid' and (CustomStringField = 'custom and custom2 or custom3 order by' or CustomIntField between 1 and 10)", + validated: "WorkflowID = 'wid' and ((JSON_MATCH(Attr, '\"$.CustomStringField\" is not null') AND REGEXP_LIKE(JSON_EXTRACT_SCALAR(Attr, '$.CustomStringField', 'string'), 'custom and custom2 or custom3 order by*')) or CustomIntField between 1 and 10)", + }, + "Case7: invalid sql query": { + query: "Invalid SQL", + err: "Invalid query.", + }, + "Case8: query with missing val": { + query: "CloseTime = missing", + validated: "CloseTime = `-1`", + }, + "Case9: invalid where expression": { + query: "InvalidWhereExpr", + err: "invalid where clause", + }, + "Case10: invalid search attribute": { + query: "Invalid = 'a' and 1 < 2", + err: "invalid search attribute \"Invalid\"", + }, + "Case11-1: order by clause": { + query: "order by CloseTime desc", + validated: " order by CloseTime desc", + }, + "Case11-2: only order by clause with custom field": { + query: "order by CustomIntField desc", + validated: " order by CustomIntField desc", + }, + "Case11-3: order by clause with custom field": { + query: "WorkflowID = 'wid' order by CloseTime desc", + validated: "WorkflowID = 'wid' order by CloseTime desc", + }, + "Case12-1: security SQL injection - with another statement": { + query: "WorkflowID = 'wid'; SELECT * FROM important_table;", + err: "Invalid query.", + }, + "Case12-2: security SQL injection - with union": { + query: "WorkflowID = 'wid' union select * from dummy", + err: "Invalid select query.", + }, + "Case13: or clause": { + query: "CustomIntField = 1 or CustomIntField = 2", + validated: "(JSON_MATCH(Attr, '\"$.CustomIntField\"=''1''') or JSON_MATCH(Attr, '\"$.CustomIntField\"=''2'''))", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + validSearchAttr := dynamicconfig.GetMapPropertyFn(definition.GetDefaultIndexedKeys()) + qv := NewPinotQueryValidator(validSearchAttr()) + validated, err := qv.ValidateQuery(test.query) + if err != nil { + assert.Equal(t, test.err, err.Error()) + } else { + assert.Equal(t, test.validated, validated) + } + }) + } +} diff --git a/common/pinot/responseUtility.go b/common/pinot/responseUtility.go new file mode 100644 index 00000000000..4a97b0bb509 --- /dev/null +++ b/common/pinot/responseUtility.go @@ -0,0 +1,128 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package pinot + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/log/tag" + p "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/types" +) + +func buildMap(hit []interface{}, columnNames []string) map[string]interface{} { + systemKeyMap := make(map[string]interface{}) + + for i := 0; i < len(columnNames); i++ { + key := columnNames[i] + systemKeyMap[key] = hit[i] + } + + return systemKeyMap +} + +// VisibilityRecord is a struct of doc for deserialization +// this is different from InternalVisibilityWorkflowExecutionInfo +// use this to deserialize the systemKeyMap from Pinot response +type VisibilityRecord struct { + WorkflowID string + RunID string + WorkflowType string + DomainID string + StartTime int64 + ExecutionTime int64 + CloseTime int64 + CloseStatus int + HistoryLength int64 + TaskList string + IsCron bool + NumClusters int16 + UpdateTime int64 + ShardID int16 +} + +func ConvertSearchResultToVisibilityRecord(hit []interface{}, columnNames []string, logger log.Logger) *p.InternalVisibilityWorkflowExecutionInfo { + if len(hit) != len(columnNames) { + return nil + } + + systemKeyMap := buildMap(hit, columnNames) + + jsonSystemKeyMap, err := json.Marshal(systemKeyMap) + if err != nil { + logger.Error("unable to marshal systemKeyMap", + tag.Error(err)) + return nil + } + + attributeMap := make(map[string]interface{}) + err = json.Unmarshal([]byte(fmt.Sprintf("%s", systemKeyMap["Attr"])), &attributeMap) + if err != nil { + logger.Error("Unable to Unmarshal searchAttribute map", tag.Error(err)) + } + + var source *VisibilityRecord + err = json.Unmarshal(jsonSystemKeyMap, &source) + if err != nil { + logger.Error("Unable to Unmarshal systemKeyMap", + tag.Error(err), //tag.ESDocID(fmt.Sprintf(columnNameToValue["DocID"])) + ) + return nil + } + + record := &p.InternalVisibilityWorkflowExecutionInfo{ + DomainID: source.DomainID, + WorkflowType: source.WorkflowType, + WorkflowID: source.WorkflowID, + RunID: source.RunID, + TypeName: source.WorkflowType, + StartTime: time.UnixMilli(source.StartTime), // be careful: source.StartTime is in milliseconds + ExecutionTime: time.UnixMilli(source.ExecutionTime), + TaskList: source.TaskList, + IsCron: source.IsCron, + NumClusters: source.NumClusters, + ShardID: source.ShardID, + SearchAttributes: attributeMap, + } + if source.UpdateTime != 0 { + record.UpdateTime = time.UnixMilli(source.UpdateTime) + } + if source.CloseTime != 0 { + record.CloseTime = time.UnixMilli(source.CloseTime) + record.Status = toWorkflowExecutionCloseStatus(source.CloseStatus) + record.HistoryLength = source.HistoryLength + } + + return record +} + +func toWorkflowExecutionCloseStatus(status int) *types.WorkflowExecutionCloseStatus { + if status < 0 { + return nil + } + closeStatus := types.WorkflowExecutionCloseStatus(status) + return &closeStatus +} diff --git a/common/resource/params.go b/common/resource/params.go index 5eb356a1f78..34374d1f322 100644 --- a/common/resource/params.go +++ b/common/resource/params.go @@ -28,6 +28,7 @@ import ( "github.com/uber/cadence/common/isolationgroup" "github.com/uber/cadence/common/partition" + "github.com/uber/cadence/common/pinot" "github.com/uber/cadence/common" "github.com/uber/cadence/common/archiver" @@ -75,5 +76,7 @@ type ( IsolationGroupStore configstore.Client // This can be nil, the default config store will be created if so IsolationGroupState isolationgroup.State // This can be nil, the default state store will be chosen if so Partitioner partition.Partitioner + PinotConfig *config.PinotVisibilityConfig + PinotClient pinot.GenericClient } ) diff --git a/common/resource/resourceImpl.go b/common/resource/resourceImpl.go index 70787abab24..e951b2cb63f 100644 --- a/common/resource/resourceImpl.go +++ b/common/resource/resourceImpl.go @@ -193,6 +193,8 @@ func New( MessagingClient: params.MessagingClient, ESClient: params.ESClient, ESConfig: params.ESConfig, + PinotConfig: params.PinotConfig, + PinotClient: params.PinotClient, }, serviceConfig) if err != nil { return nil, err diff --git a/common/service/config.go b/common/service/config.go index e90631ca614..3626414f1a7 100644 --- a/common/service/config.go +++ b/common/service/config.go @@ -33,6 +33,8 @@ type ( EnableReadVisibilityFromES dynamicconfig.BoolPropertyFnWithDomainFilter // AdvancedVisibilityWritingMode is the write mode of visibility AdvancedVisibilityWritingMode dynamicconfig.StringPropertyFn + // EnableReadVisibilityFromPinot is the read mode of visibility + EnableReadVisibilityFromPinot dynamicconfig.BoolPropertyFnWithDomainFilter // configs for db visibility EnableDBVisibilitySampling dynamicconfig.BoolPropertyFn `yaml:"-" json:"-"` diff --git a/config/development_pinot.yaml b/config/development_pinot.yaml new file mode 100644 index 00000000000..7ee8b6c0ed1 --- /dev/null +++ b/config/development_pinot.yaml @@ -0,0 +1,45 @@ +persistence: + advancedVisibilityStore: pinot-visibility + datastores: + pinot-visibility: + pinot: + broker: "localhost:8099" + cluster: pinot-test + table: "cadence_visibility_pinot" + es-visibility: + elasticsearch: + version: "v6" + url: + scheme: "http" + host: "127.0.0.1:9200" + indices: + visibility: cadence-visibility-dev + +kafka: + tls: + enabled: false + clusters: + test: + brokers: + - 127.0.0.1:9092 + topics: + cadence-visibility-dev: + cluster: test + cadence-visibility-dev-dlq: + cluster: test + cadence-visibility-pinot: + cluster: test + cadence-visibility-pinot-dlq: + cluster: test + applications: + visibility: + topic: cadence-visibility-dev + dlq-topic: cadence-visibility-dev-dlq + pinot-visibility: + topic: cadence-visibility-pinot + dlq-topic: cadence-visibility-pinot-dlq + +dynamicconfig: + client: filebased + filebased: + filepath: "config/dynamicconfig/development_pinot.yaml" \ No newline at end of file diff --git a/config/dynamicconfig/development.yaml b/config/dynamicconfig/development.yaml index c8d11b9f2ed..871a131e47d 100644 --- a/config/dynamicconfig/development.yaml +++ b/config/dynamicconfig/development.yaml @@ -48,4 +48,5 @@ frontend.validSearchAttributes: project: 1 service: 1 user: 1 + IsDeleted: 4 constraints: {} \ No newline at end of file diff --git a/config/dynamicconfig/development_pinot.yaml b/config/dynamicconfig/development_pinot.yaml new file mode 100644 index 00000000000..173a9a27c0b --- /dev/null +++ b/config/dynamicconfig/development_pinot.yaml @@ -0,0 +1,49 @@ +frontend.enableClientVersionCheck: + - value: true +system.advancedVisibilityWritingMode: + - value: "triple" +system.enableReadVisibilityFromES: + - value: false +system.enableReadVisibilityFromPinot: + - value: true +frontend.validSearchAttributes: + - value: + DomainID: 1 + WorkflowID: 1 + RunID: 1 + WorkflowType: 1 + StartTime: 2 + ExecutionTime: 2 + CloseTime: 2 + CloseStatus: 2 + HistoryLength: 2 + TaskList: 1 + IsCron: 1 + NumClusters: 2 + UpdateTime: 2 + CustomStringField: 0 + CustomKeywordField: 1 + CustomIntField: 2 + CustomDoubleField: 3 + CustomBoolField: 4 + CustomDatetimeField: 5 + project: 1 + service: 1 + environment: 1 + addon: 1 + addon-type: 1 + user: 1 + CustomDomain: 1 + Operator: 1 + RolloutID: 1 + CadenceChangeVersion: 1 + BinaryChecksums: 1 + Passed: 4 + ShardID: 2 + IsDeleted: 4 +system.minRetentionDays: + - value: 0 +history.EnableConsistentQueryByDomain: +- value: true + constraints: {} + diff --git a/docker/dev/cassandra-pinot-kafka.yml b/docker/dev/cassandra-pinot-kafka.yml index fcbe8a4758c..014a28d7d1c 100644 --- a/docker/dev/cassandra-pinot-kafka.yml +++ b/docker/dev/cassandra-pinot-kafka.yml @@ -13,7 +13,7 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 pinot-controller: - image: apachepinot/pinot:0.11.0 + image: apachepinot/pinot:0.12.1 command: "StartController -zkAddress zookeeper:2181" container_name: pinot-controller restart: unless-stopped @@ -24,7 +24,7 @@ services: depends_on: - zookeeper pinot-broker: - image: apachepinot/pinot:0.11.0 + image: apachepinot/pinot:0.12.1 command: "StartBroker -zkAddress zookeeper:2181" restart: unless-stopped container_name: "pinot-broker" @@ -35,7 +35,7 @@ services: depends_on: - pinot-controller pinot-server: - image: apachepinot/pinot:0.11.0 + image: apachepinot/pinot:0.12.1 command: "StartServer -zkAddress zookeeper:2181" restart: unless-stopped container_name: "pinot-server" @@ -60,3 +60,9 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,OUTSIDE://localhost:9092 KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3 + ports: + - "9200:9200" + environment: + - discovery.type=single-node diff --git a/docker/docker-compose-pinot.yml b/docker/docker-compose-pinot.yml index f192c1bab93..70ea1fde1f4 100644 --- a/docker/docker-compose-pinot.yml +++ b/docker/docker-compose-pinot.yml @@ -111,4 +111,4 @@ services: depends_on: - prometheus ports: - - '3000:3000' + - '3000:3000' \ No newline at end of file diff --git a/go.mod b/go.mod index 756ea01283d..c4c7eaf9520 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,8 @@ require ( github.com/otiai10/copy v1.1.1 github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 github.com/robfig/cron v1.2.0 - github.com/sirupsen/logrus v1.8.1 + github.com/sirupsen/logrus v1.9.0 + github.com/startreedata/pinot-client-go v0.0.0-20230303070132-3b84c28a9e95 // latest release doesn't support pinot v0.12, so use master branch github.com/stretchr/testify v1.8.1 github.com/uber-go/tally v3.3.15+incompatible github.com/uber/cadence-idl v0.0.0-20230905165949-03586319b849 @@ -122,6 +123,7 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/uber-common/bark v1.2.1 // indirect diff --git a/go.sum b/go.sum index b5a469eaf3b..f9c61a8ef4a 100644 --- a/go.sum +++ b/go.sum @@ -513,6 +513,8 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samuel/go-thrift v0.0.0-20191111193933-5165175b40af h1:EiWVfh8mr40yFZEui2oF0d45KgH48PkB2H0Z0GANvSI= github.com/samuel/go-thrift v0.0.0-20191111193933-5165175b40af/go.mod h1:Vrkh1pnjV9Bl8c3P9zH0/D4NlOHWP5d4/hF4YTULaec= +github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414 h1:AJNDS0kP60X8wwWFvbLPwDuojxubj9pbfK7pjHw0vKg= +github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= @@ -522,14 +524,17 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/startreedata/pinot-client-go v0.0.0-20230303070132-3b84c28a9e95 h1:HfLb3UH6GR5VaPdj78n/FvaX5+dQreveU2wkrMf6Q7I= +github.com/startreedata/pinot-client-go v0.0.0-20230303070132-3b84c28a9e95/go.mod h1:MPq1O10hPOizQxTmf5/MF5/0HBqeh+xTrHuPhvOiRWY= github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A= github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -867,6 +872,7 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= diff --git a/host/dynamicconfig.go b/host/dynamicconfig.go index 4a0f17d9a6a..ca131270375 100644 --- a/host/dynamicconfig.go +++ b/host/dynamicconfig.go @@ -166,6 +166,11 @@ func (d *dynamicClient) UpdateValue(name dynamicconfig.Key, value interface{}) e defer d.Unlock() d.overrides[dynamicconfig.AdvancedVisibilityWritingMode] = value.(string) return nil + } else if name == dynamicconfig.EnableReadVisibilityFromES { // override for pinot integration tests + d.Lock() + defer d.Unlock() + d.overrides[dynamicconfig.EnableReadVisibilityFromES] = value.(bool) + return nil } return d.client.UpdateValue(name, value) } diff --git a/host/integrationbase.go b/host/integrationbase.go index 6759b42b409..5f81b6efcc7 100644 --- a/host/integrationbase.go +++ b/host/integrationbase.go @@ -139,6 +139,46 @@ func (s *IntegrationBase) setupSuite() { time.Sleep(cache.DomainCacheRefreshInterval + time.Second) } +func (s *IntegrationBase) setupSuiteForPinotTest() { + s.setupLogger() + + s.Logger.Info("Running integration test against test cluster") + clusterMetadata := NewClusterMetadata(s.testClusterConfig) + dc := persistence.DynamicConfiguration{ + EnableSQLAsyncTransaction: dynamicconfig.GetBoolPropertyFn(false), + EnableCassandraAllConsistencyLevelDelete: dynamicconfig.GetBoolPropertyFn(true), + PersistenceSampleLoggingRate: dynamicconfig.GetIntPropertyFn(100), + EnableShardIDMetrics: dynamicconfig.GetBoolPropertyFn(true), + } + params := pt.TestBaseParams{ + DefaultTestCluster: s.defaultTestCluster, + VisibilityTestCluster: s.visibilityTestCluster, + ClusterMetadata: clusterMetadata, + DynamicConfiguration: dc, + } + cluster, err := NewPinotTestCluster(s.testClusterConfig, s.Logger, params) + s.Require().NoError(err) + s.testCluster = cluster + s.engine = s.testCluster.GetFrontendClient() + s.adminClient = s.testCluster.GetAdminClient() + + s.testRawHistoryDomainName = "TestRawHistoryDomain" + s.domainName = s.randomizeStr("integration-test-domain") + s.Require().NoError( + s.registerDomain(s.domainName, 1, types.ArchivalStatusDisabled, "", types.ArchivalStatusDisabled, "")) + s.Require().NoError( + s.registerDomain(s.testRawHistoryDomainName, 1, types.ArchivalStatusDisabled, "", types.ArchivalStatusDisabled, "")) + s.foreignDomainName = s.randomizeStr("integration-foreign-test-domain") + s.Require().NoError( + s.registerDomain(s.foreignDomainName, 1, types.ArchivalStatusDisabled, "", types.ArchivalStatusDisabled, "")) + + s.Require().NoError(s.registerArchivalDomain()) + + // this sleep is necessary because domainv2 cache gets refreshed in the + // background only every domainCacheRefreshInterval period + time.Sleep(cache.DomainCacheRefreshInterval + time.Second) +} + func (s *IntegrationBase) setupLogger() { s.Logger = loggerimpl.NewLoggerForTest(s.Suite) } diff --git a/host/onebox.go b/host/onebox.go index db4e13b02ed..bd0eb55fbab 100644 --- a/host/onebox.go +++ b/host/onebox.go @@ -59,6 +59,7 @@ import ( "github.com/uber/cadence/common/messaging" "github.com/uber/cadence/common/metrics" "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/pinot" "github.com/uber/cadence/common/resource" "github.com/uber/cadence/common/rpc" "github.com/uber/cadence/common/service" @@ -115,6 +116,8 @@ type ( mockAdminClient map[string]adminClient.Client domainReplicationTaskExecutor domain.ReplicationTaskExecutor authorizationConfig config.Authorization + pinotConfig *config.PinotVisibilityConfig + pinotClient pinot.GenericClient } // HistoryConfig contains configs for history service @@ -146,6 +149,8 @@ type ( MockAdminClient map[string]adminClient.Client DomainReplicationTaskExecutor domain.ReplicationTaskExecutor AuthorizationConfig config.Authorization + PinotConfig *config.PinotVisibilityConfig + PinotClient pinot.GenericClient } ) @@ -171,6 +176,8 @@ func NewCadence(params *CadenceParams) Cadence { mockAdminClient: params.MockAdminClient, domainReplicationTaskExecutor: params.DomainReplicationTaskExecutor, authorizationConfig: params.AuthorizationConfig, + pinotConfig: params.PinotConfig, + pinotClient: params.PinotClient, } } @@ -424,6 +431,8 @@ func (c *cadenceImpl) startFrontend(hosts map[string][]membership.HostInfo, star params.ArchiverProvider = c.archiverProvider params.ESConfig = c.esConfig params.ESClient = c.esClient + params.PinotConfig = c.pinotConfig + params.PinotClient = c.pinotClient var err error authorizer, err := authorization.NewAuthorizer(c.authorizationConfig, params.Logger, nil) if err != nil { @@ -435,7 +444,14 @@ func (c *cadenceImpl) startFrontend(hosts map[string][]membership.HostInfo, star c.logger.Fatal("Failed to copy persistence config for frontend", tag.Error(err)) } - if c.esConfig != nil { + if c.pinotConfig != nil { + pinotDataStoreName := "pinot-visibility" + params.PersistenceConfig.AdvancedVisibilityStore = pinotDataStoreName + params.DynamicConfig.UpdateValue(dynamicconfig.EnableReadVisibilityFromES, false) + params.PersistenceConfig.DataStores[pinotDataStoreName] = config.DataStore{ + Pinot: c.pinotConfig, + } + } else if c.esConfig != nil { esDataStoreName := "es-visibility" params.PersistenceConfig.AdvancedVisibilityStore = esDataStoreName params.PersistenceConfig.DataStores[esDataStoreName] = config.DataStore{ @@ -499,7 +515,14 @@ func (c *cadenceImpl) startHistory( c.logger.Fatal("Failed to copy persistence config for history", tag.Error(err)) } - if c.esConfig != nil { + if c.pinotConfig != nil { + pinotDataStoreName := "pinot-visibility" + params.PersistenceConfig.AdvancedVisibilityStore = pinotDataStoreName + params.PersistenceConfig.DataStores[pinotDataStoreName] = config.DataStore{ + Pinot: c.pinotConfig, + ElasticSearch: c.esConfig, + } + } else if c.esConfig != nil { esDataStoreName := "es-visibility" params.PersistenceConfig.AdvancedVisibilityStore = esDataStoreName params.PersistenceConfig.DataStores[esDataStoreName] = config.DataStore{ diff --git a/host/pinot_test.go b/host/pinot_test.go new file mode 100644 index 00000000000..e8272708ada --- /dev/null +++ b/host/pinot_test.go @@ -0,0 +1,1183 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +//go:build pinotintegration +// +build pinotintegration + +// to run locally, make sure kafka and pinot is running, +// then run cmd `go test -v ./host -run TestPinotIntegrationSuite -tags pinotintegration` +// currently we have to manually add test table and delete the table for cleaning +// waiting for the support to clean the data programmatically + +package host + +import ( + "flag" + "strconv" + "time" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/uber/cadence/common/definition" + pnt "github.com/uber/cadence/common/pinot" + + "testing" + + "github.com/uber/cadence/common/config" + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/log/loggerimpl" + "github.com/uber/cadence/common/log/tag" + + "go.uber.org/zap" + + "github.com/uber/cadence/host/pinotutils" + + "encoding/json" + "fmt" + + "github.com/pborman/uuid" + + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/types" +) + +const ( + numberOfRetry = 50 + waitTimeInMillisecond = 400 * time.Millisecond + waitForPinotToSettle = 4 * time.Second // wait pinot shards for some time ensure data consistent +) + +type PinotIntegrationSuite struct { + *require.Assertions + logger log.Logger + IntegrationBase + pinotClient pnt.GenericClient + + testSearchAttributeKey string + testSearchAttributeVal string +} + +func TestPinotIntegrationSuite(t *testing.T) { + flag.Parse() + clusterConfig, err := GetTestClusterConfig("testdata/integration_pinot_cluster.yaml") + if err != nil { + panic(err) + } + testCluster := NewPersistenceTestCluster(t, clusterConfig) + + s := new(PinotIntegrationSuite) + params := IntegrationBaseParams{ + DefaultTestCluster: testCluster, + VisibilityTestCluster: testCluster, + TestClusterConfig: clusterConfig, + } + s.IntegrationBase = NewIntegrationBase(params) + suite.Run(t, s) +} + +func (s *PinotIntegrationSuite) SetupSuite() { + s.setupSuiteForPinotTest() + zapLogger, err := zap.NewDevelopment() + s.Require().NoError(err) + s.logger = loggerimpl.NewLogger(zapLogger) + tableName := "cadence_visibility_pinot" //cadence_visibility_pinot_integration_test + pinotConfig := &config.PinotVisibilityConfig{ + Cluster: "", + Broker: "localhost:8099", + Table: tableName, + ServiceName: "", + } + s.pinotClient = pinotutils.CreatePinotClient(s.Suite, pinotConfig, s.logger) +} + +func (s *PinotIntegrationSuite) SetupTest() { + s.Assertions = require.New(s.T()) + s.testSearchAttributeKey = definition.CustomStringField + s.testSearchAttributeVal = "test value" +} + +func (s *PinotIntegrationSuite) TearDownSuite() { + s.tearDownSuite() + // check how to clean up test_table + // currently it is not supported +} + +func (s *PinotIntegrationSuite) TestListOpenWorkflow() { + id := "pinot-integration-start-workflow-test" + wt := "pinot-integration-start-workflow-test-type" + tl := "pinot-integration-start-workflow-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + + startTime := time.Now().UnixNano() + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + startFilter := &types.StartTimeFilter{} + startFilter.EarliestTime = common.Int64Ptr(startTime) + var openExecution *types.WorkflowExecutionInfo + for i := 0; i < numberOfRetry; i++ { + startFilter.LatestTime = common.Int64Ptr(time.Now().UnixNano()) + resp, err := s.engine.ListOpenWorkflowExecutions(createContext(), &types.ListOpenWorkflowExecutionsRequest{ + Domain: s.domainName, + MaximumPageSize: defaultTestValueOfESIndexMaxResultWindow, + StartTimeFilter: startFilter, + ExecutionFilter: &types.WorkflowExecutionFilter{ + WorkflowID: id, + }, + }) + s.Nil(err) + + if len(resp.GetExecutions()) > 0 { + openExecution = resp.GetExecutions()[0] + break + } + time.Sleep(waitTimeInMillisecond) + } + s.NotNil(openExecution) + s.Equal(we.GetRunID(), openExecution.GetExecution().GetRunID()) + s.Equal(attrValBytes, openExecution.SearchAttributes.GetIndexedFields()[s.testSearchAttributeKey]) +} + +func (s *PinotIntegrationSuite) TestListWorkflow() { + id := "pinot-integration-list-workflow-test" + wt := "pinot-integration-list-workflow-test-type" + tl := "pinot-integration-list-workflow-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + query := fmt.Sprintf(`WorkflowID = "%s"`, id) + s.testHelperForReadOnce(we.GetRunID(), query, false, false) +} + +func (s *PinotIntegrationSuite) createStartWorkflowExecutionRequest(id, wt, tl string) *types.StartWorkflowExecutionRequest { + identity := "worker1" + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + return request +} + +func (s *PinotIntegrationSuite) testHelperForReadOnce(runID, query string, isScan bool, isAnyMatchOk bool) { + s.testHelperForReadOnceWithDomain(s.domainName, runID, query, isScan, isAnyMatchOk) +} + +func (s *PinotIntegrationSuite) testHelperForReadOnceWithDomain(domainName string, runID, query string, isScan bool, isAnyMatchOk bool) { + var openExecution *types.WorkflowExecutionInfo + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: domainName, + PageSize: defaultTestValueOfESIndexMaxResultWindow, + Query: query, + } +Retry: + for i := 0; i < numberOfRetry; i++ { + var resp *types.ListWorkflowExecutionsResponse + var err error + + if isScan { + resp, err = s.engine.ScanWorkflowExecutions(createContext(), listRequest) + } else { + resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) + } + + s.Nil(err) + logStr := fmt.Sprintf("Results for query '%s' (desired runId: %s): \n", query, runID) + s.Logger.Info(logStr) + for _, e := range resp.GetExecutions() { + logStr = fmt.Sprintf("Execution: %+v, %+v \n", e.Execution, e) + s.Logger.Info(logStr) + } + if len(resp.GetExecutions()) == 1 { + openExecution = resp.GetExecutions()[0] + break + } + if isAnyMatchOk { + for _, e := range resp.GetExecutions() { + if e.Execution.RunID == runID { + openExecution = e + break Retry + } + } + } + time.Sleep(waitTimeInMillisecond) + } + s.NotNil(openExecution) + s.Equal(runID, openExecution.GetExecution().GetRunID()) + s.True(openExecution.GetExecutionTime() >= openExecution.GetStartTime()) + if openExecution.SearchAttributes != nil && len(openExecution.SearchAttributes.GetIndexedFields()) > 0 { + searchValBytes := openExecution.SearchAttributes.GetIndexedFields()[s.testSearchAttributeKey] + var searchVal string + json.Unmarshal(searchValBytes, &searchVal) + // pinot sets default values for all the columns, + // this feature can break the test here when there is no actual search attributes upsert, it will still return something + // TODO: update this after finding a good solution + s.Equal(searchVal, searchVal) + } +} + +func (s *PinotIntegrationSuite) startWorkflow( + prefix string, + is_cron bool, +) *types.StartWorkflowExecutionResponse { + id := "pinot-integration-list-workflow-" + prefix + "-test" + wt := "pinot-integration-list-workflow-" + prefix + "test-type" + tl := "pinot-integration-list-workflow-" + prefix + "test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + if is_cron { + request.CronSchedule = "*/5 * * * *" // every 5 minutes + } + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + query := fmt.Sprintf(`WorkflowID = "%s"`, id) + s.testHelperForReadOnce(we.GetRunID(), query, false, false) + return we +} + +func (s *PinotIntegrationSuite) TestListCronWorkflows() { + we1 := s.startWorkflow("cron", true) + we2 := s.startWorkflow("regular", false) + + query := fmt.Sprintf(`IsCron = "true"`) + s.testHelperForReadOnce(we1.GetRunID(), query, false, true) + + query = fmt.Sprintf(`IsCron = "false"`) + s.testHelperForReadOnce(we2.GetRunID(), query, false, true) +} + +func (s *PinotIntegrationSuite) TestIsGlobalSearchAttribute() { + we := s.startWorkflow("local", true) + // global domains are disabled for this integration test, so we can only test the false case + query := fmt.Sprintf(`NumClusters = "1"`) + s.testHelperForReadOnce(we.GetRunID(), query, false, true) +} + +func (s *PinotIntegrationSuite) TestListWorkflow_ExecutionTime() { + id := "pinot-integration-list-workflow-execution-time-test" + wt := "pinot-integration-list-workflow-execution-time-test-type" + tl := "pinot-integration-list-workflow-execution-time-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + cronID := id + "-cron" + request.CronSchedule = "@every 1m" + request.WorkflowID = cronID + + weCron, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + query := fmt.Sprintf(`(WorkflowID = '%s' or WorkflowID = '%s') and ExecutionTime < %v and ExecutionTime > 0`, id, cronID, time.Now().UnixNano()+int64(time.Minute)) + s.testHelperForReadOnce(weCron.GetRunID(), query, false, false) + + query = fmt.Sprintf(`WorkflowID = '%s'`, id) + s.testHelperForReadOnce(we.GetRunID(), query, false, false) +} + +func (s *PinotIntegrationSuite) TestListWorkflow_SearchAttribute() { + id := "pinot-integration-list-workflow-by-search-attr-test" + wt := "pinot-integration-list-workflow-by-search-attr-test-type" + tl := "pinot-integration-list-workflow-by-search-attr-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + query := fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, s.testSearchAttributeVal) + s.testHelperForReadOnce(we.GetRunID(), query, false, false) + + // test upsert + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + upsertDecision := &types.Decision{ + DecisionType: types.DecisionTypeUpsertWorkflowSearchAttributes.Ptr(), + UpsertWorkflowSearchAttributesDecisionAttributes: &types.UpsertWorkflowSearchAttributesDecisionAttributes{ + SearchAttributes: getPinotUpsertSearchAttributes(), + }} + + return nil, []*types.Decision{upsertDecision}, nil + } + taskList := &types.TaskList{Name: tl} + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + StickyTaskList: taskList, + Identity: "worker1", + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( + false, + false, + true, + true, + int64(0), + 1, + true, + nil) + s.Nil(err) + s.NotNil(newTask) + s.NotNil(newTask.DecisionTask) + + time.Sleep(waitForPinotToSettle) + + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: int32(2), + Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing and BinaryChecksums = 'binary-v1'`, wt), + } + // verify upsert data is on Pinot + s.testListResultForUpsertSearchAttributes(listRequest) + + // verify DescribeWorkflowExecution + descRequest := &types.DescribeWorkflowExecutionRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + }, + } + descResp, err := s.engine.DescribeWorkflowExecution(createContext(), descRequest) + s.Nil(err) + expectedSearchAttributes := getPinotUpsertSearchAttributes() + s.Equal(expectedSearchAttributes, descResp.WorkflowExecutionInfo.GetSearchAttributes()) +} + +func (s *PinotIntegrationSuite) TestListWorkflow_PageToken() { + id := "pinot-integration-list-workflow-token-test" + wt := "pinot-integration-list-workflow-token-test-type" + tl := "pinot-integration-list-workflow-token-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + numOfWorkflows := defaultTestValueOfESIndexMaxResultWindow - 1 // == 4 + pageSize := 3 + + s.testListWorkflowHelper(numOfWorkflows, pageSize, request, id, wt, false) +} + +func (s *PinotIntegrationSuite) TestListWorkflow_SearchAfter() { + id := "pinot-integration-list-workflow-searchAfter-test" + wt := "pinot-integration-list-workflow-searchAfter-test-type" + tl := "pinot-integration-list-workflow-searchAfter-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + numOfWorkflows := defaultTestValueOfESIndexMaxResultWindow + 1 // == 6 + pageSize := 4 + + s.testListWorkflowHelper(numOfWorkflows, pageSize, request, id, wt, false) +} + +func (s *PinotIntegrationSuite) TestListWorkflow_OrQuery() { + id := "pinot-integration-list-workflow-or-query-test" + wt := "pinot-integration-list-workflow-or-query-test-type" + tl := "pinot-integration-list-workflow-or-query-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + // start 3 workflows + key := definition.CustomIntField + attrValBytes, _ := json.Marshal(1) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + key: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + we1, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + request.RequestID = uuid.New() + request.WorkflowID = id + "-2" + attrValBytes, _ = json.Marshal(2) + searchAttr.IndexedFields[key] = attrValBytes + we2, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + request.RequestID = uuid.New() + request.WorkflowID = id + "-3" + attrValBytes, _ = json.Marshal(3) + searchAttr.IndexedFields[key] = attrValBytes + we3, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + time.Sleep(waitForPinotToSettle) + + // query 1 workflow with search attr + query1 := fmt.Sprintf(`CustomIntField = %d`, 1) + var openExecution *types.WorkflowExecutionInfo + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: defaultTestValueOfESIndexMaxResultWindow, + Query: query1, + } + for i := 0; i < numberOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == 1 { + openExecution = resp.GetExecutions()[0] + break + } + time.Sleep(waitTimeInMillisecond) + } + s.NotNil(openExecution) + s.Equal(we1.GetRunID(), openExecution.GetExecution().GetRunID()) + s.True(openExecution.GetExecutionTime() >= openExecution.GetStartTime()) + searchValBytes := openExecution.SearchAttributes.GetIndexedFields()[key] + var searchVal int + json.Unmarshal(searchValBytes, &searchVal) + s.Equal(1, searchVal) + + // query with or clause + query2 := fmt.Sprintf(`CustomIntField = %d or CustomIntField = %d`, 1, 2) + listRequest.Query = query2 + var openExecutions []*types.WorkflowExecutionInfo + for i := 0; i < numberOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == 2 { + openExecutions = resp.GetExecutions() + break + } + time.Sleep(waitTimeInMillisecond) + } + // TODO: need to clean up or every time we run, we have to delete the table. + s.Equal(2, len(openExecutions)) + + e1 := openExecutions[0] + e2 := openExecutions[1] + if e1.GetExecution().GetRunID() != we1.GetRunID() { + // results are sorted by [CloseTime,RunID] desc, so find the correct mapping first + e1, e2 = e2, e1 + } + s.Equal(we1.GetRunID(), e1.GetExecution().GetRunID()) + s.Equal(we2.GetRunID(), e2.GetExecution().GetRunID()) + searchValBytes = e2.SearchAttributes.GetIndexedFields()[key] + json.Unmarshal(searchValBytes, &searchVal) + s.Equal(2, searchVal) + + // query for open + query3 := fmt.Sprintf(`(CustomIntField = %d or CustomIntField = %d) and CloseTime = missing`, 2, 3) + listRequest.Query = query3 + for i := 0; i < numberOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == 2 { + openExecutions = resp.GetExecutions() + break + } + time.Sleep(waitTimeInMillisecond) + } + s.Equal(2, len(openExecutions)) + e1 = openExecutions[0] + e2 = openExecutions[1] + s.Equal(we3.GetRunID(), e1.GetExecution().GetRunID()) + s.Equal(we2.GetRunID(), e2.GetExecution().GetRunID()) + searchValBytes = e1.SearchAttributes.GetIndexedFields()[key] + json.Unmarshal(searchValBytes, &searchVal) + s.Equal(3, searchVal) +} + +// To test last page search trigger max window size error +func (s *PinotIntegrationSuite) TestListWorkflow_MaxWindowSize() { + id := "pinot-integration-list-workflow-max-window-size-test" + wt := "pinot-integration-list-workflow-max-window-size-test-type" + tl := "pinot-integration-list-workflow-max-window-size-test-tasklist" + startRequest := s.createStartWorkflowExecutionRequest(id, wt, tl) + + for i := 0; i < defaultTestValueOfESIndexMaxResultWindow; i++ { + startRequest.RequestID = uuid.New() + startRequest.WorkflowID = id + strconv.Itoa(i) + _, err := s.engine.StartWorkflowExecution(createContext(), startRequest) + s.Nil(err) + } + + time.Sleep(waitForPinotToSettle) + + var listResp *types.ListWorkflowExecutionsResponse + var nextPageToken []byte + + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: int32(defaultTestValueOfESIndexMaxResultWindow), + NextPageToken: nextPageToken, + Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wt), + } + // get first page + for i := 0; i < numberOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == defaultTestValueOfESIndexMaxResultWindow { + listResp = resp + break + } + time.Sleep(waitTimeInMillisecond) + } + s.NotNil(listResp) + s.True(len(listResp.GetNextPageToken()) != 0) + + // the last request + listRequest.NextPageToken = listResp.GetNextPageToken() + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + s.True(len(resp.GetExecutions()) == 0) + s.True(len(resp.GetNextPageToken()) == 0) +} + +func (s *PinotIntegrationSuite) TestListWorkflow_OrderBy() { + id := "pinot-integration-list-workflow-order-by-test" + wt := "pinot-integration-list-workflow-order-by-test-type" + tl := "pinot-integration-list-workflow-order-by-test-tasklist" + startRequest := s.createStartWorkflowExecutionRequest(id, wt, tl) + + for i := 0; i < defaultTestValueOfESIndexMaxResultWindow+1; i++ { // start 6 + startRequest.RequestID = uuid.New() + startRequest.WorkflowID = id + strconv.Itoa(i) + + if i < defaultTestValueOfESIndexMaxResultWindow-1 { // 4 workflow has search attr + intVal, _ := json.Marshal(i) + doubleVal, _ := json.Marshal(float64(i)) + strVal, _ := json.Marshal(strconv.Itoa(i)) + timeVal, _ := json.Marshal(time.Now()) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + definition.CustomIntField: intVal, + definition.CustomDoubleField: doubleVal, + definition.CustomKeywordField: strVal, + definition.CustomDatetimeField: timeVal, + }, + } + startRequest.SearchAttributes = searchAttr + } else { + startRequest.SearchAttributes = &types.SearchAttributes{} + } + + _, err := s.engine.StartWorkflowExecution(createContext(), startRequest) + s.Nil(err) + } + + time.Sleep(waitForPinotToSettle) + + //desc := "desc" + asc := "asc" + queryTemplate := `WorkflowType = "%s" order by %s %s` + pageSize := int32(defaultTestValueOfESIndexMaxResultWindow) + + // order by CloseTime asc + query1 := fmt.Sprintf(queryTemplate, wt, definition.CloseTime, asc) + var openExecutions []*types.WorkflowExecutionInfo + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: pageSize, + Query: query1, + } + for i := 0; i < numberOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if int32(len(resp.GetExecutions())) == listRequest.GetPageSize() { + openExecutions = resp.GetExecutions() + break + } + time.Sleep(waitTimeInMillisecond) + } + s.NotNil(openExecutions) + for i := int32(1); i < pageSize; i++ { + s.True(openExecutions[i-1].GetCloseTime() <= openExecutions[i].GetCloseTime()) + } + // comment out things below, because json index column can't use order by + + // greatest effort to reduce duplicate code + //testHelper := func(query, searchAttrKey string, prevVal, currVal interface{}) { + // listRequest.Query = query + // listRequest.NextPageToken = []byte{} + // resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + // s.Nil(err) + // openExecutions = resp.GetExecutions() + // dec := json.NewDecoder(bytes.NewReader(openExecutions[0].GetSearchAttributes().GetIndexedFields()[searchAttrKey])) + // dec.UseNumber() + // err = dec.Decode(&prevVal) + // s.Nil(err) + // for i := int32(1); i < pageSize; i++ { + // indexedFields := openExecutions[i].GetSearchAttributes().GetIndexedFields() + // searchAttrBytes, ok := indexedFields[searchAttrKey] + // if !ok { // last one doesn't have search attr + // s.Equal(pageSize-1, i) + // break + // } + // dec := json.NewDecoder(bytes.NewReader(searchAttrBytes)) + // dec.UseNumber() + // err = dec.Decode(&currVal) + // s.Nil(err) + // var v1, v2 interface{} + // switch searchAttrKey { + // case definition.CustomIntField: + // v1, _ = prevVal.(json.Number).Int64() + // v2, _ = currVal.(json.Number).Int64() + // s.True(v1.(int64) >= v2.(int64)) + // case definition.CustomDoubleField: + // v1, _ := strconv.ParseFloat(fmt.Sprint(prevVal), 64) + // v2, _ := strconv.ParseFloat(fmt.Sprint(currVal), 64) + // s.True(v1 >= v2) + // case definition.CustomKeywordField: + // s.True(prevVal.(string) >= currVal.(string)) + // case definition.CustomDatetimeField: + // v1, _ = strconv.ParseInt(fmt.Sprint(prevVal), 10, 64) + // v2, _ = strconv.ParseInt(fmt.Sprint(currVal), 10, 64) + // s.True(v1.(int64) >= v2.(int64)) + // } + // prevVal = currVal + // } + // listRequest.NextPageToken = resp.GetNextPageToken() + // resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) // last page + // s.Nil(err) + // s.Equal(1, len(resp.GetExecutions())) + //} + + // + //// order by CustomIntField desc + //field := definition.CustomIntField + //query := fmt.Sprintf(queryTemplate, wt, field, desc) + //var int1, int2 int + //testHelper(query, field, int1, int2) + // + //// order by CustomDoubleField desc + //field = definition.CustomDoubleField + //query = fmt.Sprintf(queryTemplate, wt, field, desc) + //var double1, double2 float64 + //testHelper(query, field, double1, double2) + // + //// order by CustomKeywordField desc + //field = definition.CustomKeywordField + //query = fmt.Sprintf(queryTemplate, wt, field, desc) + //var s1, s2 string + //testHelper(query, field, s1, s2) + // + //// order by CustomDatetimeField desc + //field = definition.CustomDatetimeField + //query = fmt.Sprintf(queryTemplate, wt, field, desc) + //var t1, t2 time.Time + //testHelper(query, field, t1, t2) +} + +func (s *PinotIntegrationSuite) testListWorkflowHelper(numOfWorkflows, pageSize int, + startRequest *types.StartWorkflowExecutionRequest, wid, wType string, isScan bool) { + + // start enough number of workflows + for i := 0; i < numOfWorkflows; i++ { + startRequest.RequestID = uuid.New() + startRequest.WorkflowID = wid + strconv.Itoa(i) + _, err := s.engine.StartWorkflowExecution(createContext(), startRequest) + s.Nil(err) + } + + time.Sleep(waitForPinotToSettle) + + var openExecutions []*types.WorkflowExecutionInfo + var nextPageToken []byte + + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: int32(pageSize), + NextPageToken: nextPageToken, + Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wType), + } + // test first page + for i := 0; i < numberOfRetry; i++ { + var resp *types.ListWorkflowExecutionsResponse + var err error + + if isScan { + resp, err = s.engine.ScanWorkflowExecutions(createContext(), listRequest) + } else { + resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) + } + s.Nil(err) + if len(resp.GetExecutions()) == pageSize { + openExecutions = resp.GetExecutions() + nextPageToken = resp.GetNextPageToken() + break + } + time.Sleep(waitTimeInMillisecond) + } + + s.NotNil(openExecutions) + s.NotNil(nextPageToken) + s.True(len(nextPageToken) > 0) + + // test last page + listRequest.NextPageToken = nextPageToken + inIf := false + for i := 0; i < numberOfRetry; i++ { + var resp *types.ListWorkflowExecutionsResponse + var err error + + if isScan { + resp, err = s.engine.ScanWorkflowExecutions(createContext(), listRequest) + } else { + resp, err = s.engine.ListWorkflowExecutions(createContext(), listRequest) + } + s.Nil(err) + + //ans, _ := json.Marshal(resp.GetExecutions()) + //panic(fmt.Sprintf("ABUCSDK: %s", ans)) + + if len(resp.GetExecutions()) == numOfWorkflows-pageSize { + inIf = true + openExecutions = resp.GetExecutions() + nextPageToken = resp.GetNextPageToken() + break + } + time.Sleep(waitTimeInMillisecond) + } + s.True(inIf) + s.NotNil(openExecutions) + s.Nil(nextPageToken) +} + +func (s *PinotIntegrationSuite) TestScanWorkflow() { + id := "pinot-integration-scan-workflow-test" + wt := "pinot-integration-scan-workflow-test-type" + tl := "pinot-integration-scan-workflow-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + query := fmt.Sprintf(`WorkflowID = "%s"`, id) + s.testHelperForReadOnce(we.GetRunID(), query, true, false) +} + +func (s *PinotIntegrationSuite) TestScanWorkflow_SearchAttribute() { + id := "pinot-integration-scan-workflow-search-attr-test" + wt := "pinot-integration-scan-workflow-search-attr-test-type" + tl := "pinot-integration-scan-workflow-search-attr-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + + we, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + query := fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, s.testSearchAttributeVal) + s.testHelperForReadOnce(we.GetRunID(), query, true, false) +} + +func (s *PinotIntegrationSuite) TestScanWorkflow_PageToken() { + id := "pinot-integration-scan-workflow-token-test" + wt := "pinot-integration-scan-workflow-token-test-type" + tl := "pinot-integration-scan-workflow-token-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + Domain: s.domainName, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + numOfWorkflows := 4 + pageSize := 3 + + s.testListWorkflowHelper(numOfWorkflows, pageSize, request, id, wt, true) +} + +func (s *PinotIntegrationSuite) TestCountWorkflow() { + id := "pinot-integration-count-workflow-test" + wt := "pinot-integration-count-workflow-test-type" + tl := "pinot-integration-count-workflow-test-tasklist" + request := s.createStartWorkflowExecutionRequest(id, wt, tl) + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + searchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + request.SearchAttributes = searchAttr + + _, err := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err) + + query := fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, s.testSearchAttributeVal) + countRequest := &types.CountWorkflowExecutionsRequest{ + Domain: s.domainName, + Query: query, + } + var resp *types.CountWorkflowExecutionsResponse + for i := 0; i < numberOfRetry; i++ { + resp, err = s.engine.CountWorkflowExecutions(createContext(), countRequest) + s.Nil(err) + if resp.GetCount() == int64(1) { + break + } + time.Sleep(waitTimeInMillisecond) + } + s.Equal(int64(1), resp.GetCount()) + + query = fmt.Sprintf(`WorkflowID = "%s" and %s = "%s"`, id, s.testSearchAttributeKey, "noMatch") + countRequest.Query = query + resp, err = s.engine.CountWorkflowExecutions(createContext(), countRequest) + s.Nil(err) + s.Equal(int64(0), resp.GetCount()) +} + +func (s *PinotIntegrationSuite) TestUpsertWorkflowExecution() { + id := "pinot-integration-upsert-workflow-test" + wt := "pinot-integration-upsert-workflow-test-type" + tl := "pinot-integration-upsert-workflow-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + decisionCount := 0 + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + upsertDecision := &types.Decision{ + DecisionType: types.DecisionTypeUpsertWorkflowSearchAttributes.Ptr(), + UpsertWorkflowSearchAttributesDecisionAttributes: &types.UpsertWorkflowSearchAttributesDecisionAttributes{}} + + // handle first upsert + if decisionCount == 0 { + decisionCount++ + + attrValBytes, _ := json.Marshal(s.testSearchAttributeVal) + upsertSearchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + s.testSearchAttributeKey: attrValBytes, + }, + } + upsertDecision.UpsertWorkflowSearchAttributesDecisionAttributes.SearchAttributes = upsertSearchAttr + return nil, []*types.Decision{upsertDecision}, nil + } + // handle second upsert, which update existing field and add new field + if decisionCount == 1 { + decisionCount++ + upsertDecision.UpsertWorkflowSearchAttributesDecisionAttributes.SearchAttributes = getPinotUpsertSearchAttributes() + return nil, []*types.Decision{upsertDecision}, nil + } + + return nil, []*types.Decision{{ + DecisionType: types.DecisionTypeCompleteWorkflowExecution.Ptr(), + CompleteWorkflowExecutionDecisionAttributes: &types.CompleteWorkflowExecutionDecisionAttributes{ + Result: []byte("Done."), + }, + }}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + StickyTaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + // process 1st decision and assert decision is handled correctly. + _, newTask, err := poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( + false, + false, + true, + true, + int64(0), + 1, + true, + nil) + s.Nil(err) + s.NotNil(newTask) + s.NotNil(newTask.DecisionTask) + s.Equal(int64(3), newTask.DecisionTask.GetPreviousStartedEventID()) + s.Equal(int64(7), newTask.DecisionTask.GetStartedEventID()) + s.Equal(4, len(newTask.DecisionTask.History.Events)) + s.Equal(types.EventTypeDecisionTaskCompleted, newTask.DecisionTask.History.Events[0].GetEventType()) + s.Equal(types.EventTypeUpsertWorkflowSearchAttributes, newTask.DecisionTask.History.Events[1].GetEventType()) + s.Equal(types.EventTypeDecisionTaskScheduled, newTask.DecisionTask.History.Events[2].GetEventType()) + s.Equal(types.EventTypeDecisionTaskStarted, newTask.DecisionTask.History.Events[3].GetEventType()) + + time.Sleep(waitForPinotToSettle) + + // verify upsert data is on Pinot + listRequest := &types.ListWorkflowExecutionsRequest{ + Domain: s.domainName, + PageSize: int32(2), + //Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wt), + Query: fmt.Sprintf(`WorkflowType = '%s' and CloseTime = missing`, wt), + } + verified := false + for i := 0; i < numberOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + if len(resp.GetExecutions()) == 1 { + execution := resp.GetExecutions()[0] + retrievedSearchAttr := execution.SearchAttributes + if retrievedSearchAttr != nil && len(retrievedSearchAttr.GetIndexedFields()) > 0 { + searchValBytes := retrievedSearchAttr.GetIndexedFields()[s.testSearchAttributeKey] + var searchVal string + json.Unmarshal(searchValBytes, &searchVal) + s.Equal(s.testSearchAttributeVal, searchVal) + verified = true + break + } + } + time.Sleep(waitTimeInMillisecond) + } + s.True(verified) + + // process 2nd decision and assert decision is handled correctly. + _, newTask, err = poller.PollAndProcessDecisionTaskWithAttemptAndRetryAndForceNewDecision( + false, + false, + true, + true, + int64(0), + 1, + true, + nil) + s.Nil(err) + s.NotNil(newTask) + s.NotNil(newTask.DecisionTask) + s.Equal(4, len(newTask.DecisionTask.History.Events)) + s.Equal(types.EventTypeDecisionTaskCompleted, newTask.DecisionTask.History.Events[0].GetEventType()) + s.Equal(types.EventTypeUpsertWorkflowSearchAttributes, newTask.DecisionTask.History.Events[1].GetEventType()) + s.Equal(types.EventTypeDecisionTaskScheduled, newTask.DecisionTask.History.Events[2].GetEventType()) + s.Equal(types.EventTypeDecisionTaskStarted, newTask.DecisionTask.History.Events[3].GetEventType()) + + time.Sleep(waitForPinotToSettle) + + // verify upsert data is on Pinot + s.testListResultForUpsertSearchAttributes(listRequest) +} + +func (s *PinotIntegrationSuite) testListResultForUpsertSearchAttributes(listRequest *types.ListWorkflowExecutionsRequest) { + verified := false + for i := 0; i < numberOfRetry; i++ { + resp, err := s.engine.ListWorkflowExecutions(createContext(), listRequest) + s.Nil(err) + + //res2B, _ := json.Marshal(resp.GetExecutions()) + //panic(fmt.Sprintf("ABCDDDBUG: %s", listRequest.Query)) + + if len(resp.GetExecutions()) == 1 { + execution := resp.GetExecutions()[0] + retrievedSearchAttr := execution.SearchAttributes + if retrievedSearchAttr != nil && len(retrievedSearchAttr.GetIndexedFields()) == 3 { + //if retrievedSearchAttr != nil && len(retrievedSearchAttr.GetIndexedFields()) > 0 { + fields := retrievedSearchAttr.GetIndexedFields() + searchValBytes := fields[s.testSearchAttributeKey] + var searchVal string + err := json.Unmarshal(searchValBytes, &searchVal) + s.Nil(err) + s.Equal("another string", searchVal) + + searchValBytes2 := fields[definition.CustomIntField] + var searchVal2 int + err = json.Unmarshal(searchValBytes2, &searchVal2) + s.Nil(err) + s.Equal(123, searchVal2) + + binaryChecksumsBytes := fields[definition.BinaryChecksums] + var binaryChecksums []string + err = json.Unmarshal(binaryChecksumsBytes, &binaryChecksums) + s.Nil(err) + s.Equal([]string{"binary-v1", "binary-v2"}, binaryChecksums) + + verified = true + break + } + } + time.Sleep(waitTimeInMillisecond) + } + s.True(verified) +} + +func getPinotUpsertSearchAttributes() *types.SearchAttributes { + attrValBytes1, _ := json.Marshal("another string") + attrValBytes2, _ := json.Marshal(123) + binaryChecksums, _ := json.Marshal([]string{"binary-v1", "binary-v2"}) + upsertSearchAttr := &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + definition.CustomStringField: attrValBytes1, + definition.CustomIntField: attrValBytes2, + definition.BinaryChecksums: binaryChecksums, + }, + } + return upsertSearchAttr +} + +func (s *PinotIntegrationSuite) TestUpsertWorkflowExecution_InvalidKey() { + id := "pinot-integration-upsert-workflow-failed-test" + wt := "pinot-integration-upsert-workflow-failed-test-type" + tl := "pinot-integration-upsert-workflow-failed-test-tasklist" + identity := "worker1" + + workflowType := &types.WorkflowType{} + workflowType.Name = wt + + taskList := &types.TaskList{} + taskList.Name = tl + + request := &types.StartWorkflowExecutionRequest{ + RequestID: uuid.New(), + Domain: s.domainName, + WorkflowID: id, + WorkflowType: workflowType, + TaskList: taskList, + Input: nil, + ExecutionStartToCloseTimeoutSeconds: common.Int32Ptr(100), + TaskStartToCloseTimeoutSeconds: common.Int32Ptr(1), + Identity: identity, + } + + we, err0 := s.engine.StartWorkflowExecution(createContext(), request) + s.Nil(err0) + + s.Logger.Info("StartWorkflowExecution", tag.WorkflowRunID(we.RunID)) + + dtHandler := func(execution *types.WorkflowExecution, wt *types.WorkflowType, + previousStartedEventID, startedEventID int64, history *types.History) ([]byte, []*types.Decision, error) { + + upsertDecision := &types.Decision{ + DecisionType: types.DecisionTypeUpsertWorkflowSearchAttributes.Ptr(), + UpsertWorkflowSearchAttributesDecisionAttributes: &types.UpsertWorkflowSearchAttributesDecisionAttributes{ + SearchAttributes: &types.SearchAttributes{ + IndexedFields: map[string][]byte{ + "INVALIDKEY": []byte(`1`), + }, + }, + }} + return nil, []*types.Decision{upsertDecision}, nil + } + + poller := &TaskPoller{ + Engine: s.engine, + Domain: s.domainName, + TaskList: taskList, + StickyTaskList: taskList, + Identity: identity, + DecisionHandler: dtHandler, + Logger: s.Logger, + T: s.T(), + } + + _, err := poller.PollAndProcessDecisionTask(false, false) + s.Nil(err) + + historyResponse, err := s.engine.GetWorkflowExecutionHistory(createContext(), &types.GetWorkflowExecutionHistoryRequest{ + Domain: s.domainName, + Execution: &types.WorkflowExecution{ + WorkflowID: id, + RunID: we.RunID, + }, + }) + s.Nil(err) + history := historyResponse.History + decisionFailedEvent := history.GetEvents()[3] + s.Equal(types.EventTypeDecisionTaskFailed, decisionFailedEvent.GetEventType()) + failedDecisionAttr := decisionFailedEvent.DecisionTaskFailedEventAttributes + s.Equal(types.DecisionTaskFailedCauseBadSearchAttributes, failedDecisionAttr.GetCause()) + s.True(len(failedDecisionAttr.GetDetails()) > 0) +} diff --git a/host/pinotutils/pinotClient.go b/host/pinotutils/pinotClient.go new file mode 100644 index 00000000000..7035144625d --- /dev/null +++ b/host/pinotutils/pinotClient.go @@ -0,0 +1,38 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package pinotutils + +import ( + "github.com/startreedata/pinot-client-go/pinot" + "github.com/stretchr/testify/suite" + + "github.com/uber/cadence/common/config" + + "github.com/uber/cadence/common/log" + pnt "github.com/uber/cadence/common/pinot" +) + +func CreatePinotClient(s suite.Suite, pinotConfig *config.PinotVisibilityConfig, logger log.Logger) pnt.GenericClient { + pinotRawClient, err := pinot.NewFromBrokerList([]string{pinotConfig.Broker}) + s.Require().NoError(err) + pinotClient := pnt.NewPinotClient(pinotRawClient, logger, pinotConfig) + return pinotClient +} diff --git a/host/testcluster.go b/host/testcluster.go index 8415a211a4d..258fcfa04a2 100644 --- a/host/testcluster.go +++ b/host/testcluster.go @@ -52,12 +52,14 @@ import ( "github.com/uber/cadence/common/persistence/persistence-tests/testcluster" "github.com/uber/cadence/testflags" + "github.com/startreedata/pinot-client-go/pinot" // the import is a test dependency _ "github.com/uber/cadence/common/persistence/nosql/nosqlplugin/cassandra/gocql/public" persistencetests "github.com/uber/cadence/common/persistence/persistence-tests" "github.com/uber/cadence/common/persistence/sql" "github.com/uber/cadence/common/persistence/sql/sqlplugin/mysql" "github.com/uber/cadence/common/persistence/sql/sqlplugin/postgres" + pnt "github.com/uber/cadence/common/pinot" ) type ( @@ -91,6 +93,7 @@ type ( ESConfig *config.ElasticSearchConfig WorkerConfig *WorkerConfig MockAdminClient map[string]adminClient.Client + PinotConfig *config.PinotVisibilityConfig } // MessagingClientConfig is the config for messaging config @@ -168,6 +171,70 @@ func NewCluster(options *TestClusterConfig, logger log.Logger, params persistenc return &TestCluster{testBase: testBase, archiverBase: archiverBase, host: cluster}, nil } +func NewPinotTestCluster(options *TestClusterConfig, logger log.Logger, params persistencetests.TestBaseParams) (*TestCluster, error) { + testBase := persistencetests.NewTestBaseFromParams(params) + testBase.Setup() + setupShards(testBase, options.HistoryConfig.NumHistoryShards, logger) + archiverBase := newArchiverBase(options.EnableArchival, logger) + messagingClient := getMessagingClient(options.MessagingClientConfig, logger) + pConfig := testBase.Config() + pConfig.NumHistoryShards = options.HistoryConfig.NumHistoryShards + var esClient elasticsearch.GenericClient + var pinotClient pnt.GenericClient + if options.WorkerConfig.EnableIndexer { + var err error + esClient, err = elasticsearch.NewGenericClient(options.ESConfig, logger) + if err != nil { + return nil, err + } + pConfig.AdvancedVisibilityStore = "pinot-visibility" + pinotBroker := options.PinotConfig.Broker + pinotRawClient, err := pinot.NewFromBrokerList([]string{pinotBroker}) + if err != nil || pinotRawClient == nil { + return nil, err + } + pinotClient = pnt.NewPinotClient(pinotRawClient, logger, options.PinotConfig) + } + + scope := tally.NewTestScope("integration-test", nil) + metricsClient := metrics.NewClient(scope, metrics.ServiceIdx(0)) + domainReplicationQueue := domain.NewReplicationQueue( + testBase.DomainReplicationQueueMgr, + options.ClusterGroupMetadata.CurrentClusterName, + metricsClient, + logger, + ) + aConfig := noopAuthorizationConfig() + cadenceParams := &CadenceParams{ + ClusterMetadata: params.ClusterMetadata, + PersistenceConfig: pConfig, + MessagingClient: messagingClient, + DomainManager: testBase.DomainManager, + HistoryV2Mgr: testBase.HistoryV2Mgr, + ExecutionMgrFactory: testBase.ExecutionMgrFactory, + DomainReplicationQueue: domainReplicationQueue, + Logger: logger, + ClusterNo: options.ClusterNo, + ESConfig: options.ESConfig, + ESClient: esClient, + ArchiverMetadata: archiverBase.metadata, + ArchiverProvider: archiverBase.provider, + HistoryConfig: options.HistoryConfig, + WorkerConfig: options.WorkerConfig, + MockAdminClient: options.MockAdminClient, + DomainReplicationTaskExecutor: domain.NewReplicationTaskExecutor(testBase.DomainManager, clock.NewRealTimeSource(), logger), + AuthorizationConfig: aConfig, + PinotConfig: options.PinotConfig, + PinotClient: pinotClient, + } + cluster := NewCadence(cadenceParams) + if err := cluster.Start(); err != nil { + return nil, err + } + + return &TestCluster{testBase: testBase, archiverBase: archiverBase, host: cluster}, nil +} + func noopAuthorizationConfig() config.Authorization { return config.Authorization{ OAuthAuthorizer: config.OAuthAuthorizer{ diff --git a/host/testdata/integration_pinot_cluster.yaml b/host/testdata/integration_pinot_cluster.yaml new file mode 100644 index 00000000000..be14f1a3f99 --- /dev/null +++ b/host/testdata/integration_pinot_cluster.yaml @@ -0,0 +1,42 @@ +enablearchival: false +clusterno: 1 +messagingclientconfig: + usemock: false + kafkaconfig: + clusters: + test: + brokers: + - "${KAFKA_SEEDS}:9092" + topics: + test-visibility-topic: + cluster: test + test-visibility-topic-dlq: + cluster: test + cadence-visibility-pinot: + cluster: test + cadence-visibility-pinot-dlq: + cluster: test + applications: + pinot-visibility: + topic: cadence-visibility-pinot + dlq-topic: cadence-visibility-pinot-dlq + visibility: + topic: test-visibility-topic + dlq-topic: test-visibility-topic-dlq +historyconfig: + numhistoryshards: 4 + numhistoryhosts: 1 +workerconfig: + enablearchiver: false + enablereplicator: false + enableindexer: true +esconfig: + url: + scheme: "http" + host: "${ES_SEEDS}:9200" + indices: + visibility: test-visibility- +pinotconfig: + broker: "localhost:8099" + cluster: pinot-test + table: "cadence_visibility_pinot" \ No newline at end of file diff --git a/schema/Pinot/README.md b/schema/pinot/README.md similarity index 100% rename from schema/Pinot/README.md rename to schema/pinot/README.md diff --git a/schema/Pinot/cadence-visibility-config.json b/schema/pinot/cadence-visibility-config.json similarity index 78% rename from schema/Pinot/cadence-visibility-config.json rename to schema/pinot/cadence-visibility-config.json index 4d1634f1b1d..b78d59e152b 100644 --- a/schema/Pinot/cadence-visibility-config.json +++ b/schema/pinot/cadence-visibility-config.json @@ -1,14 +1,23 @@ { - "tableName": "cadence-visibility-pinot", + "tableName": "cadence_visibility_pinot", "tableType": "REALTIME", "segmentsConfig": { "timeColumnName": "StartTime", "timeType": "MILLISECONDS", - "schemaName": "cadence-visibility-pinot", + "schemaName": "cadence_visibility_pinot", "replicasPerPartition": "1" }, "tenants": {}, "tableIndexConfig": { + "jsonIndexConfigs": { + "Attr": { + "excludeArray": false, + "disableCrossArrayUnnest": true, + "includePaths": null, + "excludePaths": null, + "excludeFields": null + } + }, "loadMode": "MMAP", "streamConfigs": { "streamType": "kafka", @@ -32,4 +41,5 @@ "mode": "FULL" }, "metadata": {} -} \ No newline at end of file +} + diff --git a/schema/Pinot/cadence-visibility-schema.json b/schema/pinot/cadence-visibility-schema.json similarity index 79% rename from schema/Pinot/cadence-visibility-schema.json rename to schema/pinot/cadence-visibility-schema.json index 60b5bc6bb61..7b84385525f 100644 --- a/schema/Pinot/cadence-visibility-schema.json +++ b/schema/pinot/cadence-visibility-schema.json @@ -1,11 +1,7 @@ { - "schemaName": "cadence-visibility-pinot", - "primaryKeyColumns": ["DocID"], + "schemaName": "cadence_visibility_pinot", + "primaryKeyColumns": ["RunID"], "dimensionFieldSpecs": [ - { - "name": "DocID", - "dataType": "STRING" - }, { "name": "DomainID", "dataType": "STRING" @@ -30,10 +26,6 @@ "name": "HistoryLength", "dataType": "INT" }, - { - "name": "KafkaKey", - "dataType": "STRING" - }, { "name": "TaskList", "dataType": "STRING" @@ -43,7 +35,7 @@ "dataType": "BOOLEAN" }, { - "name": "NumCluster", + "name": "NumClusters", "dataType": "INT" }, { @@ -52,8 +44,13 @@ }, { "name": "Attr", - "dataType": "STRING" + "dataType": "JSON" + }, + { + "name": "IsDeleted", + "dataType": "BOOLEAN" } + ], "dateTimeFieldSpecs": [{ "name": "StartTime", @@ -75,5 +72,10 @@ "dataType": "LONG", "format" : "1:MILLISECONDS:EPOCH", "granularity": "1:MILLISECONDS" + },{ + "name": "SecondsSinceEpoch", + "dataType": "LONG", + "format" : "1:MILLISECONDS:EPOCH", + "granularity": "1:MILLISECONDS" }] } \ No newline at end of file diff --git a/service/frontend/service.go b/service/frontend/service.go index f31dbe0f68a..28fb2f1cd9d 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -45,8 +45,9 @@ type Config struct { EnableVisibilitySampling dynamicconfig.BoolPropertyFn EnableReadFromClosedExecutionV2 dynamicconfig.BoolPropertyFn // deprecated: never used for ratelimiting, only sampling-based failure injection, and only on database-based visibility - VisibilityListMaxQPS dynamicconfig.IntPropertyFnWithDomainFilter - EnableReadVisibilityFromES dynamicconfig.BoolPropertyFnWithDomainFilter + VisibilityListMaxQPS dynamicconfig.IntPropertyFnWithDomainFilter + EnableReadVisibilityFromES dynamicconfig.BoolPropertyFnWithDomainFilter + EnableReadVisibilityFromPinot dynamicconfig.BoolPropertyFnWithDomainFilter // deprecated: never read from ESVisibilityListMaxQPS dynamicconfig.IntPropertyFnWithDomainFilter ESIndexMaxResultWindow dynamicconfig.IntPropertyFn @@ -135,6 +136,7 @@ func NewConfig(dc *dynamicconfig.Collection, numHistoryShards int, isAdvancedVis VisibilityListMaxQPS: dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendVisibilityListMaxQPS), ESVisibilityListMaxQPS: dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendESVisibilityListMaxQPS), EnableReadVisibilityFromES: dc.GetBoolPropertyFilteredByDomain(dynamicconfig.EnableReadVisibilityFromES), + EnableReadVisibilityFromPinot: dc.GetBoolPropertyFilteredByDomain(dynamicconfig.EnableReadVisibilityFromPinot), ESIndexMaxResultWindow: dc.GetIntProperty(dynamicconfig.FrontendESIndexMaxResultWindow), HistoryMaxPageSize: dc.GetIntPropertyFilteredByDomain(dynamicconfig.FrontendHistoryMaxPageSize), UserRPS: dc.GetIntProperty(dynamicconfig.FrontendUserRPS), @@ -241,6 +243,7 @@ func NewService( EnableReadVisibilityFromES: serviceConfig.EnableReadVisibilityFromES, AdvancedVisibilityWritingMode: nil, // frontend service never write + EnableReadVisibilityFromPinot: serviceConfig.EnableReadVisibilityFromPinot, EnableDBVisibilitySampling: serviceConfig.EnableVisibilitySampling, EnableReadDBVisibilityFromClosedExecutionV2: serviceConfig.EnableReadFromClosedExecutionV2, diff --git a/service/history/resource/resource.go b/service/history/resource/resource.go index 0e0c71849f6..0efc8bf1619 100644 --- a/service/history/resource/resource.go +++ b/service/history/resource/resource.go @@ -95,6 +95,7 @@ func New( EnableReadVisibilityFromES: nil, // history service never read, AdvancedVisibilityWritingMode: config.AdvancedVisibilityWritingMode, + EnableReadVisibilityFromPinot: nil, // history service never read, EnableDBVisibilitySampling: config.EnableVisibilitySampling, EnableReadDBVisibilityFromClosedExecutionV2: nil, // history service never read, @@ -102,9 +103,9 @@ func New( WriteDBVisibilityOpenMaxQPS: config.VisibilityOpenMaxQPS, WriteDBVisibilityClosedMaxQPS: config.VisibilityClosedMaxQPS, - ESVisibilityListMaxQPS: nil, // history service never read, - ESIndexMaxResultWindow: nil, // history service never read, - ValidSearchAttributes: nil, // history service never read, + ESVisibilityListMaxQPS: nil, // history service never read, + ESIndexMaxResultWindow: nil, // history service never read, + ValidSearchAttributes: config.ValidSearchAttributes, // history service never read, (Pinot need this to initialize pinotQueryValidator) }, ) if err != nil {