From ccdef32314d37eb1bc4abfcebad157223f29c965 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 1 Jul 2024 13:43:52 +0200 Subject: [PATCH 1/9] Support providing configuration from YAML files --- proxy/launch.go | 3 +- proxy/pkg/config/config.go | 174 +++++++++++++-------- proxy/pkg/config/config_dual_reads_test.go | 2 +- proxy/pkg/config/config_test.go | 92 ++++++++++- proxy/pkg/config/config_tls_test.go | 4 +- proxy/pkg/config/configtestutils.go | 23 ++- 6 files changed, 219 insertions(+), 79 deletions(-) diff --git a/proxy/launch.go b/proxy/launch.go index 593b1c2a..a4a359bd 100644 --- a/proxy/launch.go +++ b/proxy/launch.go @@ -24,7 +24,8 @@ func runSignalListener(cancelFunc context.CancelFunc) { } func launchProxy(profilingSupported bool) { - conf, err := config.New().ParseEnvVars() + conf, err := config.New().LoadConfig() + if err != nil { log.Errorf("Error loading configuration: %v. Aborting startup.", err) os.Exit(-1) diff --git a/proxy/pkg/config/config.go b/proxy/pkg/config/config.go index d5cc5c67..0ee7b466 100644 --- a/proxy/pkg/config/config.go +++ b/proxy/pkg/config/config.go @@ -6,7 +6,9 @@ import ( "github.com/datastax/zdm-proxy/proxy/pkg/common" "github.com/kelseyhightower/envconfig" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" "net" + "os" "strconv" "strings" ) @@ -16,111 +18,111 @@ type Config struct { // Global bucket - PrimaryCluster string `default:"ORIGIN" split_words:"true"` - ReadMode string `default:"PRIMARY_ONLY" split_words:"true"` - ReplaceCqlFunctions bool `default:"false" split_words:"true"` - AsyncHandshakeTimeoutMs int `default:"4000" split_words:"true"` - LogLevel string `default:"INFO" split_words:"true"` + PrimaryCluster string `default:"ORIGIN" split_words:"true" yaml:"primary_cluster"` + ReadMode string `default:"PRIMARY_ONLY" split_words:"true" yaml:"read_mode"` + ReplaceCqlFunctions bool `default:"false" split_words:"true" yaml:"replace_cql_functions"` + AsyncHandshakeTimeoutMs int `default:"4000" split_words:"true" yaml:"async_handshake_timeout_ms"` + LogLevel string `default:"INFO" split_words:"true" yaml:"log_level"` // Proxy Topology (also known as system.peers "virtualization") bucket - ProxyTopologyIndex int `default:"0" split_words:"true"` - ProxyTopologyAddresses string `split_words:"true"` - ProxyTopologyNumTokens int `default:"8" split_words:"true"` + ProxyTopologyIndex int `default:"0" split_words:"true" yaml:"proxy_topology_index"` + ProxyTopologyAddresses string `split_words:"true" yaml:"proxy_topology_addresses"` + ProxyTopologyNumTokens int `default:"8" split_words:"true" yaml:"proxy_topology_num_tokens"` // Origin bucket - OriginContactPoints string `split_words:"true"` - OriginPort int `default:"9042" split_words:"true"` - OriginSecureConnectBundlePath string `split_words:"true"` - OriginLocalDatacenter string `split_words:"true"` - OriginUsername string `required:"true" split_words:"true"` - OriginPassword string `required:"true" split_words:"true" json:"-"` - OriginConnectionTimeoutMs int `default:"30000" split_words:"true"` + OriginContactPoints string `split_words:"true" yaml:"origin_contact_points"` + OriginPort int `default:"9042" split_words:"true" yaml:"origin_port"` + OriginSecureConnectBundlePath string `split_words:"true" yaml:"origin_secure_connect_bundle_path"` + OriginLocalDatacenter string `split_words:"true" yaml:"origin_local_datacenter"` + OriginUsername string `split_words:"true" yaml:"origin_username"` + OriginPassword string `split_words:"true" json:"-" yaml:"origin_password"` + OriginConnectionTimeoutMs int `default:"30000" split_words:"true" yaml:"origin_connection_timeout_ms"` - OriginTlsServerCaPath string `split_words:"true"` - OriginTlsClientCertPath string `split_words:"true"` - OriginTlsClientKeyPath string `split_words:"true"` + OriginTlsServerCaPath string `split_words:"true" yaml:"origin_tls_server_ca_path"` + OriginTlsClientCertPath string `split_words:"true" yaml:"origin_tls_client_cert_path"` + OriginTlsClientKeyPath string `split_words:"true" yaml:"origin_tls_client_key_path"` // Target bucket - TargetContactPoints string `split_words:"true"` - TargetPort int `default:"9042" split_words:"true"` - TargetSecureConnectBundlePath string `split_words:"true"` - TargetLocalDatacenter string `split_words:"true"` - TargetUsername string `required:"true" split_words:"true"` - TargetPassword string `required:"true" split_words:"true" json:"-"` - TargetConnectionTimeoutMs int `default:"30000" split_words:"true"` + TargetContactPoints string `split_words:"true" yaml:"target_contact_points"` + TargetPort int `default:"9042" split_words:"true" yaml:"target_port"` + TargetSecureConnectBundlePath string `split_words:"true" yaml:"target_secure_connect_bundle_path"` + TargetLocalDatacenter string `split_words:"true" yaml:"target_local_datacenter"` + TargetUsername string `split_words:"true" yaml:"target_username"` + TargetPassword string `split_words:"true" json:"-" yaml:"target_password"` + TargetConnectionTimeoutMs int `default:"30000" split_words:"true" yaml:"target_connection_timeout_ms"` - TargetTlsServerCaPath string `split_words:"true"` - TargetTlsClientCertPath string `split_words:"true"` - TargetTlsClientKeyPath string `split_words:"true"` + TargetTlsServerCaPath string `split_words:"true" yaml:"target_tls_server_ca_path"` + TargetTlsClientCertPath string `split_words:"true" yaml:"target_tls_client_cert_path"` + TargetTlsClientKeyPath string `split_words:"true" yaml:"target_tls_client_key_path"` // Proxy bucket - ProxyListenAddress string `default:"localhost" split_words:"true"` - ProxyListenPort int `default:"14002" split_words:"true"` - ProxyRequestTimeoutMs int `default:"10000" split_words:"true"` - ProxyMaxClientConnections int `default:"1000" split_words:"true"` - ProxyMaxStreamIds int `default:"2048" split_words:"true"` + ProxyListenAddress string `default:"localhost" split_words:"true" yaml:"proxy_listen_address"` + ProxyListenPort int `default:"14002" split_words:"true" yaml:"proxy_listen_port"` + ProxyRequestTimeoutMs int `default:"10000" split_words:"true" yaml:"proxy_request_timeout_ms"` + ProxyMaxClientConnections int `default:"1000" split_words:"true" yaml:"proxy_max_client_connections"` + ProxyMaxStreamIds int `default:"2048" split_words:"true" yaml:"proxy_max_stream_ids"` - ProxyTlsCaPath string `split_words:"true"` - ProxyTlsCertPath string `split_words:"true"` - ProxyTlsKeyPath string `split_words:"true"` - ProxyTlsRequireClientAuth bool `split_words:"true"` + ProxyTlsCaPath string `split_words:"true" yaml:"proxy_tls_ca_path"` + ProxyTlsCertPath string `split_words:"true" yaml:"proxy_tls_cert_path"` + ProxyTlsKeyPath string `split_words:"true" yaml:"proxy_tls_key_path"` + ProxyTlsRequireClientAuth bool `split_words:"true" yaml:"proxy_tls_require_client_auth"` // Metrics bucket - MetricsEnabled bool `default:"true" split_words:"true"` - MetricsAddress string `default:"localhost" split_words:"true"` - MetricsPort int `default:"14001" split_words:"true"` - MetricsPrefix string `default:"zdm" split_words:"true"` + MetricsEnabled bool `default:"true" split_words:"true" yaml:"metrics_enabled"` + MetricsAddress string `default:"localhost" split_words:"true" yaml:"metrics_address"` + MetricsPort int `default:"14001" split_words:"true" yaml:"metrics_port"` + MetricsPrefix string `default:"zdm" split_words:"true" yaml:"metrics_prefix"` - MetricsOriginLatencyBucketsMs string `default:"1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000" split_words:"true"` - MetricsTargetLatencyBucketsMs string `default:"1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000" split_words:"true"` - MetricsAsyncReadLatencyBucketsMs string `default:"1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000" split_words:"true"` + MetricsOriginLatencyBucketsMs string `default:"1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000" split_words:"true" yaml:"metrics_origin_latency_buckets_ms"` + MetricsTargetLatencyBucketsMs string `default:"1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000" split_words:"true" yaml:"metrics_target_latency_buckets_ms"` + MetricsAsyncReadLatencyBucketsMs string `default:"1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000" split_words:"true" yaml:"metrics_async_read_latency_buckets_ms"` // Heartbeat bucket - HeartbeatIntervalMs int `default:"30000" split_words:"true"` + HeartbeatIntervalMs int `default:"30000" split_words:"true" yaml:"heartbeat_interval_ms"` - HeartbeatRetryIntervalMinMs int `default:"250" split_words:"true"` - HeartbeatRetryIntervalMaxMs int `default:"30000" split_words:"true"` - HeartbeatRetryBackoffFactor float64 `default:"2" split_words:"true"` - HeartbeatFailureThreshold int `default:"1" split_words:"true"` + HeartbeatRetryIntervalMinMs int `default:"250" split_words:"true" yaml:"heartbeat_retry_interval_min_ms"` + HeartbeatRetryIntervalMaxMs int `default:"30000" split_words:"true" yaml:"heartbeat_retry_interval_max_ms"` + HeartbeatRetryBackoffFactor float64 `default:"2" split_words:"true" yaml:"heartbeat_retry_backoff_factor"` + HeartbeatFailureThreshold int `default:"1" split_words:"true" yaml:"heartbeat_failure_threshold"` ////////////////////////////////////////////////////////////////////// /// THE SETTINGS BELOW AREN'T SUPPORTED AND MAY CHANGE AT ANY TIME /// ////////////////////////////////////////////////////////////////////// - SystemQueriesMode string `default:"ORIGIN" split_words:"true"` + SystemQueriesMode string `default:"ORIGIN" split_words:"true" yaml:"system_queries_mode"` - ForwardClientCredentialsToOrigin bool `default:"false" split_words:"true"` // only takes effect if both clusters have auth enabled + ForwardClientCredentialsToOrigin bool `default:"false" split_words:"true" yaml:"forward_client_credentials_to_origin"` // only takes effect if both clusters have auth enabled - OriginEnableHostAssignment bool `default:"true" split_words:"true"` - TargetEnableHostAssignment bool `default:"true" split_words:"true"` + OriginEnableHostAssignment bool `default:"true" split_words:"true" yaml:"origin_enable_host_assignment"` + TargetEnableHostAssignment bool `default:"true" split_words:"true" yaml:"target_enable_host_assignment"` ////////////////////////////////////////////////////////////////////////////////////////////////////////// /// THE SETTINGS BELOW ARE FOR PERFORMANCE TUNING; THEY AREN'T SUPPORTED AND MAY CHANGE AT ANY TIME ////// ////////////////////////////////////////////////////////////////////////////////////////////////////////// - RequestWriteQueueSizeFrames int `default:"128" split_words:"true"` - RequestWriteBufferSizeBytes int `default:"4096" split_words:"true"` - RequestReadBufferSizeBytes int `default:"32768" split_words:"true"` + RequestWriteQueueSizeFrames int `default:"128" split_words:"true" yaml:"request_write_queue_size_frames"` + RequestWriteBufferSizeBytes int `default:"4096" split_words:"true" yaml:"request_write_buffer_size_bytes"` + RequestReadBufferSizeBytes int `default:"32768" split_words:"true" yaml:"request_read_buffer_size_bytes"` - ResponseWriteQueueSizeFrames int `default:"128" split_words:"true"` - ResponseWriteBufferSizeBytes int `default:"8192" split_words:"true"` - ResponseReadBufferSizeBytes int `default:"32768" split_words:"true"` + ResponseWriteQueueSizeFrames int `default:"128" split_words:"true" yaml:"response_write_queue_size_frames"` + ResponseWriteBufferSizeBytes int `default:"8192" split_words:"true" yaml:"response_write_buffer_size_bytes"` + ResponseReadBufferSizeBytes int `default:"32768" split_words:"true" yaml:"response_read_buffer_size_bytes"` - RequestResponseMaxWorkers int `default:"-1" split_words:"true"` - WriteMaxWorkers int `default:"-1" split_words:"true"` - ReadMaxWorkers int `default:"-1" split_words:"true"` - ListenerMaxWorkers int `default:"-1" split_words:"true"` + RequestResponseMaxWorkers int `default:"-1" split_words:"true" yaml:"request_response_max_workers"` + WriteMaxWorkers int `default:"-1" split_words:"true" yaml:"write_max_workers"` + ReadMaxWorkers int `default:"-1" split_words:"true" yaml:"read_max_workers"` + ListenerMaxWorkers int `default:"-1" split_words:"true" yaml:"listener_max_workers"` - EventQueueSizeFrames int `default:"12" split_words:"true"` + EventQueueSizeFrames int `default:"12" split_words:"true" yaml:"event_queue_size_frames"` - AsyncConnectorWriteQueueSizeFrames int `default:"2048" split_words:"true"` - AsyncConnectorWriteBufferSizeBytes int `default:"4096" split_words:"true"` + AsyncConnectorWriteQueueSizeFrames int `default:"2048" split_words:"true" yaml:"async_connector_write_queue_size_frames"` + AsyncConnectorWriteBufferSizeBytes int `default:"4096" split_words:"true" yaml:"async_connector_write_buffer_size_bytes"` } func (c *Config) String() string { @@ -133,12 +135,46 @@ func New() *Config { return &Config{} } +func (c *Config) loadFromFiles() error { + paths := os.Getenv("ZDM_CONFIG_FILES") + for _, path := range strings.Split(paths, ",") { + if len(path) == 0 { + continue + } + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("could not read configuration file %v: %w", path, err) + } + defer file.Close() + + dec := yaml.NewDecoder(file) + if err = dec.Decode(c); err != nil { + return fmt.Errorf("could not parse yaml file %v: %w", path, err) + } + } + return nil +} + // ParseEnvVars fills out the fields of the Config struct according to envconfig rules // See: Usage @ https://github.com/kelseyhightower/envconfig -func (c *Config) ParseEnvVars() (*Config, error) { +func (c *Config) parseEnvVars() error { err := envconfig.Process("ZDM", c) if err != nil { - return nil, fmt.Errorf("could not load environment variables: %w", err) + return fmt.Errorf("could not load environment variables: %w", err) + } + + return nil +} + +func (c *Config) LoadConfig() (*Config, error) { + err := c.parseEnvVars() + if err != nil { + return nil, err + } + + err = c.loadFromFiles() + if err != nil { + return nil, err } err = c.Validate() @@ -148,7 +184,7 @@ func (c *Config) ParseEnvVars() (*Config, error) { log.Infof("Parsed configuration: %v", c) - return c, nil + return c, err } func lookupFirstIp4(host string) (net.IP, error) { diff --git a/proxy/pkg/config/config_dual_reads_test.go b/proxy/pkg/config/config_dual_reads_test.go index 96a1df06..f2202739 100644 --- a/proxy/pkg/config/config_dual_reads_test.go +++ b/proxy/pkg/config/config_dual_reads_test.go @@ -62,7 +62,7 @@ func TestConfig_ParseReadMode(t *testing.T) { setOriginContactPointsAndPortEnvVars() setTargetContactPointsAndPortEnvVars() - conf, err := New().ParseEnvVars() + conf, err := New().LoadConfig() if err != nil { if tt.errExpected { require.Equal(t, tt.errMsg, err.Error()) diff --git a/proxy/pkg/config/config_test.go b/proxy/pkg/config/config_test.go index 5265131b..3afb6bf0 100644 --- a/proxy/pkg/config/config_test.go +++ b/proxy/pkg/config/config_test.go @@ -17,7 +17,7 @@ func TestTargetConfig_WithBundleOnly(t *testing.T) { // test-specific setup setEnvVar("ZDM_TARGET_SECURE_CONNECT_BUNDLE_PATH", "/path/to/target/bundle") - conf, err := New().ParseEnvVars() + conf, err := New().LoadConfig() require.Nil(t, err) require.Equal(t, conf.TargetSecureConnectBundlePath, "/path/to/target/bundle") require.Empty(t, conf.TargetContactPoints) @@ -36,7 +36,7 @@ func TestTargetConfig_WithHostnameAndPortOnly(t *testing.T) { // test-specific setup setTargetContactPointsAndPortEnvVars() - conf, err := New().ParseEnvVars() + conf, err := New().LoadConfig() require.Nil(t, err) require.Equal(t, conf.TargetContactPoints, "target.hostname.com") require.Equal(t, conf.TargetPort, 5647) @@ -56,7 +56,7 @@ func TestTargetConfig_WithBundleAndHostname(t *testing.T) { setTargetContactPointsAndPortEnvVars() setTargetSecureConnectBundleEnvVar() - _, err := New().ParseEnvVars() + _, err := New().LoadConfig() require.Error(t, err, "TargetSecureConnectBundlePath and TargetContactPoints are "+ "mutually exclusive. Please specify only one of them.") } @@ -72,7 +72,7 @@ func TestTargetConfig_WithoutBundleAndHostname(t *testing.T) { // no test-specific setup in this case - _, err := New().ParseEnvVars() + _, err := New().LoadConfig() require.Error(t, err, "Both TargetSecureConnectBundlePath and TargetContactPoints are "+ "empty. Please specify either one of them.") } @@ -89,7 +89,89 @@ func TestTargetConfig_WithHostnameButWithoutPort(t *testing.T) { //test-specific setup setEnvVar("ZDM_TARGET_CONTACT_POINTS", "target.hostname.com") - c, err := New().ParseEnvVars() + c, err := New().LoadConfig() require.Nil(t, err) require.Equal(t, 9042, c.TargetPort) } + +func TestConfig_LoadNotExistingFile(t *testing.T) { + defer clearAllEnvVars() + clearAllEnvVars() + + setConfigFilesEnvVar("/not/existing/file") + + _, err := New().LoadConfig() + require.NotNil(t, err) + require.Contains(t, err.Error(), "could not read configuration file /not/existing/file") +} + +func TestConfig_LoadConfigFromFile(t *testing.T) { + defer clearAllEnvVars() + clearAllEnvVars() + + f, err := createConfigFile(` +primary_cluster: ORIGIN + +origin_username: foo1 +origin_password: bar1 +target_username: foo2 +target_password: bar2 + +origin_contact_points: 192.168.100.101 +origin_port: 19042 +target_contact_points: 192.168.100.102 +target_port: 29042 +proxy_listen_port: 39042 +`) + defer removeConfigFile(f) + require.Nil(t, err) + setConfigFilesEnvVar(f.Name()) + + c, err := New().LoadConfig() + require.Nil(t, err) + require.Equal(t, "ORIGIN", c.PrimaryCluster) + require.Equal(t, "foo1", c.OriginUsername) + require.Equal(t, "bar1", c.OriginPassword) + require.Equal(t, "foo2", c.TargetUsername) + require.Equal(t, "bar2", c.TargetPassword) + require.Equal(t, "192.168.100.101", c.OriginContactPoints) + require.Equal(t, 19042, c.OriginPort) + require.Equal(t, "192.168.100.102", c.TargetContactPoints) + require.Equal(t, 29042, c.TargetPort) + require.Equal(t, 39042, c.ProxyListenPort) +} + +func TestConfig_LoadConfigFromFileAndEnvVars(t *testing.T) { + defer clearAllEnvVars() + clearAllEnvVars() + + // publicly available information stored as environment variables + setOriginContactPointsAndPortEnvVars() + setTargetContactPointsAndPortEnvVars() + + // sensitive username and passwords stored inside two files + f1, err := createConfigFile(` +origin_username: foo1 +origin_password: bar1 +`) + defer removeConfigFile(f1) + require.Nil(t, err) + f2, err := createConfigFile(` +target_username: foo2 +target_password: bar2 +target_contact_points: secret.hostname.com +`) + defer removeConfigFile(f2) + require.Nil(t, err) + + setConfigFilesEnvVar(f1.Name(), f2.Name()) + + c, err := New().LoadConfig() + require.Nil(t, err) + require.Equal(t, "foo1", c.OriginUsername) + require.Equal(t, "bar1", c.OriginPassword) + require.Equal(t, "foo2", c.TargetUsername) + require.Equal(t, "bar2", c.TargetPassword) + require.Equal(t, "origin.hostname.com", c.OriginContactPoints) + require.Equal(t, "secret.hostname.com", c.TargetContactPoints) // file value overrides env var +} diff --git a/proxy/pkg/config/config_tls_test.go b/proxy/pkg/config/config_tls_test.go index 7092d18f..ab4d8b95 100644 --- a/proxy/pkg/config/config_tls_test.go +++ b/proxy/pkg/config/config_tls_test.go @@ -191,7 +191,7 @@ func TestOriginConfig_ClusterTlsConfig(t *testing.T) { setTargetContactPointsAndPortEnvVars() var tlsConf *common.ClusterTlsConfig - conf, err := New().ParseEnvVars() + conf, err := New().LoadConfig() if err != nil { if tt.errExpected { require.Equal(t, tt.errMsg, err.Error()) @@ -389,7 +389,7 @@ func TestTargetConfig_ClusterTlsConfig(t *testing.T) { setOriginContactPointsAndPortEnvVars() var tlsConf *common.ClusterTlsConfig - conf, err := New().ParseEnvVars() + conf, err := New().LoadConfig() if err != nil { if tt.errExpected { // Expected configuration validation error diff --git a/proxy/pkg/config/configtestutils.go b/proxy/pkg/config/configtestutils.go index d6eae0c2..d1504e69 100644 --- a/proxy/pkg/config/configtestutils.go +++ b/proxy/pkg/config/configtestutils.go @@ -1,6 +1,9 @@ package config -import "os" +import ( + "os" + "strings" +) type envVar struct { vName string @@ -35,6 +38,24 @@ func setTargetSecureConnectBundleEnvVar() { setEnvVar("ZDM_TARGET_SECURE_CONNECT_BUNDLE_PATH", "/path/to/origin/bundle") } +func setConfigFilesEnvVar(paths ...string) { + setEnvVar("ZDM_CONFIG_FILES", strings.Join(paths[:], ",")) +} + +func createConfigFile(content string) (*os.File, error) { + f, err := os.CreateTemp("", "config.*.yml") + if err == nil { + _, err = f.WriteString(content) + } + return f, err +} + +func removeConfigFile(f *os.File) { + if f != nil { + _ = os.Remove(f.Name()) + } +} + func setEnvVar(key string, value string) { os.Setenv(key, value) } From 7f47c41cc4c101e673a5f761f22134ddb1bbdaf8 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 1 Jul 2024 16:58:53 +0200 Subject: [PATCH 2/9] Do not allow to override configuration values --- go.mod | 3 ++- go.sum | 7 +++++- proxy/pkg/config/config.go | 41 ++++++++++++++++++++++++++++++++- proxy/pkg/config/config_test.go | 28 ++++++++++++++++++++-- 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index cd3ed9dc..e5cd8e7b 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,13 @@ require ( github.com/google/uuid v1.1.1 github.com/jpillora/backoff v1.0.0 github.com/kelseyhightower/envconfig v1.4.0 + github.com/mcuadros/go-defaults v1.2.0 github.com/prometheus/client_golang v1.3.0 github.com/prometheus/client_model v0.1.0 github.com/rs/zerolog v1.20.0 github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.8.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -33,5 +35,4 @@ require ( github.com/prometheus/procfs v0.0.8 // indirect golang.org/x/sys v0.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e28fd660..5a3c50fa 100644 --- a/go.sum +++ b/go.sum @@ -62,11 +62,15 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk56oZtc= +github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E= github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -127,8 +131,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/proxy/pkg/config/config.go b/proxy/pkg/config/config.go index 0ee7b466..b0dc358a 100644 --- a/proxy/pkg/config/config.go +++ b/proxy/pkg/config/config.go @@ -5,10 +5,12 @@ import ( "fmt" "github.com/datastax/zdm-proxy/proxy/pkg/common" "github.com/kelseyhightower/envconfig" + def "github.com/mcuadros/go-defaults" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" "net" "os" + "reflect" "strconv" "strings" ) @@ -125,6 +127,14 @@ type Config struct { AsyncConnectorWriteBufferSizeBytes int `default:"4096" split_words:"true" yaml:"async_connector_write_buffer_size_bytes"` } +var DefaultConfig *Config = nil + +func init() { + c := &Config{} + def.SetDefaults(c) + DefaultConfig = c +} + func (c *Config) String() string { serializedConfig, _ := json.Marshal(c) return string(serializedConfig) @@ -148,13 +158,42 @@ func (c *Config) loadFromFiles() error { defer file.Close() dec := yaml.NewDecoder(file) - if err = dec.Decode(c); err != nil { + pc := &Config{} + def.SetDefaults(pc) // apply default tag, YAML decoder does not support it + if err = dec.Decode(pc); err != nil { return fmt.Errorf("could not parse yaml file %v: %w", path, err) } + err = c.mergeConfig(pc) + if err != nil { + return err + } } return nil } +func (c *Config) mergeConfig(other *Config) error { + cRef := reflect.ValueOf(c).Elem() + otherRef := reflect.ValueOf(*other) + defaultRef := reflect.ValueOf(*DefaultConfig) + for i := 0; i < cRef.NumField(); i++ { + field := cRef.Type().Field(i).Tag.Get("yaml") + cVal := cRef.Field(i).Interface() + otherVal := otherRef.Field(i).Interface() + defaultVal := defaultRef.Field(i).Interface() + + if otherVal != defaultVal { + // other value is different from the default + if cVal != defaultVal && cVal != otherVal { + return fmt.Errorf("inconsistent values [%v] and [%v] for parameter %v", cVal, otherVal, field) + } else { + cRef.Field(i).Set(otherRef.Field(i)) + } + } + } + + return nil +} + // ParseEnvVars fills out the fields of the Config struct according to envconfig rules // See: Usage @ https://github.com/kelseyhightower/envconfig func (c *Config) parseEnvVars() error { diff --git a/proxy/pkg/config/config_test.go b/proxy/pkg/config/config_test.go index 3afb6bf0..88c1b05d 100644 --- a/proxy/pkg/config/config_test.go +++ b/proxy/pkg/config/config_test.go @@ -159,7 +159,6 @@ origin_password: bar1 f2, err := createConfigFile(` target_username: foo2 target_password: bar2 -target_contact_points: secret.hostname.com `) defer removeConfigFile(f2) require.Nil(t, err) @@ -173,5 +172,30 @@ target_contact_points: secret.hostname.com require.Equal(t, "foo2", c.TargetUsername) require.Equal(t, "bar2", c.TargetPassword) require.Equal(t, "origin.hostname.com", c.OriginContactPoints) - require.Equal(t, "secret.hostname.com", c.TargetContactPoints) // file value overrides env var + require.Equal(t, "target.hostname.com", c.TargetContactPoints) +} + +func TestConfig_FailOnDuplicateValuePresent(t *testing.T) { + defer clearAllEnvVars() + clearAllEnvVars() + + // publicly available information stored as environment variables + setOriginCredentialsEnvVars() + setTargetCredentialsEnvVars() + setOriginContactPointsAndPortEnvVars() + setTargetContactPointsAndPortEnvVars() + + // try to overriding values should raise error + f, err := createConfigFile(` +origin_username: different +origin_password: different +`) + defer removeConfigFile(f) + require.Nil(t, err) + + setConfigFilesEnvVar(f.Name()) + + _, err = New().LoadConfig() + require.NotNil(t, err) + require.Contains(t, err.Error(), "inconsistent values [originUser] and [different] for parameter origin_username") } From 186f92017fcfb83471436b03b3c4ff0d781c5f8d Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Mon, 1 Jul 2024 18:47:42 +0200 Subject: [PATCH 3/9] Support single configuration file --- proxy/launch.go | 4 +- proxy/main.go | 5 +- proxy/main_profiling.go | 3 +- proxy/pkg/config/config.go | 82 ++++++---------------- proxy/pkg/config/config_dual_reads_test.go | 2 +- proxy/pkg/config/config_test.go | 77 +++----------------- proxy/pkg/config/config_tls_test.go | 4 +- proxy/pkg/config/configtestutils.go | 5 -- 8 files changed, 38 insertions(+), 144 deletions(-) diff --git a/proxy/launch.go b/proxy/launch.go index a4a359bd..d769b111 100644 --- a/proxy/launch.go +++ b/proxy/launch.go @@ -23,8 +23,8 @@ func runSignalListener(cancelFunc context.CancelFunc) { }() } -func launchProxy(profilingSupported bool) { - conf, err := config.New().LoadConfig() +func launchProxy(profilingSupported bool, configFile string) { + conf, err := config.New().LoadConfig(configFile) if err != nil { log.Errorf("Error loading configuration: %v. Aborting startup.", err) diff --git a/proxy/main.go b/proxy/main.go index a564618d..bdf51ffa 100644 --- a/proxy/main.go +++ b/proxy/main.go @@ -16,7 +16,8 @@ import ( // TODO: to be managed externally const ZdmVersionString = "2.2.0" -var displayVersion = flag.Bool("version", false, "Display the ZDM proxy version and exit") +var displayVersion = flag.Bool("version", false, "display the ZDM proxy version and exit") +var configFile = flag.String("config", "", "specify path to ZDM configuration file") func main() { @@ -29,5 +30,5 @@ func main() { // Always record version information (very) early in the log log.Infof("Starting ZDM proxy version %v", ZdmVersionString) - launchProxy(false) + launchProxy(false, *configFile) } diff --git a/proxy/main_profiling.go b/proxy/main_profiling.go index 0acd9ff7..1e883bb4 100644 --- a/proxy/main_profiling.go +++ b/proxy/main_profiling.go @@ -15,6 +15,7 @@ import ( var cpuProfile = flag.String("cpu_profile", "", "write cpu profile to the specified file") var memProfile = flag.String("mem_profile", "", "write memory profile to the specified file") +var configFile = flag.String("config", "", "specify path to ZDM configuration file") func main() { @@ -61,5 +62,5 @@ func main() { }() } - launchProxy(true) + launchProxy(true, configFile) } diff --git a/proxy/pkg/config/config.go b/proxy/pkg/config/config.go index b0dc358a..b8019de6 100644 --- a/proxy/pkg/config/config.go +++ b/proxy/pkg/config/config.go @@ -10,7 +10,6 @@ import ( "gopkg.in/yaml.v3" "net" "os" - "reflect" "strconv" "strings" ) @@ -38,8 +37,8 @@ type Config struct { OriginPort int `default:"9042" split_words:"true" yaml:"origin_port"` OriginSecureConnectBundlePath string `split_words:"true" yaml:"origin_secure_connect_bundle_path"` OriginLocalDatacenter string `split_words:"true" yaml:"origin_local_datacenter"` - OriginUsername string `split_words:"true" yaml:"origin_username"` - OriginPassword string `split_words:"true" json:"-" yaml:"origin_password"` + OriginUsername string `required:"true" split_words:"true" yaml:"origin_username"` + OriginPassword string `required:"true" split_words:"true" json:"-" yaml:"origin_password"` OriginConnectionTimeoutMs int `default:"30000" split_words:"true" yaml:"origin_connection_timeout_ms"` OriginTlsServerCaPath string `split_words:"true" yaml:"origin_tls_server_ca_path"` @@ -52,8 +51,8 @@ type Config struct { TargetPort int `default:"9042" split_words:"true" yaml:"target_port"` TargetSecureConnectBundlePath string `split_words:"true" yaml:"target_secure_connect_bundle_path"` TargetLocalDatacenter string `split_words:"true" yaml:"target_local_datacenter"` - TargetUsername string `split_words:"true" yaml:"target_username"` - TargetPassword string `split_words:"true" json:"-" yaml:"target_password"` + TargetUsername string `required:"true" split_words:"true" yaml:"target_username"` + TargetPassword string `required:"true" split_words:"true" json:"-" yaml:"target_password"` TargetConnectionTimeoutMs int `default:"30000" split_words:"true" yaml:"target_connection_timeout_ms"` TargetTlsServerCaPath string `split_words:"true" yaml:"target_tls_server_ca_path"` @@ -127,14 +126,6 @@ type Config struct { AsyncConnectorWriteBufferSizeBytes int `default:"4096" split_words:"true" yaml:"async_connector_write_buffer_size_bytes"` } -var DefaultConfig *Config = nil - -func init() { - c := &Config{} - def.SetDefaults(c) - DefaultConfig = c -} - func (c *Config) String() string { serializedConfig, _ := json.Marshal(c) return string(serializedConfig) @@ -145,52 +136,18 @@ func New() *Config { return &Config{} } -func (c *Config) loadFromFiles() error { - paths := os.Getenv("ZDM_CONFIG_FILES") - for _, path := range strings.Split(paths, ",") { - if len(path) == 0 { - continue - } - file, err := os.Open(path) - if err != nil { - return fmt.Errorf("could not read configuration file %v: %w", path, err) - } - defer file.Close() - - dec := yaml.NewDecoder(file) - pc := &Config{} - def.SetDefaults(pc) // apply default tag, YAML decoder does not support it - if err = dec.Decode(pc); err != nil { - return fmt.Errorf("could not parse yaml file %v: %w", path, err) - } - err = c.mergeConfig(pc) - if err != nil { - return err - } +func (c *Config) loadFromFile(configFile string) error { + file, err := os.Open(configFile) + if err != nil { + return fmt.Errorf("could not read configuration file %v: %w", configFile, err) } - return nil -} + defer file.Close() -func (c *Config) mergeConfig(other *Config) error { - cRef := reflect.ValueOf(c).Elem() - otherRef := reflect.ValueOf(*other) - defaultRef := reflect.ValueOf(*DefaultConfig) - for i := 0; i < cRef.NumField(); i++ { - field := cRef.Type().Field(i).Tag.Get("yaml") - cVal := cRef.Field(i).Interface() - otherVal := otherRef.Field(i).Interface() - defaultVal := defaultRef.Field(i).Interface() - - if otherVal != defaultVal { - // other value is different from the default - if cVal != defaultVal && cVal != otherVal { - return fmt.Errorf("inconsistent values [%v] and [%v] for parameter %v", cVal, otherVal, field) - } else { - cRef.Field(i).Set(otherRef.Field(i)) - } - } + def.SetDefaults(c) // apply default tag, it is not supported by YAML decoder + dec := yaml.NewDecoder(file) + if err = dec.Decode(c); err != nil { + return fmt.Errorf("could not parse yaml file %v: %w", configFile, err) } - return nil } @@ -205,13 +162,14 @@ func (c *Config) parseEnvVars() error { return nil } -func (c *Config) LoadConfig() (*Config, error) { - err := c.parseEnvVars() - if err != nil { - return nil, err - } +func (c *Config) LoadConfig(configFile string) (*Config, error) { + var err error - err = c.loadFromFiles() + if configFile != "" { + err = c.loadFromFile(configFile) + } else { + err = c.parseEnvVars() + } if err != nil { return nil, err } diff --git a/proxy/pkg/config/config_dual_reads_test.go b/proxy/pkg/config/config_dual_reads_test.go index f2202739..2b4c5722 100644 --- a/proxy/pkg/config/config_dual_reads_test.go +++ b/proxy/pkg/config/config_dual_reads_test.go @@ -62,7 +62,7 @@ func TestConfig_ParseReadMode(t *testing.T) { setOriginContactPointsAndPortEnvVars() setTargetContactPointsAndPortEnvVars() - conf, err := New().LoadConfig() + conf, err := New().LoadConfig("") if err != nil { if tt.errExpected { require.Equal(t, tt.errMsg, err.Error()) diff --git a/proxy/pkg/config/config_test.go b/proxy/pkg/config/config_test.go index 88c1b05d..e211fc5d 100644 --- a/proxy/pkg/config/config_test.go +++ b/proxy/pkg/config/config_test.go @@ -17,7 +17,7 @@ func TestTargetConfig_WithBundleOnly(t *testing.T) { // test-specific setup setEnvVar("ZDM_TARGET_SECURE_CONNECT_BUNDLE_PATH", "/path/to/target/bundle") - conf, err := New().LoadConfig() + conf, err := New().LoadConfig("") require.Nil(t, err) require.Equal(t, conf.TargetSecureConnectBundlePath, "/path/to/target/bundle") require.Empty(t, conf.TargetContactPoints) @@ -36,7 +36,7 @@ func TestTargetConfig_WithHostnameAndPortOnly(t *testing.T) { // test-specific setup setTargetContactPointsAndPortEnvVars() - conf, err := New().LoadConfig() + conf, err := New().LoadConfig("") require.Nil(t, err) require.Equal(t, conf.TargetContactPoints, "target.hostname.com") require.Equal(t, conf.TargetPort, 5647) @@ -56,7 +56,7 @@ func TestTargetConfig_WithBundleAndHostname(t *testing.T) { setTargetContactPointsAndPortEnvVars() setTargetSecureConnectBundleEnvVar() - _, err := New().LoadConfig() + _, err := New().LoadConfig("") require.Error(t, err, "TargetSecureConnectBundlePath and TargetContactPoints are "+ "mutually exclusive. Please specify only one of them.") } @@ -72,7 +72,7 @@ func TestTargetConfig_WithoutBundleAndHostname(t *testing.T) { // no test-specific setup in this case - _, err := New().LoadConfig() + _, err := New().LoadConfig("") require.Error(t, err, "Both TargetSecureConnectBundlePath and TargetContactPoints are "+ "empty. Please specify either one of them.") } @@ -89,7 +89,7 @@ func TestTargetConfig_WithHostnameButWithoutPort(t *testing.T) { //test-specific setup setEnvVar("ZDM_TARGET_CONTACT_POINTS", "target.hostname.com") - c, err := New().LoadConfig() + c, err := New().LoadConfig("") require.Nil(t, err) require.Equal(t, 9042, c.TargetPort) } @@ -98,9 +98,7 @@ func TestConfig_LoadNotExistingFile(t *testing.T) { defer clearAllEnvVars() clearAllEnvVars() - setConfigFilesEnvVar("/not/existing/file") - - _, err := New().LoadConfig() + _, err := New().LoadConfig("/not/existing/file") require.NotNil(t, err) require.Contains(t, err.Error(), "could not read configuration file /not/existing/file") } @@ -125,9 +123,8 @@ proxy_listen_port: 39042 `) defer removeConfigFile(f) require.Nil(t, err) - setConfigFilesEnvVar(f.Name()) - c, err := New().LoadConfig() + c, err := New().LoadConfig(f.Name()) require.Nil(t, err) require.Equal(t, "ORIGIN", c.PrimaryCluster) require.Equal(t, "foo1", c.OriginUsername) @@ -139,63 +136,5 @@ proxy_listen_port: 39042 require.Equal(t, "192.168.100.102", c.TargetContactPoints) require.Equal(t, 29042, c.TargetPort) require.Equal(t, 39042, c.ProxyListenPort) -} - -func TestConfig_LoadConfigFromFileAndEnvVars(t *testing.T) { - defer clearAllEnvVars() - clearAllEnvVars() - - // publicly available information stored as environment variables - setOriginContactPointsAndPortEnvVars() - setTargetContactPointsAndPortEnvVars() - - // sensitive username and passwords stored inside two files - f1, err := createConfigFile(` -origin_username: foo1 -origin_password: bar1 -`) - defer removeConfigFile(f1) - require.Nil(t, err) - f2, err := createConfigFile(` -target_username: foo2 -target_password: bar2 -`) - defer removeConfigFile(f2) - require.Nil(t, err) - - setConfigFilesEnvVar(f1.Name(), f2.Name()) - - c, err := New().LoadConfig() - require.Nil(t, err) - require.Equal(t, "foo1", c.OriginUsername) - require.Equal(t, "bar1", c.OriginPassword) - require.Equal(t, "foo2", c.TargetUsername) - require.Equal(t, "bar2", c.TargetPassword) - require.Equal(t, "origin.hostname.com", c.OriginContactPoints) - require.Equal(t, "target.hostname.com", c.TargetContactPoints) -} - -func TestConfig_FailOnDuplicateValuePresent(t *testing.T) { - defer clearAllEnvVars() - clearAllEnvVars() - - // publicly available information stored as environment variables - setOriginCredentialsEnvVars() - setTargetCredentialsEnvVars() - setOriginContactPointsAndPortEnvVars() - setTargetContactPointsAndPortEnvVars() - - // try to overriding values should raise error - f, err := createConfigFile(` -origin_username: different -origin_password: different -`) - defer removeConfigFile(f) - require.Nil(t, err) - - setConfigFilesEnvVar(f.Name()) - - _, err = New().LoadConfig() - require.NotNil(t, err) - require.Contains(t, err.Error(), "inconsistent values [originUser] and [different] for parameter origin_username") + require.Equal(t, 4000, c.AsyncHandshakeTimeoutMs) // verify that defaults were applied } diff --git a/proxy/pkg/config/config_tls_test.go b/proxy/pkg/config/config_tls_test.go index ab4d8b95..ba89525d 100644 --- a/proxy/pkg/config/config_tls_test.go +++ b/proxy/pkg/config/config_tls_test.go @@ -191,7 +191,7 @@ func TestOriginConfig_ClusterTlsConfig(t *testing.T) { setTargetContactPointsAndPortEnvVars() var tlsConf *common.ClusterTlsConfig - conf, err := New().LoadConfig() + conf, err := New().LoadConfig("") if err != nil { if tt.errExpected { require.Equal(t, tt.errMsg, err.Error()) @@ -389,7 +389,7 @@ func TestTargetConfig_ClusterTlsConfig(t *testing.T) { setOriginContactPointsAndPortEnvVars() var tlsConf *common.ClusterTlsConfig - conf, err := New().LoadConfig() + conf, err := New().LoadConfig("") if err != nil { if tt.errExpected { // Expected configuration validation error diff --git a/proxy/pkg/config/configtestutils.go b/proxy/pkg/config/configtestutils.go index d1504e69..43bea35e 100644 --- a/proxy/pkg/config/configtestutils.go +++ b/proxy/pkg/config/configtestutils.go @@ -2,7 +2,6 @@ package config import ( "os" - "strings" ) type envVar struct { @@ -38,10 +37,6 @@ func setTargetSecureConnectBundleEnvVar() { setEnvVar("ZDM_TARGET_SECURE_CONNECT_BUNDLE_PATH", "/path/to/origin/bundle") } -func setConfigFilesEnvVar(paths ...string) { - setEnvVar("ZDM_CONFIG_FILES", strings.Join(paths[:], ",")) -} - func createConfigFile(content string) (*os.File, error) { f, err := os.CreateTemp("", "config.*.yml") if err == nil { From 49b874c5746a7a2fb5435977bfa36f54f0acfa9a Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Tue, 2 Jul 2024 12:16:51 +0200 Subject: [PATCH 4/9] Apply review comments --- README.md | 20 +++++++++++++++++--- proxy/launch.go | 18 ++++++++++++++++++ proxy/main.go | 19 ++----------------- proxy/main_profiling.go | 5 +++-- proxy/pkg/config/config.go | 2 +- 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 3a243d28..e1ae8bf9 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,10 @@ An overview of the proxy architecture and logical flow can be viewed [here](http ## Quick Start -In order to run the proxy, you'll need to set some environment variables to configure it properly. +In order to run the proxy, you'll need to set some environment variables or pass reference to YAML configuration file. Below you'll find a list with the most important variables along with their default values. -The required ones are marked with a comment. +The required ones are marked with a comment. Variable names for YAML configuration file do not have `ZDM_` prefix and +are lower-cased. ```shell ZDM_ORIGIN_CONTACT_POINTS=10.0.0.1 #required @@ -36,7 +37,7 @@ ZDM_READ_MODE=PRIMARY_ONLY ZDM_LOG_LEVEL=INFO ``` -The environment variables must be set and exported for the proxy to work. +The environment variables (or YAM configuration file) must be set for the proxy to work. In order to get started quickly, in your local environment, grab a copy of the binary distribution in the [Releases](https://github.com/datastax/zdm-proxy/releases) page. For the recommended installation in a production @@ -55,6 +56,19 @@ export ZDM_TARGET_PASSWORD=cassandra \ ./zdm-proxy-v2.0.0 # run the ZDM proxy executable ``` +If you prefer to use YAML configuration file, analogical setup would look like: + +```shell +$ cat zdm-config.yml +origin_contact_points: 10.0.0.1 +target_contact_points: 10.0.0.2 +origin_username: cassandra +origin_password: cassandra +target_username: cassandra +target_password: cassandra +$ ./zdm-proxy-v2.0.0 --config=./zdm-config.yml # run the ZDM proxy executable +``` + At this point, you should be able to connect some client such as [CQLSH](https://downloads.datastax.com/#cqlsh) to the proxy and write data to it and the proxy will take care of forwarding the requests to both clusters concurrently. diff --git a/proxy/launch.go b/proxy/launch.go index d769b111..36a0e093 100644 --- a/proxy/launch.go +++ b/proxy/launch.go @@ -2,6 +2,8 @@ package main import ( "context" + "flag" + "fmt" "github.com/datastax/zdm-proxy/proxy/pkg/config" "github.com/datastax/zdm-proxy/proxy/pkg/runner" log "github.com/sirupsen/logrus" @@ -10,6 +12,12 @@ import ( "syscall" ) +// TODO: to be managed externally +const ZdmVersionString = "2.2.0" + +var displayVersionOpt = flag.Bool("version", false, "display the ZDM proxy version and exit") +var configFileOpt = flag.String("config", "", "specify path to ZDM configuration file") + func runSignalListener(cancelFunc context.CancelFunc) { sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) @@ -23,6 +31,16 @@ func runSignalListener(cancelFunc context.CancelFunc) { }() } +func displayVersion() { + if *displayVersionOpt { + fmt.Printf("ZDM proxy version %v\n", ZdmVersionString) + os.Exit(0) + } + + // Always record version information (very) early in the log + log.Infof("Starting ZDM proxy version %v", ZdmVersionString) +} + func launchProxy(profilingSupported bool, configFile string) { conf, err := config.New().LoadConfig(configFile) diff --git a/proxy/main.go b/proxy/main.go index bdf51ffa..c219c82d 100644 --- a/proxy/main.go +++ b/proxy/main.go @@ -7,28 +7,13 @@ package main import ( "flag" - "fmt" - "os" - - log "github.com/sirupsen/logrus" ) -// TODO: to be managed externally -const ZdmVersionString = "2.2.0" - -var displayVersion = flag.Bool("version", false, "display the ZDM proxy version and exit") -var configFile = flag.String("config", "", "specify path to ZDM configuration file") - func main() { flag.Parse() - if *displayVersion { - fmt.Printf("ZDM proxy version %v\n", ZdmVersionString) - os.Exit(0) - } - // Always record version information (very) early in the log - log.Infof("Starting ZDM proxy version %v", ZdmVersionString) + displayVersion() - launchProxy(false, *configFile) + launchProxy(false, *configFileOpt) } diff --git a/proxy/main_profiling.go b/proxy/main_profiling.go index 1e883bb4..bb019d73 100644 --- a/proxy/main_profiling.go +++ b/proxy/main_profiling.go @@ -15,12 +15,13 @@ import ( var cpuProfile = flag.String("cpu_profile", "", "write cpu profile to the specified file") var memProfile = flag.String("mem_profile", "", "write memory profile to the specified file") -var configFile = flag.String("config", "", "specify path to ZDM configuration file") func main() { flag.Parse() + displayVersion() + // the cpu profiling is enabled at startup and is periodically collected while the proxy is running // if cpu profiling is requested, any error configuring or starting it will cause the proxy startup to fail if *cpuProfile != "" { @@ -62,5 +63,5 @@ func main() { }() } - launchProxy(true, configFile) + launchProxy(true, configFileOpt) } diff --git a/proxy/pkg/config/config.go b/proxy/pkg/config/config.go index b8019de6..afd2f220 100644 --- a/proxy/pkg/config/config.go +++ b/proxy/pkg/config/config.go @@ -181,7 +181,7 @@ func (c *Config) LoadConfig(configFile string) (*Config, error) { log.Infof("Parsed configuration: %v", c) - return c, err + return c, nil } func lookupFirstIp4(host string) (net.IP, error) { From c359db621539f0d4fcad9a397f3a02197191be62 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Tue, 2 Jul 2024 12:42:05 +0200 Subject: [PATCH 5/9] Apply review comments --- README.md | 2 +- RELEASE_PROCESS.md | 2 +- proxy/launch.go | 14 ++++++-------- proxy/main.go | 5 +---- proxy/main_profiling.go | 5 +---- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e1ae8bf9..917a4c83 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ export ZDM_TARGET_PASSWORD=cassandra \ ./zdm-proxy-v2.0.0 # run the ZDM proxy executable ``` -If you prefer to use YAML configuration file, analogical setup would look like: +If you prefer to use YAML configuration file, an equivalent setup would look like: ```shell $ cat zdm-config.yml diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 02e01b7e..811053de 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -8,7 +8,7 @@ All published container images can be found at [https://hub.docker.com/r/datasta Before triggering the build and publish process for an official/stable release, three files need to be updated, the `RELEASE_NOTES`, `CHANGELOG` and `main.go`. -Please update the ZDM version displayed during component startup in `main.go`: +Please update the ZDM version displayed during component startup in `launch.go`: ```go const ZdmVersionString = "2.0.0" ``` diff --git a/proxy/launch.go b/proxy/launch.go index 36a0e093..1edcbfa2 100644 --- a/proxy/launch.go +++ b/proxy/launch.go @@ -15,8 +15,8 @@ import ( // TODO: to be managed externally const ZdmVersionString = "2.2.0" -var displayVersionOpt = flag.Bool("version", false, "display the ZDM proxy version and exit") -var configFileOpt = flag.String("config", "", "specify path to ZDM configuration file") +var displayVersion = flag.Bool("version", false, "display the ZDM proxy version and exit") +var configFile = flag.String("config", "", "specify path to ZDM configuration file") func runSignalListener(cancelFunc context.CancelFunc) { sigCh := make(chan os.Signal, 1) @@ -31,18 +31,16 @@ func runSignalListener(cancelFunc context.CancelFunc) { }() } -func displayVersion() { - if *displayVersionOpt { +func launchProxy(profilingSupported bool) { + if *displayVersion { fmt.Printf("ZDM proxy version %v\n", ZdmVersionString) - os.Exit(0) + return } // Always record version information (very) early in the log log.Infof("Starting ZDM proxy version %v", ZdmVersionString) -} -func launchProxy(profilingSupported bool, configFile string) { - conf, err := config.New().LoadConfig(configFile) + conf, err := config.New().LoadConfig(*configFile) if err != nil { log.Errorf("Error loading configuration: %v. Aborting startup.", err) diff --git a/proxy/main.go b/proxy/main.go index c219c82d..d9361b4a 100644 --- a/proxy/main.go +++ b/proxy/main.go @@ -10,10 +10,7 @@ import ( ) func main() { - flag.Parse() - displayVersion() - - launchProxy(false, *configFileOpt) + launchProxy(false) } diff --git a/proxy/main_profiling.go b/proxy/main_profiling.go index bb019d73..95896076 100644 --- a/proxy/main_profiling.go +++ b/proxy/main_profiling.go @@ -17,11 +17,8 @@ var cpuProfile = flag.String("cpu_profile", "", "write cpu profile to the specif var memProfile = flag.String("mem_profile", "", "write memory profile to the specified file") func main() { - flag.Parse() - displayVersion() - // the cpu profiling is enabled at startup and is periodically collected while the proxy is running // if cpu profiling is requested, any error configuring or starting it will cause the proxy startup to fail if *cpuProfile != "" { @@ -63,5 +60,5 @@ func main() { }() } - launchProxy(true, configFileOpt) + launchProxy(true) } From b88d2fcc9e88441d01a51dbb149105e59ea49855 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Tue, 2 Jul 2024 15:52:40 +0200 Subject: [PATCH 6/9] Reference configuration file --- docs/assets/zdm-config-reference.yml | 164 +++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 docs/assets/zdm-config-reference.yml diff --git a/docs/assets/zdm-config-reference.yml b/docs/assets/zdm-config-reference.yml new file mode 100644 index 00000000..b76e3c1d --- /dev/null +++ b/docs/assets/zdm-config-reference.yml @@ -0,0 +1,164 @@ +# This variable determines which cluster is currently considered the primary cluster. +# At the start of the migration, the primary cluster is Origin, as it contains all the data. +# In Phase 4 of the migration, once all the existing data has been transferred and any validation/reconciliation +# step has been successfully executed, you can switch the primary cluster to be Target. +# Valid values: ORIGIN, TARGET. +primary_cluster: ORIGIN + +# This variable determines how reads are handled by the ZDM Proxy. Valid values: +# PRIMARY_ONLY - reads are only sent synchronously to the primary cluster. This is the default behavior. +# DUAL_ASYNC_ON_SECONDARY - reads are sent synchronously to the primary cluster and also asynchronously +# to the secondary cluster. See Phase 3: Enable asynchronous dual reads. +read_mode: PRIMARY_ONLY + +# Whether the ZDM Proxy should replace standard CQL function calls in write +# requests with a value computed at proxy level. Currently, only the replacement +# of now() is supported. Disabled by default. Enabling this will have a noticeable performance impact. +replace_cql_functions: false + +# Timeout (in ms) when performing the initialization (handshake) of a proxy-to-secondary cluster +# connection that will be used solely for asynchronous dual reads. If this timeout occurs, the asynchronous +# reads will not be sent. This has no impact on the handling of synchronous requests: the ZDM Proxy will +# continue to handle all synchronous reads and writes normally. +async_handshake_timeout_ms: 4000 + +# Specifies logging level. +log_level: INFO + +# TODO: ProxyTopologyIndex +# TODO: ProxyTopologyAddresses +# TODO: ProxyTopologyNumTokens + +# Comma separated list of origin cluster contact points. +origin_contact_points: origin.datastax.com + +# Port used when connecting to nodes from origin cluster. +origin_port: 9042 + +# If origin cluster is DataStax Astra, path to secure connection bundle. +# origin_secure_connect_bundle_path: + +# Local data center for origin cluster. +# origin_local_datacenter: + +# Origin cluster username. +origin_username: user1 + +# Origin cluster password. +origin_password: pass1 + +# Timeout (in ms) when attempting to establish a connection from the proxy to origin cluster. +origin_connection_timeout_ms: 30000 + +# CA certificate used when verifying identity of origin nodes. +# origin_tls_server_ca_path: + +# Public key used when establishing connectivity with origin cluster. +# origin_tls_client_cert_path: + +# Private key used to secure communication with origin cluster. +# origin_tls_client_key_path: + +# Comma separated ist of target cluster contact points. +target_contact_points: target.datastax.com + +# If target cluster is DataStax Astra, path to secure connection bundle. +# target_secure_connect_bundle_path: + +# Local data center for target cluster. +# target_local_datacenter: DC1 + +# Port used when connecting to nodes from target cluster. +target_port: 9042 + +# Target cluster username. +target_username: user2 + +# Target cluster password. +target_password: pass2 + +# Timeout (in ms) when attempting to establish a connection from the proxy to target cluster. +target_connection_timeout_ms: 30000 + +# CA certificate used when verifying identity of target nodes. +# target_tls_server_ca_path: + +# Public key used when establishing connectivity with target cluster. +# target_tls_client_cert_path: + +# Private key used to secure communication with target cluster. +# target_tls_client_key_path: + +# Listen address of ZDM proxy. +proxy_listen_address: localhost + +# Port number on which ZDM proxy is listening. +proxy_listen_port: 14002 + +# Global timeout (in ms) of a request at proxy level. This variable determines how long the +# ZDM Proxy will wait for one cluster (in case of reads) or both clusters (in case of writes) +# to reply to a request. If this timeout is reached, the ZDM Proxy will abandon that request +# and no longer consider it as pending, thus freeing up the corresponding internal resources. +# Note that, in this case, the ZDM Proxy will not return any result or error: when the client +# application’s own timeout is reached, the driver will time out the request on its side. +proxy_request_timeout_ms: 10000 + +# Defines hot many clients may connect to single ZDM proxy instance. ZDM proxy closes +# connection if threshold is reached. +proxy_max_client_connections: 1000 + +# In the CQL protocol every request has a unique id, named stream id. This variable allows +# you to tune the maximum pool size of the available stream ids managed by the ZDM Proxy +# per client connection. In the application client, the stream ids are managed internally +# by the driver, and in most drivers the max number is 2048 (the same default value used +# in the proxy). If you have a custom driver configuration with a higher value, you should +# change this property accordingly. +proxy_max_stream_ids: 2048 + +# CA certificate used when verifying identity of connecting client applications. +# proxy_tls_ca_path: + +# Public key used when establishing connectivity with client applications. +# proxy_tls_cert_path: + +# Private key used by ZDM proxy to encrypt connection between itself and client applications +# proxy_tls_key_path: + +# If true enforces mutual TLS between proxy and client applications +# proxy_tls_require_client_auth: false + +# If true ZDM proxy exposes performance metrics in Prometheus format. +metrics_enabled: true + +# Network interface used to expose Prometheus metrics. +metrics_address: localhost + +# Port used to expose Prometheus metrics. +metrics_port: 14001 + +# Prefix prepended to each metric name. +metrics_prefix: zdm + +# List of histogram buckets for measuring latency of origin cluster +metrics_origin_latency_buckets_ms: 1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000 + +# List of histogram buckets for measuring latency of target cluster +metrics_target_latency_buckets_ms: 1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000 + +# List of histogram buckets for measuring latency of asynchronous +# read requests routed to target cluster. See parameter read_mode. +metrics_async_read_latency_buckets_ms: 1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000 + +# Frequency (in ms) with which heartbeats will be sent on cluster connections +# (i.e. all control and request connections to Origin and Target). Heartbeats +# keep idle connections alive. +heartbeat_interval_ms: 30000 + +# Below properties define reconnection strategy for establishing control connection. +heartbeat_retry_interval_min_ms: 250 +heartbeat_retry_interval_max_ms: 30000 +heartbeat_retry_backoff_factor: 2 + +# Control connection failure threshold. If threshold is exceeded, +# readiness probe of ZDM will report failure and pod will be recreated. +heartbeat_failure_threshold: 1 \ No newline at end of file From 13ba5380cc815f99561d7151196a36afcf93ec14 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Tue, 2 Jul 2024 16:46:05 +0200 Subject: [PATCH 7/9] Reference configuration file --- docs/assets/zdm-config-reference.yml | 58 +++++++++++++++------------- proxy/pkg/config/configtestutils.go | 4 +- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/docs/assets/zdm-config-reference.yml b/docs/assets/zdm-config-reference.yml index b76e3c1d..9509f34b 100644 --- a/docs/assets/zdm-config-reference.yml +++ b/docs/assets/zdm-config-reference.yml @@ -14,23 +14,29 @@ read_mode: PRIMARY_ONLY # Whether the ZDM Proxy should replace standard CQL function calls in write # requests with a value computed at proxy level. Currently, only the replacement # of now() is supported. Disabled by default. Enabling this will have a noticeable performance impact. -replace_cql_functions: false +# replace_cql_functions: false # Timeout (in ms) when performing the initialization (handshake) of a proxy-to-secondary cluster # connection that will be used solely for asynchronous dual reads. If this timeout occurs, the asynchronous # reads will not be sent. This has no impact on the handling of synchronous requests: the ZDM Proxy will # continue to handle all synchronous reads and writes normally. -async_handshake_timeout_ms: 4000 +# async_handshake_timeout_ms: 4000 # Specifies logging level. -log_level: INFO +# log_level: INFO -# TODO: ProxyTopologyIndex -# TODO: ProxyTopologyAddresses -# TODO: ProxyTopologyNumTokens +# List of peer ZDM proxy instances. This configuration parameter should be identical through all ZDM proxies. +# proxy_topology_addresses: 127.0.1.1, 127.0.1.2, 127.0.1.3 + +# Index of local ZDM proxy instance within "proxy_topology_addresses" list. +# proxy_topology_index: 0 + +# Number of tokens each proxy isntance owns. The default value of 8 should workf for majority of use-case. +# To learn more about this concept, look into "virtual nodes" in Apache Cassandra. +# proxy_topology_num_tokens: 8 # Comma separated list of origin cluster contact points. -origin_contact_points: origin.datastax.com +origin_contact_points: 127.0.0.1 # Port used when connecting to nodes from origin cluster. origin_port: 9042 @@ -48,7 +54,7 @@ origin_username: user1 origin_password: pass1 # Timeout (in ms) when attempting to establish a connection from the proxy to origin cluster. -origin_connection_timeout_ms: 30000 +# origin_connection_timeout_ms: 30000 # CA certificate used when verifying identity of origin nodes. # origin_tls_server_ca_path: @@ -60,7 +66,7 @@ origin_connection_timeout_ms: 30000 # origin_tls_client_key_path: # Comma separated ist of target cluster contact points. -target_contact_points: target.datastax.com +target_contact_points: 127.0.0.2 # If target cluster is DataStax Astra, path to secure connection bundle. # target_secure_connect_bundle_path: @@ -78,7 +84,7 @@ target_username: user2 target_password: pass2 # Timeout (in ms) when attempting to establish a connection from the proxy to target cluster. -target_connection_timeout_ms: 30000 +# target_connection_timeout_ms: 30000 # CA certificate used when verifying identity of target nodes. # target_tls_server_ca_path: @@ -101,11 +107,11 @@ proxy_listen_port: 14002 # and no longer consider it as pending, thus freeing up the corresponding internal resources. # Note that, in this case, the ZDM Proxy will not return any result or error: when the client # application’s own timeout is reached, the driver will time out the request on its side. -proxy_request_timeout_ms: 10000 +# proxy_request_timeout_ms: 10000 # Defines hot many clients may connect to single ZDM proxy instance. ZDM proxy closes # connection if threshold is reached. -proxy_max_client_connections: 1000 +# proxy_max_client_connections: 1000 # In the CQL protocol every request has a unique id, named stream id. This variable allows # you to tune the maximum pool size of the available stream ids managed by the ZDM Proxy @@ -113,7 +119,7 @@ proxy_max_client_connections: 1000 # by the driver, and in most drivers the max number is 2048 (the same default value used # in the proxy). If you have a custom driver configuration with a higher value, you should # change this property accordingly. -proxy_max_stream_ids: 2048 +# proxy_max_stream_ids: 2048 # CA certificate used when verifying identity of connecting client applications. # proxy_tls_ca_path: @@ -128,37 +134,37 @@ proxy_max_stream_ids: 2048 # proxy_tls_require_client_auth: false # If true ZDM proxy exposes performance metrics in Prometheus format. -metrics_enabled: true +# metrics_enabled: true # Network interface used to expose Prometheus metrics. -metrics_address: localhost +# metrics_address: localhost # Port used to expose Prometheus metrics. -metrics_port: 14001 +# metrics_port: 14001 # Prefix prepended to each metric name. -metrics_prefix: zdm +# metrics_prefix: zdm # List of histogram buckets for measuring latency of origin cluster -metrics_origin_latency_buckets_ms: 1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000 +# metrics_origin_latency_buckets_ms: 1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000 # List of histogram buckets for measuring latency of target cluster -metrics_target_latency_buckets_ms: 1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000 +# metrics_target_latency_buckets_ms: 1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000 # List of histogram buckets for measuring latency of asynchronous -# read requests routed to target cluster. See parameter read_mode. -metrics_async_read_latency_buckets_ms: 1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000 +# read requests routed to target cluster. See parameter "read_mode". +# metrics_async_read_latency_buckets_ms: 1, 4, 7, 10, 25, 40, 60, 80, 100, 150, 250, 500, 1000, 2500, 5000, 10000, 15000 # Frequency (in ms) with which heartbeats will be sent on cluster connections # (i.e. all control and request connections to Origin and Target). Heartbeats # keep idle connections alive. -heartbeat_interval_ms: 30000 +# heartbeat_interval_ms: 30000 # Below properties define reconnection strategy for establishing control connection. -heartbeat_retry_interval_min_ms: 250 -heartbeat_retry_interval_max_ms: 30000 -heartbeat_retry_backoff_factor: 2 +# heartbeat_retry_interval_min_ms: 250 +# heartbeat_retry_interval_max_ms: 30000 +# heartbeat_retry_backoff_factor: 2 # Control connection failure threshold. If threshold is exceeded, # readiness probe of ZDM will report failure and pod will be recreated. -heartbeat_failure_threshold: 1 \ No newline at end of file +# heartbeat_failure_threshold: 1 diff --git a/proxy/pkg/config/configtestutils.go b/proxy/pkg/config/configtestutils.go index 43bea35e..7f8525c1 100644 --- a/proxy/pkg/config/configtestutils.go +++ b/proxy/pkg/config/configtestutils.go @@ -1,8 +1,6 @@ package config -import ( - "os" -) +import "os" type envVar struct { vName string From 439763ee534d98d1b31dd265d4426735ab6488b7 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Tue, 2 Jul 2024 17:15:17 +0200 Subject: [PATCH 8/9] Reference configuration file --- docs/assets/zdm-config-reference.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/assets/zdm-config-reference.yml b/docs/assets/zdm-config-reference.yml index 9509f34b..bfbec8c0 100644 --- a/docs/assets/zdm-config-reference.yml +++ b/docs/assets/zdm-config-reference.yml @@ -25,14 +25,18 @@ read_mode: PRIMARY_ONLY # Specifies logging level. # log_level: INFO -# List of peer ZDM proxy instances. This configuration parameter should be identical through all ZDM proxies. +# List of peer ZDM proxy instances. This configuration parameter should be *identical* +# (elements form the list placed in the same order) through all ZDM proxies. # proxy_topology_addresses: 127.0.1.1, 127.0.1.2, 127.0.1.3 # Index of local ZDM proxy instance within "proxy_topology_addresses" list. +# Given "proxy_topology_addresses: 127.0.1.1, 127.0.1.2, 127.0.1.3", value of +# "proxy_topology_index" should equal "0" in the configuration file present on server +# 127.0.1.1, "1" on 127.0.1.2 and "2" on 127.0.1.3. # proxy_topology_index: 0 -# Number of tokens each proxy isntance owns. The default value of 8 should workf for majority of use-case. -# To learn more about this concept, look into "virtual nodes" in Apache Cassandra. +# Number of tokens each proxy instance owns. The default value of 8 should work for +# the majority of use case. To learn more about this concept, look into "virtual nodes" in Apache Cassandra. # proxy_topology_num_tokens: 8 # Comma separated list of origin cluster contact points. From b1b08c05faa1b912c46875918240814652324023 Mon Sep 17 00:00:00 2001 From: Lukasz Antoniak Date: Tue, 2 Jul 2024 17:34:49 +0200 Subject: [PATCH 9/9] Reference configuration file --- docs/assets/zdm-config-reference.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/assets/zdm-config-reference.yml b/docs/assets/zdm-config-reference.yml index bfbec8c0..e9d239f0 100644 --- a/docs/assets/zdm-config-reference.yml +++ b/docs/assets/zdm-config-reference.yml @@ -40,12 +40,16 @@ read_mode: PRIMARY_ONLY # proxy_topology_num_tokens: 8 # Comma separated list of origin cluster contact points. +# When this configuration is present, "origin_secure_connect_bundle_path" +# should be left blank. origin_contact_points: 127.0.0.1 # Port used when connecting to nodes from origin cluster. origin_port: 9042 # If origin cluster is DataStax Astra, path to secure connection bundle. +# Users do not need to list contact points ("origin_contact_points") when +# they leverage connection bundle mechanism. # origin_secure_connect_bundle_path: # Local data center for origin cluster. @@ -70,9 +74,13 @@ origin_password: pass1 # origin_tls_client_key_path: # Comma separated ist of target cluster contact points. +# When this configuration is present, "target_secure_connect_bundle_path" +# should be left blank. target_contact_points: 127.0.0.2 # If target cluster is DataStax Astra, path to secure connection bundle. +# Users do not need to list contact points ("target_contact_points") when +# they leverage connection bundle mechanism. # target_secure_connect_bundle_path: # Local data center for target cluster.