Skip to content

Commit

Permalink
Improve connection doctor and validation of connection URLs
Browse files Browse the repository at this point in the history
This PR improves the validation of connection URLs in configuration,
giving clearer error messages to the user and the expected format of the
connection string.

Related to that, RemoteLogDrainUrl and QuesmaInternalTelemetryUrl now
can't be provided in the config - they were overwritten by
hardcoded constants either way.

Additionally, ClickHouse connection doctor is improved:
- If connection succeeded it suggests making sure that username/password
  is correct
- It now skips TLS trial connections if the user disabled TLS in the
  config (and informs of that fact)
- It tries default ports in more cases

This PR is extracted from larger PR QuesmaOrg#938.
  • Loading branch information
avelanarius committed Nov 4, 2024
1 parent 3b7cce6 commit bc58fb1
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 34 deletions.
51 changes: 32 additions & 19 deletions quesma/clickhouse/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,7 @@ func InitDBConnectionPool(c *config.QuesmaConfiguration) *sql.DB {
// in case of misconfigured ClickHouse connection. In the future, we might rethink how do we manage this and perhaps
// move some parts to InitDBConnectionPool, but for now this should already provide some useful feedback.
func RunClickHouseConnectionDoctor(c *config.QuesmaConfiguration) {
timeout := 1 * time.Second
defaultNativeProtocolPort := "9000"
defaultNativeProtocolPortEncrypted := "9440"
timeout := 5 * time.Second

logger.Info().Msgf("[connection-doctor] Starting ClickHouse connection doctor")
hostName, port := c.ClickHouse.Url.Hostname(), c.ClickHouse.Url.Port()
Expand All @@ -112,24 +110,39 @@ func RunClickHouseConnectionDoctor(c *config.QuesmaConfiguration) {
connTcp, errTcp := net.DialTimeout("tcp", address, timeout)
if errTcp != nil {
logger.Info().Msgf("[connection-doctor] Failed dialing with %s, err=[%v], no service listening at configured host/port, make sure to specify reachable ClickHouse address", address, errTcp)
logger.Info().Msgf("[connection-doctor] Trying default ClickHouse native ports...")
if conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", hostName, defaultNativeProtocolPort), timeout); err == nil {
logger.Info().Msgf("[connection-doctor] Default ClickHouse plaintext port is reachable, consider changing ClickHouse port to %s in Quesma configuration", defaultNativeProtocolPort)
conn.Close()
}
if conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", hostName, defaultNativeProtocolPortEncrypted), timeout); err == nil {
logger.Info().Msgf("[connection-doctor] Default ClickHouse TLS port is reachable, consider changing ClickHouse port to %s in Quesma conbfiguration", defaultNativeProtocolPortEncrypted)
conn.Close()
}
tryDefaultPorts(hostName, timeout)
return
}
logger.Info().Msgf("[connection-doctor] Successfully dialed host/port (%s)...", address)
defer connTcp.Close()
logger.Info().Msgf("[connection-doctor] Trying to establish TLS connection with configured host/port (%s)", address)
connTls, errTls := tls.DialWithDialer(&net.Dialer{Timeout: timeout}, "tcp", address, &tls.Config{InsecureSkipVerify: true})
if errTls != nil {
logger.Info().Msgf("[connection-doctor] Failed establishing TLS connection with %s, err=[%v], please use `clickhouse.disableTLS: true` in Quesma configuration", address, errTls)
return

if !c.ClickHouse.DisableTLS {
logger.Info().Msgf("[connection-doctor] Trying to establish TLS connection with configured host/port (%s)", address)
connTls, errTls := tls.DialWithDialer(&net.Dialer{Timeout: timeout}, "tcp", address, &tls.Config{InsecureSkipVerify: true})
if errTls != nil {
logger.Info().Msgf("[connection-doctor] Failed establishing TLS connection with %s, err=[%v], please use `config.disableTLS: true` in Quesma configuration of ClickHouse backend connector", address, errTls)
return
}
defer connTls.Close()
logger.Info().Msgf("[connection-doctor] TLS connection (handshake) with %s established successfully", address)
} else {
logger.Info().Msgf("[connection-doctor] TLS connection is disabled in Quesma configuration (consider trying `config.disableTLS: false` in Quesma configuration of ClickHouse backend connector), skipping TLS connection tests.")
}
logger.Info().Msgf("[connection-doctor] Make sure you are using the correct protocol (currently: %s), correct username/password and correct database (currently: '%s')", c.ClickHouse.Url.Scheme, c.ClickHouse.Database)
tryDefaultPorts(hostName, timeout)
}

func tryDefaultPorts(hostName string, timeout time.Duration) {
defaultNativeProtocolPort := "9000"
defaultNativeProtocolPortEncrypted := "9440"

logger.Info().Msgf("[connection-doctor] Trying default ClickHouse ports...")
if conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", hostName, defaultNativeProtocolPort), timeout); err == nil {
logger.Info().Msgf("[connection-doctor] Default ClickHouse plaintext port is reachable, consider changing the ClickHouse URL in Quesma configuration to clickhouse://%s:%s", hostName, defaultNativeProtocolPort)
conn.Close()
}
if conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", hostName, defaultNativeProtocolPortEncrypted), timeout); err == nil {
logger.Info().Msgf("[connection-doctor] Default ClickHouse TLS port is reachable, consider changing the ClickHouse URL in Quesma configuration to clickhouse://%s:%s", hostName, defaultNativeProtocolPortEncrypted)
conn.Close()
}
defer connTls.Close()
logger.Info().Msgf("[connection-doctor] TLS connection (handshake) with %s established successfully", address)
}
24 changes: 10 additions & 14 deletions quesma/quesma/config/config_v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,23 +39,22 @@ const (
)

type QuesmaNewConfiguration struct {
BackendConnectors []BackendConnector `koanf:"backendConnectors"`
FrontendConnectors []FrontendConnector `koanf:"frontendConnectors"`
InstallationId string `koanf:"installationId"`
LicenseKey string `koanf:"licenseKey"`
Logging LoggingConfiguration `koanf:"logging"`
IngestStatistics bool `koanf:"ingestStatistics"`
QuesmaInternalTelemetryUrl *Url `koanf:"internalTelemetryUrl"`
Processors []Processor `koanf:"processors"`
Pipelines []Pipeline `koanf:"pipelines"`
DisableTelemetry bool `koanf:"disableTelemetry"`
BackendConnectors []BackendConnector `koanf:"backendConnectors"`
FrontendConnectors []FrontendConnector `koanf:"frontendConnectors"`
InstallationId string `koanf:"installationId"`
LicenseKey string `koanf:"licenseKey"`
Logging LoggingConfiguration `koanf:"logging"`
IngestStatistics bool `koanf:"ingestStatistics"`
Processors []Processor `koanf:"processors"`
Pipelines []Pipeline `koanf:"pipelines"`
DisableTelemetry bool `koanf:"disableTelemetry"`
}

type LoggingConfiguration struct {
Path string `koanf:"path"`
Level *zerolog.Level `koanf:"level"`
RemoteLogDrainUrl *Url `koanf:"remoteUrl"`
FileLogging bool `koanf:"fileLogging"`
RemoteLogDrainUrl *Url
}

type Pipeline struct {
Expand Down Expand Up @@ -118,9 +117,6 @@ type QuesmaProcessorConfig struct {

func LoadV2Config() QuesmaNewConfiguration {
var v2config QuesmaNewConfiguration
v2config.QuesmaInternalTelemetryUrl = telemetryUrl
v2config.Logging.RemoteLogDrainUrl = telemetryUrl

loadConfigFile()
// We have to use custom env provider to allow array overrides
if err := k.Load(Env2JsonProvider("QUESMA_", "_", nil), json.Parser(), koanf.WithMergeFunc(mergeDictFunc)); err != nil {
Expand Down
11 changes: 10 additions & 1 deletion quesma/quesma/config/url.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
// SPDX-License-Identifier: Elastic-2.0
package config

import "net/url"
import (
"fmt"
"net/url"
)

type Url url.URL

Expand All @@ -15,6 +18,12 @@ func (u *Url) UnmarshalText(text []byte) error {
if err != nil {
return err
}
if len(urlValue.Scheme) == 0 || len(urlValue.Host) == 0 {
return fmt.Errorf("invalid URL (missing protocol and/or hostname). Expected URL in a format 'protocol://hostname:port' (e.g. 'http://localhost:8123'), but instead got: %s", urlValue)
}
if len(urlValue.Port()) == 0 {
return fmt.Errorf("URL port (e.g. 8123 in 'http://localhost:8123') is missing from the provided URL: %s", urlValue)
}
*u = Url(*urlValue)
return nil
}
Expand Down

0 comments on commit bc58fb1

Please sign in to comment.