From 2938f4e4ce526cd43db2cc70213d130f28cf350a Mon Sep 17 00:00:00 2001 From: ideascf Date: Fri, 1 Mar 2024 18:06:31 +0800 Subject: [PATCH] support run tiflash pod with ConfigMap mounted in tiflash container (#5552) --- pkg/apis/label/label.go | 4 + pkg/apis/pingcap/v1alpha1/tidbcluster.go | 4 + .../member/startscript/v1/render_script.go | 30 ++ pkg/manager/member/startscript/v1/template.go | 72 ++++ .../member/startscript/v1/template_test.go | 333 ++++++++++++++++++ .../startscript/v2/tiflash_start_script.go | 107 ++++++ .../v2/tiflash_start_script_test.go | 279 +++++++++++++++ pkg/manager/member/tiflash_member_manager.go | 76 ++-- pkg/manager/member/tiflash_util.go | 30 +- pkg/manager/member/tiflash_util_test.go | 43 +++ tests/e2e/tidbcluster/tidbcluster.go | 22 ++ 11 files changed, 960 insertions(+), 40 deletions(-) diff --git a/pkg/apis/label/label.go b/pkg/apis/label/label.go index 1cd280c4f2..b73cd40035 100644 --- a/pkg/apis/label/label.go +++ b/pkg/apis/label/label.go @@ -104,6 +104,10 @@ const ( AnnTiCDCGracefulShutdownBeginTime = "tidb.pingcap.com/ticdc-graceful-shutdown-begin-time" // AnnStsLastSyncTimestamp is sts annotation key to indicate the last timestamp the operator sync the sts AnnStsLastSyncTimestamp = "tidb.pingcap.com/sync-timestamp" + // AnnTiflashMountCMInTiflashContainer is tiflash pod annotation key to indicate whether directly mount ConfigMap + // in tiflash container instead of init container for tiflash. With it annotated, the tiflash container will directly + // read config from files mounted by ConfigMap and that enables tiflash support hot-reload config. + AnnTiflashMountCMInTiflashContainer = "tiflash.tidb.pingcap.com/mount-cm-in-tiflash-container" // AnnPVCScaleInTime is pvc scaled in time key used in PVC for e2e test only AnnPVCScaleInTime = "tidb.pingcap.com/scale-in-time" diff --git a/pkg/apis/pingcap/v1alpha1/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/tidbcluster.go index a092b457ec..985e6ff4fc 100644 --- a/pkg/apis/pingcap/v1alpha1/tidbcluster.go +++ b/pkg/apis/pingcap/v1alpha1/tidbcluster.go @@ -1158,6 +1158,10 @@ func (tiflash *TiFlashSpec) GetScaleOutParallelism() int { return int(*(tiflash.ScalePolicy.ScaleOutParallelism)) } +func (tiflash *TiFlashSpec) DoesMountCMInTiflashContainer() bool { + return tiflash.Annotations[label.AnnTiflashMountCMInTiflashContainer] == "true" +} + func (tidbSvc *TiDBServiceSpec) ShouldExposeStatus() bool { exposeStatus := tidbSvc.ExposeStatus if exposeStatus == nil { diff --git a/pkg/manager/member/startscript/v1/render_script.go b/pkg/manager/member/startscript/v1/render_script.go index 767bf7f7ae..808614ad55 100644 --- a/pkg/manager/member/startscript/v1/render_script.go +++ b/pkg/manager/member/startscript/v1/render_script.go @@ -233,9 +233,39 @@ done } func RenderTiFlashStartScript(tc *v1alpha1.TidbCluster) (string, error) { + if tc.Spec.TiFlash.DoesMountCMInTiflashContainer() { + return RenderTiFlashStartScriptWithStartArgs(tc) + } return "/tiflash/tiflash server --config-file /data0/config.toml", nil } +func RenderTiFlashStartScriptWithStartArgs(tc *v1alpha1.TidbCluster) (string, error) { + model := &TiflashStartScriptModel{ + CommonModel: CommonModel{ + AcrossK8s: tc.AcrossK8s(), + ClusterDomain: tc.Spec.ClusterDomain, + }, + AdvertiseAddr: fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashProxyPort), + EnableAdvertiseStatusAddr: false, + } + // only the tiflash learner supports dynamic configuration + if tc.Spec.EnableDynamicConfiguration != nil && *tc.Spec.EnableDynamicConfiguration { + model.AdvertiseStatusAddr = fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashProxyStatusPort) + model.EnableAdvertiseStatusAddr = true + } + + model.PDAddress = fmt.Sprintf("%s://${CLUSTER_NAME}-pd:%d", tc.Scheme(), v1alpha1.DefaultPDClientPort) + if tc.AcrossK8s() { + model.PDAddress = fmt.Sprintf("%s://${CLUSTER_NAME}-pd:%d", tc.Scheme(), v1alpha1.DefaultPDClientPort) // get pd addr from discovery in startup script + } else if tc.Heterogeneous() && tc.WithoutLocalPD() { + model.PDAddress = fmt.Sprintf("%s://%s:%d", tc.Scheme(), controller.PDMemberName(tc.Spec.Cluster.Name), v1alpha1.DefaultPDClientPort) // use pd of reference cluster + } + + model.Addr = fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashFlashPort) + + return renderTemplateFunc(tiflashStartScriptTpl, model) +} + func RenderTiFlashInitScript(tc *v1alpha1.TidbCluster) (string, error) { tcName := tc.GetName() diff --git a/pkg/manager/member/startscript/v1/template.go b/pkg/manager/member/startscript/v1/template.go index b9dc14e4d7..9aa41d8374 100644 --- a/pkg/manager/member/startscript/v1/template.go +++ b/pkg/manager/member/startscript/v1/template.go @@ -742,3 +742,75 @@ type DMWorkerStartScriptModel struct { func RenderDMWorkerStartScript(model *DMWorkerStartScriptModel) (string, error) { return renderTemplateFunc(dmWorkerStartScriptTpl, model) } + +var tiflashStartScriptTplText = `#!/bin/sh + +# This script is used to start tiflash containers in kubernetes cluster + +# Use DownwardAPIVolumeFiles to store informations of the cluster: +# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api +# +# runmode="normal/debug" +# + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" + +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="{{ .PDAddress }}"{{ if .AcrossK8s }} +pd_url="{{ .PDAddress }}" +encoded_domain_url=$(echo $pd_url | base64 | tr "\n" " " | sed "s/ //g") +discovery_url="${CLUSTER_NAME}-discovery.${NAMESPACE}:10261" + +until result=$(wget -qO- -T 3 http://${discovery_url}/verify/${encoded_domain_url} 2>/dev/null); do +echo "waiting for the verification of PD endpoints ..." +sleep $((RANDOM % 5)) +done +PD_ADDRESS=${result} +{{ end }} + +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr={{ .AdvertiseAddr }} \{{if .EnableAdvertiseStatusAddr }} +--flash.proxy.advertise-status-addr={{ .AdvertiseStatusAddr }} \{{end}} +--flash.service_addr={{ .Addr }} \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +` + +var tiflashStartScriptTpl = template.Must(template.New("tiflash-start-script").Parse(tiflashStartScriptTplText)) + +type TiflashStartScriptModel struct { + CommonModel + + AdvertiseAddr string + EnableAdvertiseStatusAddr bool + AdvertiseStatusAddr string + Addr string + PDAddress string +} diff --git a/pkg/manager/member/startscript/v1/template_test.go b/pkg/manager/member/startscript/v1/template_test.go index 03cb99bb63..19a396348f 100644 --- a/pkg/manager/member/startscript/v1/template_test.go +++ b/pkg/manager/member/startscript/v1/template_test.go @@ -1688,6 +1688,339 @@ func TestRenderTiFlashStartScript(t *testing.T) { } } +func TestRenderTiFlashStartScriptWithStartArgs(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + type testcase struct { + name string + + modifyTC func(tc *v1alpha1.TidbCluster) + expectScript string + } + + cases := []testcase{ + { + name: "basic", + modifyTC: func(tc *v1alpha1.TidbCluster) {}, + expectScript: `#!/bin/sh + +# This script is used to start tiflash containers in kubernetes cluster + +# Use DownwardAPIVolumeFiles to store informations of the cluster: +# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api +# +# runmode="normal/debug" +# + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" + +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="http://${CLUSTER_NAME}-pd:2379" + +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20170 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + { + name: "enable AdvertiseAddr", + modifyTC: func(tc *v1alpha1.TidbCluster) { + enable := true + tc.Spec.EnableDynamicConfiguration = &enable + }, + expectScript: `#!/bin/sh + +# This script is used to start tiflash containers in kubernetes cluster + +# Use DownwardAPIVolumeFiles to store informations of the cluster: +# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api +# +# runmode="normal/debug" +# + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" + +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="http://${CLUSTER_NAME}-pd:2379" + +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20170 \ +--flash.proxy.advertise-status-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20292 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + { + name: "across k8s", + modifyTC: func(tc *v1alpha1.TidbCluster) { + tc.Spec.ClusterDomain = "cluster.local" + tc.Spec.AcrossK8s = true + }, + expectScript: `#!/bin/sh + +# This script is used to start tiflash containers in kubernetes cluster + +# Use DownwardAPIVolumeFiles to store informations of the cluster: +# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api +# +# runmode="normal/debug" +# + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" + +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="http://${CLUSTER_NAME}-pd:2379" +pd_url="http://${CLUSTER_NAME}-pd:2379" +encoded_domain_url=$(echo $pd_url | base64 | tr "\n" " " | sed "s/ //g") +discovery_url="${CLUSTER_NAME}-discovery.${NAMESPACE}:10261" + +until result=$(wget -qO- -T 3 http://${discovery_url}/verify/${encoded_domain_url} 2>/dev/null); do +echo "waiting for the verification of PD endpoints ..." +sleep $((RANDOM % 5)) +done +PD_ADDRESS=${result} + + +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc.cluster.local:20170 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc.cluster.local:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + { + name: "across k8s with tls enabled", + modifyTC: func(tc *v1alpha1.TidbCluster) { + tc.Spec.ClusterDomain = "cluster.local" + tc.Spec.AcrossK8s = true + tc.Spec.TLSCluster = &v1alpha1.TLSCluster{Enabled: true} + }, + expectScript: `#!/bin/sh + +# This script is used to start tiflash containers in kubernetes cluster + +# Use DownwardAPIVolumeFiles to store informations of the cluster: +# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api +# +# runmode="normal/debug" +# + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" + +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="https://${CLUSTER_NAME}-pd:2379" +pd_url="https://${CLUSTER_NAME}-pd:2379" +encoded_domain_url=$(echo $pd_url | base64 | tr "\n" " " | sed "s/ //g") +discovery_url="${CLUSTER_NAME}-discovery.${NAMESPACE}:10261" + +until result=$(wget -qO- -T 3 http://${discovery_url}/verify/${encoded_domain_url} 2>/dev/null); do +echo "waiting for the verification of PD endpoints ..." +sleep $((RANDOM % 5)) +done +PD_ADDRESS=${result} + + +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc.cluster.local:20170 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc.cluster.local:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + { + name: "basic with ipv6", + modifyTC: func(tc *v1alpha1.TidbCluster) { + tc.Spec.PreferIPv6 = true + }, + expectScript: `#!/bin/sh + +# This script is used to start tiflash containers in kubernetes cluster + +# Use DownwardAPIVolumeFiles to store informations of the cluster: +# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api +# +# runmode="normal/debug" +# + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" + +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="http://${CLUSTER_NAME}-pd:2379" + +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20170 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + t.Logf("test case: %s", c.name) + + tc := &v1alpha1.TidbCluster{ + Spec: v1alpha1.TidbClusterSpec{ + TiFlash: &v1alpha1.TiFlashSpec{}, + }, + } + tc.Name = "start-script-test" + tc.Namespace = "start-script-test-ns" + + if c.modifyTC != nil { + c.modifyTC(tc) + } + + script, err := RenderTiFlashStartScriptWithStartArgs(tc) + g.Expect(err).Should(gomega.Succeed()) + if diff := cmp.Diff(c.expectScript, script); diff != "" { + t.Errorf("unexpected (-want, +got): %s", diff) + t.Logf("got: %s", script) + } + g.Expect(validateScript(script)).Should(gomega.Succeed()) + }) + } +} + func TestRenderTiFlashInitScript(t *testing.T) { g := gomega.NewGomegaWithT(t) diff --git a/pkg/manager/member/startscript/v2/tiflash_start_script.go b/pkg/manager/member/startscript/v2/tiflash_start_script.go index b2878f157e..80a754b468 100644 --- a/pkg/manager/member/startscript/v2/tiflash_start_script.go +++ b/pkg/manager/member/startscript/v2/tiflash_start_script.go @@ -14,9 +14,13 @@ package v2 import ( + "fmt" + "slices" + "strings" "text/template" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/controller" ) // TiFlashStartScriptModel contain fields for rendering TiFlash start script @@ -26,6 +30,10 @@ type TiFlashStartScriptModel struct { // RenderTiFlashStartScript renders TiFlash start script from TidbCluster func RenderTiFlashStartScript(tc *v1alpha1.TidbCluster) (string, error) { + if tc.Spec.TiFlash.DoesMountCMInTiflashContainer() { + return RenderTiFlashStartScriptWithStartArgs(tc) + } + m := &TiFlashStartScriptModel{} m.ExtraArgs = "" @@ -33,6 +41,56 @@ func RenderTiFlashStartScript(tc *v1alpha1.TidbCluster) (string, error) { return renderTemplateFunc(tiflashStartScriptTpl, m) } +// TiFlashStartScriptWithStartArgsModel is an alternative of TiFlashStartScriptModel that enable tiflash pod run without +// initContainer +type TiFlashStartScriptWithStartArgsModel struct { + AdvertiseAddr string + AdvertiseStatusAddr string + Addr string + PDAddresses string + ExtraArgs string + + AcrossK8s *AcrossK8sScriptModel +} + +func RenderTiFlashStartScriptWithStartArgs(tc *v1alpha1.TidbCluster) (string, error) { + m := &TiFlashStartScriptWithStartArgsModel{ + AdvertiseAddr: fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashProxyPort), + ExtraArgs: "", + } + tcName := tc.Name + tcNS := tc.Namespace + + // only the tiflash learner supports dynamic configuration + if tc.Spec.EnableDynamicConfiguration != nil && *tc.Spec.EnableDynamicConfiguration { + m.AdvertiseStatusAddr = fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashProxyStatusPort) + } + + preferPDAddressesOverDiscovery := slices.Contains( + tc.Spec.StartScriptV2FeatureFlags, v1alpha1.StartScriptV2FeatureFlagPreferPDAddressesOverDiscovery) + if preferPDAddressesOverDiscovery { + pdAddressesWithSchemeAndPort := addressesWithSchemeAndPort(tc.Spec.PDAddresses, "", v1alpha1.DefaultPDClientPort) + m.PDAddresses = strings.Join(pdAddressesWithSchemeAndPort, ",") + } + if len(m.PDAddresses) == 0 { + if tc.AcrossK8s() { + m.AcrossK8s = &AcrossK8sScriptModel{ + PDAddr: fmt.Sprintf("%s:%d", controller.PDMemberName(tcName), v1alpha1.DefaultPDClientPort), + DiscoveryAddr: fmt.Sprintf("%s-discovery.%s:10261", tcName, tcNS), + } + m.PDAddresses = "${result}" // get pd addr in subscript + } else if tc.Heterogeneous() && tc.WithoutLocalPD() { + m.PDAddresses = fmt.Sprintf("%s:%d", controller.PDMemberName(tc.Spec.Cluster.Name), v1alpha1.DefaultPDClientPort) // use pd of reference cluster + } else { + m.PDAddresses = fmt.Sprintf("%s:%d", controller.PDMemberName(tcName), v1alpha1.DefaultPDClientPort) + } + } + + m.Addr = fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashFlashPort) + + return renderTemplateFunc(tiflashStartScriptWithStartArgsTpl, m) +} + const ( // tiflashStartSubScript contains optional subscripts used in start script. tiflashStartSubScript = `` @@ -49,6 +107,49 @@ ARGS="${ARGS} {{ .ExtraArgs }}" echo "starting tiflash-server ..." echo "/tiflash/tiflash ${ARGS}" exec /tiflash/tiflash server ${ARGS} +` + + tiflashStartSubScriptWithStartArgs = ` +{{ define "AcrossK8sSubscript" }} +pd_url={{ .AcrossK8s.PDAddr }} +encoded_domain_url=$(echo $pd_url | base64 | tr "\n" " " | sed "s/ //g") +discovery_url={{ .AcrossK8s.DiscoveryAddr }} +until result=$(wget -qO- -T 3 http://${discovery_url}/verify/${encoded_domain_url} 2>/dev/null | sed 's/http:\/\///g' | sed 's/https:\/\///g'); do + echo "waiting for the verification of PD endpoints ..." + sleep $((RANDOM % 5)) +done +PD_ADDRESS=${result} +{{- end}} +` + + // tiflashStartScriptWithStartArgs is the new way to launch tiflash, that enable tiflash pod run without init container. + tiflashStartScriptWithStartArgs = ` +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="{{ .PDAddresses }}" + +{{- if .AcrossK8s -}} {{ template "AcrossK8sSubscript" . }} {{- end }} +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr={{ .AdvertiseAddr }} \{{- if .AdvertiseStatusAddr }} +--flash.proxy.advertise-status-addr={{ .AdvertiseStatusAddr }} \ +{{- end }} +--flash.service_addr={{ .Addr }} \ +--raft.pd_addr=${PD_ADDRESS} +" + +{{- if .ExtraArgs }} +ARGS="${ARGS} {{ .ExtraArgs }}" +{{- end }} + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} ` ) @@ -57,3 +158,9 @@ var tiflashStartScriptTpl = template.Must( template.New("tiflash-start-script").Parse(tiflashStartSubScript), ).Parse(componentCommonScript + tiflashStartScript), ) + +var tiflashStartScriptWithStartArgsTpl = template.Must( + template.Must( + template.New("tiflash-start-script-with-start-args").Parse(tiflashStartSubScriptWithStartArgs), + ).Parse(componentCommonScript + tiflashStartScriptWithStartArgs), +) diff --git a/pkg/manager/member/startscript/v2/tiflash_start_script_test.go b/pkg/manager/member/startscript/v2/tiflash_start_script_test.go index 81c0dd4c1e..2dec2505ce 100644 --- a/pkg/manager/member/startscript/v2/tiflash_start_script_test.go +++ b/pkg/manager/member/startscript/v2/tiflash_start_script_test.go @@ -87,3 +87,282 @@ exec /tiflash/tiflash server ${ARGS} g.Expect(validateScript(script)).Should(gomega.Succeed()) } } + +func TestRenderTiFlashStartScriptWithStartArgs(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + type testcase struct { + name string + + modifyTC func(tc *v1alpha1.TidbCluster) + expectScript string + } + + cases := []testcase{ + { + name: "basic", + modifyTC: func(tc *v1alpha1.TidbCluster) {}, + expectScript: `#!/bin/sh + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="start-script-test-pd:2379" +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20170 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + { + name: "enable AdvertiseAddr", + modifyTC: func(tc *v1alpha1.TidbCluster) { + enable := true + tc.Spec.EnableDynamicConfiguration = &enable + }, + expectScript: `#!/bin/sh + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="start-script-test-pd:2379" +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20170 \ +--flash.proxy.advertise-status-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20292 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + { + name: "across k8s", + modifyTC: func(tc *v1alpha1.TidbCluster) { + tc.Spec.ClusterDomain = "cluster.local" + tc.Spec.AcrossK8s = true + }, + expectScript: `#!/bin/sh + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="${result}" +pd_url=start-script-test-pd:2379 +encoded_domain_url=$(echo $pd_url | base64 | tr "\n" " " | sed "s/ //g") +discovery_url=start-script-test-discovery.start-script-test-ns:10261 +until result=$(wget -qO- -T 3 http://${discovery_url}/verify/${encoded_domain_url} 2>/dev/null | sed 's/http:\/\///g' | sed 's/https:\/\///g'); do + echo "waiting for the verification of PD endpoints ..." + sleep $((RANDOM % 5)) +done +PD_ADDRESS=${result} +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc.cluster.local:20170 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc.cluster.local:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + { + name: "across k8s with tls enabled", + modifyTC: func(tc *v1alpha1.TidbCluster) { + tc.Spec.ClusterDomain = "cluster.local" + tc.Spec.AcrossK8s = true + tc.Spec.TLSCluster = &v1alpha1.TLSCluster{Enabled: true} + }, + expectScript: `#!/bin/sh + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="${result}" +pd_url=start-script-test-pd:2379 +encoded_domain_url=$(echo $pd_url | base64 | tr "\n" " " | sed "s/ //g") +discovery_url=start-script-test-discovery.start-script-test-ns:10261 +until result=$(wget -qO- -T 3 http://${discovery_url}/verify/${encoded_domain_url} 2>/dev/null | sed 's/http:\/\///g' | sed 's/https:\/\///g'); do + echo "waiting for the verification of PD endpoints ..." + sleep $((RANDOM % 5)) +done +PD_ADDRESS=${result} +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc.cluster.local:20170 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc.cluster.local:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + { + name: "basic with ipv6", + modifyTC: func(tc *v1alpha1.TidbCluster) { + tc.Spec.PreferIPv6 = true + }, + expectScript: `#!/bin/sh + +set -uo pipefail + +ANNOTATIONS="/etc/podinfo/annotations" +if [[ ! -f "${ANNOTATIONS}" ]] +then + echo "${ANNOTATIONS} does't exist, exiting." + exit 1 +fi +source ${ANNOTATIONS} 2>/dev/null + +runmode=${runmode:-normal} +if [[ X${runmode} == Xdebug ]] +then + echo "entering debug mode." + tail -f /dev/null +fi + +# Use HOSTNAME if POD_NAME is unset for backward compatibility. +POD_NAME=${POD_NAME:-$HOSTNAME} +PD_ADDRESS="start-script-test-pd:2379" +ARGS="server --config-file /etc/tiflash/config_templ.toml \ +-- \ +--flash.proxy.advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20170 \ +--flash.service_addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:3930 \ +--raft.pd_addr=${PD_ADDRESS} +" + +if [ ! -z "${STORE_LABELS:-}" ]; then + LABELS=" --labels ${STORE_LABELS} " + ARGS="${ARGS}${LABELS}" +fi + +echo "starting tiflash-server ..." +echo "/tiflash/tiflash ${ARGS}" +exec /tiflash/tiflash ${ARGS} +`, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + t.Logf("test case: %s", c.name) + + tc := &v1alpha1.TidbCluster{ + Spec: v1alpha1.TidbClusterSpec{ + TiFlash: &v1alpha1.TiFlashSpec{}, + }, + } + tc.Name = "start-script-test" + tc.Namespace = "start-script-test-ns" + + if c.modifyTC != nil { + c.modifyTC(tc) + } + + script, err := RenderTiFlashStartScriptWithStartArgs(tc) + g.Expect(err).Should(gomega.Succeed()) + if diff := cmp.Diff(c.expectScript, script); diff != "" { + t.Errorf("unexpected (-want, +got): %s", diff) + t.Logf("got: %s", script) + } + g.Expect(validateScript(script)).Should(gomega.Succeed()) + }) + } +} diff --git a/pkg/manager/member/tiflash_member_manager.go b/pkg/manager/member/tiflash_member_manager.go index c343a78ec0..6c2b231ffc 100644 --- a/pkg/manager/member/tiflash_member_manager.go +++ b/pkg/manager/member/tiflash_member_manager.go @@ -373,6 +373,7 @@ func getNewStatefulSet(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*apps.St tcName := tc.GetName() baseTiFlashSpec := tc.BaseTiFlashSpec() spec := tc.Spec.TiFlash + mountCMInTiflashContainer := spec.DoesMountCMInTiflashContainer() tiflashConfigMap := controller.MemberConfigMapName(tc, v1alpha1.TiFlashMemberType) if cm != nil { @@ -402,6 +403,13 @@ func getNewStatefulSet(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*apps.St Name: tiflashCertVolumeName, ReadOnly: true, MountPath: tiflashCertPath, }) } + // with mountCMInTiflashContainer enabled, the tiflash container should directly read config from `/etc/tiflash/xxx.toml` mounted from ConfigMap + // rather than `/data0/xxx.toml` created by initContainer. + if mountCMInTiflashContainer { + volMounts = append(volMounts, corev1.VolumeMount{ + Name: "config", ReadOnly: true, MountPath: "/etc/tiflash", + }) + } vols := []corev1.Volume{ annoVolume, @@ -463,43 +471,47 @@ func getNewStatefulSet(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*apps.St podSecurityContext.Sysctls = []corev1.Sysctl{} } - // Append init container for config files initialization - initVolMounts := []corev1.VolumeMount{ - {Name: "data0", MountPath: "/data0"}, - {Name: "config", ReadOnly: true, MountPath: "/etc/tiflash"}, - } - initEnv := []corev1.EnvVar{ - { - Name: "POD_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.name", + // the work of initContainer is substituted by new start scripts which moves the configuration items depending on running env + // to command args. + if !mountCMInTiflashContainer { + // Append init container for config files initialization + initVolMounts := []corev1.VolumeMount{ + {Name: "data0", MountPath: "/data0"}, + {Name: "config", ReadOnly: true, MountPath: "/etc/tiflash"}, + } + initEnv := []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, }, }, - }, - } + } - // NOTE: the config content should respect the init script - initScript, err := startscript.RenderTiFlashInitScript(tc) - if err != nil { - return nil, fmt.Errorf("render start-script for tc %s/%s failed: %v", tc.Namespace, tc.Name, err) - } + // NOTE: the config content should respect the init script + initScript, err := startscript.RenderTiFlashInitScript(tc) + if err != nil { + return nil, fmt.Errorf("render start-script for tc %s/%s failed: %v", tc.Namespace, tc.Name, err) + } - initializer := corev1.Container{ - Name: "init", - Image: tc.HelperImage(), - Command: []string{ - "sh", - "-c", - initScript, - }, - Env: initEnv, - VolumeMounts: initVolMounts, - } - if spec.Initializer != nil { - initializer.Resources = controller.ContainerResource(spec.Initializer.ResourceRequirements) + initializer := corev1.Container{ + Name: "init", + Image: tc.HelperImage(), + Command: []string{ + "sh", + "-c", + initScript, + }, + Env: initEnv, + VolumeMounts: initVolMounts, + } + if spec.Initializer != nil { + initializer.Resources = controller.ContainerResource(spec.Initializer.ResourceRequirements) + } + initContainers = append(initContainers, initializer) } - initContainers = append(initContainers, initializer) stsLabels := labelTiFlash(tc) setName := controller.TiFlashMemberName(tcName) diff --git a/pkg/manager/member/tiflash_util.go b/pkg/manager/member/tiflash_util.go index f10c54c0c0..eda519dcc7 100644 --- a/pkg/manager/member/tiflash_util.go +++ b/pkg/manager/member/tiflash_util.go @@ -129,6 +129,7 @@ func getTiFlashConfigV2(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper if config.Proxy == nil { config.Proxy = v1alpha1.NewTiFlashProxyConfig() } + mountCMInTiflashContainer := tc.Spec.TiFlash.DoesMountCMInTiflashContainer() common := config.Common proxy := config.Proxy @@ -186,13 +187,21 @@ func getTiFlashConfigV2(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper } } common.SetIfNil("flash.tidb_status_addr", tidbStatusAddr) - common.SetIfNil("flash.service_addr", fmt.Sprintf("%s:%d", listenHost, v1alpha1.DefaultTiFlashFlashPort)) + if !mountCMInTiflashContainer { + common.SetIfNil("flash.service_addr", fmt.Sprintf("%s:%d", listenHost, v1alpha1.DefaultTiFlashFlashPort)) + } common.SetIfNil("flash.flash_cluster.log", defaultClusterLog) common.SetIfNil("flash.proxy.addr", fmt.Sprintf("%s:%d", listenHost, v1alpha1.DefaultTiFlashProxyPort)) - common.SetIfNil("flash.proxy.advertise-addr", fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:%d", controller.TiFlashMemberName(name), - controller.TiFlashPeerMemberName(name), ns, controller.FormatClusterDomain(clusterDomain), v1alpha1.DefaultTiFlashProxyPort)) + if !mountCMInTiflashContainer { + common.SetIfNil("flash.proxy.advertise-addr", fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:%d", controller.TiFlashMemberName(name), + controller.TiFlashPeerMemberName(name), ns, controller.FormatClusterDomain(clusterDomain), v1alpha1.DefaultTiFlashProxyPort)) + } common.SetIfNil("flash.proxy.data-dir", "/data0/proxy") - common.SetIfNil("flash.proxy.config", "/data0/proxy.toml") + if !mountCMInTiflashContainer { + common.SetIfNil("flash.proxy.config", "/data0/proxy.toml") + } else { + common.SetIfNil("flash.proxy.config", "/etc/tiflash/proxy_templ.toml") + } // logger common.SetIfNil("logger.errorlog", defaultErrorLog) @@ -206,6 +215,9 @@ func getTiFlashConfigV2(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper pdAddr = fmt.Sprintf("%s.%s.svc%s:%d", controller.PDMemberName(ref.Name), ref.Namespace, controller.FormatClusterDomain(ref.ClusterDomain), v1alpha1.DefaultPDClientPort) // use pd of reference cluster } + // tiflash require at least one configuration item in ["raft"] config group, otherwise + // tiflash with version less than v7.1.0 may encounter schema sync problems. So we keep this item + // even if this item is configured via command line args. common.SetIfNil("raft.pd_addr", pdAddr) if listenHost == listenHostForIPv6 { @@ -217,11 +229,13 @@ func getTiFlashConfigV2(tc *v1alpha1.TidbCluster) *v1alpha1.TiFlashConfigWraper // proxy { proxy.SetIfNil("log-level", "info") - proxy.SetIfNil("server.engine-addr", fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:%d", controller.TiFlashMemberName(name), controller.TiFlashPeerMemberName(name), ns, - controller.FormatClusterDomain(clusterDomain), v1alpha1.DefaultTiFlashFlashPort)) + if !mountCMInTiflashContainer { + proxy.SetIfNil("server.engine-addr", fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:%d", controller.TiFlashMemberName(name), controller.TiFlashPeerMemberName(name), ns, + controller.FormatClusterDomain(clusterDomain), v1alpha1.DefaultTiFlashFlashPort)) + proxy.SetIfNil("server.advertise-status-addr", fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:%d", controller.TiFlashMemberName(name), controller.TiFlashPeerMemberName(name), ns, + controller.FormatClusterDomain(clusterDomain), v1alpha1.DefaultTiFlashProxyStatusPort)) + } proxy.SetIfNil("server.status-addr", fmt.Sprintf("%s:%d", listenHost, v1alpha1.DefaultTiFlashProxyStatusPort)) - proxy.SetIfNil("server.advertise-status-addr", fmt.Sprintf("%s-POD_NUM.%s.%s.svc%s:%d", controller.TiFlashMemberName(name), controller.TiFlashPeerMemberName(name), ns, - controller.FormatClusterDomain(clusterDomain), v1alpha1.DefaultTiFlashProxyStatusPort)) } // Note the config of tiflash use "_" by convention, others(proxy) use "-". diff --git a/pkg/manager/member/tiflash_util_test.go b/pkg/manager/member/tiflash_util_test.go index cff1648c33..3b178e5a66 100644 --- a/pkg/manager/member/tiflash_util_test.go +++ b/pkg/manager/member/tiflash_util_test.go @@ -21,6 +21,7 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" + "github.com/pingcap/tidb-operator/pkg/apis/label" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -1583,6 +1584,48 @@ func TestTestGetTiFlashConfig(t *testing.T) { engine-addr = "test-tiflash-POD_NUM.test-tiflash-peer.default.svc:3930" status-addr = "0.0.0.0:20292"`, }, + { + name: "config is nil and cluster enable tls and annotate mountCMInTiflash", + setTC: func(tc *v1alpha1.TidbCluster) { + tc.Spec.TiFlash.Config = nil + tc.Spec.TiFlash.Annotations = map[string]string{ + label.AnnTiflashMountCMInTiflashContainer: "true", + } + tc.Spec.TLSCluster = &v1alpha1.TLSCluster{Enabled: true} + }, + expectCommonCfg: ` + tmp_path = "/data0/tmp" + [flash] + tidb_status_addr = "test-tidb.default.svc:10080" + [flash.flash_cluster] + log = "/data0/logs/flash_cluster_manager.log" + [flash.proxy] + addr = "0.0.0.0:20170" + config = "/etc/tiflash/proxy_templ.toml" + data-dir = "/data0/proxy" + [logger] + errorlog = "/data0/logs/error.log" + log = "/data0/logs/server.log" + [raft] + pd_addr = "test-pd.default.svc:2379" + [security] + ca_path = "/var/lib/tiflash-tls/ca.crt" + cert_path = "/var/lib/tiflash-tls/tls.crt" + key_path = "/var/lib/tiflash-tls/tls.key" + [storage] + [storage.main] + dir = ["/data0/db"] + [storage.raft] + dir = ["/data0/kvstore"]`, + expectProxyCfg: ` + log-level = "info" + [security] + ca-path = "/var/lib/tiflash-tls/ca.crt" + cert-path = "/var/lib/tiflash-tls/tls.crt" + key-path = "/var/lib/tiflash-tls/tls.key" + [server] + status-addr = "0.0.0.0:20292"`, + }, { name: "config contain cert-allowed-cn and cluster enable tls", setTC: func(tc *v1alpha1.TidbCluster) { diff --git a/tests/e2e/tidbcluster/tidbcluster.go b/tests/e2e/tidbcluster/tidbcluster.go index 9030c73161..a1b191dd71 100644 --- a/tests/e2e/tidbcluster/tidbcluster.go +++ b/tests/e2e/tidbcluster/tidbcluster.go @@ -3197,6 +3197,28 @@ var _ = ginkgo.Describe("TiDBCluster", func() { } }) + + ginkgo.Describe("Mount ConfigMap in tiflash container", func() { + ginkgo.It("deploy tidb cluster", func() { + ginkgo.By("Deploy initial tc") + tc := fixture.GetTidbCluster(ns, "mount-cm-in-tiflash-container", utilimage.TiDBLatest) + fixture.AddTiFlashForTidbCluster(tc) + tc.Spec.PD.Replicas = 1 + tc.Spec.TiKV.Replicas = 1 + tc.Spec.TiDB.Replicas = 1 + tc.Spec.TiFlash.Replicas = 2 + if tc.Spec.TiFlash.Annotations == nil { + tc.Spec.TiFlash.Annotations = map[string]string{} + } + tc.Spec.TiFlash.Annotations[label.AnnTiflashMountCMInTiflashContainer] = "true" + + tc, err := cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Create(context.TODO(), tc, metav1.CreateOptions{}) + framework.ExpectNoError(err, "Expected create tidbcluster") + err = oa.WaitForTidbClusterReady(tc, 30*time.Minute, 5*time.Second) + framework.ExpectNoError(err, "Expected get tidbcluster") + }) + }) + }) // checkPumpStatus check there are onlineNum online pump instance running now.