support run tiflash pod with ConfigMap mounted in tiflash container (#…

…5552)

pkg/apis/label/label.go

@@ -104,6 +104,10 @@ const (
 	AnnTiCDCGracefulShutdownBeginTime = "tidb.pingcap.com/ticdc-graceful-shutdown-begin-time"
 	// AnnStsLastSyncTimestamp is sts annotation key to indicate the last timestamp the operator sync the sts
 	AnnStsLastSyncTimestamp = "tidb.pingcap.com/sync-timestamp"
+	// AnnTiflashMountCMInTiflashContainer is tiflash pod annotation key to indicate whether directly mount ConfigMap
+	// in tiflash container instead of init container for tiflash. With it annotated, the tiflash container will directly
+	// read config from files mounted by ConfigMap and that enables tiflash support hot-reload config.
+	AnnTiflashMountCMInTiflashContainer = "tiflash.tidb.pingcap.com/mount-cm-in-tiflash-container"
 
 	// AnnPVCScaleInTime is pvc scaled in time key used in PVC for e2e test only
 	AnnPVCScaleInTime = "tidb.pingcap.com/scale-in-time"

pkg/apis/pingcap/v1alpha1/tidbcluster.go

@@ -1158,6 +1158,10 @@ func (tiflash *TiFlashSpec) GetScaleOutParallelism() int {
 	return int(*(tiflash.ScalePolicy.ScaleOutParallelism))
 }
 
+func (tiflash *TiFlashSpec) DoesMountCMInTiflashContainer() bool {
+	return tiflash.Annotations[label.AnnTiflashMountCMInTiflashContainer] == "true"
+}
+
 func (tidbSvc *TiDBServiceSpec) ShouldExposeStatus() bool {
 	exposeStatus := tidbSvc.ExposeStatus
 	if exposeStatus == nil {

pkg/manager/member/startscript/v1/render_script.go

@@ -233,9 +233,39 @@ done
 }
 
 func RenderTiFlashStartScript(tc *v1alpha1.TidbCluster) (string, error) {
+	if tc.Spec.TiFlash.DoesMountCMInTiflashContainer() {
+		return RenderTiFlashStartScriptWithStartArgs(tc)
+	}
 	return "/tiflash/tiflash server --config-file /data0/config.toml", nil
 }
 
+func RenderTiFlashStartScriptWithStartArgs(tc *v1alpha1.TidbCluster) (string, error) {
+	model := &TiflashStartScriptModel{
+		CommonModel: CommonModel{
+			AcrossK8s:     tc.AcrossK8s(),
+			ClusterDomain: tc.Spec.ClusterDomain,
+		},
+		AdvertiseAddr:             fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashProxyPort),
+		EnableAdvertiseStatusAddr: false,
+	}
+	// only the tiflash learner supports dynamic configuration
+	if tc.Spec.EnableDynamicConfiguration != nil && *tc.Spec.EnableDynamicConfiguration {
+		model.AdvertiseStatusAddr = fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashProxyStatusPort)
+		model.EnableAdvertiseStatusAddr = true
+	}
+
+	model.PDAddress = fmt.Sprintf("%s://${CLUSTER_NAME}-pd:%d", tc.Scheme(), v1alpha1.DefaultPDClientPort)
+	if tc.AcrossK8s() {
+		model.PDAddress = fmt.Sprintf("%s://${CLUSTER_NAME}-pd:%d", tc.Scheme(), v1alpha1.DefaultPDClientPort) // get pd addr from discovery in startup script
+	} else if tc.Heterogeneous() && tc.WithoutLocalPD() {
+		model.PDAddress = fmt.Sprintf("%s://%s:%d", tc.Scheme(), controller.PDMemberName(tc.Spec.Cluster.Name), v1alpha1.DefaultPDClientPort) // use pd of reference cluster
+	}
+
+	model.Addr = fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashFlashPort)
+
+	return renderTemplateFunc(tiflashStartScriptTpl, model)
+}
+
 func RenderTiFlashInitScript(tc *v1alpha1.TidbCluster) (string, error) {
 	tcName := tc.GetName()
 

pkg/manager/member/startscript/v1/template.go

@@ -742,3 +742,75 @@ type DMWorkerStartScriptModel struct {
 func RenderDMWorkerStartScript(model *DMWorkerStartScriptModel) (string, error) {
 	return renderTemplateFunc(dmWorkerStartScriptTpl, model)
 }
+
+var tiflashStartScriptTplText = `#!/bin/sh
+
+# This script is used to start tiflash containers in kubernetes cluster
+
+# Use DownwardAPIVolumeFiles to store informations of the cluster:
+# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api
+#
+#   runmode="normal/debug"
+#
+
+set -uo pipefail
+
+ANNOTATIONS="/etc/podinfo/annotations"
+
+if [[ ! -f "${ANNOTATIONS}" ]]
+then
+    echo "${ANNOTATIONS} does't exist, exiting."
+    exit 1
+fi
+source ${ANNOTATIONS} 2>/dev/null
+
+runmode=${runmode:-normal}
+if [[ X${runmode} == Xdebug ]]
+then
+    echo "entering debug mode."
+    tail -f /dev/null
+fi
+
+# Use HOSTNAME if POD_NAME is unset for backward compatibility.
+POD_NAME=${POD_NAME:-$HOSTNAME}
+PD_ADDRESS="{{ .PDAddress }}"{{ if .AcrossK8s }}
+pd_url="{{ .PDAddress }}"
+encoded_domain_url=$(echo $pd_url | base64 | tr "\n" " " | sed "s/ //g")
+discovery_url="${CLUSTER_NAME}-discovery.${NAMESPACE}:10261"
+
+until result=$(wget -qO- -T 3 http://${discovery_url}/verify/${encoded_domain_url} 2>/dev/null); do
+echo "waiting for the verification of PD endpoints ..."
+sleep $((RANDOM % 5))
+done
+PD_ADDRESS=${result}
+{{ end }}
+
+ARGS="server --config-file /etc/tiflash/config_templ.toml \
+-- \
+--flash.proxy.advertise-addr={{ .AdvertiseAddr }} \{{if .EnableAdvertiseStatusAddr }}
+--flash.proxy.advertise-status-addr={{ .AdvertiseStatusAddr }} \{{end}}
+--flash.service_addr={{ .Addr }} \
+--raft.pd_addr=${PD_ADDRESS}
+"
+
+if [ ! -z "${STORE_LABELS:-}" ]; then
+  LABELS=" --labels ${STORE_LABELS} "
+  ARGS="${ARGS}${LABELS}"
+fi
+
+echo "starting tiflash-server ..."
+echo "/tiflash/tiflash ${ARGS}"
+exec /tiflash/tiflash ${ARGS}
+`
+
+var tiflashStartScriptTpl = template.Must(template.New("tiflash-start-script").Parse(tiflashStartScriptTplText))
+
+type TiflashStartScriptModel struct {
+	CommonModel
+
+	AdvertiseAddr             string
+	EnableAdvertiseStatusAddr bool
+	AdvertiseStatusAddr       string
+	Addr                      string
+	PDAddress                 string
+}