Skip to content

Commit

Permalink
merge master
Browse files Browse the repository at this point in the history
Signed-off-by: husharp <[email protected]>
  • Loading branch information
HuSharp committed Mar 4, 2024
2 parents 25ba435 + 2938f4e commit 5185b22
Show file tree
Hide file tree
Showing 11 changed files with 960 additions and 40 deletions.
4 changes: 4 additions & 0 deletions pkg/apis/label/label.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,10 @@ const (
AnnTiCDCGracefulShutdownBeginTime = "tidb.pingcap.com/ticdc-graceful-shutdown-begin-time"
// AnnStsLastSyncTimestamp is sts annotation key to indicate the last timestamp the operator sync the sts
AnnStsLastSyncTimestamp = "tidb.pingcap.com/sync-timestamp"
// AnnTiflashMountCMInTiflashContainer is tiflash pod annotation key to indicate whether directly mount ConfigMap
// in tiflash container instead of init container for tiflash. With it annotated, the tiflash container will directly
// read config from files mounted by ConfigMap and that enables tiflash support hot-reload config.
AnnTiflashMountCMInTiflashContainer = "tiflash.tidb.pingcap.com/mount-cm-in-tiflash-container"

// AnnPVCScaleInTime is pvc scaled in time key used in PVC for e2e test only
AnnPVCScaleInTime = "tidb.pingcap.com/scale-in-time"
Expand Down
4 changes: 4 additions & 0 deletions pkg/apis/pingcap/v1alpha1/tidbcluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -1158,6 +1158,10 @@ func (tiflash *TiFlashSpec) GetScaleOutParallelism() int {
return int(*(tiflash.ScalePolicy.ScaleOutParallelism))
}

func (tiflash *TiFlashSpec) DoesMountCMInTiflashContainer() bool {
return tiflash.Annotations[label.AnnTiflashMountCMInTiflashContainer] == "true"
}

func (tidbSvc *TiDBServiceSpec) ShouldExposeStatus() bool {
exposeStatus := tidbSvc.ExposeStatus
if exposeStatus == nil {
Expand Down
30 changes: 30 additions & 0 deletions pkg/manager/member/startscript/v1/render_script.go
Original file line number Diff line number Diff line change
Expand Up @@ -233,9 +233,39 @@ done
}

func RenderTiFlashStartScript(tc *v1alpha1.TidbCluster) (string, error) {
if tc.Spec.TiFlash.DoesMountCMInTiflashContainer() {
return RenderTiFlashStartScriptWithStartArgs(tc)
}
return "/tiflash/tiflash server --config-file /data0/config.toml", nil
}

func RenderTiFlashStartScriptWithStartArgs(tc *v1alpha1.TidbCluster) (string, error) {
model := &TiflashStartScriptModel{
CommonModel: CommonModel{
AcrossK8s: tc.AcrossK8s(),
ClusterDomain: tc.Spec.ClusterDomain,
},
AdvertiseAddr: fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashProxyPort),
EnableAdvertiseStatusAddr: false,
}
// only the tiflash learner supports dynamic configuration
if tc.Spec.EnableDynamicConfiguration != nil && *tc.Spec.EnableDynamicConfiguration {
model.AdvertiseStatusAddr = fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashProxyStatusPort)
model.EnableAdvertiseStatusAddr = true
}

model.PDAddress = fmt.Sprintf("%s://${CLUSTER_NAME}-pd:%d", tc.Scheme(), v1alpha1.DefaultPDClientPort)
if tc.AcrossK8s() {
model.PDAddress = fmt.Sprintf("%s://${CLUSTER_NAME}-pd:%d", tc.Scheme(), v1alpha1.DefaultPDClientPort) // get pd addr from discovery in startup script
} else if tc.Heterogeneous() && tc.WithoutLocalPD() {
model.PDAddress = fmt.Sprintf("%s://%s:%d", tc.Scheme(), controller.PDMemberName(tc.Spec.Cluster.Name), v1alpha1.DefaultPDClientPort) // use pd of reference cluster
}

model.Addr = fmt.Sprintf("${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc%s:%d", controller.FormatClusterDomain(tc.Spec.ClusterDomain), v1alpha1.DefaultTiFlashFlashPort)

return renderTemplateFunc(tiflashStartScriptTpl, model)
}

func RenderTiFlashInitScript(tc *v1alpha1.TidbCluster) (string, error) {
tcName := tc.GetName()

Expand Down
72 changes: 72 additions & 0 deletions pkg/manager/member/startscript/v1/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -742,3 +742,75 @@ type DMWorkerStartScriptModel struct {
func RenderDMWorkerStartScript(model *DMWorkerStartScriptModel) (string, error) {
return renderTemplateFunc(dmWorkerStartScriptTpl, model)
}

var tiflashStartScriptTplText = `#!/bin/sh
# This script is used to start tiflash containers in kubernetes cluster
# Use DownwardAPIVolumeFiles to store informations of the cluster:
# https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api
#
# runmode="normal/debug"
#
set -uo pipefail
ANNOTATIONS="/etc/podinfo/annotations"
if [[ ! -f "${ANNOTATIONS}" ]]
then
echo "${ANNOTATIONS} does't exist, exiting."
exit 1
fi
source ${ANNOTATIONS} 2>/dev/null
runmode=${runmode:-normal}
if [[ X${runmode} == Xdebug ]]
then
echo "entering debug mode."
tail -f /dev/null
fi
# Use HOSTNAME if POD_NAME is unset for backward compatibility.
POD_NAME=${POD_NAME:-$HOSTNAME}
PD_ADDRESS="{{ .PDAddress }}"{{ if .AcrossK8s }}
pd_url="{{ .PDAddress }}"
encoded_domain_url=$(echo $pd_url | base64 | tr "\n" " " | sed "s/ //g")
discovery_url="${CLUSTER_NAME}-discovery.${NAMESPACE}:10261"
until result=$(wget -qO- -T 3 http://${discovery_url}/verify/${encoded_domain_url} 2>/dev/null); do
echo "waiting for the verification of PD endpoints ..."
sleep $((RANDOM % 5))
done
PD_ADDRESS=${result}
{{ end }}
ARGS="server --config-file /etc/tiflash/config_templ.toml \
-- \
--flash.proxy.advertise-addr={{ .AdvertiseAddr }} \{{if .EnableAdvertiseStatusAddr }}
--flash.proxy.advertise-status-addr={{ .AdvertiseStatusAddr }} \{{end}}
--flash.service_addr={{ .Addr }} \
--raft.pd_addr=${PD_ADDRESS}
"
if [ ! -z "${STORE_LABELS:-}" ]; then
LABELS=" --labels ${STORE_LABELS} "
ARGS="${ARGS}${LABELS}"
fi
echo "starting tiflash-server ..."
echo "/tiflash/tiflash ${ARGS}"
exec /tiflash/tiflash ${ARGS}
`

var tiflashStartScriptTpl = template.Must(template.New("tiflash-start-script").Parse(tiflashStartScriptTplText))

type TiflashStartScriptModel struct {
CommonModel

AdvertiseAddr string
EnableAdvertiseStatusAddr bool
AdvertiseStatusAddr string
Addr string
PDAddress string
}
Loading

0 comments on commit 5185b22

Please sign in to comment.