deployments(ticdc): improve the docker build time (#8757)

close #8758

.github/workflows/ticdc_integration.yaml

@@ -26,6 +26,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Check Docker Version
+        run: docker --version
+      - name: Install Latest Docker
+        run: |
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+          sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+          sudo apt-get update
+          sudo apt-get install docker-ce
+      - name: Check Docker Version
+        run: docker --version
+
       - name: Free disk space
         run: |
           sudo rm -rf /usr/local/lib/android
@@ -78,6 +89,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Check Docker Version
+        run: docker --version
+      - name: Install Latest Docker
+        run: |
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+          sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+          sudo apt-get update
+          sudo apt-get install docker-ce
+      - name: Check Docker Version
+        run: docker --version
+
       - name: Free disk space
         run: |
           sudo rm -rf /usr/local/lib/android
@@ -130,6 +152,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Check Docker Version
+        run: docker --version
+      - name: Install Latest Docker
+        run: |
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+          sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+          sudo apt-get update
+          sudo apt-get install docker-ce
+      - name: Check Docker Version
+        run: docker --version
+
       - name: Free disk space
         run: |
           sudo rm -rf /usr/local/lib/android

Makefile

@@ -140,7 +140,8 @@ format-makefiles: $(MAKE_FILES)
 bank:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/bank ./tests/bank/bank.go ./tests/bank/case.go
 
-build-failpoint: check_failpoint_ctl
+build-cdc-with-failpoint: check_failpoint_ctl
+build-cdc-with-failpoint: ## Build cdc with failpoint enabled.
 	$(FAILPOINT_ENABLE)
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc ./cmd/cdc/main.go
 	$(FAILPOINT_DISABLE)
@@ -154,10 +155,6 @@ kafka_consumer:
 storage_consumer:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go
 
-cdc_test_image: 
-	@which docker || (echo "docker not found in ${PATH}"; exit 1)
-	docker build --platform linux/amd64 -f deployments/ticdc/docker/test.Dockerfile -t cdc:test ./ 
-
 install:
 	go install ./...
 

deployments/ticdc/docker-compose/docker-compose-avro.yml

@@ -1,5 +1,5 @@
 ---
-version: '2.1'
+version: '3.5'
 
 services:
   controller:

deployments/ticdc/docker-compose/docker-compose-canal.yml

@@ -1,5 +1,5 @@
 ---
-version: '2.1'
+version: '3.5'
 
 services:
   controller:

deployments/ticdc/docker-compose/docker-compose-kafka-integration.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '3.5'
 
 services:
   zookeeper:

deployments/ticdc/docker-compose/docker-compose-mysql-integration.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '3.5'
 
 services:
   mysql_integration_test:

deployments/ticdc/docker-compose/docker-compose-mysql.yml

@@ -1,5 +1,5 @@
 ---
-    version: '2.1'
+    version: '3.5'
 
     services:
       controller:

deployments/ticdc/docker-compose/docker-compose-storage-integration.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '3.5'
 
 services:
   storage_integration_test:

deployments/ticdc/docker/dev.Dockerfile

@@ -1,12 +1,9 @@
 FROM golang:1.20-alpine as builder
-RUN apk add --no-cache git make bash
+RUN apk add --no-cache git make bash findutils
 WORKDIR /go/src/github.com/pingcap/tiflow
 COPY . .
-ENV CDC_ENABLE_VENDOR=1
-RUN go mod vendor
-RUN make failpoint-enable
-RUN make cdc
-RUN make failpoint-disable
+
+RUN --mount=type=cache,target=/root/.cache/go-build,target=/go/pkg/mod make build-cdc-with-failpoint
 
 FROM alpine:3.15
 RUN apk add --no-cache tzdata bash curl socat

deployments/ticdc/docker/kafka-consumer.Dockerfile

@@ -3,7 +3,8 @@ RUN apk add --no-cache make bash
 WORKDIR /go/src/github.com/pingcap/tiflow
 COPY . .
 
-RUN make kafka_consumer
+RUN --mount=type=cache,target=/go/pkg/mod go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build make kafka_consumer
 
 FROM alpine:3.15
 COPY --from=builder  /go/src/github.com/pingcap/tiflow/bin/cdc_kafka_consumer /cdc_kafka_consumer

deployments/ticdc/docker/storage-consumer.Dockerfile

@@ -2,8 +2,8 @@ FROM golang:1.20-alpine as builder
 RUN apk add --no-cache make bash
 WORKDIR /go/src/github.com/pingcap/tiflow
 COPY . .
-
-RUN make storage_consumer
+RUN --mount=type=cache,target=/go/pkg/mod go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build make storage_consumer
 
 FROM alpine:3.15
 COPY --from=builder  /go/src/github.com/pingcap/tiflow/bin/cdc_storage_consumer /cdc_storage_consumer