From b7464567b406a3c0adc67eabaa8751c0efac7caf Mon Sep 17 00:00:00 2001
From: Rustin <rustin.liu@gmail.com>
Date: Sun, 9 Apr 2023 14:10:59 +0800
Subject: [PATCH] deployments(ticdc): improve the docker build time (#8757)

close pingcap/tiflow#8758
---
 .../docker/.dockerignore => .dockerignore     |  0
 .github/workflows/ticdc_integration.yaml      | 33 +++++++++++++++++++
 Makefile                                      |  7 ++--
 .../docker-compose/docker-compose-avro.yml    |  2 +-
 .../docker-compose/docker-compose-canal.yml   |  2 +-
 .../docker-compose-kafka-integration.yml      |  2 +-
 .../docker-compose-mysql-integration.yml      |  2 +-
 .../docker-compose/docker-compose-mysql.yml   |  2 +-
 .../docker-compose-storage-integration.yml    |  2 +-
 deployments/ticdc/docker/dev.Dockerfile       |  9 ++---
 .../ticdc/docker/kafka-consumer.Dockerfile    |  3 +-
 .../ticdc/docker/storage-consumer.Dockerfile  |  4 +--
 deployments/ticdc/docker/test.Dockerfile      |  6 ----
 13 files changed, 48 insertions(+), 26 deletions(-)
 rename deployments/ticdc/docker/.dockerignore => .dockerignore (100%)
 delete mode 100644 deployments/ticdc/docker/test.Dockerfile

diff --git a/deployments/ticdc/docker/.dockerignore b/.dockerignore
similarity index 100%
rename from deployments/ticdc/docker/.dockerignore
rename to .dockerignore
diff --git a/.github/workflows/ticdc_integration.yaml b/.github/workflows/ticdc_integration.yaml
index ef8651f8e5e..7cf07f7adf0 100644
--- a/.github/workflows/ticdc_integration.yaml
+++ b/.github/workflows/ticdc_integration.yaml
@@ -26,6 +26,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Check Docker Version
+        run: docker --version
+      - name: Install Latest Docker
+        run: |
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+          sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+          sudo apt-get update
+          sudo apt-get install docker-ce
+      - name: Check Docker Version
+        run: docker --version
+
       - name: Free disk space
         run: |
           sudo rm -rf /usr/local/lib/android
@@ -78,6 +89,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Check Docker Version
+        run: docker --version
+      - name: Install Latest Docker
+        run: |
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+          sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+          sudo apt-get update
+          sudo apt-get install docker-ce
+      - name: Check Docker Version
+        run: docker --version
+
       - name: Free disk space
         run: |
           sudo rm -rf /usr/local/lib/android
@@ -130,6 +152,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Check Docker Version
+        run: docker --version
+      - name: Install Latest Docker
+        run: |
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+          sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+          sudo apt-get update
+          sudo apt-get install docker-ce
+      - name: Check Docker Version
+        run: docker --version
+
       - name: Free disk space
         run: |
           sudo rm -rf /usr/local/lib/android
diff --git a/Makefile b/Makefile
index 9dc1b099637..c5d78e1ea9d 100644
--- a/Makefile
+++ b/Makefile
@@ -140,7 +140,8 @@ format-makefiles: $(MAKE_FILES)
 bank:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/bank ./tests/bank/bank.go ./tests/bank/case.go
 
-build-failpoint: check_failpoint_ctl
+build-cdc-with-failpoint: check_failpoint_ctl
+build-cdc-with-failpoint: ## Build cdc with failpoint enabled.
 	$(FAILPOINT_ENABLE)
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc ./cmd/cdc/main.go
 	$(FAILPOINT_DISABLE)
@@ -154,10 +155,6 @@ kafka_consumer:
 storage_consumer:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go
 
-cdc_test_image: 
-	@which docker || (echo "docker not found in ${PATH}"; exit 1)
-	docker build --platform linux/amd64 -f deployments/ticdc/docker/test.Dockerfile -t cdc:test ./ 
-
 install:
 	go install ./...
 
diff --git a/deployments/ticdc/docker-compose/docker-compose-avro.yml b/deployments/ticdc/docker-compose/docker-compose-avro.yml
index 95b0ed8aa74..0097361478f 100644
--- a/deployments/ticdc/docker-compose/docker-compose-avro.yml
+++ b/deployments/ticdc/docker-compose/docker-compose-avro.yml
@@ -1,5 +1,5 @@
 ---
-version: '2.1'
+version: '3.5'
 
 services:
   controller:
diff --git a/deployments/ticdc/docker-compose/docker-compose-canal.yml b/deployments/ticdc/docker-compose/docker-compose-canal.yml
index b1587f40966..ebe42d5c111 100644
--- a/deployments/ticdc/docker-compose/docker-compose-canal.yml
+++ b/deployments/ticdc/docker-compose/docker-compose-canal.yml
@@ -1,5 +1,5 @@
 ---
-version: '2.1'
+version: '3.5'
 
 services:
   controller:
diff --git a/deployments/ticdc/docker-compose/docker-compose-kafka-integration.yml b/deployments/ticdc/docker-compose/docker-compose-kafka-integration.yml
index 4168432197b..22e9809ab51 100644
--- a/deployments/ticdc/docker-compose/docker-compose-kafka-integration.yml
+++ b/deployments/ticdc/docker-compose/docker-compose-kafka-integration.yml
@@ -1,4 +1,4 @@
-version: '2.1'
+version: '3.5'
 
 services:
   zookeeper:
diff --git a/deployments/ticdc/docker-compose/docker-compose-mysql-integration.yml b/deployments/ticdc/docker-compose/docker-compose-mysql-integration.yml
index d30d8956b6c..adbc4b6bbbe 100644
--- a/deployments/ticdc/docker-compose/docker-compose-mysql-integration.yml
+++ b/deployments/ticdc/docker-compose/docker-compose-mysql-integration.yml
@@ -1,4 +1,4 @@
-version: '2.1'
+version: '3.5'
 
 services:
   mysql_integration_test:
diff --git a/deployments/ticdc/docker-compose/docker-compose-mysql.yml b/deployments/ticdc/docker-compose/docker-compose-mysql.yml
index 1e79a68210b..781bcaf2542 100644
--- a/deployments/ticdc/docker-compose/docker-compose-mysql.yml
+++ b/deployments/ticdc/docker-compose/docker-compose-mysql.yml
@@ -1,5 +1,5 @@
 ---
-    version: '2.1'
+    version: '3.5'
 
     services:
       controller:
diff --git a/deployments/ticdc/docker-compose/docker-compose-storage-integration.yml b/deployments/ticdc/docker-compose/docker-compose-storage-integration.yml
index 90968020ebc..6056d6e19a9 100644
--- a/deployments/ticdc/docker-compose/docker-compose-storage-integration.yml
+++ b/deployments/ticdc/docker-compose/docker-compose-storage-integration.yml
@@ -1,4 +1,4 @@
-version: '2.1'
+version: '3.5'
 
 services:
   storage_integration_test:
diff --git a/deployments/ticdc/docker/dev.Dockerfile b/deployments/ticdc/docker/dev.Dockerfile
index c34d37ff999..bb3ac4c67c4 100644
--- a/deployments/ticdc/docker/dev.Dockerfile
+++ b/deployments/ticdc/docker/dev.Dockerfile
@@ -1,12 +1,9 @@
 FROM golang:1.20-alpine as builder
-RUN apk add --no-cache git make bash
+RUN apk add --no-cache git make bash findutils
 WORKDIR /go/src/github.com/pingcap/tiflow
 COPY . .
-ENV CDC_ENABLE_VENDOR=1
-RUN go mod vendor
-RUN make failpoint-enable
-RUN make cdc
-RUN make failpoint-disable
+
+RUN --mount=type=cache,target=/root/.cache/go-build,target=/go/pkg/mod make build-cdc-with-failpoint
 
 FROM alpine:3.15
 RUN apk add --no-cache tzdata bash curl socat
diff --git a/deployments/ticdc/docker/kafka-consumer.Dockerfile b/deployments/ticdc/docker/kafka-consumer.Dockerfile
index 105745a999c..ed96e67d37d 100644
--- a/deployments/ticdc/docker/kafka-consumer.Dockerfile
+++ b/deployments/ticdc/docker/kafka-consumer.Dockerfile
@@ -3,7 +3,8 @@ RUN apk add --no-cache make bash
 WORKDIR /go/src/github.com/pingcap/tiflow
 COPY . .
 
-RUN make kafka_consumer
+RUN --mount=type=cache,target=/go/pkg/mod go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build make kafka_consumer
 
 FROM alpine:3.15
 COPY --from=builder  /go/src/github.com/pingcap/tiflow/bin/cdc_kafka_consumer /cdc_kafka_consumer
diff --git a/deployments/ticdc/docker/storage-consumer.Dockerfile b/deployments/ticdc/docker/storage-consumer.Dockerfile
index db12ab7fbed..57141408bef 100644
--- a/deployments/ticdc/docker/storage-consumer.Dockerfile
+++ b/deployments/ticdc/docker/storage-consumer.Dockerfile
@@ -2,8 +2,8 @@ FROM golang:1.20-alpine as builder
 RUN apk add --no-cache make bash
 WORKDIR /go/src/github.com/pingcap/tiflow
 COPY . .
-
-RUN make storage_consumer
+RUN --mount=type=cache,target=/go/pkg/mod go mod download
+RUN --mount=type=cache,target=/root/.cache/go-build make storage_consumer
 
 FROM alpine:3.15
 COPY --from=builder  /go/src/github.com/pingcap/tiflow/bin/cdc_storage_consumer /cdc_storage_consumer
diff --git a/deployments/ticdc/docker/test.Dockerfile b/deployments/ticdc/docker/test.Dockerfile
deleted file mode 100644
index 7c1c07ce717..00000000000
--- a/deployments/ticdc/docker/test.Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM alpine:3.15
-RUN apk add --no-cache tzdata bash curl socat
-COPY ./bin/cdc /cdc
-EXPOSE 8300
-CMD [ "/cdc" ]
-