Merge pull request #328 from Shopify/mysql8_arm64_branch

add bits for running arm64 mysql8

.github/workflows/start-mysql.sh

@@ -1,35 +1,49 @@
 #!/bin/bash
 set -xe
 
-DOCKER_COMPOSE_VERSION=1.29.2
+DOCKER_COMPOSE_VERSION=v2.2.3
 
 sudo apt-get update
 sudo apt-get install -y netcat-openbsd make gcc
 
 sudo curl -o /usr/local/bin/docker-compose -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m`
 sudo chmod +x /usr/local/bin/docker-compose
 
-docker-compose up -d mysql-1 mysql-2
+if [ "$MYSQL_VERSION" == "8.0" ]; then
+  docker-compose -f docker-compose_8.0.yml up -d mysql-1 mysql-2
+else
+  docker-compose up -d mysql-1 mysql-2
+fi
 
-# We need a way to check if the mysql servers have booted or not before running
-# the tests and this way is slightly faster than installing mysql-client
+MAX_ATTEMPTS=60
 
-wait_for_mysql() {
-  port=$1
-  echo "Waiting for MySQL at port $port..."
+function wait_for_version () {
   attempts=0
-  while ! nc -w 1 localhost $port | grep -q "mysql"; do
+  until docker exec -t $1 mysql -N -s -u root -e "select @@version"; do
     sleep 1
     attempts=$((attempts + 1))
-    if (( attempts > 60 )); then
-      echo "ERROR: mysql $port was not started." >&2
-      exit 1
+    if (( attempts > $MAX_ATTEMPTS )); then
+      echo "ERROR: $1 was not started." >&2
+    exit 1
     fi
   done
-  echo "MySQL at port $port has started!"
 }
 
-wait_for_mysql 29291
-wait_for_mysql 29292
+wait_for_configuration () {
+  attempts=0
+  # we do need to see the "root@%" user configured, so wait for that
+  until mysql --port $1 --protocol tcp --skip-password -N -s -u root -e "select host from mysql.user where user = 'root';" 2>/dev/null | grep -q '%'; do
+    sleep 1
+    attempts=$((attempts + 1))
+    if (( attempts > $MAX_ATTEMPTS )); then
+      echo "ERROR: $1 was not started." >&2
+    exit 1
+    fi
+  done
+}
+
+wait_for_version "ghostferry-mysql-1-1"
+wait_for_version "ghostferry-mysql-2-1"
 
-docker-compose exec -T mysql-1 mysql -u root -e "select @@version"
+wait_for_configuration 29291
+wait_for_configuration 29292

.github/workflows/tests.yml

@@ -1,4 +1,4 @@
-name: Ghostferry tests
+name: Ghostferry MySQL 8.0 tests
 
 on:
   push:
@@ -12,6 +12,7 @@ jobs:
     timeout-minutes: 15
     env:
       CI: "true"
+      MYSQL_VERSION: "8.0"
     steps:
       - uses: actions/checkout@v2
 
@@ -30,6 +31,7 @@ jobs:
     timeout-minutes: 15
     env:
       CI: "true"
+      MYSQL_VERSION: "8.0"
     steps:
       - uses: actions/checkout@v2
 
@@ -49,6 +51,7 @@ jobs:
     env:
       CI: "true"
       BUNDLE_WITHOUT: "development"
+      MYSQL_VERSION: "8.0"
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/tests_5.7.yml

@@ -0,0 +1,74 @@
+name: Ghostferry MySQL 5.7 tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  gh-285:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      CI: "true"
+      MYSQL_VERSION: "5.7"
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+
+      - name: Starting up MySQL
+        run: .github/workflows/start-mysql.sh
+
+      - name: Running GH-285 test
+        run: ./examples/gh-285/bugreport.sh
+  go-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      CI: "true"
+      MYSQL_VERSION: "5.7"
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+
+      - name: Starting up MySQL
+        run: .github/workflows/start-mysql.sh
+
+      - name: Running Golang tests
+        run: make test-go
+  ruby-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      CI: "true"
+      MYSQL_VERSION: "5.7"
+      BUNDLE_WITHOUT: "development"
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Golang
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+          bundler-cache: true
+
+      - name: Starting up MySQL
+        run: .github/workflows/start-mysql.sh
+
+      - name: Running Ruby tests
+        run: bundle exec ruby test/main.rb
+

dev.yml

@@ -1,8 +1,14 @@
 name: ghostferry
 
+env:
+  MYSQL_VERSION: "8.0"
+
 up:
   - homebrew:
-    - mysql
+      - [email protected]:
+          or: [[email protected]]
+          conflicts: [mysql-connector-c, mysql, mysql-client]
+
   - ruby: "2.7.3"
   - bundler
   - go:
@@ -20,11 +26,17 @@ up:
       meet: echo 'go mod failed to download dependencies'; false
   - custom:
       name: MySQL
-      met?: docker-compose up -d mysql-1 mysql-2
+      met?: docker-compose -f docker-compose_8.0.yml up -d mysql-1 mysql-2
       meet: echo 'mysql failed to start'; false
-      down: docker-compose stop mysql-1 mysql-2
+      down: docker-compose -f docker-compose_8.0.yml stop mysql-1 mysql-2
 
 commands:
   test:
-    desc: Run the test suite.
+    desc: Run all the tests.
     run: make test
+  test-go:
+    desc: Run the golang test suite.
+    run: make test-go
+  test-ruby:
+    desc: Run the ruby test suite.
+    run: make test-ruby

dml_events.go

@@ -423,6 +423,10 @@ func appendEscapedValue(buffer []byte, value interface{}, column schema.TableCol
 
 	switch v := value.(type) {
 	case string:
+		// since https://github.com/go-mysql-org/go-mysql/pull/658/files merged, go-mysql returns JSON events as a string, but we would prefer them as []byte for consistency with other types
+		if column.Type == schema.TYPE_JSON {
+			return appendEscapedBuffer(buffer, []byte(v), true)
+		}
 		var rightPadLengthForBinaryColumn int = 0
 		// see appendEscapedString() for details why we need special
 		// handling of BINARY column types
@@ -432,7 +436,8 @@ func appendEscapedValue(buffer []byte, value interface{}, column schema.TableCol
 
 		return appendEscapedString(buffer, v, rightPadLengthForBinaryColumn)
 	case []byte:
-		return appendEscapedBuffer(buffer, v, column.Type == schema.TYPE_JSON)
+		// schema type cannot be JSON at this point because all JSON results are strings
+		return appendEscapedBuffer(buffer, v, false)
 	case bool:
 		if v {
 			return append(buffer, '1')

docker-compose.yml

@@ -2,7 +2,6 @@ version: "3"
 services:
   mysql-1:
     image: percona:5.7
-    platform: linux/x86_64
     command: --server-id=1
       --log-bin=mysql-bin
       --max-binlog-size=4096
@@ -18,14 +17,14 @@ services:
       --binlog-rows-query-log-events=ON
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+      MYSQL_ROOT_HOST: "%"
     volumes:
       - /var/lib/mysql
     ports:
       - "29291:3306"
 
   mysql-2:
     image: percona:5.7
-    platform: linux/x86_64
     command: --server-id=2
       --log-bin=mysql-bin
       --binlog-format=ROW
@@ -40,6 +39,7 @@ services:
       --binlog-rows-query-log-events=ON
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+      MYSQL_ROOT_HOST: "%"
     volumes:
       - /var/lib/mysql
     ports:
@@ -61,6 +61,7 @@ services:
       --binlog-rows-query-log-events=ON
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+      MYSQL_ROOT_HOST: "%"
     volumes:
       - /var/lib/mysql
     ports:

docker-compose_8.0.yml

@@ -0,0 +1,68 @@
+version: "3"
+services:
+  mysql-1:
+    image: docker.io/mysql/mysql-server:8.0
+    command: --server-id=1
+      --log-bin=mysql-bin
+      --max-binlog-size=4096
+      --binlog-format=ROW
+      --sync-binlog=1
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --character-set-server=utf8mb4
+      --collation-server=utf8mb4_unicode_ci
+      --max-connections=1000
+      --read-only=OFF
+      --binlog-rows-query-log-events=ON
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+      MYSQL_ROOT_HOST: "%"
+    volumes:
+      - /var/lib/mysql
+    ports:
+      - "29291:3306"
+
+  mysql-2:
+    image: docker.io/mysql/mysql-server:8.0
+    command: --server-id=2
+      --log-bin=mysql-bin
+      --binlog-format=ROW
+      --max-binlog-size=4096
+      --sync-binlog=1
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --character-set-server=utf8mb4
+      --collation-server=utf8mb4_unicode_ci
+      --max-connections=1000
+      --binlog-rows-query-log-events=ON
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+      MYSQL_ROOT_HOST: "%"
+    volumes:
+      - /var/lib/mysql
+    ports:
+      - "29292:3306"
+
+  mysql-3:
+    image: docker.io/mysql/mysql-server:8.0
+    command: --server-id=3
+      --log-bin=mysql-bin
+      --binlog-format=ROW
+      --max-binlog-size=4096
+      --sync-binlog=1
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --character-set-server=utf8mb4
+      --collation-server=utf8mb4_unicode_ci
+      --max-connections=1000
+      --binlog-rows-query-log-events=ON
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+      MYSQL_ROOT_HOST: "%"
+    volumes:
+      - /var/lib/mysql
+    ports:
+      - "29293:3306"

examples/gh-285/bugreport.sh

@@ -2,8 +2,6 @@
 
 set -e
 
-docker-compose up -d
-
 mysql -h 127.0.0.1 -u root -P 29291 -e 'DROP DATABASE IF EXISTS `abc`'
 mysql -h 127.0.0.1 -u root -P 29292 -e 'DROP DATABASE IF EXISTS `abc`'
 

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/DataDog/datadog-go v4.8.2+incompatible
 	github.com/Masterminds/squirrel v0.0.0-20180620232226-b127ed9be034
 	github.com/Microsoft/go-winio v0.5.0 // indirect
-	github.com/go-mysql-org/go-mysql v1.3.0
+	github.com/go-mysql-org/go-mysql v1.4.1-0.20220112102103-b3f1a27311d8
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db
 	github.com/gorilla/context v1.1.1 // indirect

go.sum

@@ -23,6 +23,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-mysql-org/go-mysql v1.3.0 h1:lpNqkwdPzIrYSZGdqt8HIgAXZaK6VxBNfr8f7Z4FgGg=
 github.com/go-mysql-org/go-mysql v1.3.0/go.mod h1:3lFZKf7l95Qo70+3XB2WpiSf9wu2s3na3geLMaIIrqQ=
+github.com/go-mysql-org/go-mysql v1.4.1-0.20220112102103-b3f1a27311d8 h1:ViPrQui89RKKSFi0FwonSGGCMksRaYFZm+mu8nQpM84=
+github.com/go-mysql-org/go-mysql v1.4.1-0.20220112102103-b3f1a27311d8/go.mod h1:3lFZKf7l95Qo70+3XB2WpiSf9wu2s3na3geLMaIIrqQ=
 github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=