From 04cf1ed114ac4c928b1d5f97d41bfceec8226097 Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Thu, 11 Jan 2024 22:18:20 +0530 Subject: [PATCH] Bump up the minimum required MySQL version (#25465) MySQL 5.7 is at end of life. https://mattermost.atlassian.net/browse/MM-55589 ```release-note We bump up minimum MySQL version to be 8.0.0 ``` Co-authored-by: Mattermost Build Co-authored-by: Ibrahim Serdar Acikgoz --- .github/workflows/esrupgrade-common.yml | 2 +- .github/workflows/migration.yml | 22 +++++++++++++++++++-- .github/workflows/server-ci-template.yml | 2 +- server/Makefile | 2 +- server/build/docker-compose.common.yml | 4 ++-- server/build/docker/mysql.conf.d/custom.cnf | 1 + server/build/dotenv/migration.env | 4 ++-- server/build/dotenv/test.env | 2 +- server/build/gitlab-dc.mysql.yml | 2 +- server/build/gitlab-dc.schemamysql.yml | 2 +- server/channels/store/sqlstore/store.go | 6 +----- server/channels/store/storetest/settings.go | 4 ++-- server/scripts/mysql-migration-test.sh | 4 ++-- server/tests/test-config.json | 2 +- 14 files changed, 37 insertions(+), 22 deletions(-) diff --git a/.github/workflows/esrupgrade-common.yml b/.github/workflows/esrupgrade-common.yml index b32d500ae89..c4b7587b89c 100644 --- a/.github/workflows/esrupgrade-common.yml +++ b/.github/workflows/esrupgrade-common.yml @@ -60,7 +60,7 @@ jobs: --ulimit nofile=8096:8096 \ --env-file=dotenv/test.env \ --env MM_SQLSETTINGS_DRIVERNAME="mysql" \ - --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true&maxAllowedPacket=4194304" \ + --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4&multiStatements=true&maxAllowedPacket=4194304" \ -v ~/work/mattermost:/mm \ -w /mm \ $BUILD_IMAGE & diff --git a/.github/workflows/migration.yml b/.github/workflows/migration.yml index d6b950542b8..5b49e8e606e 100644 --- a/.github/workflows/migration.yml +++ b/.github/workflows/migration.yml @@ -8,6 +8,7 @@ jobs: env: COMPOSE_PROJECT_NAME: ghactions BUILD_IMAGE: mattermost/mattermost-build-server:20230904_golang-1.20.7 + TEST_IMAGE: migration-test-image defaults: run: working-directory: server @@ -20,6 +21,23 @@ jobs: docker compose --ansi never run --rm start_dependencies docker compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; docker compose --ansi never ps + - name: Fix MySQL user permissions + run: | + cd build + docker exec ${COMPOSE_PROJECT_NAME}-mysql-1 sh -c "sed -i '/# default-authentication-plugin/c\default-authentication-plugin=mysql_native_password' /etc/my.cnf" + docker exec ${COMPOSE_PROJECT_NAME}-mysql-1 sh -c "mysql -u root -pmostest -e \"ALTER USER 'mmuser'@'%' IDENTIFIED WITH mysql_native_password BY 'mostest';\"" + docker exec ${COMPOSE_PROJECT_NAME}-mysql-1 sh -c "mysql -u root -pmostest -e \"ALTER USER 'root'@'%' IDENTIFIED WITH mysql_native_password BY 'mostest';\"" + docker restart ${COMPOSE_PROJECT_NAME}-mysql-1 && sleep 10 + - name: Build test image + run: | + echo -e "FROM $BUILD_IMAGE\n\nRUN apt update && \ + apt remove -y pgloader && \ + apt install -y sbcl unzip libsqlite3-dev make curl gawk freetds-dev libzip-dev && \ + git clone https://github.com/dimitri/pgloader.git && \ + cd pgloader && \ + make && \ + mv build/bin/pgloader /bin" >> dockerfile_tmp + docker build -t $TEST_IMAGE - < dockerfile_tmp - name: Generate test-data run: | docker run --net ${COMPOSE_PROJECT_NAME}_mm-test \ @@ -29,7 +47,7 @@ jobs: -e GOCACHE=/go/cache \ -v $PWD:/mattermost \ -w /mattermost \ - $BUILD_IMAGE \ + $TEST_IMAGE \ make test-data - name: Migrate the DB and compare run: | @@ -40,7 +58,7 @@ jobs: -e GOCACHE=/go/cache \ -v $PWD:/mattermost \ -w /mattermost \ - $BUILD_IMAGE \ + $TEST_IMAGE \ make test-migration - name: Upload artifacts uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 diff --git a/.github/workflows/server-ci-template.yml b/.github/workflows/server-ci-template.yml index 6de027f9aa5..a8317e6c952 100644 --- a/.github/workflows/server-ci-template.yml +++ b/.github/workflows/server-ci-template.yml @@ -285,7 +285,7 @@ jobs: uses: ./.github/workflows/server-test-template.yml with: name: MySQL - datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true&maxAllowedPacket=4194304 + datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4&multiStatements=true&maxAllowedPacket=4194304 drivername: mysql logsartifact: mysql-server-test-logs secrets: inherit diff --git a/server/Makefile b/server/Makefile index ccc01bd83c3..f3312a06553 100644 --- a/server/Makefile +++ b/server/Makefile @@ -828,7 +828,7 @@ test-migration: @sed -i'' -e 's|{{ .source_schema }}|${MYSQL_SCHEMA}|g' tests/temp.load # run pgloader and save the logs - pgloader tests/temp.load > migration.log + pgloader --debug tests/temp.load > migration.log $(GO) install github.com/mattermost/dbcmp/cmd/dbcmp@latest # compare two database contents diff --git a/server/build/docker-compose.common.yml b/server/build/docker-compose.common.yml index d3935e515d2..d11dc8a7df9 100644 --- a/server/build/docker-compose.common.yml +++ b/server/build/docker-compose.common.yml @@ -1,7 +1,7 @@ version: '2.4' services: mysql: - image: "mysql/mysql-server:5.7.12" + image: "mysql/mysql-server:8.0.32" restart: always networks: - mm-test @@ -19,7 +19,7 @@ services: volumes: - ./docker/mysql.conf.d/source.cnf:/etc/mysql/conf.d/mysql.cnf mysql-read-replica: - image: "mysql/mysql-server:5.7.12" + image: "mysql/mysql-server:8.0.32" restart: always networks: - mm-test diff --git a/server/build/docker/mysql.conf.d/custom.cnf b/server/build/docker/mysql.conf.d/custom.cnf index c7202552820..ee186841506 100644 --- a/server/build/docker/mysql.conf.d/custom.cnf +++ b/server/build/docker/mysql.conf.d/custom.cnf @@ -1,6 +1,7 @@ [mysqld] bind-address = 0.0.0.0 +default-authentication-plugin=mysql_native_password log-output = NONE slow-query-log = 0 innodb_flush_log_at_trx_commit = 2 diff --git a/server/build/dotenv/migration.env b/server/build/dotenv/migration.env index 20339301d90..1a36e6871ca 100644 --- a/server/build/dotenv/migration.env +++ b/server/build/dotenv/migration.env @@ -6,11 +6,11 @@ MYSQL_USERNAME=root MYSQL_PASSWORD=mostest MYSQL_SCHEMA=mattermost_test POSTGRES_MIGRATIONS_PATH=./channels/db/migrations/postgres -MYSQL_DSN=${MYSQL_USERNAME}:${MYSQL_PASSWORD}@tcp(mysql:3306)/${MYSQL_SCHEMA}?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true +MYSQL_DSN=${MYSQL_USERNAME}:${MYSQL_PASSWORD}@tcp(mysql:3306)/${MYSQL_SCHEMA}?charset=utf8mb4&readTimeout=30s&writeTimeout=30s&multiStatements=true POSTGRES_DSN=postgres://${PG_USERNAME}:${PG_PASSWORD}@postgres:5432/${PG_SCHEMA}?sslmode=disable MM_SERVICESETTINGS_ENABLELOCALMODE=true MM_SQLSETTINGS_DRIVERNAME=mysql -MM_SQLSETTINGS_DATASOURCE=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304 +MM_SQLSETTINGS_DATASOURCE=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304 MAX_WAIT_SECONDS=500 IS_CI=true diff --git a/server/build/dotenv/test.env b/server/build/dotenv/test.env index 929c5607de4..9b4eb175fdd 100644 --- a/server/build/dotenv/test.env +++ b/server/build/dotenv/test.env @@ -1,4 +1,4 @@ -TEST_DATABASE_MYSQL_DSN=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304 +TEST_DATABASE_MYSQL_DSN=mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304 TEST_DATABASE_POSTGRESQL_DSN=postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10 TEST_DATABASE_MYSQL_ROOT_PASSWD=mostest GOBIN=/mattermost/server/bin diff --git a/server/build/gitlab-dc.mysql.yml b/server/build/gitlab-dc.mysql.yml index 2c102abe5a9..641807e5075 100644 --- a/server/build/gitlab-dc.mysql.yml +++ b/server/build/gitlab-dc.mysql.yml @@ -1,7 +1,7 @@ version: '2.4' services: mysql: - image: mattermostdevelopment/mirrored-mysql:5.7.12 + image: mattermostdevelopment/mirrored-mysql:8.0.32 restart: always environment: MYSQL_ROOT_HOST: "%" diff --git a/server/build/gitlab-dc.schemamysql.yml b/server/build/gitlab-dc.schemamysql.yml index d25001e3365..181565b4875 100644 --- a/server/build/gitlab-dc.schemamysql.yml +++ b/server/build/gitlab-dc.schemamysql.yml @@ -1,7 +1,7 @@ version: '2.4' services: mysql: - image: mattermostdevelopment/mirrored-mysql:5.7.12 + image: mattermostdevelopment/mirrored-mysql:8.0.32 restart: always environment: MYSQL_ROOT_HOST: "%" diff --git a/server/channels/store/sqlstore/store.go b/server/channels/store/sqlstore/store.go index 2b0c831fc1e..4369e6edd8a 100644 --- a/server/channels/store/sqlstore/store.go +++ b/server/channels/store/sqlstore/store.go @@ -52,8 +52,7 @@ const ( // 9.6.3 would be 90603. minimumRequiredPostgresVersion = 110000 // major*1000 + minor*100 + patch - minimumRequiredMySQLVersion = 5712 - minimumRecommendedMySQLVersion = 8000 + minimumRequiredMySQLVersion = 8000 migrationsDirectionUp migrationDirection = "up" migrationsDirectionDown migrationDirection = "down" @@ -1232,9 +1231,6 @@ func (ss *SqlStore) ensureMinimumDBVersion(ver string) (bool, error) { return false, fmt.Errorf("cannot parse MySQL DB version: %w", err2) } intVer := majorVer*1000 + minorVer*100 + patchVer - if intVer < minimumRecommendedMySQLVersion { - mlog.Warn("The MySQL version being used is EOL. Please upgrade to a later version.", mlog.Int("current_version", intVer), mlog.Int("minimum_recommended_version", minimumRecommendedMySQLVersion)) - } if intVer < minimumRequiredMySQLVersion { return false, fmt.Errorf("Minimum MySQL version requirements not met. Found: %s, Wanted: %s", versionString(intVer, *ss.settings.DriverName), versionString(minimumRequiredMySQLVersion, *ss.settings.DriverName)) } diff --git a/server/channels/store/storetest/settings.go b/server/channels/store/storetest/settings.go index 49a9deed43a..de30ef35441 100644 --- a/server/channels/store/storetest/settings.go +++ b/server/channels/store/storetest/settings.go @@ -22,10 +22,10 @@ import ( ) const ( - defaultMysqlDSN = "mmuser:mostest@tcp(localhost:3306)/mattermost_test?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304" + defaultMysqlDSN = "mmuser:mostest@tcp(localhost:3306)/mattermost_test?charset=utf8mb4&readTimeout=30s&writeTimeout=30s&multiStatements=true&maxAllowedPacket=4194304" defaultPostgresqlDSN = "postgres://mmuser:mostest@localhost:5432/mattermost_test?sslmode=disable&connect_timeout=10" defaultMysqlRootPWD = "mostest" - defaultMysqlReplicaDSN = "root:mostest@tcp(localhost:3307)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s" + defaultMysqlReplicaDSN = "root:mostest@tcp(localhost:3307)/mattermost_test?charset=utf8mb4\u0026readTimeout=30s" ) func getEnv(name, defaultValue string) string { diff --git a/server/scripts/mysql-migration-test.sh b/server/scripts/mysql-migration-test.sh index d2051a41a1f..67b3b099a0d 100755 --- a/server/scripts/mysql-migration-test.sh +++ b/server/scripts/mysql-migration-test.sh @@ -14,7 +14,7 @@ docker exec -i mattermost-mysql mysql -D migrated -uroot -pmostest -e "INSERT IN echo "Setting up config for db migration" cat config/config.json | \ - jq '.SqlSettings.DataSource = "mmuser:mostest@tcp(localhost:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s"' | \ + jq '.SqlSettings.DataSource = "mmuser:mostest@tcp(localhost:3306)/migrated?charset=utf8mb4&readTimeout=30s&writeTimeout=30s"' | \ jq '.SqlSettings.DriverName = "mysql"' > $TMPDIR/config.json echo "Running the migration" @@ -22,7 +22,7 @@ make ARGS="db migrate --config $TMPDIR/config.json" run-cli echo "Setting up config for fresh db setup" cat config/config.json | \ - jq '.SqlSettings.DataSource = "mmuser:mostest@tcp(localhost:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s"' | \ + jq '.SqlSettings.DataSource = "mmuser:mostest@tcp(localhost:3306)/latest?charset=utf8mb4&readTimeout=30s&writeTimeout=30s"' | \ jq '.SqlSettings.DriverName = "mysql"' > $TMPDIR/config.json echo "Setting up fresh db" diff --git a/server/tests/test-config.json b/server/tests/test-config.json index 36a8fe8b0d4..bbdb1d92d73 100644 --- a/server/tests/test-config.json +++ b/server/tests/test-config.json @@ -90,7 +90,7 @@ }, "SqlSettings": { "DriverName": "mysql", - "DataSource": "mmuser:mostest@tcp(localhost:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s\u0026maxAllowedPacket=4194304", + "DataSource": "mmuser:mostest@tcp(localhost:3306)/mattermost_test?charset=utf8mb4\u0026readTimeout=30s\u0026writeTimeout=30s\u0026maxAllowedPacket=4194304", "DataSourceReplicas": [], "DataSourceSearchReplicas": [], "MaxIdleConns": 20,