From 7b7bb775056d1965506d6ce8558ddcfbed0a629a Mon Sep 17 00:00:00 2001 From: husharp Date: Mon, 6 Nov 2023 13:24:05 +0800 Subject: [PATCH] add ci Signed-off-by: husharp --- .github/workflows/real_cluster.yaml | 35 ++++++ Makefile | 3 + go.mod | 4 +- go.sum | 8 +- pkg/member/member.go | 4 +- server/util.go | 6 +- tests/integrations/client/go.mod | 2 +- tests/integrations/mcs/go.mod | 2 +- tests/integrations/real/Makefile | 57 ++++++++++ tests/integrations/real/api_test.go | 47 ++++++++ tests/integrations/real/deploy.sh | 17 +++ tests/integrations/real/go.mod | 30 +++++ tests/integrations/real/go.sum | 105 ++++++++++++++++++ .../integrations/real/min_resolved_ts_test.go | 42 +++++++ tests/integrations/real/mock_db.go | 91 +++++++++++++++ tests/integrations/real/tikv.toml | 2 + tests/integrations/real/util.go | 32 ++++++ tests/integrations/real/wait_tiup.sh | 23 ++++ tests/integrations/tso/go.mod | 2 +- tools/pd-api-bench/go.mod | 2 +- 20 files changed, 499 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/real_cluster.yaml create mode 100644 tests/integrations/real/Makefile create mode 100644 tests/integrations/real/api_test.go create mode 100755 tests/integrations/real/deploy.sh create mode 100644 tests/integrations/real/go.mod create mode 100644 tests/integrations/real/go.sum create mode 100644 tests/integrations/real/min_resolved_ts_test.go create mode 100644 tests/integrations/real/mock_db.go create mode 100644 tests/integrations/real/tikv.toml create mode 100644 tests/integrations/real/util.go create mode 100755 tests/integrations/real/wait_tiup.sh diff --git a/.github/workflows/real_cluster.yaml b/.github/workflows/real_cluster.yaml new file mode 100644 index 000000000000..f60772ab2d2d --- /dev/null +++ b/.github/workflows/real_cluster.yaml @@ -0,0 +1,35 @@ +name: PD Real Test +on: + push: + branches: + - master + - release-* + pull_request: + branches: + - master + - release-* +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true +jobs: + real-cluster: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v3 + with: + go-version: '1.21' + - name: Checkout code + uses: actions/checkout@v3 + - name: Restore cache + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + **/.tools + **/.dashboard_download_cache + key: ${{ runner.os }}-go-${{ matrix.worker_id }}-${{ hashFiles('**/go.sum') }} + + - name: Test + run: make check + working-directory: tests/integrations/real diff --git a/Makefile b/Makefile index 54ad331aea49..d3cbae441926 100644 --- a/Makefile +++ b/Makefile @@ -236,6 +236,9 @@ ci-test-job: install-tools dashboard-ui for mod in $(shell ./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX)); do cd $$mod && $(MAKE) ci-test-job && cd $(ROOT_PATH) > /dev/null && cat $$mod/covprofile >> covprofile; done; \ fi +ci-real-test-job: + cd tests/integrations/real && $(MAKE) check + TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso test-tso-function: install-tools diff --git a/go.mod b/go.mod index e8da2542be20..0e11737c5839 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.1.12 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a + golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 golang.org/x/text v0.13.0 golang.org/x/time v0.1.0 golang.org/x/tools v0.6.0 @@ -179,7 +179,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/mod v0.8.0 // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect diff --git a/go.sum b/go.sum index 28e210ef1cd0..7af474e72a8b 100644 --- a/go.sum +++ b/go.sum @@ -655,8 +655,8 @@ golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU= -golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= +golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.5.0 h1:5JMiNunQeQw++mMOz48/ISeNu3Iweh/JaZU8ZLqHRrI= golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= @@ -672,8 +672,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/pkg/member/member.go b/pkg/member/member.go index 80332a65f941..1427ce0aaf1b 100644 --- a/pkg/member/member.go +++ b/pkg/member/member.go @@ -54,7 +54,7 @@ type EmbeddedEtcdMember struct { id uint64 // etcd server id. member *pdpb.Member // current PD's info. rootPath string - // memberValue is the serialized string of `member`. It will be save in + // memberValue is the serialized string of `member`. It will be saved in // etcd leader key when the PD node is successfully elected as the PD leader // of the cluster. Every write will use it to check PD leadership. memberValue string @@ -186,7 +186,7 @@ func (m *EmbeddedEtcdMember) KeepLeader(ctx context.Context) { m.leadership.Keep(ctx) } -// PreCheckLeader does some pre-check before checking whether or not it's the leader. +// PreCheckLeader does some pre-check before checking whether it's the leader. func (m *EmbeddedEtcdMember) PreCheckLeader() error { if m.GetEtcdLeader() == 0 { return errs.ErrEtcdLeaderNotFound diff --git a/server/util.go b/server/util.go index f88d0146a7f3..84a7e3b7bf9b 100644 --- a/server/util.go +++ b/server/util.go @@ -36,9 +36,9 @@ import ( // CheckPDVersion checks if PD needs to be upgraded. func CheckPDVersion(opt *config.PersistOptions) { pdVersion := versioninfo.MinSupportedVersion(versioninfo.Base) - if versioninfo.PDReleaseVersion != "None" { - pdVersion = versioninfo.MustParseVersion(versioninfo.PDReleaseVersion) - } + //if versioninfo.PDReleaseVersion != "None" { + // pdVersion = versioninfo.MustParseVersion(versioninfo.PDReleaseVersion) + //} clusterVersion := *opt.GetClusterVersion() log.Info("load cluster version", zap.Stringer("cluster-version", clusterVersion)) if pdVersion.LessThan(clusterVersion) { diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index b9b868cf8e35..794f72ee6daf 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -17,7 +17,7 @@ require ( github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/stretchr/testify v1.8.2 github.com/tikv/pd v0.0.0-00010101000000-000000000000 - github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 go.uber.org/goleak v1.1.12 go.uber.org/zap v1.24.0 diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index c2dfdbe96ef4..486c7ea91500 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -17,7 +17,7 @@ require ( github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/stretchr/testify v1.8.2 github.com/tikv/pd v0.0.0-00010101000000-000000000000 - github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd go.etcd.io/etcd v0.5.0-alpha.5.0.20220915004622-85b640cee793 go.uber.org/goleak v1.1.12 go.uber.org/zap v1.24.0 diff --git a/tests/integrations/real/Makefile b/tests/integrations/real/Makefile new file mode 100644 index 000000000000..bbbb410d77a3 --- /dev/null +++ b/tests/integrations/real/Makefile @@ -0,0 +1,57 @@ +# Copyright 2023 TiKV Project Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT_PATH := ../../.. +GO_TOOLS_BIN_PATH := $(ROOT_PATH)/.tools/bin +PATH := $(GO_TOOLS_BIN_PATH):$(PATH) +SHELL := env PATH='$(PATH)' GOBIN='$(GO_TOOLS_BIN_PATH)' $(shell which bash) + +check: static deploy test clean + +static: install-tools + @ echo "gofmt ..." + @ gofmt -s -l -d . 2>&1 | awk '{ print } END { if (NR > 0) { exit 1 } }' + @ echo "golangci-lint ..." + @ golangci-lint run -c $(ROOT_PATH)/.golangci.yml --verbose ./... --allow-parallel-runners + @ echo "revive ..." + @ revive -formatter friendly -config $(ROOT_PATH)/revive.toml ./... + +tidy: + @ go mod tidy + git diff go.mod go.sum | cat + git diff --quiet go.mod go.sum + +deploy: clean + echo "deploying..." + ./deploy.sh + echo "wait tiup cluster ready..." + ./wait_tiup.sh 15 20 + echo "check cluster status..." + pid=$$(ps -ef | grep 'tiup' | grep -v grep | awk '{print $$2}' | head -n 1); \ + echo $$pid; + lsof -i:4000 + +test: + CGO_ENABLED=1 go test ./... -v -tags deadlock -race -cover || { exit 1; } + +clean: + pid=$$(ps -ef | grep 'tiup' | grep -v grep | awk '{print $$2}' | head -n 1); \ + if [ ! -z "$$pid" ]; then \ + echo $$pid; \ + kill $$pid; \ + fi + rm -rf playground.log + +install-tools: + cd $(ROOT_PATH) && $(MAKE) install-tools diff --git a/tests/integrations/real/api_test.go b/tests/integrations/real/api_test.go new file mode 100644 index 000000000000..739206be7565 --- /dev/null +++ b/tests/integrations/real/api_test.go @@ -0,0 +1,47 @@ +// Copyright 2023 TiKV Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package real + +import ( + flag "github.com/spf13/pflag" + "github.com/stretchr/testify/suite" + "testing" +) + +var ( + withTiKV = flag.Bool("with-tikv", false, "run tests with TiKV cluster started. (not use the mock server)") + // pdAddrs = flag.String("pd-addrs", "127.0.0.1:2379", "pd addrs") +) + +func TestPDAPI(t *testing.T) { + if !*withTiKV { + t.Skip("skipping TestPDAPI because with-tikv is not enabled") + } + suite.Run(t, new(apiTestSuite)) +} + +type apiTestSuite struct { + suite.Suite +} + +func (s *apiTestSuite) SetupTest() { + // addrs := strings.Split(*pdAddrs, ",") + // pdClient, err := pd.NewClient(addrs, pd.SecurityOption{}) + +} + +func (s *apiTestSuite) TestGetStoresMinResolvedTS() { + +} diff --git a/tests/integrations/real/deploy.sh b/tests/integrations/real/deploy.sh new file mode 100755 index 000000000000..b78b0408d5d6 --- /dev/null +++ b/tests/integrations/real/deploy.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +TIUP_BIN_DIR=$HOME/.tiup/bin/tiup + +make clean +cd ../../../ +rm -rf bin +make pd-server-basic + +# Install TiUP +echo "install tiup" +curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh +$TIUP_BIN_DIR update playground + +# Run TiUP +#tiup playground nightly --kv 3 --tiflash 0 --db 3 --without-monitor --pd.binpath ./bin/pd-server +$TIUP_BIN_DIR playground nightly --kv 3 --tiflash 0 --db 1 --without-monitor --pd.binpath ./bin/pd-server > ./tests/integrations/real/playground.log 2>&1 & diff --git a/tests/integrations/real/go.mod b/tests/integrations/real/go.mod new file mode 100644 index 000000000000..210674c8c1c2 --- /dev/null +++ b/tests/integrations/real/go.mod @@ -0,0 +1,30 @@ +module github.com/tikv/pd/tests/integrations/real + +go 1.21 + +require ( + github.com/DATA-DOG/go-sqlmock v1.5.0 + github.com/go-sql-driver/mysql v1.7.1 + github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 + github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.8.4 + gorm.io/driver/mysql v1.5.2 + gorm.io/gorm v1.25.5 + moul.io/zapgorm2 v1.3.0 +) + +require ( + github.com/benbjohnson/clock v1.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.24.0 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/tests/integrations/real/go.sum b/tests/integrations/real/go.sum new file mode 100644 index 000000000000..a6ae6e02b3f3 --- /dev/null +++ b/tests/integrations/real/go.sum @@ -0,0 +1,105 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= +github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= +github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.5.2 h1:QC2HRskSE75wBuOxe0+iCkyJZ+RqpudsQtqkp+IMuXs= +gorm.io/driver/mysql v1.5.2/go.mod h1:pQLhh1Ut/WUAySdTHwBpBv6+JKcj+ua4ZFx1QQTBzb8= +gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +moul.io/zapgorm2 v1.3.0 h1:+CzUTMIcnafd0d/BvBce8T4uPn6DQnpIrz64cyixlkk= +moul.io/zapgorm2 v1.3.0/go.mod h1:nPVy6U9goFKHR4s+zfSo1xVFaoU7Qgd5DoCdOfzoCqs= diff --git a/tests/integrations/real/min_resolved_ts_test.go b/tests/integrations/real/min_resolved_ts_test.go new file mode 100644 index 000000000000..d8aee117d02e --- /dev/null +++ b/tests/integrations/real/min_resolved_ts_test.go @@ -0,0 +1,42 @@ +// Copyright 2023 TiKV Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package real + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTS(t *testing.T) { + re := require.New(t) + _, cancel := context.WithCancel(context.Background()) + defer cancel() + + db := OpenTestDB(t) + db.MustExec("use test") + db.MustExec("drop table if exists t") + db.MustExec("create table t(a int, index i(a))") + db.MustExec("insert t values (1), (2), (3)") + + var ts uint64 + err := db.inner.Begin().Raw("select @@tidb_current_ts").Scan(&ts).Rollback() + re.NoError(err.Error) + + fmt.Printf("ts: %s\n", GetTimeFromTS(ts)) + db.MustClose() +} diff --git a/tests/integrations/real/mock_db.go b/tests/integrations/real/mock_db.go new file mode 100644 index 000000000000..235e632aa4fc --- /dev/null +++ b/tests/integrations/real/mock_db.go @@ -0,0 +1,91 @@ +// Copyright 2023 TiKV Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package real + +import ( + "gorm.io/driver/mysql" + "gorm.io/gorm" + "moul.io/zapgorm2" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + mysqldriver "github.com/go-sql-driver/mysql" + "github.com/pingcap/log" + "github.com/stretchr/testify/require" +) + +// TestDB is a test database +type TestDB struct { + inner *gorm.DB + require *require.Assertions + + isUnderlyingMocked bool + mock sqlmock.Sqlmock +} + +// OpenTestDB opens a test database +func OpenTestDB(t *testing.T, configModifier ...func(*mysqldriver.Config, *gorm.Config)) *TestDB { + r := require.New(t) + + dsn := mysqldriver.NewConfig() + dsn.Net = "tcp" + dsn.Addr = "127.0.0.1:4000" + dsn.Params = map[string]string{"time_zone": "'+00:00'"} + dsn.ParseTime = true + dsn.Loc = time.UTC + dsn.User = "root" + dsn.DBName = "test" + + config := &gorm.Config{ + Logger: zapgorm2.New(log.L()), + } + + for _, m := range configModifier { + m(dsn, config) + } + + db, err := gorm.Open(mysql.Open(dsn.FormatDSN()), config) + r.NoError(err) + + return &TestDB{ + inner: db.Debug(), + require: r, + } +} + +// MustClose closes the test database +func (db *TestDB) MustClose() { + if db.isUnderlyingMocked { + db.mock.ExpectClose() + } + + d, err := db.inner.DB() + db.require.NoError(err) + + err = d.Close() + db.require.NoError(err) +} + +// Gorm returns the underlying gorm.DB +func (db *TestDB) Gorm() *gorm.DB { + return db.inner +} + +// MustExec executes a query +func (db *TestDB) MustExec(sql string, values ...interface{}) { + err := db.inner.Exec(sql, values...).Error + db.require.NoError(err) +} diff --git a/tests/integrations/real/tikv.toml b/tests/integrations/real/tikv.toml new file mode 100644 index 000000000000..e6dd2afd3343 --- /dev/null +++ b/tests/integrations/real/tikv.toml @@ -0,0 +1,2 @@ +[storage] +reserve-space = "1KB" diff --git a/tests/integrations/real/util.go b/tests/integrations/real/util.go new file mode 100644 index 000000000000..74a5ae072600 --- /dev/null +++ b/tests/integrations/real/util.go @@ -0,0 +1,32 @@ +// Copyright 2023 TiKV Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package real + +import ( + "time" +) + +const physicalShiftBits = 18 + +// GetTimeFromTS extracts time.Time from a timestamp. +func GetTimeFromTS(ts uint64) time.Time { + ms := ExtractPhysical(ts) + return time.Unix(ms/1e3, (ms%1e3)*1e6) +} + +// ExtractPhysical returns a ts's physical part. +func ExtractPhysical(ts uint64) int64 { + return int64(ts >> physicalShiftBits) +} diff --git a/tests/integrations/real/wait_tiup.sh b/tests/integrations/real/wait_tiup.sh new file mode 100755 index 000000000000..eb210933cd18 --- /dev/null +++ b/tests/integrations/real/wait_tiup.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Wait unitl `tiup playground` command runs success + +INTERVAL=$1 +MAX_TIMES=$2 + +if ([ -z "${INTERVAL}" ] || [ -z "${MAX_TIMES}" ]); then + echo "Usage: command " + exit 1 +fi + +TIUP_BIN_DIR=$HOME/.tiup/bin/tiup + +for ((i=0; i<${MAX_TIMES}; i++)); do + sleep ${INTERVAL} + $TIUP_BIN_DIR playground display + if [ $? -eq 0 ]; then + exit 0 + fi + cat ./playground.log +done + +exit 1 diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index e5131f15d911..48fa009523ab 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -16,7 +16,7 @@ require ( github.com/pingcap/kvproto v0.0.0-20231018065736-c0689aded40c github.com/stretchr/testify v1.8.4 github.com/tikv/pd v0.0.0-00010101000000-000000000000 - github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd github.com/tikv/pd/tests/integrations/mcs v0.0.0-00010101000000-000000000000 google.golang.org/grpc v1.54.0 ) diff --git a/tools/pd-api-bench/go.mod b/tools/pd-api-bench/go.mod index e6e896a07978..d4c2d5f3297f 100644 --- a/tools/pd-api-bench/go.mod +++ b/tools/pd-api-bench/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/tikv/pd v0.0.0-00010101000000-000000000000 - github.com/tikv/pd/client v0.0.0-00010101000000-000000000000 + github.com/tikv/pd/client v0.0.0-20231101084237-a1a1eea8dafd go.uber.org/zap v1.24.0 google.golang.org/grpc v1.54.0 )