diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a358117d0d86f..67889161d3760 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -24,11 +24,11 @@ By default, **CHOOSE MASTER ONLY** so your changes will be applied to the next T For details, see [tips for choosing the affected versions](https://github.com/pingcap/docs/blob/master/CONTRIBUTING.md#guideline-for-choosing-the-affected-versions). - [ ] master (the latest development version) +- [ ] v8.2 (TiDB 8.2 versions) - [ ] v8.1 (TiDB 8.1 versions) - [ ] v8.0 (TiDB 8.0 versions) - [ ] v7.6 (TiDB 7.6 versions) - [ ] v7.5 (TiDB 7.5 versions) -- [ ] v7.4 (TiDB 7.4 versions) - [ ] v7.1 (TiDB 7.1 versions) - [ ] v6.5 (TiDB 6.5 versions) - [ ] v6.1 (TiDB 6.1 versions) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1a93b303192fd..45d885dc8abae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,8 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: "18" - name: Verify duplicated file names @@ -32,8 +32,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: "18" - name: Check TOC-tidb-cloud.md existence @@ -57,7 +57,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Vale Linter uses: errata-ai/vale-action@v2.0.1 with: diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 252ed8cab9bbc..7755a1e489898 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -10,17 +10,17 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Download translator repo with: repository: "shczhen/markdown-translator" path: "markdown-translator" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Download docs repo and specified branch with: ref: "i18n-ja-release-7.5" path: "docs" - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 name: Setup node 18 with: node-version: 18 @@ -68,17 +68,17 @@ jobs: # runs-on: ubuntu-latest # steps: -# - uses: actions/checkout@v3 +# - uses: actions/checkout@v4 # name: Download translator repo # with: # repository: "shczhen/markdown-translator" # path: "markdown-translator" -# - uses: actions/checkout@v3 +# - uses: actions/checkout@v4 # name: Download docs repo and specified branch # with: # ref: "i18n-ja-release-7.1" # path: "docs" -# - uses: actions/setup-node@v3 +# - uses: actions/setup-node@v4 # name: Setup node 18 # with: # node-version: 18 diff --git a/.github/workflows/ja-full-translation-google.yaml b/.github/workflows/ja-full-translation-google.yaml index 18c2d3edeef8c..f0eb5bfa1564e 100644 --- a/.github/workflows/ja-full-translation-google.yaml +++ b/.github/workflows/ja-full-translation-google.yaml @@ -7,17 +7,17 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Download translator repo with: repository: "shczhen/markdown-translator" path: "markdown-translator" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Download specified branch of docs repo with: - ref: "release-7.5" + ref: "release-8.1" path: "docs" - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 name: Setup node 18 with: node-version: 18 @@ -45,7 +45,7 @@ jobs: - name: Copy translated files to docs repo run: | cd docs - git checkout -b i18n-ja-release-7.5 + git checkout -b i18n-ja-release-8.1 cd .. cp -r markdown-translator/output/markdowns/* docs/ @@ -61,7 +61,7 @@ jobs: else echo "No changes detected, skipped" fi - git push --set-upstream origin i18n-ja-release-7.5 + git push --set-upstream origin i18n-ja-release-8.1 dispatch: runs-on: ubuntu-latest @@ -75,4 +75,4 @@ jobs: -H "Accept: application/vnd.github+json" \ -H "Authorization: token ${{ secrets.DOCS_STAGING }}" \ https://api.github.com/repos/pingcap/docs-staging/actions/workflows/update.yml/dispatches \ - -d '{"ref":"main","inputs":{"full": "false", "repo":"${{ github.repository }}","branch":"i18n-ja-release-7.5"}}' + -d '{"ref":"main","inputs":{"full": "false", "repo":"${{ github.repository }}","branch":"i18n-ja-release-8.1"}}' diff --git a/.github/workflows/link-fail-fast.yaml b/.github/workflows/link-fail-fast.yaml index 146f4d0a21f02..a5e4677d08972 100644 --- a/.github/workflows/link-fail-fast.yaml +++ b/.github/workflows/link-fail-fast.yaml @@ -7,7 +7,7 @@ jobs: linkChecker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 diff --git a/.github/workflows/link.yaml b/.github/workflows/link.yaml index 4536d939fe79d..5a31c2df21dbd 100644 --- a/.github/workflows/link.yaml +++ b/.github/workflows/link.yaml @@ -10,7 +10,7 @@ jobs: linkChecker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download Exclude Path run: | diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml index 71ae0a603ac6b..bb69844588170 100644 --- a/.github/workflows/media.yml +++ b/.github/workflows/media.yml @@ -11,7 +11,7 @@ jobs: name: Upload media files runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Must use at least depth 2! fetch-depth: 2 diff --git a/.github/workflows/prevent-deletion.yaml b/.github/workflows/prevent-deletion.yaml index 62b6c32c9a506..8e4a9994fa0f7 100644 --- a/.github/workflows/prevent-deletion.yaml +++ b/.github/workflows/prevent-deletion.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout base - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Fetch head run: | git remote add head ${{ github.event.pull_request.head.repo.clone_url }} diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml index 73cce3c0fa894..7d3f25c248dcd 100644 --- a/.github/workflows/rebase.yml +++ b/.github/workflows/rebase.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the latest code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: token: ${{ secrets.REBASE_SECRET_KEY }} fetch-depth: 0 # otherwise, you will fail to push refs to dest repo diff --git a/.github/workflows/translation.yaml b/.github/workflows/translation.yaml index fd8ec27d49a84..e9c8c488b4646 100644 --- a/.github/workflows/translation.yaml +++ b/.github/workflows/translation.yaml @@ -8,18 +8,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Download translator repo with: repository: "shczhen/markdown-translator" ref: "openai" path: "markdown-translator" - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Download docs repo and specified branch with: ref: "i18n-ja-release-7.1" path: "docs" - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 name: Setup node 18 with: node-version: 18 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d9ccc98b1ad2..a95ce9e6ca663 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ Please check out these templates before you submit a pull request: We use separate branches to maintain different versions of TiDB documentation. - The [documentation under development](https://docs.pingcap.com/tidb/dev) is maintained in the `master` branch. -- The [published documentation](https://docs.pingcap.com/tidb/stable/) is maintained in the corresponding `release-` branch. For example, TiDB v7.5 documentation is maintained in the `release-7.5` branch. +- The [published documentation](https://docs.pingcap.com/tidb/stable/) is maintained in the corresponding `release-` branch. For example, TiDB v7.5 documentation is maintained in the `release-7.5` branch. - The [archived documentation](https://docs-archive.pingcap.com/) is no longer maintained and does not receive any further updates. ### Use cherry-pick labels diff --git a/README.md b/README.md index 9c4f8e55edeca..69b20ba79eb64 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,11 @@ Currently, we maintain the following versions of TiDB documentation in different | Branch name | TiDB docs version | | :---------|:----------| | [`master`](https://github.com/pingcap/docs/tree/master) | The latest development version | +| [`release-8.1`](https://github.com/pingcap/docs/tree/release-8.1) | 8.1 LTS (Long-Term Support) | | [`release-8.0`](https://github.com/pingcap/docs/tree/release-8.0) | 8.0 Development Milestone Release | | [`release-7.6`](https://github.com/pingcap/docs/tree/release-7.6) | 7.6 Development Milestone Release | | [`release-7.5`](https://github.com/pingcap/docs/tree/release-7.5) | 7.5 LTS (Long-Term Support) | -| [`release-7.4`](https://github.com/pingcap/docs/tree/release-7.4) | 7.4 Development Milestone Release | +| [`release-7.4`](https://github.com/pingcap/docs/tree/release-7.4) | 7.4 Development Milestone Release (Archived documentation, no longer updated) | | [`release-7.3`](https://github.com/pingcap/docs/tree/release-7.3) | 7.3 Development Milestone Release (Archived documentation, no longer updated) | | [`release-7.2`](https://github.com/pingcap/docs/tree/release-7.2) | 7.2 Development Milestone Release (Archived documentation, no longer updated) | | [`release-7.1`](https://github.com/pingcap/docs/tree/release-7.1) | 7.1 LTS (Long-Term Support) version | @@ -47,8 +48,8 @@ Currently, we maintain the following versions of TiDB documentation in different | [`release-5.3`](https://github.com/pingcap/docs/tree/release-5.3) | 5.3 stable version | | [`release-5.2`](https://github.com/pingcap/docs/tree/release-5.2) | 5.2 stable version | | [`release-5.1`](https://github.com/pingcap/docs/tree/release-5.1) | 5.1 stable version | -| [`release-5.0`](https://github.com/pingcap/docs/tree/release-5.0) | 5.0 stable version | -| [`release-4.0`](https://github.com/pingcap/docs/tree/release-4.0) | 4.0 stable version | +| [`release-5.0`](https://github.com/pingcap/docs/tree/release-5.0) | 5.0 stable version (Archived documentation, no longer updated) | +| [`release-4.0`](https://github.com/pingcap/docs/tree/release-4.0) | 4.0 stable version (Archived documentation, no longer updated) | | [`release-3.1`](https://github.com/pingcap/docs/tree/release-3.1) | 3.1 stable version (Archived documentation, no longer updated) | | [`release-3.0`](https://github.com/pingcap/docs/tree/release-3.0) | 3.0 stable version (Archived documentation, no longer updated) | | [`release-2.1`](https://github.com/pingcap/docs/tree/release-2.1) | 2.1 stable version (Archived documentation, no longer updated) | diff --git a/TOC-tidb-cloud.md b/TOC-tidb-cloud.md index 7fb4dcaede862..0257bcb95ac52 100644 --- a/TOC-tidb-cloud.md +++ b/TOC-tidb-cloud.md @@ -26,6 +26,8 @@ - [MySQL Workbench](/develop/dev-guide-gui-mysql-workbench.md) - [Navicat](/develop/dev-guide-gui-navicat.md) - [Choose Driver or ORM](/develop/dev-guide-choose-driver-or-orm.md) + - BI + - [Looker Studio](/tidb-cloud/dev-guide-bi-looker-studio.md) - Java - [JDBC](/develop/dev-guide-sample-application-java-jdbc.md) - [MyBatis](/develop/dev-guide-sample-application-java-mybatis.md) @@ -253,6 +255,11 @@ - [To Kafka Sink](/tidb-cloud/changefeed-sink-to-apache-kafka.md) - [To TiDB Cloud Sink](/tidb-cloud/changefeed-sink-to-tidb-cloud.md) - [To Cloud Storage](/tidb-cloud/changefeed-sink-to-cloud-storage.md) +- Disaster Recovery + - [Recovery Group Overview](/tidb-cloud/recovery-group-overview.md) + - [Get Started](/tidb-cloud/recovery-group-get-started.md) + - [Failover and Reprotect Databases](/tidb-cloud/recovery-group-failover.md) + - [Delete a Recovery Group](/tidb-cloud/recovery-group-delete.md) - Security - Identity Access Control - [Password Authentication](/tidb-cloud/tidb-cloud-password-authentication.md) @@ -283,6 +290,7 @@ - [Billing from AWS or GCP Marketplace](/tidb-cloud/tidb-cloud-billing.md#billing-from-aws-marketplace-or-google-cloud-marketplace) - [Billing for Changefeed](/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md) - [Billing for Data Migration](/tidb-cloud/tidb-cloud-billing-dm.md) + - [Billing for Recovery Groups](/tidb-cloud/tidb-cloud-billing-recovery-group.md) - API - API - [API Overview](/tidb-cloud/api-overview.md) @@ -348,8 +356,7 @@ - [Expression Syntax](/expression-syntax.md) - [Comment Syntax](/comment-syntax.md) - SQL Statements - - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) - - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) + - [Overview](/sql-statements/sql-statement-overview.md) - [`ADMIN`](/sql-statements/sql-statement-admin.md) - [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) - [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) @@ -360,20 +367,28 @@ - [`ADMIN RESUME DDL`](/sql-statements/sql-statement-admin-resume-ddl.md) - [`ADMIN SHOW DDL [JOBS|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) - [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) - - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) - [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) - [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md) - [`ALTER RANGE`](/sql-statements/sql-statement-alter-range.md) - [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) - - [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) - - [`ALTER TABLE COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) + - [`ALTER SEQUENCE`](/sql-statements/sql-statement-alter-sequence.md) + - `ALTER TABLE` + - [Overview](/sql-statements/sql-statement-alter-table.md) + - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) + - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) + - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) + - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) + - [`COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) + - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) + - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) + - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) + - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) - [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) - [`BACKUP`](/sql-statements/sql-statement-backup.md) - [`BATCH`](/sql-statements/sql-statement-batch.md) - [`BEGIN`](/sql-statements/sql-statement-begin.md) - [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) - - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) - [`COMMIT`](/sql-statements/sql-statement-commit.md) - [`CREATE [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-create-binding.md) - [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) @@ -392,7 +407,6 @@ - [`DESCRIBE`](/sql-statements/sql-statement-describe.md) - [`DO`](/sql-statements/sql-statement-do.md) - [`DROP [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-drop-binding.md) - - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) - [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) - [`DROP PLACEMENT POLICY`](/sql-statements/sql-statement-drop-placement-policy.md) @@ -421,11 +435,9 @@ - [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) - [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md) - [`LOCK TABLES` and `UNLOCK TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) - - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) - [`PREPARE`](/sql-statements/sql-statement-prepare.md) - [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) - [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) - - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) - [`RENAME USER`](/sql-statements/sql-statement-rename-user.md) - [`REPLACE`](/sql-statements/sql-statement-replace.md) @@ -448,7 +460,8 @@ - [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) - [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) - [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) - - [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) + - [`SHOW COLUMN_STATS_USAGE`](/sql-statements/sql-statement-show-column-stats-usage.md) + - [`SHOW COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) - [`SHOW CREATE DATABASE`](/sql-statements/sql-statement-show-create-database.md) - [`SHOW CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-show-create-placement-policy.md) - [`SHOW CREATE RESOURCE GROUP`](/sql-statements/sql-statement-show-create-resource-group.md) @@ -458,7 +471,7 @@ - [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) - [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) - [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) - - [`SHOW [FULL] FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) + - [`SHOW FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) - [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) - [`SHOW IMPORT JOB`](/sql-statements/sql-statement-show-import-job.md) - [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md) @@ -468,18 +481,20 @@ - [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md) - [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) - [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) - - [`SHOW [FULL] PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) + - [`SHOW PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) - [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) - [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) + - [`SHOW STATS_BUCKETS`](/sql-statements/sql-statement-show-stats-buckets.md) - [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) - - [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-histograms.md) + - [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-stats-histograms.md) - [`SHOW STATS_LOCKED`](/sql-statements/sql-statement-show-stats-locked.md) - [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) + - [`SHOW STATS_TOPN`](/sql-statements/sql-statement-show-stats-topn.md) - [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) - [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) - [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) - [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) - - [`SHOW [FULL] TABLES`](/sql-statements/sql-statement-show-tables.md) + - [`SHOW TABLES`](/sql-statements/sql-statement-show-tables.md) - [`SHOW [GLOBAL|SESSION] VARIABLES`](/sql-statements/sql-statement-show-variables.md) - [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) - [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) @@ -518,6 +533,7 @@ - [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) - [Precision Math](/functions-and-operators/precision-math.md) - [Set Operations](/functions-and-operators/set-operators.md) + - [Sequence Functions](/functions-and-operators/sequence-functions.md) - [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) - [TiDB Specific Functions](/functions-and-operators/tidb-functions.md) - [Clustered Indexes](/clustered-indexes.md) diff --git a/TOC.md b/TOC.md index ebd0a4eca8d6f..6b9c7a0f98d8d 100644 --- a/TOC.md +++ b/TOC.md @@ -4,7 +4,7 @@ - [Docs Home](https://docs.pingcap.com/) - About TiDB - [TiDB Introduction](/overview.md) - - [TiDB 8.0 Release Notes](/releases/release-8.0.0.md) + - [TiDB 8.1 Release Notes](/releases/release-8.1.0.md) - [Features](/basic-features.md) - [MySQL Compatibility](/mysql-compatibility.md) - [TiDB Limitations](/tidb-limitations.md) @@ -521,15 +521,20 @@ - [Release Notes](/dm/dm-release-notes.md) - TiDB Lightning - [Overview](/tidb-lightning/tidb-lightning-overview.md) + - [`IMPORT INTO` vs. TiDB Lightning](/tidb-lightning/import-into-vs-tidb-lightning.md) + - [Compatibility of TiDB Lightning and `IMPORT INTO` with TiCDC and Log Backup](/tidb-lightning/tidb-lightning-compatibility-and-scenarios.md) - [Get Started](/get-started-with-tidb-lightning.md) - [Deploy TiDB Lightning](/tidb-lightning/deploy-tidb-lightning.md) - [Target Database Requirements](/tidb-lightning/tidb-lightning-requirements.md) - Data Sources - [Data Match Rules](/tidb-lightning/tidb-lightning-data-source.md) + - [Rename databases and tables](/tidb-lightning/tidb-lightning-data-source.md#rename-databases-and-tables) - [CSV](/tidb-lightning/tidb-lightning-data-source.md#csv) - [SQL](/tidb-lightning/tidb-lightning-data-source.md#sql) - [Parquet](/tidb-lightning/tidb-lightning-data-source.md#parquet) + - [Compressed files](/tidb-lightning/tidb-lightning-data-source.md#compressed-files) - [Customized File](/tidb-lightning/tidb-lightning-data-source.md#match-customized-files) + - [Import data from Amazon S3](/tidb-lightning/tidb-lightning-data-source.md#import-data-from-amazon-s3) - Physical Import Mode - [Requirements and Limitations](/tidb-lightning/tidb-lightning-physical-import-mode.md) - [Use Physical Import Mode](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md) @@ -561,6 +566,7 @@ - [Replicate Data to Pulsar](/ticdc/ticdc-sink-to-pulsar.md) - [Replicate Data to Storage Services](/ticdc/ticdc-sink-to-cloud-storage.md) - [Manage Changefeeds](/ticdc/ticdc-manage-changefeed.md) + - [TiCDC Client Authentication](/ticdc/ticdc-client-authentication.md) - [Log Filter](/ticdc/ticdc-filter.md) - [DDL Replication](/ticdc/ticdc-ddl.md) - [Bidirectional Replication](/ticdc/ticdc-bidirectional-replication.md) @@ -628,6 +634,7 @@ - [Configuration](/tiproxy/tiproxy-configuration.md) - [Command Line Parameters](/tiproxy/tiproxy-command-line-flags.md) - [Monitoring Metrics](/tiproxy/tiproxy-grafana.md) + - [API](/tiproxy/tiproxy-api.md) - [Troubleshooting](/tiproxy/troubleshoot-tiproxy.md) - [Performance Test](/tiproxy/tiproxy-performance-test.md) - Reference @@ -714,8 +721,7 @@ - [Expression Syntax](/expression-syntax.md) - [Comment Syntax](/comment-syntax.md) - SQL Statements - - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) - - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) + - [Overview](/sql-statements/sql-statement-overview.md) - [`ADMIN`](/sql-statements/sql-statement-admin.md) - [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) - [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) @@ -728,13 +734,22 @@ - [`ADMIN SHOW DDL [JOBS|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) - [`ADMIN SHOW TELEMETRY`](/sql-statements/sql-statement-admin-show-telemetry.md) - [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) - - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) - [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) - [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md) - [`ALTER RANGE`](/sql-statements/sql-statement-alter-range.md) - [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) - - [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) - - [`ALTER TABLE COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) + - [`ALTER SEQUENCE`](/sql-statements/sql-statement-alter-sequence.md) + - `ALTER TABLE` + - [Overview](/sql-statements/sql-statement-alter-table.md) + - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) + - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) + - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) + - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) + - [`COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) + - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) + - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) + - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) + - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) - [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) - [`BACKUP`](/sql-statements/sql-statement-backup.md) @@ -742,7 +757,6 @@ - [`BEGIN`](/sql-statements/sql-statement-begin.md) - [`CALIBRATE RESOURCE`](/sql-statements/sql-statement-calibrate-resource.md) - [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) - - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) - [`COMMIT`](/sql-statements/sql-statement-commit.md) - [`CHANGE DRAINER`](/sql-statements/sql-statement-change-drainer.md) - [`CHANGE PUMP`](/sql-statements/sql-statement-change-pump.md) @@ -763,9 +777,7 @@ - [`DESCRIBE`](/sql-statements/sql-statement-describe.md) - [`DO`](/sql-statements/sql-statement-do.md) - [`DROP BINDING`](/sql-statements/sql-statement-drop-binding.md) - - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) - [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) - - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) - [`DROP PLACEMENT POLICY`](/sql-statements/sql-statement-drop-placement-policy.md) - [`DROP RESOURCE GROUP`](/sql-statements/sql-statement-drop-resource-group.md) - [`DROP ROLE`](/sql-statements/sql-statement-drop-role.md) @@ -792,12 +804,10 @@ - [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) - [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md) - [`[LOCK|UNLOCK] TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) - - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) - [`PREPARE`](/sql-statements/sql-statement-prepare.md) - [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) - [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) - [`RENAME USER`](/sql-statements/sql-statement-rename-user.md) - - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) - [`REPLACE`](/sql-statements/sql-statement-replace.md) - [`RESTORE`](/sql-statements/sql-statement-restore.md) @@ -819,6 +829,7 @@ - [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) - [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) - [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) + - [`SHOW COLUMN_STATS_USAGE`](/sql-statements/sql-statement-show-column-stats-usage.md) - [`SHOW COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) - [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) - [`SHOW CREATE DATABASE`](/sql-statements/sql-statement-show-create-database.md) @@ -845,10 +856,12 @@ - [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) - [`SHOW PUMP STATUS`](/sql-statements/sql-statement-show-pump-status.md) - [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) + - [`SHOW STATS_BUCKETS`](/sql-statements/sql-statement-show-stats-buckets.md) - [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) - - [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-histograms.md) + - [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-stats-histograms.md) - [`SHOW STATS_LOCKED`](/sql-statements/sql-statement-show-stats-locked.md) - [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) + - [`SHOW STATS_TOPN`](/sql-statements/sql-statement-show-stats-topn.md) - [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) - [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) - [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) @@ -893,6 +906,7 @@ - [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) - [Precision Math](/functions-and-operators/precision-math.md) - [Set Operations](/functions-and-operators/set-operators.md) + - [Sequence Functions](/functions-and-operators/sequence-functions.md) - [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) - [TiDB Specific Functions](/functions-and-operators/tidb-functions.md) - [Comparisons between Functions and Syntax of Oracle and TiDB](/oracle-functions-to-tidb.md) @@ -1041,6 +1055,8 @@ - [Release Timeline](/releases/release-timeline.md) - [TiDB Versioning](/releases/versioning.md) - [TiDB Installation Packages](/binary-package.md) + - v8.1 + - [8.1.0](/releases/release-8.1.0.md) - v8.0 - [8.0.0-DMR](/releases/release-8.0.0.md) - v7.6 @@ -1055,6 +1071,7 @@ - v7.2 - [7.2.0-DMR](/releases/release-7.2.0.md) - v7.1 + - [7.1.5](/releases/release-7.1.5.md) - [7.1.4](/releases/release-7.1.4.md) - [7.1.3](/releases/release-7.1.3.md) - [7.1.2](/releases/release-7.1.2.md) @@ -1065,6 +1082,7 @@ - v6.6 - [6.6.0-DMR](/releases/release-6.6.0.md) - v6.5 + - [6.5.9](/releases/release-6.5.9.md) - [6.5.8](/releases/release-6.5.8.md) - [6.5.7](/releases/release-6.5.7.md) - [6.5.6](/releases/release-6.5.6.md) diff --git a/_docHome.md b/_docHome.md index cb0ac6782d1ce..db74f2c1a4b1c 100644 --- a/_docHome.md +++ b/_docHome.md @@ -76,7 +76,7 @@ Learn how to deploy TiDB locally in a production environment. The open-source TiDB platform is released under the Apache 2.0 license and is supported by the community. -Download +View on GitHub diff --git a/alert-rules.md b/alert-rules.md index d3b7eef544875..61fba8e2b4b31 100644 --- a/alert-rules.md +++ b/alert-rules.md @@ -121,7 +121,7 @@ This section gives the alert rules for the TiDB component. * Description: - The latency of handling a request in TiDB. If the ninety-ninth percentile latency exceeds 1 second, an alert is triggered. + The latency of handling a request in TiDB. The response time for 99% of requests should be within 1 second; otherwise, an alert is triggered. * Solution: diff --git a/auto-increment.md b/auto-increment.md index a227a8e5e1e22..001e1e4ecb090 100644 --- a/auto-increment.md +++ b/auto-increment.md @@ -373,7 +373,7 @@ CREATE TABLE t(a int AUTO_INCREMENT key) AUTO_ID_CACHE 1; > - Since TiDB v6.4.0, the modification of the `AUTO_INCREMENT` value is faster because it is only an in-memory operation in the TiDB process as the centralized allocating service is introduced. > - Setting `AUTO_ID_CACHE` to `0` means that TiDB uses the default cache size `30000`. -After you enable the MySQL compatibility mode, the allocated IDs are **unique** and **monotonically increasing**, and the behavior is almost the same as MySQL. Even if you access across TiDB instances, the IDs will keep monotonic. Only when the primary instance of the centralized service crashes, there might be a few IDs that are not continuous. This is because the secondary instance discards some IDs that are supposed to have been allocated by the primary instance during the failover to ensure ID uniqueness. +After you enable the MySQL compatibility mode, the allocated IDs are **unique** and **monotonically increasing**, and the behavior is almost the same as MySQL. Even if you access across TiDB instances, the IDs will keep monotonic. Only when the primary instance of the centralized auto-increment ID allocating service exits (for example, during the TiDB node restart) or crashes, there might be some non-consecutive IDs. This is because the secondary instance discards some IDs that are allocated by the primary instance during the failover to ensure ID uniqueness. ## Restrictions diff --git a/basic-features.md b/basic-features.md index da84de79bd3b6..77d3dd6565c44 100644 --- a/basic-features.md +++ b/basic-features.md @@ -22,245 +22,244 @@ You can try out TiDB features on [TiDB Playground](https://play.tidbcloud.com/?u ## Data types, functions, and operators -| Data types, functions, and operators | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Numeric types](/data-type-numeric.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Date and time types](/data-type-date-and-time.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [String types](/data-type-string.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [JSON type](/data-type-json.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Control flow functions](/functions-and-operators/control-flow-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [String functions](/functions-and-operators/string-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Date and time functions](/functions-and-operators/date-and-time-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Bit functions and operators](/functions-and-operators/bit-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Information functions](/functions-and-operators/information-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [JSON functions](/functions-and-operators/json-functions.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Window functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Miscellaneous functions](/functions-and-operators/miscellaneous-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Operators](/functions-and-operators/operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Character sets and collations](/character-set-and-collation.md) [^1] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [User-level lock](/functions-and-operators/locking-functions.md) | Y | Y | Y | Y | Y | Y | N | N | N | N | N | N | +| Data types, functions, and operators | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Numeric types](/data-type-numeric.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Date and time types](/data-type-date-and-time.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [String types](/data-type-string.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [JSON type](/data-type-json.md) | Y | Y | Y | Y | E | E | E | E | E | +| [Control flow functions](/functions-and-operators/control-flow-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [String functions](/functions-and-operators/string-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Date and time functions](/functions-and-operators/date-and-time-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Bit functions and operators](/functions-and-operators/bit-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Information functions](/functions-and-operators/information-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [JSON functions](/functions-and-operators/json-functions.md) | Y | Y | Y | Y | E | E | E | E | E | +| [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Window functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Miscellaneous functions](/functions-and-operators/miscellaneous-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Operators](/functions-and-operators/operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Character sets and collations](/character-set-and-collation.md) [^1] | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [User-level lock](/functions-and-operators/locking-functions.md) | Y | Y | Y | Y | Y | N | N | N | N | ## Indexing and constraints -| Indexing and constraints | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) [^2] | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Columnar storage (TiFlash)](/tiflash/tiflash-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Use FastScan to accelerate queries in OLAP scenarios](/tiflash/use-fastscan.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [RocksDB engine](/storage-engine/rocksdb-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Titan plugin](/storage-engine/titan-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Titan Level Merge](/storage-engine/titan-configuration.md#level-merge-experimental) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Use buckets to improve scan concurrency](/tune-region-performance.md#use-bucket-to-increase-concurrency) | E | E | E | E | E | E | N | N | N | N | N | N | -| [Invisible indexes](/sql-statements/sql-statement-create-index.md#invisible-index) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [Composite `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`CHECK` constraints](/constraints.md#check) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [Unique indexes](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Clustered index on integer `PRIMARY KEY`](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Clustered index on composite or non-integer key](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Foreign key](/constraints.md#foreign-key) | E | E | E | E | N | N | N | N | N | N | N | N | -| [TiFlash late materialization](/tiflash/tiflash-late-materialization.md) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | +| Indexing and constraints | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) [^2] | Y | Y | Y | Y | E | E | E | E | E | +| [Columnar storage (TiFlash)](/tiflash/tiflash-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Use FastScan to accelerate queries in OLAP scenarios](/tiflash/use-fastscan.md) | Y | Y | Y | E | N | N | N | N | N | +| [RocksDB engine](/storage-engine/rocksdb-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Titan plugin](/storage-engine/titan-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Titan Level Merge](/storage-engine/titan-configuration.md#level-merge-experimental) | E | E | E | E | E | E | E | E | E | +| [Use buckets to improve scan concurrency](/tune-region-performance.md#use-bucket-to-increase-concurrency) | E | E | E | E | E | N | N | N | N | +| [Invisible indexes](/sql-statements/sql-statement-create-index.md#invisible-index) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Composite `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`CHECK` constraints](/constraints.md#check) | Y | Y | N | N | N | N | N | N | N | +| [Unique indexes](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Clustered index on integer `PRIMARY KEY`](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Clustered index on composite or non-integer key](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) | Y | Y | Y | N | N | N | N | N | N | +| [Foreign key](/constraints.md#foreign-key) | E | E | E | N | N | N | N | N | N | +| [TiFlash late materialization](/tiflash/tiflash-late-materialization.md) | Y | Y | Y | N | N | N | N | N | N | ## SQL statements -| SQL statements [^3] | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Basic `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `REPLACE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `INSERT ON DUPLICATE KEY UPDATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `LOAD DATA INFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `SELECT INTO OUTFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `INNER JOIN`, LEFT\|RIGHT [OUTER] JOIN | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `UNION`, `UNION ALL` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`EXCEPT` and `INTERSECT` operators](/functions-and-operators/set-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| `GROUP BY`, `ORDER BY` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Window Functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| `START TRANSACTION`, `COMMIT`, `ROLLBACK` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [User-defined variables](/user-defined-variables.md) | E | E | E | E | E | E | E | E | E | E | E | E | -| [`BATCH [ON COLUMN] LIMIT INTEGER DELETE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | Y | Y | N | N | N | N | N | N | -| [`BATCH [ON COLUMN] LIMIT INTEGER INSERT/UPDATE/REPLACE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) | Y | Y | Y | Y | Y | E | N | N | N | N | N | N | -| [Table Lock](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | E | E | E | E | E | E | E | E | E | E | E | E | -| [TiFlash Query Result Materialization](/tiflash/tiflash-results-materialization.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | +| SQL statements [^3] | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| Basic `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `REPLACE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `INSERT ON DUPLICATE KEY UPDATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `LOAD DATA INFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `SELECT INTO OUTFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `INNER JOIN`, LEFT\|RIGHT [OUTER] JOIN | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `UNION`, `UNION ALL` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`EXCEPT` and `INTERSECT` operators](/functions-and-operators/set-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `GROUP BY`, `ORDER BY` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Window Functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `START TRANSACTION`, `COMMIT`, `ROLLBACK` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [User-defined variables](/user-defined-variables.md) | E | E | E | E | E | E | E | E | E | +| [`BATCH [ON COLUMN] LIMIT INTEGER DELETE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | Y | N | N | N | N | +| [`BATCH [ON COLUMN] LIMIT INTEGER INSERT/UPDATE/REPLACE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | N | N | N | N | N | +| [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) | Y | Y | Y | Y | E | N | N | N | N | +| [Table Lock](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | E | E | E | E | E | E | E | E | E | +| [TiFlash Query Result Materialization](/tiflash/tiflash-results-materialization.md) | Y | Y | Y | E | N | N | N | N | N | ## Advanced SQL features -| Advanced SQL features | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Prepared statement cache](/sql-prepared-plan-cache.md) | Y | Y | Y | Y | Y | Y | Y | Y | E | E | E | E | -| [Non-prepared statement cache](/sql-non-prepared-plan-cache.md) | Y | Y | Y | E | N | N | N | N | N | N | N | N | -| [SQL binding](/sql-plan-management.md#sql-binding) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Cross-database binding](/sql-plan-management.md#cross-database-binding) | Y | Y | N | N | N | N | N | N | N | N | N | N | -| [Create bindings according to historical execution plans](/sql-plan-management.md#create-a-binding-according-to-a-historical-execution-plan) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [Coprocessor cache](/coprocessor-cache.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | E | -| [Stale Read](/stale-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Follower reads](/follower-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Read historical data (tidb_snapshot)](/read-historical-data.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Optimizer hints](/optimizer-hints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [MPP execution engine](/explain-mpp.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [MPP execution engine - compression exchange](/explain-mpp.md#mpp-version-and-exchange-data-compression) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [TiFlash Pipeline Model](/tiflash/tiflash-pipeline-model.md) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [TiFlash replica selection strategy](/system-variables.md#tiflash_replica_read-new-in-v730) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [Index Merge](/explain-index-merge.md) | Y | Y | Y | Y | Y | Y | Y | E | E | E | E | E | -| [Placement Rules in SQL](/placement-rules-in-sql.md) | Y | Y | Y | Y | Y | Y | E | E | N | N | N | N | -| [Cascades Planner](/system-variables.md#tidb_enable_cascades_planner) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Runtime Filter](/runtime-filter.md) | Y | Y | Y | N | N | N | N | N | N | N | N | N | +| Advanced SQL features | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Prepared statement cache](/sql-prepared-plan-cache.md) | Y | Y | Y | Y | Y | Y | Y | E | E | +| [Non-prepared statement cache](/sql-non-prepared-plan-cache.md) | Y | Y | E | N | N | N | N | N | N | +| [SQL binding](/sql-plan-management.md#sql-binding) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Cross-database binding](/sql-plan-management.md#cross-database-binding) | Y | N | N | N | N | N | N | N | N | +| [Create bindings according to historical execution plans](/sql-plan-management.md#create-a-binding-according-to-a-historical-execution-plan) | Y | Y | Y | E | N | N | N | N | N | +| [Coprocessor cache](/coprocessor-cache.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Stale Read](/stale-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Follower reads](/follower-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Read historical data (tidb_snapshot)](/read-historical-data.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Optimizer hints](/optimizer-hints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [MPP execution engine](/explain-mpp.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [MPP execution engine - compression exchange](/explain-mpp.md#mpp-version-and-exchange-data-compression) | Y | Y | Y | N | N | N | N | N | N | +| [TiFlash Pipeline Model](/tiflash/tiflash-pipeline-model.md) | Y | Y | N | N | N | N | N | N | N | +| [TiFlash replica selection strategy](/system-variables.md#tiflash_replica_read-new-in-v730) | Y | Y | N | N | N | N | N | N | N | +| [Index Merge](/explain-index-merge.md) | Y | Y | Y | Y | Y | Y | E | E | E | +| [Placement Rules in SQL](/placement-rules-in-sql.md) | Y | Y | Y | Y | Y | E | E | N | N | +| [Cascades Planner](/system-variables.md#tidb_enable_cascades_planner) | E | E | E | E | E | E | E | E | E | +| [Runtime Filter](/runtime-filter.md) | Y | Y | N | N | N | N | N | N | N | ## Data definition language (DDL) -| Data definition language (DDL) | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Basic `CREATE`, `DROP`, `ALTER`, `RENAME`, `TRUNCATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Generated columns](/generated-columns.md) | Y | Y | Y | Y | E | E | E | E | E | E | E | E | -| [Views](/views.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Sequences](/sql-statements/sql-statement-create-sequence.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Auto increment](/auto-increment.md) | Y | Y | Y | Y | Y[^4] | Y | Y | Y | Y | Y | Y | Y | -| [Auto random](/auto-random.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TTL (Time to Live)](/time-to-live.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [DDL algorithm assertions](/sql-statements/sql-statement-alter-table.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| Multi-schema change: add columns | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Change column type](/sql-statements/sql-statement-modify-column.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Temporary tables](/temporary-tables.md) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | N | N | -| Concurrent DDL statements | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Acceleration of `ADD INDEX` and `CREATE INDEX`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Metadata lock](/metadata-lock.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Pause](/sql-statements/sql-statement-admin-pause-ddl.md)/[Resume](/sql-statements/sql-statement-admin-resume-ddl.md) DDL | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [TiDB Accelerated Table Creation](/accelerated-table-creation.md) | E | N | N | N | N | N | N | N | N | N | N | N | -| [Configure BDR role to replicate DDL statements in BDR mode](/sql-statements/sql-statement-admin-bdr-role.md#admin-setshowunset-bdr-role) | E | E | N | N | N | N | N | N | N | N | N | N | +| Data definition language (DDL) | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| Basic `CREATE`, `DROP`, `ALTER`, `RENAME`, `TRUNCATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Generated columns](/generated-columns.md) | Y | Y | Y | E | E | E | E | E | E | +| [Views](/views.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Sequences](/sql-statements/sql-statement-create-sequence.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Auto increment](/auto-increment.md) | Y | Y | Y | Y[^4] | Y | Y | Y | Y | Y | +| [Auto random](/auto-random.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TTL (Time to Live)](/time-to-live.md) | Y | Y | Y | E | N | N | N | N | N | +| [DDL algorithm assertions](/sql-statements/sql-statement-alter-table.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| Multi-schema change: add columns | Y | Y | Y | Y | E | E | E | E | E | +| [Change column type](/sql-statements/sql-statement-modify-column.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Temporary tables](/temporary-tables.md) | Y | Y | Y | Y | Y | Y | Y | N | N | +| Concurrent DDL statements | Y | Y | Y | Y | N | N | N | N | N | +| [Acceleration of `ADD INDEX` and `CREATE INDEX`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) | Y | Y | Y | Y | N | N | N | N | N | +| [Metadata lock](/metadata-lock.md) | Y | Y | Y | Y | N | N | N | N | N | +| [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) | Y | Y | Y | Y | N | N | N | N | N | +| [Pause](/sql-statements/sql-statement-admin-pause-ddl.md)/[Resume](/sql-statements/sql-statement-admin-resume-ddl.md) DDL | Y | Y | N | N | N | N | N | N | N | +| [TiDB Accelerated Table Creation](/accelerated-table-creation.md) | E | N | N | N | N | N | N | N | N | +| [Configure BDR role to replicate DDL statements in BDR mode](/sql-statements/sql-statement-admin-bdr-role.md#admin-setshowunset-bdr-role) | E | N | N | N | N | N | N | N | N | ## Transactions -| Transactions | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Async commit](/system-variables.md#tidb_enable_async_commit-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [1PC](/system-variables.md#tidb_enable_1pc-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [Large transactions (10GB)](/transaction-overview.md#transaction-size-limit) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Pessimistic transactions](/pessimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Optimistic transactions](/optimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Repeatable-read isolation (snapshot isolation)](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Read-committed isolation](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Automatically terminating long-running idle transactions](/system-variables.md#tidb_idle_transaction_timeout-new-in-v760) | Y | Y | N | N | N | N | N | N | N | N | N | N | +| Transactions | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Async commit](/system-variables.md#tidb_enable_async_commit-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [1PC](/system-variables.md#tidb_enable_1pc-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Large transactions (10GB)](/transaction-overview.md#transaction-size-limit) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Pessimistic transactions](/pessimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Optimistic transactions](/optimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Repeatable-read isolation (snapshot isolation)](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Read-committed isolation](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Automatically terminating long-running idle transactions](/system-variables.md#tidb_idle_transaction_timeout-new-in-v760) | Y | N | N | N | N | N | N | N | N | ## Partitioning -| Partitioning | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Range partitioning](/partitioned-table.md#range-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Hash partitioning](/partitioned-table.md#hash-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Key partitioning](/partitioned-table.md#key-partitioning) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [List partitioning](/partitioned-table.md#list-partitioning) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | N | -| [List COLUMNS partitioning](/partitioned-table.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | N | -| [Default partition for List and List COLUMNS partitioned tables](/partitioned-table.md#default-list-partition) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [`EXCHANGE PARTITION`](/partitioned-table.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | N | -| [`REORGANIZE PARTITION`](/partitioned-table.md#reorganize-partitions) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [`COALESCE PARTITION`](/partitioned-table.md#decrease-the-number-of-partitions) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | Y | E | E | E | E | N | N | -| [Range COLUMNS partitioning](/partitioned-table.md#range-columns-partitioning) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Range INTERVAL partitioning](/partitioned-table.md#range-interval-partitioning) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [Convert a partitioned table to a non-partitioned table](/partitioned-table.md#convert-a-partitioned-table-to-a-non-partitioned-table) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [Partition an existing table](/partitioned-table.md#partition-an-existing-table) | Y | Y | Y | N | N | N | N | N | N | N | N | N | +| Partitioning | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Range partitioning](/partitioned-table.md#range-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Hash partitioning](/partitioned-table.md#hash-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Key partitioning](/partitioned-table.md#key-partitioning) | Y | Y | Y | N | N | N | N | N | N | +| [List partitioning](/partitioned-table.md#list-partitioning) | Y | Y | Y | Y | Y | E | E | E | E | +| [List COLUMNS partitioning](/partitioned-table.md) | Y | Y | Y | Y | Y | E | E | E | E | +| [Default partition for List and List COLUMNS partitioned tables](/partitioned-table.md#default-list-partition) | Y | Y | N | N | N | N | N | N | N | +| [`EXCHANGE PARTITION`](/partitioned-table.md) | Y | Y | Y | Y | E | E | E | E | E | +| [`REORGANIZE PARTITION`](/partitioned-table.md#reorganize-partitions) | Y | Y | Y | N | N | N | N | N | N | +| [`COALESCE PARTITION`](/partitioned-table.md#decrease-the-number-of-partitions) | Y | Y | Y | N | N | N | N | N | N | +| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | E | E | E | E | +| [Range COLUMNS partitioning](/partitioned-table.md#range-columns-partitioning) | Y | Y | Y | Y | N | N | N | N | N | +| [Range INTERVAL partitioning](/partitioned-table.md#range-interval-partitioning) | Y | Y | Y | E | N | N | N | N | N | +| [Convert a partitioned table to a non-partitioned table](/partitioned-table.md#convert-a-partitioned-table-to-a-non-partitioned-table) | Y | Y | N | N | N | N | N | N | N | +| [Partition an existing table](/partitioned-table.md#partition-an-existing-table) | Y | Y | N | N | N | N | N | N | N | ## Statistics -| Statistics | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [CMSketch](/statistics.md) | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Y | Y | Y |Y | -| [Histograms](/statistics.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Extended statistics](/extended-statistics.md) | E | E | E | E | E | E | E | E | E | E | E | N | -| Statistics feedback | N | N | N | N | N | Deprecated | Deprecated | E | E | E | E | E | -| [Automatically update statistics](/statistics.md#automatic-update) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Fast Analyze](/system-variables.md#tidb_enable_fast_analyze) | Deprecated | Deprecated | Deprecated | E | E | E | E | E | E | E | E | E | -| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | Y | E | E | E | E | N | N | -| [Collect statistics for `PREDICATE COLUMNS`](/statistics.md#collect-statistics-on-some-columns) | E | E | E | E | E | E | E | N | N | N | N | N | -| [Control the memory quota for collecting statistics](/statistics.md#the-memory-quota-for-collecting-statistics) | E | E | E | E | E | N | N | N | N | N | N | N | -| [Randomly sample about 10000 rows of data to quickly build statistics](/system-variables.md#tidb_enable_fast_analyze) | Deprecated | Deprecated | Deprecated | E | E | E | E | E | E | E | E | E | -| [Lock statistics](/statistics.md#lock-statistics) | Y | Y | Y | E | E | N | N | N | N | N | N | N | -| [Lightweight statistics initialization](/statistics.md#load-statistics) | Y | Y | Y | E | N | N | N | N | N | N | N | N | -| [Show the progress of collecting statistics](/sql-statements/sql-statement-show-analyze-status.md) | Y | Y | Y | N | N | N | N | N | N | N | N | N | +| Statistics | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|---|---|---|---|---|---|---|---|---| +| [CMSketch](/statistics.md) | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Y | Y | +| [Histograms](/statistics.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Extended statistics](/extended-statistics.md) | E | E | E | E | E | E | E | E | E | +| Statistics feedback | N | N | N | N | Deprecated | Deprecated | E | E | E | +| [Automatically update statistics](/statistics.md#automatic-update) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | E | E | E | E | +| [Collect statistics for `PREDICATE COLUMNS`](/statistics.md#collect-statistics-on-some-columns) | E | E | E | E | E | E | N | N | N | +| [Control the memory quota for collecting statistics](/statistics.md#the-memory-quota-for-collecting-statistics) | E | E | E | E | N | N | N | N | N | +| [Randomly sample about 10000 rows of data to quickly build statistics](/system-variables.md#tidb_enable_fast_analyze) | Deprecated | Deprecated | E | E | E | E | E | E | E | +| [Lock statistics](/statistics.md#lock-statistics) | Y | Y | E | E | N | N | N | N | N | +| [Lightweight statistics initialization](/statistics.md#load-statistics) | Y | Y | E | N | N | N | N | N | N | +| [Show the progress of collecting statistics](/sql-statements/sql-statement-show-analyze-status.md) | Y | Y | N | N | N | N | N | N | N | ## Security -| Security | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Transparent layer security (TLS)](/enable-tls-between-clients-and-servers.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Encryption at rest (TDE)](/encryption-at-rest.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Role-based authentication (RBAC)](/role-based-access-control.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Certificate-based authentication](/certificate-authentication.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`caching_sha2_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | N | -| [`tidb_sm3_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`tidb_auth_token` authentication](/security-compatibility-with-mysql.md#tidb_auth_token) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`authentication_ldap_sasl` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | N | N | N | N | N | N | N | N | -| [`authentication_ldap_simple` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Password management](/password-management.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [MySQL compatible `GRANT` system](/privilege-management.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Dynamic Privileges](/privilege-management.md#dynamic-privileges) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Security Enhanced Mode](/system-variables.md#tidb_enable_enhanced_security) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Redacted Log Files](/log-redaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | +| Security | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Transparent layer security (TLS)](/enable-tls-between-clients-and-servers.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Encryption at rest (TDE)](/encryption-at-rest.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Role-based authentication (RBAC)](/role-based-access-control.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Certificate-based authentication](/certificate-authentication.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`caching_sha2_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | Y | Y | Y | N | +| [`tidb_sm3_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | N | N | N | N | N | +| [`tidb_auth_token` authentication](/security-compatibility-with-mysql.md#tidb_auth_token) | Y | Y | Y | Y | N | N | N | N | N | +| [`authentication_ldap_sasl` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | N | N | N | N | N | N | N | +| [`authentication_ldap_simple` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | N | N | N | N | N | N | +| [Password management](/password-management.md) | Y | Y | Y | Y | N | N | N | N | N | +| [MySQL compatible `GRANT` system](/privilege-management.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Dynamic Privileges](/privilege-management.md#dynamic-privileges) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Security Enhanced Mode](/system-variables.md#tidb_enable_enhanced_security) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Redacted Log Files](/log-redaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | ## Data import and export -| Data import and export | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Fast import using TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Fast import using the `IMPORT INTO` statement](/sql-statements/sql-statement-import-into.md) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| mydumper logical dumper | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | -| [Dumpling logical dumper](/dumpling-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Transactional `LOAD DATA`](/sql-statements/sql-statement-load-data.md) [^5] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N [^6] | -| [Database migration toolkit (DM)](/migration-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) [^7] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Change data capture (CDC)](/ticdc/ticdc-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Stream data to Amazon S3, GCS, Azure Blob Storage, and NFS through TiCDC](/ticdc/ticdc-sink-to-cloud-storage.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [TiCDC supports bidirectional replication between two TiDB clusters](/ticdc/ticdc-bidirectional-replication.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [TiCDC OpenAPI v2](/ticdc/ticdc-open-api-v2.md) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [DM](/dm/dm-overview.md) supports migrating MySQL 8.0 | Y | Y | E | E | E | E | N | N | N | N | N | N | +| Data import and export | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|---|---|---|---|---|---|---|---|---| +| [Fast import using TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Fast import using the `IMPORT INTO` statement](/sql-statements/sql-statement-import-into.md) | Y | Y | N | N | N | N | N | N | N | +| mydumper logical dumper | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | +| [Dumpling logical dumper](/dumpling-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Transactional `LOAD DATA`](/sql-statements/sql-statement-load-data.md) [^5] | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Database migration toolkit (DM)](/migration-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) [^6] | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Change data capture (CDC)](/ticdc/ticdc-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Stream data to Amazon S3, GCS, Azure Blob Storage, and NFS through TiCDC](/ticdc/ticdc-sink-to-cloud-storage.md) | Y | Y | Y | E | N | N | N | N | N | +| [TiCDC supports bidirectional replication between two TiDB clusters](/ticdc/ticdc-bidirectional-replication.md) | Y | Y | Y | Y | N | N | N | N | N | +| [TiCDC OpenAPI v2](/ticdc/ticdc-open-api-v2.md) | Y | Y | Y | N | N | N | N | N | N | +| [DM](/dm/dm-overview.md) supports migrating MySQL 8.0 | Y | E | E | E | E | N | N | N | N | ## Management, observability, and tools -| Management, observability, and tools | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [TiDB Dashboard UI](/dashboard/dashboard-intro.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TiDB Dashboard Continuous Profiling](/dashboard/continuous-profiling.md) | Y | Y | Y | Y | Y | Y | E | E | N | N | N | N | -| [TiDB Dashboard Top SQL](/dashboard/top-sql.md) | Y | Y | Y | Y | Y | Y | E | N | N | N | N | N | -| [TiDB Dashboard SQL Diagnostics](/information-schema/information-schema-sql-diagnostics.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [TiDB Dashboard Cluster Diagnostics](/dashboard/dashboard-diagnostics-access.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [TiKV-FastTune dashboard](/grafana-tikv-dashboard.md#tikv-fasttune-dashboard) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Information schema](/information-schema/information-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Metrics schema](/metrics-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Statements summary tables](/statement-summary-tables.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Statements summary tables - summary persistence](/statement-summary-tables.md#persist-statements-summary) | E | E | E | E | N | N | N | N | N | N | N | N | -| [Slow query log](/identify-slow-queries.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TiUP deployment](/tiup/tiup-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Kubernetes operator](https://docs.pingcap.com/tidb-in-kubernetes/) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Built-in physical backup](/br/backup-and-restore-use-cases.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Global Kill](/sql-statements/sql-statement-kill.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | E | -| [Lock View](/information-schema/information-schema-data-lock-waits.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | E | E | E | -| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`SET CONFIG`](/dynamic-config.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | E | -| [DM WebUI](/dm/dm-webui-guide.md) | E | E | E | E | E | E | N | N | N | N | N | N | -| [Foreground Quota Limiter](/tikv-configuration-file.md#foreground-quota-limiter) | Y | Y | Y | Y | Y | E | N | N | N | N | N | N | -| [Background Quota Limiter](/tikv-configuration-file.md#background-quota-limiter) | E | E | E | E | E | N | N | N | N | N | N | N | -| [EBS volume snapshot backup and restore](https://docs.pingcap.com/tidb-in-kubernetes/v1.4/backup-to-aws-s3-by-snapshot) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [PITR](/br/backup-and-restore-overview.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Global memory control](/configure-memory-usage.md#configure-the-memory-usage-threshold-of-a-tidb-server-instance) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Cross-cluster RawKV replication](/tikv-configuration-file.md#api-version-new-in-v610) | E | E | E | E | E | N | N | N | N | N | N | N | -| [Green GC](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) | E | E | E | E | E | E | E | E | E | E | E | N | -| [Resource control](/tidb-resource-control.md) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Runaway Queries management](/tidb-resource-control.md#manage-queries-that-consume-more-resources-than-expected-runaway-queries) | E | E | E | N | N | N | N | N | N | N | N | N | -| [Background tasks management](/tidb-resource-control.md#manage-background-tasks) | E | E | E | N | N | N | N | N | N | N | N | N | -| [TiFlash Disaggregated Storage and Compute Architecture and S3 Support](/tiflash/tiflash-disaggregated-and-s3.md) | Y | Y | Y | E | N | N | N | N | N | N | N | N | -| [Selecting TiDB nodes for the Distributed eXecution Framework (DXF) tasks](/system-variables.md#tidb_service_scope-new-in-v740) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| PD Follower Proxy (controlled by [`tidb_enable_tso_follower_proxy`](/system-variables.md#tidb_enable_tso_follower_proxy-new-in-v530)) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | N | N | -| [Active PD Follower](/tune-region-performance.md#use-the-active-pd-follower-feature-to-enhance-the-scalability-of-pds-region-information-query-service) (controlled by [`pd_enable_follower_handle_region`](/system-variables.md#pd_enable_follower_handle_region-new-in-v760)) | E | E | N | N | N | N | N | N | N | N | N | N | -| [PD microservices](/pd-microservices.md) | Y | N | N | N | N | N | N | N | N | N | N | N | -| [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) | Y | Y | Y | E | N | N | N | N | N | N | N | N | -| [Global Sort](/tidb-global-sort.md) | Y | E | E | N | N | N | N | N | N | N | N | N | +| Management, observability, and tools | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [TiDB Dashboard UI](/dashboard/dashboard-intro.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TiDB Dashboard Continuous Profiling](/dashboard/continuous-profiling.md) | Y | Y | Y | Y | Y | E | E | N | N | +| [TiDB Dashboard Top SQL](/dashboard/top-sql.md) | Y | Y | Y | Y | Y | E | N | N | N | +| [TiDB Dashboard SQL Diagnostics](/information-schema/information-schema-sql-diagnostics.md) | Y | Y | Y | Y | E | E | E | E | E | +| [TiDB Dashboard Cluster Diagnostics](/dashboard/dashboard-diagnostics-access.md) | Y | Y | Y | Y | E | E | E | E | E | +| [TiKV-FastTune dashboard](/grafana-tikv-dashboard.md#tikv-fasttune-dashboard) | E | E | E | E | E | E | E | E | E | +| [Information schema](/information-schema/information-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Metrics schema](/metrics-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Statements summary tables](/statement-summary-tables.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Statements summary tables - summary persistence](/statement-summary-tables.md#persist-statements-summary) | E | E | E | N | N | N | N | N | N | +| [Slow query log](/identify-slow-queries.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TiUP deployment](/tiup/tiup-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Kubernetes operator](https://docs.pingcap.com/tidb-in-kubernetes/) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Built-in physical backup](/br/backup-and-restore-use-cases.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Global Kill](/sql-statements/sql-statement-kill.md) | Y | Y | Y | Y | Y | E | E | E | E | +| [Lock View](/information-schema/information-schema-data-lock-waits.md) | Y | Y | Y | Y | Y | Y | Y | Y | E | +| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`SET CONFIG`](/dynamic-config.md) | Y | Y | Y | Y | Y | E | E | E | E | +| [DM WebUI](/dm/dm-webui-guide.md) | E | E | E | E | E | N | N | N | N | +| [Foreground Quota Limiter](/tikv-configuration-file.md#foreground-quota-limiter) | Y | Y | Y | Y | E | N | N | N | N | +| [Background Quota Limiter](/tikv-configuration-file.md#background-quota-limiter) | E | E | E | E | N | N | N | N | N | +| [EBS volume snapshot backup and restore](https://docs.pingcap.com/tidb-in-kubernetes/v1.4/backup-to-aws-s3-by-snapshot) | Y | Y | Y | Y | N | N | N | N | N | +| [PITR](/br/backup-and-restore-overview.md) | Y | Y | Y | Y | N | N | N | N | N | +| [Global memory control](/configure-memory-usage.md#configure-the-memory-usage-threshold-of-a-tidb-server-instance) | Y | Y | Y | Y | N | N | N | N | N | +| [Cross-cluster RawKV replication](/tikv-configuration-file.md#api-version-new-in-v610) | E | E | E | E | N | N | N | N | N | +| [Green GC](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) | E | E | E | E | E | E | E | E | E | +| [Resource control](/tidb-resource-control.md) | Y | Y | Y | N | N | N | N | N | N | +| [Runaway Queries management](/tidb-resource-control.md#manage-queries-that-consume-more-resources-than-expected-runaway-queries) | Y | E | N | N | N | N | N | N | N | +| [Background tasks management](/tidb-resource-control.md#manage-background-tasks) | E | E | N | N | N | N | N | N | N | +| [TiFlash Disaggregated Storage and Compute Architecture and S3 Support](/tiflash/tiflash-disaggregated-and-s3.md) | Y | Y | E | N | N | N | N | N | N | +| [Selecting TiDB nodes for the Distributed eXecution Framework (DXF) tasks](/system-variables.md#tidb_service_scope-new-in-v740) | Y | Y | N | N | N | N | N | N | N | +| PD Follower Proxy (controlled by [`tidb_enable_tso_follower_proxy`](/system-variables.md#tidb_enable_tso_follower_proxy-new-in-v530)) | Y | Y | Y | Y | Y | Y | Y | N | N | +| [Active PD Follower](/tune-region-performance.md#use-the-active-pd-follower-feature-to-enhance-the-scalability-of-pds-region-information-query-service) (controlled by [`pd_enable_follower_handle_region`](/system-variables.md#pd_enable_follower_handle_region-new-in-v760)) | E | N | N | N | N | N | N | N | N | +| [PD microservices](/pd-microservices.md) | E | N | N | N | N | N | N | N | N | +| [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) | Y | Y | E | N | N | N | N | N | N | +| [Global Sort](/tidb-global-sort.md) | Y | E | N | N | N | N | N | N | N | [^1]: TiDB incorrectly treats latin1 as a subset of utf8. See [TiDB #18955](https://github.com/pingcap/tidb/issues/18955) for more details. @@ -272,6 +271,4 @@ You can try out TiDB features on [TiDB Playground](https://play.tidbcloud.com/?u [^5]: Starting from [TiDB v7.0.0](/releases/release-7.0.0.md), the new parameter `FIELDS DEFINED NULL BY` and support for importing data from S3 and GCS are experimental features. Starting from [v7.6.0](/releases/release-7.6.0.md), TiDB processes `LOAD DATA` in transactions in the same way as MySQL. The `LOAD DATA` statement in a transaction no longer automatically commits the current transaction or starts a new transaction. Moreover, you can explicitly commit or roll back the `LOAD DATA` statement in a transaction. Additionally, the `LOAD DATA` statement is affected by the TiDB transaction mode setting (optimistic or pessimistic transaction). -[^6]: For TiDB v4.0, the `LOAD DATA` transaction does not guarantee atomicity. - -[^7]: Starting from TiDB v7.5.0, technical support for the data replication feature of [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) is no longer provided. It is strongly recommended to use [TiCDC](/ticdc/ticdc-overview.md) as an alternative solution for data replication. Although TiDB Binlog v7.5.0 still supports the Point-in-Time Recovery (PITR) scenario, this component will be completely deprecated in future versions. It is recommended to use [PITR](/br/br-pitr-guide.md) as an alternative solution for data recovery. +[^6]: Starting from TiDB v7.5.0, technical support for the data replication feature of [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) is no longer provided. It is strongly recommended to use [TiCDC](/ticdc/ticdc-overview.md) as an alternative solution for data replication. Although TiDB Binlog v7.5.0 still supports the Point-in-Time Recovery (PITR) scenario, this component will be completely deprecated in future versions. It is recommended to use [PITR](/br/br-pitr-guide.md) as an alternative solution for data recovery. diff --git a/basic-sql-operations.md b/basic-sql-operations.md index 6bd04f4324776..ff9ee11fe451a 100644 --- a/basic-sql-operations.md +++ b/basic-sql-operations.md @@ -14,7 +14,7 @@ To experiment with SQL and test out TiDB compatibility with MySQL queries, you c -This page walks you through the basic TiDB SQL statements such as DDL, DML and CRUD operations. For a complete list of TiDB statements, see [TiDB SQL Syntax Diagram](https://pingcap.github.io/sqlgram/). +This page walks you through the basic TiDB SQL statements such as DDL, DML and CRUD operations. For a complete list of TiDB statements, see [TiDB SQL Syntax](/sql-statements/sql-statement-overview.md)). ## Category diff --git a/benchmark/benchmark-tidb-using-sysbench.md b/benchmark/benchmark-tidb-using-sysbench.md index 5977f46af152e..f821bdb1ff03f 100644 --- a/benchmark/benchmark-tidb-using-sysbench.md +++ b/benchmark/benchmark-tidb-using-sysbench.md @@ -20,7 +20,7 @@ server_configs: log.level: "error" ``` -It is also recommended to make sure [`tidb_enable_prepared_plan_cache`](/system-variables.md#tidb_enable_prepared_plan_cache-new-in-v610) is enabled and that you allow sysbench to use prepared statements by using `--db-ps-mode=auto`. See the [SQL Prepared Execution Plan Cache](/sql-prepared-plan-cache.md) for documetnation about what the SQL plan cache does and how to monitor it. +It is also recommended to make sure [`tidb_enable_prepared_plan_cache`](/system-variables.md#tidb_enable_prepared_plan_cache-new-in-v610) is enabled and that you allow sysbench to use prepared statements by using `--db-ps-mode=auto`. See the [SQL Prepared Execution Plan Cache](/sql-prepared-plan-cache.md) for documentation about what the SQL plan cache does and how to monitor it. > **Note:** > diff --git a/best-practices-on-public-cloud.md b/best-practices-on-public-cloud.md index d4677de975273..06497bfaf2f3b 100644 --- a/best-practices-on-public-cloud.md +++ b/best-practices-on-public-cloud.md @@ -180,7 +180,7 @@ To reduce the number of Regions and alleviate the heartbeat overhead on the syst ## After tuning -After the tunning, the following effects can be observed: +After the tuning, the following effects can be observed: - The TSO requests per second are decreased to 64,800. - The CPU utilization is significantly reduced from approximately 4,600% to 1,400%. diff --git a/best-practices/readonly-nodes.md b/best-practices/readonly-nodes.md index 9597e8ddbf6cf..80faadb4da59f 100644 --- a/best-practices/readonly-nodes.md +++ b/best-practices/readonly-nodes.md @@ -127,5 +127,5 @@ spark.tispark.replica_read learner To read data from read-only nodes when backing up cluster data, you can specify the `--replica-read-label` option in the br command line. Note that when running the following command in shell, you need to use single quotes to wrap the label to prevent `$` from being parsed. ```shell -br backup full ... --replica-read-label '$mode:readonly' +tiup br backup full ... --replica-read-label '$mode:readonly' ``` diff --git a/best-practices/uuid.md b/best-practices/uuid.md index 406831b742ad3..04b420b2973f6 100644 --- a/best-practices/uuid.md +++ b/best-practices/uuid.md @@ -7,17 +7,17 @@ summary: UUIDs, when used as primary keys, offer benefits such as reduced networ ## Overview of UUIDs -When used as a primary key, instead of an `AUTO_INCREMENT` integer value, a universally unique identifier (UUID) delivers the following benefits: +When used as a primary key, instead of an [`AUTO_INCREMENT`](/auto-increment.md) integer value, a universally unique identifier (UUID) delivers the following benefits: - UUIDs can be generated on multiple systems without risking conflicts. In some cases, this means that the number of network trips to TiDB can be reduced, leading to improved performance. - UUIDs are supported by most programming languages and database systems. -- When used as a part of a URL, a UUID is not vulnerable to enumeration attacks. In comparison, with an `auto_increment` number, it is possible to guess the invoice IDs or user IDs. +- When used as a part of a URL, a UUID is not vulnerable to enumeration attacks. In comparison, with an `AUTO_INCREMENT` number, it is possible to guess the invoice IDs or user IDs. ## Best practices ### Store as binary -The textual UUID format looks like this: `ab06f63e-8fe7-11ec-a514-5405db7aad56`, which is a string of 36 characters. By using `UUID_TO_BIN()`, the textual format can be converted into a binary format of 16 bytes. This allows you to store the text in a `BINARY(16)` column. When retrieving the UUID, you can use the `BIN_TO_UUID()` function to get back to the textual format. +The textual UUID format looks like this: `ab06f63e-8fe7-11ec-a514-5405db7aad56`, which is a string of 36 characters. By using [`UUID_TO_BIN()`](/functions-and-operators/miscellaneous-functions.md#uuid_to_bin), the textual format can be converted into a binary format of 16 bytes. This allows you to store the text in a [`BINARY(16)`](/data-type-string.md#binary-type) column. When retrieving the UUID, you can use the [`BIN_TO_UUID()`](/functions-and-operators/miscellaneous-functions.md#bin_to_uuid) function to get back to the textual format. ### UUID format binary order and a clustered PK diff --git a/binary-package.md b/binary-package.md index b73e1db15b7b6..bd4f3f135c3ad 100644 --- a/binary-package.md +++ b/binary-package.md @@ -5,7 +5,7 @@ summary: Learn about TiDB installation packages and the specific components incl # TiDB Installation Packages -Before [deploying TiUP offline](/production-deployment-using-tiup.md#deploy-tiup-offline), you need to download the binary packages of TiDB at the [official download page](https://en.pingcap.com/download/). +Before [deploying TiUP offline](/production-deployment-using-tiup.md#deploy-tiup-offline), you need to download the binary packages of TiDB as described in [Prepare the TiUP offline component package](/production-deployment-using-tiup.md#prepare-the-tiup-offline-component-package). TiDB binary packages are available in amd64 and arm64 architectures. In either architecture, TiDB provides two binary packages: `TiDB-community-server` and `TiDB-community-toolkit`. diff --git a/br/backup-and-restore-storages.md b/br/backup-and-restore-storages.md index 891769137e6e6..8f589075995ef 100644 --- a/br/backup-and-restore-storages.md +++ b/br/backup-and-restore-storages.md @@ -19,7 +19,7 @@ By default, BR sends a credential to each TiKV node when using Amazon S3, GCS, o Note that this operation is not applicable to cloud environments. If you use IAM Role authorization, each node has its own role and permissions. In this case, you need to configure `--send-credentials-to-tikv=false` (or `-c=0` in short) to disable sending credentials: ```bash -./br backup full -c=0 -u pd-service:2379 --storage 's3://bucket-name/prefix' +tiup br backup full -c=0 -u pd-service:2379 --storage 's3://bucket-name/prefix' ``` If you back up or restore data using the [`BACKUP`](/sql-statements/sql-statement-backup.md) and [`RESTORE`](/sql-statements/sql-statement-restore.md) statements, you can add the `SEND_CREDENTIALS_TO_TIKV = FALSE` option: @@ -50,14 +50,14 @@ This section provides some URI examples by using `external` as the `host` parame **Back up snapshot data to Amazon S3** ```shell -./br backup full -u "${PD_IP}:2379" \ +tiup br backup full -u "${PD_IP}:2379" \ --storage "s3://external/backup-20220915?access-key=${access-key}&secret-access-key=${secret-access-key}" ``` **Restore snapshot data from Amazon S3** ```shell -./br restore full -u "${PD_IP}:2379" \ +tiup br restore full -u "${PD_IP}:2379" \ --storage "s3://external/backup-20220915?access-key=${access-key}&secret-access-key=${secret-access-key}" ``` @@ -67,14 +67,14 @@ This section provides some URI examples by using `external` as the `host` parame **Back up snapshot data to GCS** ```shell -./br backup full --pd "${PD_IP}:2379" \ +tiup br backup full --pd "${PD_IP}:2379" \ --storage "gcs://external/backup-20220915?credentials-file=${credentials-file-path}" ``` **Restore snapshot data from GCS** ```shell -./br restore full --pd "${PD_IP}:2379" \ +tiup br restore full --pd "${PD_IP}:2379" \ --storage "gcs://external/backup-20220915?credentials-file=${credentials-file-path}" ``` @@ -84,14 +84,14 @@ This section provides some URI examples by using `external` as the `host` parame **Back up snapshot data to Azure Blob Storage** ```shell -./br backup full -u "${PD_IP}:2379" \ +tiup br backup full -u "${PD_IP}:2379" \ --storage "azure://external/backup-20220915?account-name=${account-name}&account-key=${account-key}" ``` **Restore the `test` database from snapshot backup data in Azure Blob Storage** ```shell -./br restore db --db test -u "${PD_IP}:2379" \ +tiup br restore db --db test -u "${PD_IP}:2379" \ --storage "azure://external/backup-20220915account-name=${account-name}&account-key=${account-key}" ``` @@ -107,8 +107,8 @@ When storing backup data in a cloud storage system, you need to configure authen Before backup, configure the following privileges to access the backup directory on S3. -- Minimum privileges for TiKV and Backup & Restore (BR) to access the backup directories during backup: `s3:ListBucket`, `s3:PutObject`, and `s3:AbortMultipartUpload` -- Minimum privileges for TiKV and BR to access the backup directories during restore: `s3:ListBucket`, `s3:GetObject`, and `s3:PutObject`. BR writes checkpoint information to the `./checkpoints` subdirectory under the backup directory. When restoring log backup data, BR writes the table ID mapping relationship of the restored cluster to the `./pitr_id_maps` subdirectory under the backup directory. +- Minimum privileges for TiKV and Backup & Restore (BR) to access the backup directories during backup: `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, `s3:PutObject`, and `s3:AbortMultipartUpload` +- Minimum privileges for TiKV and BR to access the backup directories during restore: `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, and `s3:PutObject`. BR writes checkpoint information to the `./checkpoints` subdirectory under the backup directory. When restoring log backup data, BR writes the table ID mapping relationship of the restored cluster to the `./pitr_id_maps` subdirectory under the backup directory. If you have not yet created a backup directory, refer to [Create a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket in the specified region. If necessary, you can also create a folder in the bucket by referring to [Create a folder](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-folders.html). @@ -128,7 +128,7 @@ It is recommended that you configure access to S3 using either of the following Associate an IAM role that can access S3 with EC2 instances where the TiKV and BR nodes run. After the association, BR can directly access the backup directories in S3 without additional settings. ```shell - br backup full --pd "${PD_IP}:2379" \ + tiup br backup full --pd "${PD_IP}:2379" \ --storage "s3://${host}/${path}" ``` @@ -195,7 +195,7 @@ You can configure the account used to access GCS by specifying the access key. I - Use BR to back up data to Azure Blob Storage: ```shell - ./br backup full -u "${PD_IP}:2379" \ + tiup br backup full -u "${PD_IP}:2379" \ --storage "azure://external/backup-20220915?account-name=${account-name}" ``` diff --git a/br/backup-and-restore-use-cases.md b/br/backup-and-restore-use-cases.md index 4f99d71de9f3b..6f3868dbdc257 100644 --- a/br/backup-and-restore-use-cases.md +++ b/br/backup-and-restore-use-cases.md @@ -1,6 +1,6 @@ --- title: TiDB Backup and Restore Use Cases -summary: TiDB provides snapshot and log backup solutions for specific use cases, such as timely data recovery and business audits. To use point-in-time recovery (PITR), deploy a TiDB cluster >= v6.2.0 and update BR to v8.0.0. Configure backup storage on Amazon S3 and set a backup policy to meet data loss and recovery requirements. Run log and snapshot backups, and use PITR to restore data to a specific time point. Clean up outdated data regularly. For detailed steps, refer to TiDB documentation. +summary: TiDB provides snapshot and log backup solutions for specific use cases, such as timely data recovery and business audits. To use point-in-time recovery (PITR), deploy a TiDB cluster >= v6.2.0 and update BR to the same version as the TiDB cluster. Configure backup storage on Amazon S3 and set a backup policy to meet data loss and recovery requirements. Run log and snapshot backups, and use PITR to restore data to a specific time point. Clean up outdated data regularly. For detailed steps, refer to TiDB documentation. aliases: ['/docs/dev/br/backup-and-restore-use-cases/','/docs/dev/reference/tools/br/use-cases/','/tidb/dev/backup-and-restore-use-cases-for-maintain/'] --- @@ -17,7 +17,7 @@ With PITR, you can satisfy the preceding requirements. ## Deploy the TiDB cluster and BR -To use PITR, you need to deploy a TiDB cluster >= v6.2.0 and update BR to the same version as the TiDB cluster. This document uses v8.0.0 as an example. +To use PITR, you need to deploy a TiDB cluster >= v6.2.0 and update BR to the same version as the TiDB cluster. This document uses v8.1.0 as an example. The following table shows the recommended hardware resources for using PITR in a TiDB cluster. @@ -44,13 +44,13 @@ Install or upgrade BR using TiUP: - Install: ```shell - tiup install br:v8.0.0 + tiup install br:v8.1.0 ``` - Upgrade: ```shell - tiup update br:v8.0.0 + tiup update br:v8.1.0 ``` ## Configure backup storage (Amazon S3) @@ -70,8 +70,8 @@ The detailed steps are as follows: 2. Configure permissions for BR and TiKV to access the S3 directory. It is recommended to grant permissions using the IAM method, which is the most secure way to access the S3 bucket. For detailed steps, refer to [AWS documentation: Controlling access to a bucket with user policies](https://docs.aws.amazon.com/AmazonS3/latest/userguide/walkthrough1.html). The required permissions are as follows: - - TiKV and BR in the backup cluster need `s3:ListBucket`, `s3:PutObject`, and `s3:AbortMultipartUpload` permissions of the `s3://tidb-pitr-bucket/backup-data` directory. - - TiKV and BR in the restore cluster need `s3:ListBucket`, `s3:GetObject`, and `s3:PutObject` permissions of the `s3://tidb-pitr-bucket/backup-data` directory. + - TiKV and BR in the backup cluster need `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, `s3:PutObject`, and `s3:AbortMultipartUpload` permissions of the `s3://tidb-pitr-bucket/backup-data` directory. + - TiKV and BR in the restore cluster need `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, and `s3:PutObject` permissions of the `s3://tidb-pitr-bucket/backup-data` directory. 3. Plan the directory structure that stores the backup data, including the snapshot (full) backup and the log backup. diff --git a/br/br-auto-tune.md b/br/br-auto-tune.md index 2c15b1ea675d0..479dee8c597b4 100644 --- a/br/br-auto-tune.md +++ b/br/br-auto-tune.md @@ -13,7 +13,7 @@ To reduce the impact of backup tasks on the cluster, TiDB v5.4.0 introduces the If you want to reduce the impact of backup tasks on the cluster, you can enable the auto-tune feature. With this feature enabled, TiDB performs backup tasks as fast as possible without excessively affecting the cluster. -Alternatively, you can limit the backup speed by using the TiKV configuration item [`backup.num-threads`](/tikv-configuration-file.md#num-threads-1) or using the parameter `--ratelimit`. +Alternatively, you can limit the backup speed by using the TiKV configuration item [`backup.num-threads`](/tikv-configuration-file.md#num-threads-1) or using the parameter `--ratelimit`. When `--ratelimit` is set, to avoid too many tasks causing the speed limit to fail, the `concurrency` parameter of br is automatically adjusted to `1`. ## Use auto-tune diff --git a/br/br-batch-create-table.md b/br/br-batch-create-table.md index 842bad3b156ad..9be3c57f838d5 100644 --- a/br/br-batch-create-table.md +++ b/br/br-batch-create-table.md @@ -27,7 +27,7 @@ BR enables the Batch Create Table feature by default, with the default configura To disable this feature, you can set `--ddl-batch-size` to `1`. See the following example command: ```shell -br restore full \ +tiup br restore full \ --storage local:///br_data/ --pd "${PD_IP}:2379" --log-file restore.log \ --ddl-batch-size=1 ``` diff --git a/br/br-checkpoint-backup.md b/br/br-checkpoint-backup.md index aaf9f7e836db6..286cbbbdd486b 100644 --- a/br/br-checkpoint-backup.md +++ b/br/br-checkpoint-backup.md @@ -35,7 +35,7 @@ To avoid this situation, `br` keeps the `gc-safepoint` for about one hour by def The following example sets `gcttl` to 15 hours (54000 seconds) to extend the retention period of `gc-safepoint`: ```shell -br backup full \ +tiup br backup full \ --storage local:///br_data/ --pd "${PD_IP}:2379" \ --gcttl 54000 ``` diff --git a/br/br-incremental-guide.md b/br/br-incremental-guide.md index 14469e18a08be..56f159b5ef102 100644 --- a/br/br-incremental-guide.md +++ b/br/br-incremental-guide.md @@ -1,6 +1,6 @@ --- title: TiDB Incremental Backup and Restore Guide -summary: Incremental data is the differentiated data between starting and end snapshots, along with DDLs. It reduces backup volume and requires setting `tidb_gc_life_time` for incremental backup. Use `br backup` with `--lastbackupts` for incremental backup and ensure all previous data is restored before restoring incremental data. +summary: Incremental data is the differentiated data between starting and end snapshots, along with DDLs. It reduces backup volume and requires setting `tidb_gc_life_time` for incremental backup. Use `tiup br backup` with `--lastbackupts` for incremental backup and ensure all previous data is restored before restoring incremental data. --- # TiDB Incremental Backup and Restore Guide @@ -11,9 +11,15 @@ Incremental data of a TiDB cluster is differentiated data between the starting s > > Development for this feature has stopped. It is recommended that you use [log backup and PITR](/br/br-pitr-guide.md) as an alternative. +## Limitations + +Because restoring the incremental backup relies on the snapshot of the database table at the backup time point to filter incremental DDL statements, the tables deleted during the incremental backup process might still exist after the data restore and need to be manually deleted. + +The incremental backup does not support batch renaming of tables. If batch renaming of tables occurs during the incremental backup process, the data restore might fail. It is recommended to perform a full backup after batch renaming tables, and use the latest full backup to replace the incremental data during restore. + ## Back up incremental data -To back up incremental data, run the `br backup` command with **the last backup timestamp** `--lastbackupts` specified. In this way, br command-line tool automatically backs up incremental data generated between `lastbackupts` and the current time. To get `--lastbackupts`, run the `validate` command. The following is an example: +To back up incremental data, run the `tiup br backup` command with **the last backup timestamp** `--lastbackupts` specified. In this way, br command-line tool automatically backs up incremental data generated between `lastbackupts` and the current time. To get `--lastbackupts`, run the `validate` command. The following is an example: ```shell LAST_BACKUP_TS=`tiup br validate decode --field="end-version" --storage "s3://backup-101/snapshot-202209081330?access-key=${access-key}&secret-access-key=${secret-access-key}"| tail -n1` diff --git a/br/br-pitr-guide.md b/br/br-pitr-guide.md index 68d3d9f94ccc9..64418cd1a7198 100644 --- a/br/br-pitr-guide.md +++ b/br/br-pitr-guide.md @@ -19,7 +19,7 @@ Before you back up or restore data using the br command-line tool (hereinafter r > - The following examples assume that Amazon S3 access keys and secret keys are used to authorize permissions. If IAM roles are used to authorize permissions, you need to set `--send-credentials-to-tikv` to `false`. > - If other storage systems or authorization methods are used to authorize permissions, adjust the parameter settings according to [Backup Storages](/br/backup-and-restore-storages.md). -To start a log backup, run `br log start`. A cluster can only run one log backup task each time. +To start a log backup, run `tiup br log start`. A cluster can only run one log backup task each time. ```shell tiup br log start --task-name=pitr --pd "${PD_IP}:2379" \ @@ -48,7 +48,7 @@ checkpoint[global]: 2022-05-13 11:31:47.2 +0800; gap=4m53s ### Run full backup regularly -The snapshot backup can be used as a method of full backup. You can run `br backup full` to back up the cluster snapshot to the backup storage according to a fixed schedule (for example, every 2 days). +The snapshot backup can be used as a method of full backup. You can run `tiup br backup full` to back up the cluster snapshot to the backup storage according to a fixed schedule (for example, every 2 days). ```shell tiup br backup full --pd "${PD_IP}:2379" \ @@ -57,10 +57,10 @@ tiup br backup full --pd "${PD_IP}:2379" \ ## Run PITR -To restore the cluster to any point in time within the backup retention period, you can use `br restore point`. When you run this command, you need to specify the **time point you want to restore**, **the latest snapshot backup data before the time point**, and the **log backup data**. BR will automatically determine and read data needed for the restore, and then restore these data to the specified cluster in order. +To restore the cluster to any point in time within the backup retention period, you can use `tiup br restore point`. When you run this command, you need to specify the **time point you want to restore**, **the latest snapshot backup data before the time point**, and the **log backup data**. BR will automatically determine and read data needed for the restore, and then restore these data to the specified cluster in order. ```shell -br restore point --pd "${PD_IP}:2379" \ +tiup br restore point --pd "${PD_IP}:2379" \ --storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}' \ --full-backup-storage='s3://backup-101/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}' \ --restored-ts '2022-05-15 18:00:00+0800' @@ -80,7 +80,7 @@ Restore KV Files <-------------------------------------------------------------- As described in the [Usage Overview of TiDB Backup and Restore](/br/br-use-overview.md): -To perform PITR, you need to restore the full backup before the restore point, and the log backup between the full backup point and the restore point. Therefore, for log backups that exceed the backup retention period, you can use `br log truncate` to delete the backup before the specified time point. **It is recommended to only delete the log backup before the full snapshot**. +To perform PITR, you need to restore the full backup before the restore point, and the log backup between the full backup point and the restore point. Therefore, for log backups that exceed the backup retention period, you can use `tiup br log truncate` to delete the backup before the specified time point. **It is recommended to only delete the log backup before the full snapshot**. The following steps describe how to clean up backup data that exceeds the backup retention period: @@ -100,7 +100,7 @@ The following steps describe how to clean up backup data that exceeds the backup 4. Delete snapshot data earlier than the snapshot backup `FULL_BACKUP_TS`: ```shell - rm -rf s3://backup-101/snapshot-${date} + aws s3 rm --recursive s3://backup-101/snapshot-${date} ``` ## Performance capabilities of PITR diff --git a/br/br-pitr-manual.md b/br/br-pitr-manual.md index b1cad71f51289..d4c9076ea1949 100644 --- a/br/br-pitr-manual.md +++ b/br/br-pitr-manual.md @@ -1,6 +1,6 @@ --- title: TiDB Log Backup and PITR Command Manual -summary: TiDB Log Backup and PITR Command Manual describes commands for log backup and point-in-time recovery. Use `br log` command to start, pause, resume, stop, truncate, and query log backup tasks. Specify parameters like `start-ts`, `task-name`, `--storage`, and `--pd` for log backup. Use `br log metadata` to view backup metadata and `br restore point` for PITR. Be cautious when pausing and resuming backup tasks. +summary: Introduce the commands used in TiDB log backup and point-in-time recovery (PITR). aliases: ['/tidb/dev/br-log-command-line/'] --- @@ -15,10 +15,10 @@ For more information about log backup and PITR, refer to: ## Perform log backup -You can start and manage log backup using the `br log` command. +You can start and manage log backup using the `tiup br log` command. ```shell -./br log --help +tiup br log --help backup stream log from TiDB/TiKV cluster @@ -37,22 +37,22 @@ Available Commands: Each subcommand is described as follows: -- `br log start`: start a log backup task. -- `br log status`: query the status of the log backup task. -- `br log pause`: pause a log backup task. -- `br log resume`: resume a paused log backup task. -- `br log stop`: stop a log backup task and delete the task metadata. -- `br log truncate`: clean up the log backup data from the backup storage. -- `br log metadata`: query the metadata of the log backup data. +- `tiup br log start`: start a log backup task. +- `tiup br log status`: query the status of the log backup task. +- `tiup br log pause`: pause a log backup task. +- `tiup br log resume`: resume a paused log backup task. +- `tiup br log stop`: stop a log backup task and delete the task metadata. +- `tiup br log truncate`: clean up the log backup data from the backup storage. +- `tiup br log metadata`: query the metadata of the log backup data. ### Start a backup task -You can run the `br log start` command to start a log backup task. This task runs in the background of your TiDB cluster and automatically backs up the change log of KV storage to the backup storage. +You can run the `tiup br log start` command to start a log backup task. This task runs in the background of your TiDB cluster and automatically backs up the change log of KV storage to the backup storage. -Run `br log start --help` to see the help information: +Run `tiup br log start --help` to see the help information: ```shell -./br log start --help +tiup br log start --help start a log backup task Usage: @@ -83,18 +83,18 @@ The example output only shows the common parameters. These parameters are descri Usage example: ```shell -./br log start --task-name=pitr --pd="${PD_IP}:2379" \ +tiup br log start --task-name=pitr --pd="${PD_IP}:2379" \ --storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' ``` ### Query the backup status -You can run the `br log status` command to query the backup status. +You can run the `tiup br log status` command to query the backup status. -Run `br log status --help` to see the help information: +Run `tiup br log status --help` to see the help information: ```shell -./br log status --help +tiup br log status --help get status for the log backup task Usage: @@ -118,7 +118,7 @@ In the example output, `task-name` is used to specify the name of the backup tas Usage example: ```shell -./br log status --task-name=pitr --pd="${PD_IP}:2379" +tiup br log status --task-name=pitr --pd="${PD_IP}:2379" ``` Expected output: @@ -146,12 +146,12 @@ The output fields are described as follows: ### Pause and resume a backup task -You can run the `br log pause` command to pause a running backup task. +You can run the `tiup br log pause` command to pause a running backup task. -Run `br log pause --help` to see the help information: +Run `tiup br log pause --help` to see the help information: ```shell -./br log pause --help +tiup br log pause --help pause a log backup task Usage: @@ -177,15 +177,15 @@ Global Flags: Usage example: ```shell -./br log pause --task-name=pitr --pd="${PD_IP}:2379" +tiup br log pause --task-name=pitr --pd="${PD_IP}:2379" ``` -You can run the `br log resume` command to resume a paused backup task. +You can run the `tiup br log resume` command to resume a paused backup task. -Run `br log resume --help` to see the help information: +Run `tiup br log resume --help` to see the help information: ```shell -./br log resume --help +tiup br log resume --help resume a log backup task Usage: @@ -202,26 +202,26 @@ Global Flags: -u, --pd strings PD address (default [127.0.0.1:2379]) ``` -After the backup task is paused for more than 24 hours, running `br log resume` reports an error, and BR prompts that backup data is lost. To handle this error, refer to [Backup & Restore FAQs](/faq/backup-and-restore-faq.md#what-should-i-do-if-the-error-message-errbackupgcsafepointexceeded-is-returned-when-using-the-br-log-resume-command-to-resume-a-suspended-task). +After the backup task is paused for more than 24 hours, running `tiup br log resume` reports an error, and BR prompts that backup data is lost. To handle this error, refer to [Backup & Restore FAQs](/faq/backup-and-restore-faq.md#what-should-i-do-if-the-error-message-errbackupgcsafepointexceeded-is-returned-when-using-the-br-log-resume-command-to-resume-a-suspended-task). Usage example: ```shell -./br log resume --task-name=pitr --pd="${PD_IP}:2379" +tiup br log resume --task-name=pitr --pd="${PD_IP}:2379" ``` ### Stop and restart a backup task -You can stop a log backup task by running the `br log stop` command and restart a backup task that is stopped by using the original `--storage` directory. +You can stop a log backup task by running the `tiup br log stop` command and restart a backup task that is stopped by using the original `--storage` directory. #### Stop a backup task -You can run the `br log stop` command to stop a log backup task. This command cleans up the task metadata in the backup cluster. +You can run the `tiup br log stop` command to stop a log backup task. This command cleans up the task metadata in the backup cluster. -Run `br log stop --help` to see the help information: +Run `tiup br log stop --help` to see the help information: ```shell -./br log stop --help +tiup br log stop --help stop a log backup task Usage: @@ -240,17 +240,17 @@ Global Flags: > **Note:** > -> Use this command with caution. If you need to pause a log backup task, use `br log pause` and `br log resume` instead. +> Use this command with caution. If you need to pause a log backup task, use `tiup br log pause` and `tiup br log resume` instead. Usage example: ```shell -./br log stop --task-name=pitr --pd="${PD_IP}:2379" +tiup br log stop --task-name=pitr --pd="${PD_IP}:2379" ``` #### Restart a backup task -After running the `br log stop` command to stop a log backup task, you can create a new log backup task in another `--storage` directory or restart the log backup task in the original `--storage` directory by running the `br log start` command. If you restart the task in the original `--storage` directory, pay attention to the following points: +After running the `tiup br log stop` command to stop a log backup task, you can create a new log backup task in another `--storage` directory or restart the log backup task in the original `--storage` directory by running the `tiup br log start` command. If you restart the task in the original `--storage` directory, pay attention to the following points: - Parameters of the `--storage` directory for restarting a task must be the same as the task that is stopped. - The `--start-ts` does not need to be specified. BR automatically starts the backup from the last backup checkpoint. @@ -258,12 +258,12 @@ After running the `br log stop` command to stop a log backup task, you can creat ### Clean up backup data -You can run the `br log truncate` command to clean up the outdated or no longer needed log backup data. +You can run the `tiup br log truncate` command to clean up the outdated or no longer needed log backup data. -Run `br log truncate --help` to see the help information: +Run `tiup br log truncate --help` to see the help information: ```shell -./br log truncate --help +tiup br log truncate --help truncate the incremental log until sometime. Usage: @@ -289,7 +289,7 @@ This command only accesses the backup storage and does not access the TiDB clust Usage example: ```shell -./br log truncate --until='2022-07-26 21:20:00+0800' \ +tiup br log truncate --until='2022-07-26 21:20:00+0800' \ –-storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' ``` @@ -305,12 +305,12 @@ Removing metadata... DONE; take = 24.038962ms ### View the backup metadata -You can run the `br log metadata` command to view the backup metadata in the storage system, such as the earliest and latest timestamp that can be restored. +You can run the `tiup br log metadata` command to view the backup metadata in the storage system, such as the earliest and latest timestamp that can be restored. -Run `br log metadata --help` to see the help information: +Run `tiup br log metadata --help` to see the help information: ```shell -./br log metadata --help +tiup br log metadata --help get the metadata of log backup storage Usage: @@ -330,7 +330,7 @@ The `--storage` parameter is used to specify the backup storage address. Current Usage example: ```shell -./br log metadata –-storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' +tiup br log metadata –-storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' ``` Expected output: @@ -341,12 +341,12 @@ Expected output: ## Restore to a specified point in time (PITR) -You can run the `br restore point` command to perform a PITR on a new cluster or just restore the log backup data. +You can run the `tiup br restore point` command to perform a PITR on a new cluster or just restore the log backup data. -Run `br restore point --help` to see the help information: +Run `tiup br restore point --help` to see the help information: ```shell -./br restore point --help +tiup br restore point --help restore data from log until specify commit timestamp Usage: @@ -379,7 +379,7 @@ The example output only shows the common parameters. These parameters are descri Usage example: ```shell -./br restore point --pd="${PD_IP}:2379" +tiup br restore point --pd="${PD_IP}:2379" --storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' --full-backup-storage='s3://backup-101/snapshot-202205120000?access-key=${access-key}&secret-access-key=${secret-access-key}"' diff --git a/br/br-snapshot-guide.md b/br/br-snapshot-guide.md index 7d6b3ed6873f2..b000385957575 100644 --- a/br/br-snapshot-guide.md +++ b/br/br-snapshot-guide.md @@ -22,7 +22,7 @@ Besides basic backup and restore, snapshot backup and restore also provides the > - The following examples assume that Amazon S3 access keys and secret keys are used to authorize permissions. If IAM roles are used to authorize permissions, you need to set `--send-credentials-to-tikv` to `false`. > - If other storage systems or authorization methods are used to authorize permissions, adjust the parameter settings according to [Backup Storages](/br/backup-and-restore-storages.md). -You can back up a TiDB cluster snapshot by running the `br backup full` command. Run `br backup full --help` to see the help information: +You can back up a TiDB cluster snapshot by running the `tiup br backup full` command. Run `tiup br backup full --help` to see the help information: ```shell tiup br backup full --pd "${PD_IP}:2379" \ @@ -33,7 +33,7 @@ tiup br backup full --pd "${PD_IP}:2379" \ In the preceding command: -- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23`. If the data of this snapshot is garbage collected, the `br backup` command returns an error and `br` exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. +- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23`. If the data of this snapshot is garbage collected, the `tiup br backup` command returns an error and `br` exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. - `--storage`: The storage address of the backup data. Snapshot backup supports Amazon S3, Google Cloud Storage, and Azure Blob Storage as backup storage. The preceding command uses Amazon S3 as an example. For more details, see [URI Formats of External Storage Services](/external-storage-uri.md). - `--ratelimit`: The maximum speed **per TiKV** performing backup tasks. The unit is in MiB/s. @@ -68,7 +68,7 @@ The output is as follows, corresponding to the physical time `2022-09-08 13:30:0 > - Starting from BR v7.6.0, to address potential restore bottlenecks in scenarios with large-scale Regions, BR supports accelerating restore through the coarse-grained Region scattering algorithm (experimental). You can enable this feature by specifying the command-line parameter `--granularity="coarse-grained"`. > - Starting from BR v8.0.0, the snapshot restore through the coarse-grained Region scattering algorithm is generally available (GA) and enabled by default. BR improves the snapshot restore speed significantly by implementing various optimizations such as adopting the coarse-grained Region scattering algorithm, creating databases and tables in batches, reducing the mutual impact between SST file downloads and ingest operations, and accelerating the restore of table statistics. According to test results from real-world cases, the SST file download speed for snapshot restore is improved by approximately up to 10 times, the data restore speed per TiKV node stabilizes at 1.2 GiB/s, the end-to-end restore speed is improved by approximately 1.5 to 3 times, and 100 TiB of data can be restored within one hour. -You can restore a snapshot backup by running the `br restore full` command. Run `br restore full --help` to see the help information: +You can restore a snapshot backup by running the `tiup br restore full` command. Run `tiup br restore full --help` to see the help information: The following example restores the [preceding backup snapshot](#back-up-cluster-snapshots) to a target cluster: @@ -77,12 +77,6 @@ tiup br restore full --pd "${PD_IP}:2379" \ --storage "s3://backup-101/snapshot-202209081330?access-key=${access-key}&secret-access-key=${secret-access-key}" ``` -> **Warning:** -> -> The coarse-grained Region scatter algorithm (enabled by setting `--granularity="coarse-grained"`) is experimental. It is recommended that you use this feature to accelerate data recovery in clusters with up to 1,000 tables. Note that this feature does not support checkpoint restore. - -To further improve the restore speed of large clusters, starting from v7.6.0, BR supports a coarse-grained Region scatter algorithm (experimental) for faster parallel recovery. You can enable this algorithm by specifying `--granularity="coarse-grained"`. After it is enabled, BR can quickly split the restore task into a large number of small tasks and scatter them to all TiKV nodes in batches, thus fully utilizing the resources of each TiKV node for fast recovery in parallel. - During restore, a progress bar is displayed in the terminal as shown below. When the progress bar advances to 100%, the restore task is completed and statistics such as total restore time, average restore speed, and total data size are displayed. ```shell @@ -96,7 +90,7 @@ BR supports restoring partial data of a specified database or table from backup **Restore a database** -To restore a database to a cluster, run the `br restore db` command. The following example restores the `test` database from the backup data to the target cluster: +To restore a database to a cluster, run the `tiup br restore db` command. The following example restores the `test` database from the backup data to the target cluster: ```shell tiup br restore db \ @@ -109,7 +103,7 @@ In the preceding command, `--db` specifies the name of the database to be restor **Restore a table** -To restore a single table to a cluster, run the `br restore table` command. The following example restores the `test.usertable` table from the backup data to the target cluster: +To restore a single table to a cluster, run the `tiup br restore table` command. The following example restores the `test.usertable` table from the backup data to the target cluster: ```shell tiup br restore table --pd "${PD_IP}:2379" \ @@ -122,7 +116,7 @@ In the preceding command, `--db` specifies the name of the database to be restor **Restore multiple tables with table filter** -To restore multiple tables with more complex filter rules, run the `br restore full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. The following example restores tables that match the `db*.tbl*` filter rule from the backup data to the target cluster: +To restore multiple tables with more complex filter rules, run the `tiup br restore full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. The following example restores tables that match the `db*.tbl*` filter rule from the backup data to the target cluster: ```shell tiup br restore full \ @@ -157,7 +151,7 @@ tiup br restore full \ - Statistics tables (`mysql.stat_*`). But statistics can be restored. See [Back up statistics](/br/br-snapshot-manual.md#back-up-statistics). - System variable tables (`mysql.tidb` and `mysql.global_variables`) -- [Other system tables](https://github.com/pingcap/tidb/blob/master/br/pkg/restore/systable_restore.go#L31) +- [Other system tables](https://github.com/pingcap/tidb/blob/master/br/pkg/restore/snap_client/systable_restore.go#L31) ``` +-----------------------------------------------------+ @@ -181,14 +175,11 @@ tiup br restore full \ +-----------------------------------------------------+ ``` -When you restore data related to system privilege, note the following: - -- Before v7.6.0, BR does not restore user data with `user` as `cloud_admin` and `host` as `'%'`. This user is reserved for TiDB Cloud. Starting from v7.6.0, BR supports restoring all user data (including `cloud_admin`) by default. -- Before restoring data, BR checks whether the system tables in the target cluster are compatible with those in the backup data. "Compatible" means that all the following conditions are met: +When you restore data related to system privilege, note that before restoring data, BR checks whether the system tables in the target cluster are compatible with those in the backup data. "Compatible" means that all the following conditions are met: - - The target cluster has the same system tables as the backup data. - - The **number of columns** in the system privilege table of the target cluster is the same as that in the backup data. The column order is not important. - - The columns in the system privilege table of the target cluster are compatible with that in the backup data. If the data type of the column is a type with a length (such as integer and string), the length in the target cluster must be >= the length in the backup data. If the data type of the column is an `ENUM` type, the number of `ENUM` values in the target cluster must be a superset of that in the backup data. +- The target cluster has the same system tables as the backup data. +- The **number of columns** in the system privilege table of the target cluster is the same as that in the backup data. The column order is not important. +- The columns in the system privilege table of the target cluster are compatible with that in the backup data. If the data type of the column is a type with a length (such as integer and string), the length in the target cluster must be >= the length in the backup data. If the data type of the column is an `ENUM` type, the number of `ENUM` values in the target cluster must be a superset of that in the backup data. ## Performance and impact @@ -205,7 +196,7 @@ To illustrate the impact of backup, this document lists the test conclusions of You can use the following methods to manually control the impact of backup tasks on cluster performance. However, these two methods also reduce the speed of backup tasks while reducing the impact of backup tasks on the cluster. -- Use the `--ratelimit` parameter to limit the speed of backup tasks. Note that this parameter limits the speed of **saving backup files to external storage**. When calculating the total size of backup files, use the `backup data size(after compressed)` as a benchmark. +- Use the `--ratelimit` parameter to limit the speed of backup tasks. Note that this parameter limits the speed of **saving backup files to external storage**. When calculating the total size of backup files, use the `backup data size(after compressed)` as a benchmark. When `--ratelimit` is set, to avoid too many tasks causing the speed limit to fail, the `concurrency` parameter of br is automatically adjusted to `1`. - Adjust the TiKV configuration item [`backup.num-threads`](/tikv-configuration-file.md#num-threads-1) to limit the number of threads used by backup tasks. According to internal tests, when BR uses no more than `8` threads for backup tasks, and the total CPU utilization of the cluster does not exceed 60%, the backup tasks have little impact on the cluster, regardless of the read and write workload. The impact of backup on cluster performance can be reduced by limiting the backup threads number, but this affects the backup performance. The preceding tests show that the backup speed is proportional to the number of backup threads. When the number of threads is small, the backup speed is about 20 MiB/thread. For example, 5 backup threads on a single TiKV node can reach a backup speed of 100 MiB/s. @@ -217,7 +208,7 @@ The impact of backup on cluster performance can be reduced by limiting the backu - BR provides a coarse-grained Region scattering algorithm to accelerate Region restore in large-scale Region scenarios. The algorithm is controlled by the command-line parameter `--granularity="coarse-grained"` and is enabled by default. This algorithm ensures that each TiKV node receives stable and evenly distributed download tasks, thus fully utilizing the resources of each TiKV node and achieving a rapid parallel recovery. In several real-world cases, the snapshot restore speed of the cluster is improved by about 3 times in large-scale Region scenarios. The following is an example: ```bash - br restore full \ + tiup br restore full \ --pd "${PDIP}:2379" \ --storage "s3://${Bucket}/${Folder}" \ --s3.region "${region}" \ diff --git a/br/br-snapshot-manual.md b/br/br-snapshot-manual.md index 8631205949ace..cd157cfc6bc1a 100644 --- a/br/br-snapshot-manual.md +++ b/br/br-snapshot-manual.md @@ -29,10 +29,10 @@ For more information about snapshot backup and restore, refer to: ## Back up cluster snapshots -You can back up the latest or specified snapshot of the TiDB cluster using the `br backup full` command. For more information about the command, run the `br backup full --help` command. +You can back up the latest or specified snapshot of the TiDB cluster using the `tiup br backup full` command. For more information about the command, run the `tiup br backup full --help` command. ```shell -br backup full \ +tiup br backup full \ --pd "${PD_IP}:2379" \ --backupts '2022-09-08 13:30:00' \ --storage "s3://${backup_collection_addr}/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}" \ @@ -42,7 +42,7 @@ br backup full \ In the preceding command: -- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23`. If the data of this snapshot is garbage collected, the `br backup` command returns an error and 'br' exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. +- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23`. If the data of this snapshot is garbage collected, the `tiup br backup` command returns an error and 'br' exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. - `--ratelimit`: The maximum speed **per TiKV** performing backup tasks. The unit is in MiB/s. - `--log-file`: The target file where `br` log is written. @@ -62,12 +62,12 @@ Backup & Restore (BR) supports backing up partial data of a specified database o ### Back up a database -To back up a database in a cluster, run the `br backup db` command. +To back up a database in a cluster, run the `tiup br backup db` command. The following example backs up the `test` database to Amazon S3: ```shell -br backup db \ +tiup br backup db \ --pd "${PD_IP}:2379" \ --db test \ --storage "s3://${backup_collection_addr}/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}" \ @@ -79,12 +79,12 @@ In the preceding command, `--db` specifies the database name, and other paramete ### Back up a table -To back up a table in a cluster, run the `br backup table` command. +To back up a table in a cluster, run the `tiup br backup table` command. The following example backs up the `test.usertable` table to Amazon S3: ```shell -br backup table \ +tiup br backup table \ --pd "${PD_IP}:2379" \ --db test \ --table usertable \ @@ -97,12 +97,12 @@ In the preceding command, `--db` and `--table` specify the database name and tab ### Back up multiple tables with table filter -To back up multiple tables with more criteria, run the `br backup full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. +To back up multiple tables with more criteria, run the `tiup br backup full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. The following example backs up tables that match the `db*.tbl*` filter rule to Amazon S3: ```shell -br backup full \ +tiup br backup full \ --pd "${PD_IP}:2379" \ --filter 'db*.tbl*' \ --storage "s3://${backup_collection_addr}/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}" \ @@ -119,7 +119,7 @@ If you do not set this parameter to `false`, the `br` command-line tool uses the The following is an example of backing up cluster snapshot data and backing up table statistics with `--ignore-stats=false`: ```shell -br backup full \ +tiup br backup full \ --storage local:///br_data/ --pd "${PD_IP}:2379" --log-file restore.log \ --ignore-stats=false ``` @@ -127,7 +127,7 @@ br backup full \ After backing up data with the preceding configuration, when you restore data, the `br` command-line tool automatically restores table statistics if table statistics are included in the backup (Starting from v8.0.0, the `br` command-line tool introduces the `--load-stats` parameter, which controls whether to restore backup statistics. The default behavior is to restore backup statistics. There is no need to set it to `false` in most cases): ```shell -br restore full \ +tiup br restore full \ --storage local:///br_data/ --pd "${PD_IP}:2379" --log-file restore.log ``` @@ -150,7 +150,7 @@ Since TiDB v5.3.0, you can encrypt backup data by configuring the following para The following is an example: ```shell -br backup full\ +tiup br backup full\ --pd ${PD_IP}:2379 \ --storage "s3://${backup_collection_addr}/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}" \ --crypter.method aes128-ctr \ @@ -164,10 +164,10 @@ br backup full\ ## Restore cluster snapshots -You can restore a TiDB cluster snapshot by running the `br restore full` command. +You can restore a TiDB cluster snapshot by running the `tiup br restore full` command. ```shell -br restore full \ +tiup br restore full \ --pd "${PD_IP}:2379" \ --with-sys-table \ --storage "s3://${backup_collection_addr}/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}" \ @@ -178,7 +178,7 @@ br restore full \ In the preceding command: - `--with-sys-table`: BR restores **data in some system tables**, including account permission data and SQL bindings, and statistics (see [Back up statistics](/br/br-snapshot-manual.md#back-up-statistics)). However, it does not restore statistics tables (`mysql.stat_*`) and system variable tables (`mysql.tidb` and `mysql.global_variables`). For more information, see [Restore tables in the `mysql` schema](/br/br-snapshot-guide.md#restore-tables-in-the-mysql-schema). -- `--ratelimit`: The maximum speed **per TiKV** performing backup tasks. The unit is in MiB/s. +- `--ratelimit`: The maximum speed **per TiKV** performing restore tasks. The unit is in MiB/s. - `--log-file`: The target file where the `br` log is written. During restore, a progress bar is displayed in the terminal as shown below. When the progress bar advances to 100%, the restore task is completed. Then `br` will verify the restored data to ensure data security. @@ -193,12 +193,12 @@ You can use `br` to restore partial data of a specified database or table from b ### Restore a database -To restore a database to a cluster, run the `br restore db` command. +To restore a database to a cluster, run the `tiup br restore db` command. The following example restores the `test` database from the backup data to the target cluster: ```shell -br restore db \ +tiup br restore db \ --pd "${PD_IP}:2379" \ --db "test" \ --ratelimit 128 \ @@ -214,12 +214,12 @@ In the preceding command, `--db` specifies the name of the database to be restor ### Restore a table -To restore a single table to a cluster, run the `br restore table` command. +To restore a single table to a cluster, run the `tiup br restore table` command. The following example restores the `test.usertable` table from Amazon S3 to the target cluster: ```shell -br restore table \ +tiup br restore table \ --pd "${PD_IP}:2379" \ --db "test" \ --table "usertable" \ @@ -232,12 +232,12 @@ In the preceding command, `--table` specifies the name of the table to be restor ### Restore multiple tables with table filter -To restore multiple tables with more complex filter rules, run the `br restore full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. +To restore multiple tables with more complex filter rules, run the `tiup br restore full` command and specify the [table filters](/table-filter.md) with `--filter` or `-f`. The following example restores tables that match the `db*.tbl*` filter rule from Amazon S3 to the target cluster: ```shell -br restore full \ +tiup br restore full \ --pd "${PD_IP}:2379" \ --filter 'db*.tbl*' \ --storage "s3://${backup_collection_addr}/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}" \ @@ -246,12 +246,12 @@ br restore full \ ### Restore execution plan bindings from the `mysql` schema -To restore execution plan bindings of a cluster, you can run the `br restore full` command, including the `--with-sys-table` option and also the `--filter` or `-f` option to specify the `mysql` schema to be restored. +To restore execution plan bindings of a cluster, you can run the `tiup br restore full` command, including the `--with-sys-table` option and also the `--filter` or `-f` option to specify the `mysql` schema to be restored. The following is an example of restoring the `mysql.bind_info` table: ```shell -br restore full \ +tiup br restore full \ --pd "${PD_IP}:2379" \ --filter 'mysql.bind_info' \ --with-sys-table \ @@ -286,7 +286,7 @@ ADMIN RELOAD BINDINGS; After encrypting the backup data, you need to pass in the corresponding decryption parameters to restore the data. Ensure that the decryption algorithm and key are correct. If the decryption algorithm or key is incorrect, the data cannot be restored. The following is an example: ```shell -br restore full\ +tiup br restore full\ --pd "${PD_IP}:2379" \ --storage "s3://${backup_collection_addr}/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}" \ --crypter.method aes128-ctr \ diff --git a/br/br-use-overview.md b/br/br-use-overview.md index 4bacceb740063..dd28eadd7b2d9 100644 --- a/br/br-use-overview.md +++ b/br/br-use-overview.md @@ -16,8 +16,8 @@ Before using TiDB backup and restore features, it is recommended that you unders **TiDB provides two types of backup. Which one should I use?** Full backup contains the full data of a cluster at a certain point in time. Log backup contains the data changes written to TiDB. It is recommended to use both types of backup at the same time: -- **[Start log backup](/br/br-pitr-guide.md#start-log-backup)**: Run the `br log start` command to start the log backup task. After that, the task keeps running on all TiKV nodes and backs up TiDB data changes to the specified storage in small batches regularly. -- **Perform [snapshot (full) backup](/br/br-snapshot-guide.md#back-up-cluster-snapshots) regularly**: Run the `br backup full` command to back up the snapshot of the cluster to the specified storage. For example, back up the cluster snapshot at 0:00 AM every day. +- **[Start log backup](/br/br-pitr-guide.md#start-log-backup)**: Run the `tiup br log start` command to start the log backup task. After that, the task keeps running on all TiKV nodes and backs up TiDB data changes to the specified storage in small batches regularly. +- **Perform [snapshot (full) backup](/br/br-snapshot-guide.md#back-up-cluster-snapshots) regularly**: Run the `tiup br backup full` command to back up the snapshot of the cluster to the specified storage. For example, back up the cluster snapshot at 0:00 AM every day. ### How to manage backup data? @@ -53,13 +53,13 @@ If the TiDB cluster is deployed in a self-built data center, the following pract Assume that you need to set the life cycle for each backup data, for example, 7 days. Such a life cycle is called **backup retention period**, which will also be mentioned in backup tutorials. -* To perform PITR, you need to restore the full backup before the restore point, and the log backup between the full backup and the restore point. Therefore, **It is recommended to only delete the log backup before the full snapshot**. For log backups that exceed the backup retention period, you can use `br log truncate` command to delete the backup before the specified time point. +* To perform PITR, you need to restore the full backup before the restore point, and the log backup between the full backup and the restore point. Therefore, **It is recommended to only delete the log backup before the full snapshot**. For log backups that exceed the backup retention period, you can use `tiup br log truncate` command to delete the backup before the specified time point. * For backup data that exceeds the retention period, you can delete or archive the backup directory. ### How to restore data? -- To restore only full backup data, you can use `br restore` to perform a full restore of the specified backup. -- If you have started log backup and regularly performed a full backup, you can run the `br restore point` command to restore data to any time point within the backup retention period. +- To restore only full backup data, you can use `tiup br restore` to perform a full restore of the specified backup. +- If you have started log backup and regularly performed a full backup, you can run the `tiup br restore point` command to restore data to any time point within the backup retention period. ## Deploy and use BR diff --git a/br/use-br-command-line-tool.md b/br/use-br-command-line-tool.md index 40ef11381c5b0..3cfcc43cde70b 100644 --- a/br/use-br-command-line-tool.md +++ b/br/use-br-command-line-tool.md @@ -1,6 +1,6 @@ --- title: br Command-line Manual -summary: The `br` command-line tool is used for snapshot backup, log backup, and point-in-time recovery (PITR) in TiDB clusters. It consists of sub-commands, options, and parameters, with common options like `--pd` for PD service address and `-s` for storage path. Sub-commands include `br backup`, `br log`, and `br restore`, each with specific functionalities. Backup commands include `full`, `db`, and `table` options, while log backup and restore commands have various tasks for managing backup operations. +summary: The `br` command-line tool is used for snapshot backup, log backup, and point-in-time recovery (PITR) in TiDB clusters. It consists of sub-commands, options, and parameters, with common options like `--pd` for PD service address and `-s` for storage path. Sub-commands include `tiup br backup`, `tiup br log`, and `tiup br restore`, each with specific functionalities. Backup commands include `full`, `db`, and `table` options, while log backup and restore commands have various tasks for managing backup operations. --- # br Command-line Manual @@ -14,26 +14,26 @@ A `br` command consists of sub-commands, options, and parameters. A sub-command The following is a complete `br` command: ```shell -br backup full --pd "${PD_IP}:2379" \ +tiup br backup full --pd "${PD_IP}:2379" \ --storage "s3://backup-data/snapshot-202209081330/" ``` Explanations for the preceding command are as follows: -* `backup`: the sub-command of `br`. -* `full`: the sub-command of `br backup`. +* `backup`: the sub-command of `tiup br`. +* `full`: the sub-command of `tiup br backup`. * `-s` (or `--storage`): the option that specifies the path where the backup files are stored. `"s3://backup-data/snapshot-202209081330/"` is the parameter of `-s`. * `--pd`: the option that specifies the PD service address. `"${PD_IP}:2379"` is the parameter of `--pd`. ### Commands and sub-commands -A `br` command consists of multiple layers of sub-commands. Currently, br command-line tool has the following sub-commands: +A `tiup br` command consists of multiple layers of sub-commands. Currently, br command-line tool has the following sub-commands: -* `br backup`: used to back up the data of the TiDB cluster. -* `br log`: used to start and manage log backup tasks. -* `br restore`: used to restore backup data of the TiDB cluster. +* `tiup br backup`: used to back up the data of the TiDB cluster. +* `tiup br log`: used to start and manage log backup tasks. +* `tiup br restore`: used to restore backup data of the TiDB cluster. -`br backup` and `br restore` include the following sub-commands: +`tiup br backup` and `tiup br restore` include the following sub-commands: * `full`: used to back up or restore all the cluster data. * `db`: used to back up or restore a specified database of the cluster. @@ -53,7 +53,7 @@ A `br` command consists of multiple layers of sub-commands. Currently, br comman ## Commands of full backup -To back up cluster data, run the `br backup` command. You can add the `full` or `table` sub-command to specify the scope of your backup operation: the whole cluster (`full`) or a single table (`table`). +To back up cluster data, run the `tiup br backup` command. You can add the `full` or `table` sub-command to specify the scope of your backup operation: the whole cluster (`full`) or a single table (`table`). - [Back up TiDB cluster snapshots](/br/br-snapshot-manual.md#back-up-cluster-snapshots) - [Back up a database](/br/br-snapshot-manual.md#back-up-a-database) @@ -63,7 +63,7 @@ To back up cluster data, run the `br backup` command. You can add the `full` or ## Commands of log backup -To start log backup and manage log backup tasks, run the `br log` command. +To start log backup and manage log backup tasks, run the `tiup br log` command. - [Start a log backup task](/br/br-pitr-manual.md#start-a-backup-task) - [Query the backup status](/br/br-pitr-manual.md#query-the-backup-status) @@ -74,7 +74,7 @@ To start log backup and manage log backup tasks, run the `br log` command. ## Commands of restoring backup data -To restore cluster data, run the `br restore` command. You can add the `full`, `db`, or `table` sub-command to specify the scope of your restore: the whole cluster (`full`), a single database (`db`), or a single table (`table`). +To restore cluster data, run the `tiup br restore` command. You can add the `full`, `db`, or `table` sub-command to specify the scope of your restore: the whole cluster (`full`), a single database (`db`), or a single table (`table`). - [Point-in-time recovery](/br/br-pitr-manual.md#restore-to-a-specified-point-in-time-pitr) - [Restore cluster snapshots](/br/br-snapshot-manual.md#restore-cluster-snapshots) diff --git a/check-before-deployment.md b/check-before-deployment.md index 38a5e3b52b935..0b8f957ca52e3 100644 --- a/check-before-deployment.md +++ b/check-before-deployment.md @@ -269,7 +269,7 @@ To check whether the NTP service is installed and whether it synchronizes with t Unable to talk to NTP daemon. Is it running? ``` -3. Run the `chronyc tracking` command to check wheter the Chrony service synchronizes with the NTP server. +3. Run the `chronyc tracking` command to check whether the Chrony service synchronizes with the NTP server. > **Note:** > @@ -630,9 +630,17 @@ Take the following steps to check the current operating system configuration and echo "net.ipv4.tcp_tw_recycle = 0">> /etc/sysctl.conf echo "net.ipv4.tcp_syncookies = 0">> /etc/sysctl.conf echo "vm.overcommit_memory = 1">> /etc/sysctl.conf + echo "vm.min_free_kbytes = 1048576">> /etc/sysctl.conf sysctl -p ``` + > **Note:** + > + > - `vm.min_free_kbytes` is a Linux kernel parameter that controls the minimum amount of free memory reserved by the system, measured in KiB. + > - The setting of `vm.min_free_kbytes` affects the memory reclaim mechanism. Setting it too large reduces the available memory, while setting it too small might cause memory request speeds to exceed background reclaim speeds, leading to memory reclamation and consequent delays in memory allocation. + > - It is recommended to set `vm.min_free_kbytes` to `1048576` KiB (1 GiB) at least. If [NUMA is installed](/check-before-deployment.md#install-the-numactl-tool), it is recommended to set it to `number of NUMA nodes * 1048576` KiB. + > - For servers with memory sizes less than 16 GiB, it is recommended to keep the default value of `vm.min_free_kbytes` unchanged. + 10. Execute the following command to configure the user's `limits.conf` file: {{< copyable "shell-regular" >}} diff --git a/choose-index.md b/choose-index.md index 3f8a4860eda67..19bf0f2f49d53 100644 --- a/choose-index.md +++ b/choose-index.md @@ -123,15 +123,15 @@ According to these factors and the cost model, the optimizer selects an index wi 1. The estimated number of rows is not accurate? - This is usually due to stale or inaccurate statistics. You can re-execute the `analyze table` statement or modify the parameters of the `analyze table` statement. + This is usually due to stale or inaccurate statistics. You can re-execute the `ANALYZE TABLE` statement or modify the parameters of the `ANALYZE TABLE` statement. 2. Statistics are accurate, and reading from TiFlash is faster, but why does the optimizer choose to read from TiKV? - At present, the cost model of distinguishing TiFlash from TiKV is still rough. You can decrease the value of `tidb_opt_seek_factor` parameter, then the optimizer prefers to choose TiFlash. + At present, the cost model of distinguishing TiFlash from TiKV is still rough. You can decrease the value of [`tidb_opt_seek_factor`](/system-variables.md#tidb_opt_seek_factor) parameter, then the optimizer prefers to choose TiFlash. 3. The statistics are accurate. Index A needs to retrieve rows from tables, but it actually executes faster than Index B that does not retrieve rows from tables. Why does the optimizer choose Index B? - In this case, the cost estimation may be too large for retrieving rows from tables. You can decrease the value of `tidb_opt_network_factor` parameter to reduce the cost of retrieving rows from tables. + In this case, the cost estimation may be too large for retrieving rows from tables. You can decrease the value of [`tidb_opt_network_factor`](/system-variables.md#tidb_opt_network_factor) parameter to reduce the cost of retrieving rows from tables. ## Control index selection @@ -143,7 +143,7 @@ The index selection can be controlled by a single query through [Optimizer Hints ## Use multi-valued indexes -[Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) are different from normal indexes. TiDB currently only uses [IndexMerge](/explain-index-merge.md) to access multi-valued indexes. Therefore, to use multi-valued indexes for data access, make sure that the value of the system variable `tidb_enable_index_merge` is set to `ON`. +[Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) are different from normal indexes. TiDB currently only uses [IndexMerge](/explain-index-merge.md) to access multi-valued indexes. Therefore, to use multi-valued indexes for data access, make sure that the value of the system variable [`tidb_enable_index_merge`](/system-variables.md#tidb_enable_index_merge-new-in-v40) is set to `ON`. For the limitations of multi-valued indexes, refer to [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md#limitations). diff --git a/command-line-flags-for-tidb-configuration.md b/command-line-flags-for-tidb-configuration.md index 1981b1c94f778..c56c6b41c7575 100644 --- a/command-line-flags-for-tidb-configuration.md +++ b/command-line-flags-for-tidb-configuration.md @@ -193,6 +193,11 @@ When you start the TiDB cluster, you can use command-line options or environment - The temporary directory of TiDB - Default: `"/tmp/tidb"` +## `--tidb-service-scope` + ++ Specifies the initial value of [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) for the current TiDB instance. ++ Default: `""` + ## `--token-limit` - The number of sessions allowed to run concurrently in TiDB. It is used for traffic control. @@ -219,6 +224,12 @@ When you start the TiDB cluster, you can use command-line options or environment + Sets the CPU affinity of TiDB servers, which is separated by commas. For example, "1,2,3". + Default: `""` +## `--redact` + ++ Determines whether the TiDB server desensitizes log files when using the subcommand `collect-log`. ++ Default: false ++ When the value is `true`, it is a masking operation, and all fields wrapped in `‹ ›` mark symbols are replaced with `?`. When the value is `false`, it is a restore operation, and all mark symbols are removed. To use this feature, execute `./tidb-server --redact=xxx collect-log ` to desensitize or restore the TiDB server log file specified by `` and output it to ``. For more information, see the system variable [`tidb_redact_log`](/system-variables.md#tidb_redact_log). + ## `--repair-mode` + Determines whether to enable the repair mode, which is only used in the data repair scenario. diff --git a/configure-memory-usage.md b/configure-memory-usage.md index de90b3dbde9d9..71e03b7661184 100644 --- a/configure-memory-usage.md +++ b/configure-memory-usage.md @@ -57,7 +57,7 @@ Currently, the memory limit set by `tidb_server_memory_limit` **DOES NOT** termi > > + During the startup process, TiDB does not guarantee that the [`tidb_server_memory_limit`](/system-variables.md#tidb_server_memory_limit-new-in-v640) limit is enforced. If the free memory of the operating system is insufficient, TiDB might still encounter OOM. You need to ensure that the TiDB instance has enough available memory. > + In the process of memory control, the total memory usage of TiDB might slightly exceed the limit set by `tidb_server_memory_limit`. -> + Since v6.5.0, the configruation item `server-memory-quota` is deprecated. To ensure compatibility, after you upgrade your cluster to v6.5.0 or a later version, `tidb_server_memory_limit` will inherit the value of `server-memory-quota`. If you have not configured `server-memory-quota` before the upgrade, the default value of `tidb_server_memory_limit` is used, which is `80%`. +> + Since v6.5.0, the configuration item `server-memory-quota` is deprecated. To ensure compatibility, after you upgrade your cluster to v6.5.0 or a later version, `tidb_server_memory_limit` will inherit the value of `server-memory-quota`. If you have not configured `server-memory-quota` before the upgrade, the default value of `tidb_server_memory_limit` is used, which is `80%`. When the memory usage of a tidb-server instance reaches a certain proportion of the total memory (the proportion is controlled by the system variable [`tidb_server_memory_limit_gc_trigger`](/system-variables.md#tidb_server_memory_limit_gc_trigger-new-in-v640)), tidb-server will try to trigger a Golang GC to relieve memory stress. To avoid frequent GCs that cause performance issues due to the instance memory fluctuating around the threshold, this GC method will trigger GC at most once every minute. diff --git a/daily-check.md b/daily-check.md index 6d82b984dd4c9..beda3f86b57ea 100644 --- a/daily-check.md +++ b/daily-check.md @@ -38,12 +38,15 @@ You can locate the slow SQL statement executed in the cluster. Then you can opti ![Region panel](/media/region-panel.png) -+ `miss-peer-region-count`: The number of Regions without enough replicas. This value is not always greater than `0`. ++ `down-peer-region-count`: The number of Regions with an unresponsive peer reported by the Raft leader. ++ `empty-region-count`: The number of empty Regions, with a size of smaller than 1 MiB. These Regions are generated by executing the `TRUNCATE TABLE`/`DROP TABLE` statement. If this number is large, you can consider enabling `Region Merge` to merge Regions across tables. + `extra-peer-region-count`: The number of Regions with extra replicas. These Regions are generated during the scheduling process. -+ `empty-region-count`: The number of empty Regions, generated by executing the `TRUNCATE TABLE`/`DROP TABLE` statement. If this number is large, you can consider enabling `Region Merge` to merge Regions across tables. ++ `learner-peer-region-count`: The number of Regions with the learner peer. The sources of learner peers can be various, for example, the learner peers in TiFlash, and the learner peers included in the configured Placement Rules. ++ `miss-peer-region-count`: The number of Regions without enough replicas. This value is not always greater than `0`. ++ `offline-peer-region-count`: The number of Regions during the peer offline process. ++ `oversized-region-count`: The number of Regions with a size larger than `region-max-size` or `region-max-keys`. + `pending-peer-region-count`: The number of Regions with outdated Raft logs. It is normal that a few pending peers are generated in the scheduling process. However, it is not normal if this value is large for a period of time (longer than 30 minutes). -+ `down-peer-region-count`: The number of Regions with an unresponsive peer reported by the Raft leader. -+ `offline-peer-region-count`: The number of Regions during the offline process. ++ `undersized-region-count`: The number of Regions with a size smaller than `max-merge-region-size` or `max-merge-region-keys`. Generally, it is normal that these values are not `0`. However, it is not normal that they are not `0` for quite a long time. diff --git a/dashboard/dashboard-session-sso.md b/dashboard/dashboard-session-sso.md index 27ccf6bce77e8..900c1944824d7 100644 --- a/dashboard/dashboard-session-sso.md +++ b/dashboard/dashboard-session-sso.md @@ -104,7 +104,7 @@ First, create an Okta Application Integration to integrate SSO. ![Sample Step](/media/dashboard/dashboard-session-sso-okta-1.png) -4. In the poped up dialog, choose **OIDC - OpenID Connect** in **Sign-in method**. +4. In the popped up dialog, choose **OIDC - OpenID Connect** in **Sign-in method**. 5. Choose **Single-Page Application** in **Application Type**. diff --git a/data-type-default-values.md b/data-type-default-values.md index 7ac7fa12a2188..7dbdaf2b35a09 100644 --- a/data-type-default-values.md +++ b/data-type-default-values.md @@ -6,9 +6,15 @@ aliases: ['/docs/dev/data-type-default-values/','/docs/dev/reference/sql/data-ty # Default Values -The `DEFAULT` value clause in a data type specification indicates a default value for a column. The default value must be a constant and cannot be a function or an expression. But for the time type, you can specify the `NOW`, `CURRENT_TIMESTAMP`, `LOCALTIME`, and `LOCALTIMESTAMP` functions as the default for `TIMESTAMP` and `DATETIME` columns. +The `DEFAULT` value clause in a data type specification indicates a default value for a column. -Starting from v8.0.0, TiDB supports [specifying expressions as default values](#specify-expressions-as-default-values) for [`BLOB`](/data-type-string.md#blob-type), [`TEXT`](/data-type-string.md#text-type), and [`JSON`](/data-type-json.md#json-type) data types. +You can set default values for all data types. Typically, default values must be constants and cannot be functions or expressions, but there are some exceptions: + +- For time types, you can use `NOW`, `CURRENT_TIMESTAMP`, `LOCALTIME`, and `LOCALTIMESTAMP` functions as default values for `TIMESTAMP` and `DATETIME` columns. +- For integer types, you can use the `NEXT VALUE FOR` function to set the next value of a sequence as the default value for a column, and use the [`RAND()`](/functions-and-operators/numeric-functions-and-operators.md) function to generate a random floating-point value as the default value for a column. +- For string types, you can use the [`UUID()`](/functions-and-operators/miscellaneous-functions.md) function to generate a [universally unique identifier (UUID)](/best-practices/uuid.md) as the default value for a column. +- For binary types, you can use the [`UUID_TO_BIN()`](/functions-and-operators/miscellaneous-functions.md) function to convert a UUID to the binary format and set the converted value as the default value for a column. +- Starting from v8.0.0, TiDB additionally supports [specifying the default values](#specify-expressions-as-default-values) for [`BLOB`](/data-type-string.md#blob-type), [`TEXT`](/data-type-string.md#text-type), and [`JSON`](/data-type-json.md#json-type) data types, but you can only use expressions to set the [default values](#default-values) for them. If a column definition includes no explicit `DEFAULT` value, TiDB determines the default value as follows: @@ -28,39 +34,21 @@ Implicit defaults are defined as follows: ## Specify expressions as default values -> **Warning:** -> -> Currently, this feature is experimental. It is not recommended that you use it in production environments. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. - -Starting from 8.0.13, MySQL supports specifying expressions as default values in the `DEFAULT` clause. For more information, see [Explicit default handling as of MySQL 8.0.13](https://dev.mysql.com/doc/refman/8.0/en/data-type-defaults.html#data-type-defaults-explicit). - -TiDB has implemented this feature and supports specifying some expressions as default values in the `DEFAULT` clause. Starting from v8.0.0, TiDB supports assigning default values to `BLOB`, `TEXT`, and `JSON` data types. However, these default values can only be set as expressions. The following is an example of `BLOB`: - -```sql -CREATE TABLE t2 (b BLOB DEFAULT (RAND())); -``` - -TiDB currently supports the following expressions: - -* [`RAND()`](/functions-and-operators/numeric-functions-and-operators.md) -* [`UUID()`](/functions-and-operators/miscellaneous-functions.md) -* [`UUID_TO_BIN()`](/functions-and-operators/miscellaneous-functions.md) +Starting from 8.0.13, MySQL supports specifying expressions as default values in the `DEFAULT` clause. For more information, see [Explicit default handling as of MySQL 8.0.13](https://dev.mysql.com/doc/refman/8.0/en/data-type-defaults.html#data-type-defaults-explicit). -Starting from TiDB v8.0.0, the `DEFAULT` clause supports using the following expressions to set default values. +Starting from v8.0.0, TiDB additionally supports specifying the following expressions as default values in the `DEFAULT` clause. * `UPPER(SUBSTRING_INDEX(USER(), '@', 1))` - * `REPLACE(UPPER(UUID()), '-', '')` - -* The `DATE_FORMAT` supports the following formats: - +* `DATE_FORMAT` expressions in the following formats: * `DATE_FORMAT(NOW(), '%Y-%m')` * `DATE_FORMAT(NOW(), '%Y-%m-%d')` * `DATE_FORMAT(NOW(), '%Y-%m-%d %H.%i.%s')` * `DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')` - * `STR_TO_DATE('1980-01-01', '%Y-%m-%d')` -> **Note:** -> -> Currently, the `ADD COLUMN` statement does not support using expressions as default values. +Starting from v8.0.0, TiDB additionally supports assigning default values to `BLOB`, `TEXT`, and `JSON` data types. However, you can only use expressions to set the default values for these data types. The following is an example of `BLOB`: + +```sql +CREATE TABLE t2 (b BLOB DEFAULT (RAND())); +``` diff --git a/ddl-introduction.md b/ddl-introduction.md index 31ec8f0234410..7658ccfae57e7 100644 --- a/ddl-introduction.md +++ b/ddl-introduction.md @@ -77,7 +77,7 @@ absent -> delete only -> write only -> write reorg -> public For users, the newly created index is unavailable before the `public` state. -
+
Before v6.2.0, the process of handling asynchronous schema changes in the TiDB SQL layer is as follows: diff --git a/develop/dev-guide-create-table.md b/develop/dev-guide-create-table.md index 058828e5a82f5..3187e6d1fe3e5 100644 --- a/develop/dev-guide-create-table.md +++ b/develop/dev-guide-create-table.md @@ -183,7 +183,7 @@ CREATE TABLE `bookshop`.`ratings` ( ); ``` -In addition, if the current time is also filled in by default when the data is being updated, the following statements can be used (but only the [current time related statements](https://pingcap.github.io/sqlgram/#NowSymOptionFraction) can be filled in after `ON UPDATE`, and [more options](https://pingcap.github.io/sqlgram/#DefaultValueExpr) are supported after `DEFAULT`): +In addition, if the current time is also filled in by default when the data is being updated, the following statements can be used (but only the current-time related expressions can be filled in after `ON UPDATE`): ```sql CREATE TABLE `bookshop`.`ratings` ( @@ -195,6 +195,8 @@ CREATE TABLE `bookshop`.`ratings` ( ); ``` +For more information on default values of different data types, see [default values](/data-type-default-values.md). + ### Prevent duplicate values If you need to prevent duplicate values in a column, you can use the `UNIQUE` constraint. diff --git a/develop/dev-guide-gui-datagrip.md b/develop/dev-guide-gui-datagrip.md index 30e56a9f613bc..8d84368558cd2 100644 --- a/develop/dev-guide-gui-datagrip.md +++ b/develop/dev-guide-gui-datagrip.md @@ -93,7 +93,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-gui-dbeaver.md b/develop/dev-guide-gui-dbeaver.md index 2348a23325f56..c036b2862d601 100644 --- a/develop/dev-guide-gui-dbeaver.md +++ b/develop/dev-guide-gui-dbeaver.md @@ -90,7 +90,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere**, and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere**, and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-golang-gorm.md b/develop/dev-guide-sample-application-golang-gorm.md index 2e5f15a4e98dc..40b7f7451a940 100644 --- a/develop/dev-guide-sample-application-golang-gorm.md +++ b/develop/dev-guide-sample-application-golang-gorm.md @@ -113,7 +113,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-golang-sql-driver.md b/develop/dev-guide-sample-application-golang-sql-driver.md index b59c38451c4fe..e08c455d4b309 100644 --- a/develop/dev-guide-sample-application-golang-sql-driver.md +++ b/develop/dev-guide-sample-application-golang-sql-driver.md @@ -114,7 +114,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-java-hibernate.md b/develop/dev-guide-sample-application-java-hibernate.md index 40958421dcbbe..c9c0b0bbe3b81 100644 --- a/develop/dev-guide-sample-application-java-hibernate.md +++ b/develop/dev-guide-sample-application-java-hibernate.md @@ -114,7 +114,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-java-jdbc.md b/develop/dev-guide-sample-application-java-jdbc.md index bd3b7373e6557..d6ba6893121e9 100644 --- a/develop/dev-guide-sample-application-java-jdbc.md +++ b/develop/dev-guide-sample-application-java-jdbc.md @@ -119,7 +119,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-java-mybatis.md b/develop/dev-guide-sample-application-java-mybatis.md index 60eeb2d8ecc40..ecaf12348f581 100644 --- a/develop/dev-guide-sample-application-java-mybatis.md +++ b/develop/dev-guide-sample-application-java-mybatis.md @@ -114,7 +114,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-java-spring-boot.md b/develop/dev-guide-sample-application-java-spring-boot.md index 24dfe5678b7bc..fa4f2426d62fb 100644 --- a/develop/dev-guide-sample-application-java-spring-boot.md +++ b/develop/dev-guide-sample-application-java-spring-boot.md @@ -115,7 +115,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-nodejs-mysql2.md b/develop/dev-guide-sample-application-nodejs-mysql2.md index 612e04f72211d..62223c4d881ac 100644 --- a/develop/dev-guide-sample-application-nodejs-mysql2.md +++ b/develop/dev-guide-sample-application-nodejs-mysql2.md @@ -122,7 +122,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). @@ -189,7 +189,7 @@ npm start If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.0) ⏳ Loading sample game data... ✅ Loaded sample game data. diff --git a/develop/dev-guide-sample-application-nodejs-mysqljs.md b/develop/dev-guide-sample-application-nodejs-mysqljs.md index e050831f0900c..bf3a16da2da55 100644 --- a/develop/dev-guide-sample-application-nodejs-mysqljs.md +++ b/develop/dev-guide-sample-application-nodejs-mysqljs.md @@ -122,7 +122,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). @@ -189,7 +189,7 @@ npm start If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.0) ⏳ Loading sample game data... ✅ Loaded sample game data. diff --git a/develop/dev-guide-sample-application-nodejs-prisma.md b/develop/dev-guide-sample-application-nodejs-prisma.md index e3fe11a0416f7..911d095776349 100644 --- a/develop/dev-guide-sample-application-nodejs-prisma.md +++ b/develop/dev-guide-sample-application-nodejs-prisma.md @@ -125,7 +125,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-nodejs-sequelize.md b/develop/dev-guide-sample-application-nodejs-sequelize.md index e9648b2d921a0..4e25791adb21d 100644 --- a/develop/dev-guide-sample-application-nodejs-sequelize.md +++ b/develop/dev-guide-sample-application-nodejs-sequelize.md @@ -123,7 +123,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-nodejs-typeorm.md b/develop/dev-guide-sample-application-nodejs-typeorm.md index 681da4620d895..27a05af0f4903 100644 --- a/develop/dev-guide-sample-application-nodejs-typeorm.md +++ b/develop/dev-guide-sample-application-nodejs-typeorm.md @@ -130,7 +130,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). @@ -231,7 +231,7 @@ npm start If the connection is successful, the terminal will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.0) 🆕 Created a new player with ID 2. ℹ️ Got Player 2: Player { id: 2, coins: 100, goods: 100 } 🔢 Added 50 coins and 50 goods to player 2, now player 2 has 100 coins and 150 goods. diff --git a/develop/dev-guide-sample-application-python-django.md b/develop/dev-guide-sample-application-python-django.md index 6d603709dd5ba..3224f99ac984f 100644 --- a/develop/dev-guide-sample-application-python-django.md +++ b/develop/dev-guide-sample-application-python-django.md @@ -132,7 +132,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-python-mysql-connector.md b/develop/dev-guide-sample-application-python-mysql-connector.md index 2debeb43cf2c9..8c1f85e8b9570 100644 --- a/develop/dev-guide-sample-application-python-mysql-connector.md +++ b/develop/dev-guide-sample-application-python-mysql-connector.md @@ -120,7 +120,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-python-mysqlclient.md b/develop/dev-guide-sample-application-python-mysqlclient.md index a90d0bfa88e5d..d6dde6f430e9c 100644 --- a/develop/dev-guide-sample-application-python-mysqlclient.md +++ b/develop/dev-guide-sample-application-python-mysqlclient.md @@ -123,7 +123,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-python-peewee.md b/develop/dev-guide-sample-application-python-peewee.md index 9b9868dae14e8..8ef835375446e 100644 --- a/develop/dev-guide-sample-application-python-peewee.md +++ b/develop/dev-guide-sample-application-python-peewee.md @@ -123,7 +123,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-python-pymysql.md b/develop/dev-guide-sample-application-python-pymysql.md index 500a0b18075a7..2c144798e6ef7 100644 --- a/develop/dev-guide-sample-application-python-pymysql.md +++ b/develop/dev-guide-sample-application-python-pymysql.md @@ -119,7 +119,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-python-sqlalchemy.md b/develop/dev-guide-sample-application-python-sqlalchemy.md index 6b10c1b31d73b..4e02f230af014 100644 --- a/develop/dev-guide-sample-application-python-sqlalchemy.md +++ b/develop/dev-guide-sample-application-python-sqlalchemy.md @@ -77,6 +77,10 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele
+> **Note:** +> +> Currently, TiDB Serverless clusters have a limitation: if there are no active connections for 5 minutes, they will shut down, which closes all connections. Therefore, when using SQLAlchemy with TiDB Serverless clusters, pooled connections might encounter `OperationalError` such as `Lost connection to MySQL server during query` or `MySQL Connection not available`. To avoid this error, you can set the `pool_recycle` parameter to `300`. For more information, see [Dealing with Disconnects](https://docs.sqlalchemy.org/en/20/core/pooling.html#dealing-with-disconnects) in SQLAlchemy documentation. + 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. @@ -126,7 +130,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). diff --git a/develop/dev-guide-sample-application-ruby-mysql2.md b/develop/dev-guide-sample-application-ruby-mysql2.md index 7c60da4a442d4..87bd15ad3d5f1 100644 --- a/develop/dev-guide-sample-application-ruby-mysql2.md +++ b/develop/dev-guide-sample-application-ruby-mysql2.md @@ -123,7 +123,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). @@ -190,7 +190,7 @@ ruby app.rb If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.0) ⏳ Loading sample game data... ✅ Loaded sample game data. diff --git a/develop/dev-guide-sample-application-ruby-rails.md b/develop/dev-guide-sample-application-ruby-rails.md index 142bac8ec1988..3e865204b52ee 100644 --- a/develop/dev-guide-sample-application-ruby-rails.md +++ b/develop/dev-guide-sample-application-ruby-rails.md @@ -113,7 +113,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. Click **Allow Access from Anywhere** and then click **Download CA cert** to download the CA certificate. For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). @@ -183,7 +183,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.0) ⏳ Loading sample game data... ✅ Loaded sample game data. diff --git a/develop/dev-guide-third-party-support.md b/develop/dev-guide-third-party-support.md index 7e0a9637767a0..1f57f94a22190 100644 --- a/develop/dev-guide-third-party-support.md +++ b/develop/dev-guide-third-party-support.md @@ -184,8 +184,9 @@ If you encounter problems when connecting to TiDB using the tools listed in this ## GUI -| GUI | Latest tested version | Support level | Tutorial | -|-----------------------------------------------------------|-----------------------|---------------|-------------------------------------------------------------------------------| -| [JetBrains DataGrip](https://www.jetbrains.com/datagrip/) | 2023.2.1 | Full | [Connect to TiDB with JetBrains DataGrip](/develop/dev-guide-gui-datagrip.md) | -| [DBeaver](https://dbeaver.io/) | 23.0.3 | Full | [Connect to TiDB with DBeaver](/develop/dev-guide-gui-dbeaver.md) | -| [Visual Studio Code](https://code.visualstudio.com/) | 1.72.0 | Full | [Connect to TiDB with Visual Studio Code](/develop/dev-guide-gui-vscode-sqltools.md) | +| GUI | Latest tested version | Support level | Tutorial | +|-----------------------------------------------------------|-----------------------|---------------|--------------------------------------------------------------------------------------| +| [Beekeeper Studio](https://www.beekeeperstudio.io/) | 4.3.0 | Full | N/A | +| [JetBrains DataGrip](https://www.jetbrains.com/datagrip/) | 2023.2.1 | Full | [Connect to TiDB with JetBrains DataGrip](/develop/dev-guide-gui-datagrip.md) | +| [DBeaver](https://dbeaver.io/) | 23.0.3 | Full | [Connect to TiDB with DBeaver](/develop/dev-guide-gui-dbeaver.md) | +| [Visual Studio Code](https://code.visualstudio.com/) | 1.72.0 | Full | [Connect to TiDB with Visual Studio Code](/develop/dev-guide-gui-vscode-sqltools.md) | diff --git a/develop/dev-guide-third-party-tools-compatibility.md b/develop/dev-guide-third-party-tools-compatibility.md index 41f5d1078a5a3..32dc4a0a48749 100644 --- a/develop/dev-guide-third-party-tools-compatibility.md +++ b/develop/dev-guide-third-party-tools-compatibility.md @@ -153,27 +153,27 @@ To ensure data consistency by transaction, you can use `UPDATE` statements to up **Description** -When the `useLocalTransactionState` and `rewriteBatchedStatements` parameters are set to `true` at the same time, the transaction might fail to commit. You can reproduce with [this code](https://github.com/Icemap/tidb-java-gitpod/tree/reproduction-local-transaction-state-txn-error). +When using MySQL Connector/J 8.0.32 or an earlier version, if the `useLocalTransactionState` and `rewriteBatchedStatements` parameters are set to `true` at the same time, the transaction might fail to commit. You can reproduce with [this code](https://github.com/Icemap/tidb-java-gitpod/tree/reproduction-local-transaction-state-txn-error). **Way to avoid** > **Note:** > -> This bug has been reported to MySQL JDBC. To keep track of the process, you can follow this [Bug Report](https://bugs.mysql.com/bug.php?id=108643). +> `useConfigs=maxPerformance` includes a group of configurations. For detailed configurations in MySQL Connector/J 8.0 and MySQL Connector/J 5.1, see [mysql-connector-j 8.0](https://github.com/mysql/mysql-connector-j/blob/release/8.0/src/main/resources/com/mysql/cj/configurations/maxPerformance.properties) and [mysql-connector-j 5.1](https://github.com/mysql/mysql-connector-j/blob/release/5.1/src/com/mysql/jdbc/configs/maxPerformance.properties) respectively. You need to disable `useLocalTransactionState` when using `maxPerformance`. That is, use `useConfigs=maxPerformance&useLocalTransactionState=false`. -**DO NOT** turn on `useLocalTransactionState` as this might prevent transactions from being committed or rolled back. +This bug has been fixed in MySQL Connector/J 8.0.33. Considering updates for the 8.0.x series have ceased, it is strongly recommended to upgrade your MySQL Connector/J to [the latest General Availability (GA) version](https://dev.mysql.com/downloads/connector/j/) for improved stability and performance. ### Connector is incompatible with the server version earlier than 5.7.5 **Description** -The database connection might hang under certain conditions when using MySQL Connector/J 8.0.29 with a MySQL server < 5.7.5 or a database using the MySQL server < 5.7.5 protocol (such as TiDB earlier than v6.3.0). For more details, see the [Bug Report](https://bugs.mysql.com/bug.php?id=106252). +The database connection might hang under certain conditions when using MySQL Connector/J 8.0.31 or an earlier version with a MySQL server < 5.7.5 or a database using the MySQL server < 5.7.5 protocol (such as TiDB earlier than v6.3.0). For more details, see the [Bug Report](https://bugs.mysql.com/bug.php?id=106252). **Way to avoid** -This is a known issue. As of October 12, 2022, MySQL Connector/J has not fixed the issue. +This bug has been fixed in MySQL Connector/J 8.0.32. Considering updates for the 8.0.x series have ceased, it is strongly recommended to upgrade your MySQL Connector/J to [the latest General Availability (GA) version](https://dev.mysql.com/downloads/connector/j/) for improved stability and performance. -TiDB fixes it in the following ways: +TiDB also fixes it in the following ways: - Client side: This bug has been fixed in **pingcap/mysql-connector-j** and you can use the [pingcap/mysql-connector-j](https://github.com/pingcap/mysql-connector-j) instead of the official MySQL Connector/J. - Server side: This compatibility issue has been fixed since TiDB v6.3.0 and you can upgrade the server to v6.3.0 or later versions. diff --git a/develop/dev-guide-tidb-crud-sql.md b/develop/dev-guide-tidb-crud-sql.md index 9fa4493e245f6..edbb34937b0c3 100644 --- a/develop/dev-guide-tidb-crud-sql.md +++ b/develop/dev-guide-tidb-crud-sql.md @@ -21,7 +21,7 @@ TiDB is compatible with MySQL, you can use MySQL statements directly in most cas To experiment with SQL and test out TiDB compatibility with MySQL queries, you can try [TiDB Playground](https://play.tidbcloud.com/?utm_source=docs&utm_medium=basic-sql-operations). You can also first deploy a TiDB cluster and then run SQL statements in it. -This page walks you through the basic TiDB SQL statements such as DDL, DML, and CRUD operations. For a complete list of TiDB statements, see [TiDB SQL Syntax Diagram](https://pingcap.github.io/sqlgram/). +This page walks you through the basic TiDB SQL statements such as DDL, DML, and CRUD operations. For a complete list of TiDB statements, see [SQL Statement Overview](/sql-statements/sql-statement-overview.md). ## Category @@ -35,7 +35,7 @@ SQL is divided into the following 4 types according to their functions: - **DCL (Data Control Language)**: It is used to define access privileges and security levels. -The following mainly introduces DML and DQL. For more information about DDL and DCL, see [Explore SQL with TiDB](/basic-sql-operations.md) or [TiDB SQL syntax detailed explanation](https://pingcap.github.io/sqlgram/). +The following mainly introduces DML and DQL. For more information about DDL and DCL, see [Explore SQL with TiDB](/basic-sql-operations.md) or [SQL Statement Overview](/sql-statements/sql-statement-overview.md). ## Data Manipulation Language diff --git a/develop/dev-guide-timeouts-in-tidb.md b/develop/dev-guide-timeouts-in-tidb.md index 8b91fc78c7626..632391c9bf836 100644 --- a/develop/dev-guide-timeouts-in-tidb.md +++ b/develop/dev-guide-timeouts-in-tidb.md @@ -11,9 +11,54 @@ This document describes various timeouts in TiDB to help you troubleshoot errors TiDB's transaction implementation uses the MVCC (Multiple Version Concurrency Control) mechanism. When the newly written data overwrites the old data, the old data will not be replaced, but kept together with the newly written data. The versions are distinguished by the timestamp. TiDB uses the mechanism of periodic Garbage Collection (GC) to clean up the old data that is no longer needed. -By default, each MVCC version (consistency snapshots) is kept for 10 minutes. Transactions that take longer than 10 minutes to read will receive an error `GC life time is shorter than transaction duration`. +- For TiDB versions earlier than v4.0: -If you need longer read time, for example, when you are using **Dumpling** for full backups (**Dumpling** backs up consistent snapshots), you can adjust the value of `tikv_gc_life_time` in the `mysql.tidb` table in TiDB to increase the MVCC version retention time. Note that `tikv_gc_life_time` takes effect globally and immediately. Increasing the value will increase the life time of all existing snapshots, and decreasing it will immediately shorten the life time of all snapshots. Too many MVCC versions will impact TiKV's processing efficiency. So you need to change `tikv_gc_life_time` back to the previous setting in time after doing a full backup with **Dumpling**. + By default, each MVCC version (consistency snapshots) is kept for 10 minutes. Transactions that take longer than 10 minutes to read will receive an error `GC life time is shorter than transaction duration`. + +- For TiDB v4.0 and later versions: + + For running transactions that do not exceed a duration of 24 hours, garbage collection (GC) are blocked during the transaction execution. The error `GC life time is shorter than transaction duration` does not occur. + +If you need longer read time temporarily in some cases, you can increase the retention time of MVCC versions: + +- For TiDB versions earlier than v5.0: adjust `tikv_gc_life_time` in the `mysql.tidb` table in TiDB. +- For TiDB v5.0 and later versions: adjust the system variable [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50). + +Note that the system variable configuration takes effect globally and immediately. Increasing its value will increase the life time of all existing snapshots, and decreasing it will immediately shorten the life time of all snapshots. Too many MVCC versions will impact the performance of the TiDB cluster. So you need to change this variable back to the previous setting in time. + + + +> **Tip:** +> +> Specifically, when Dumpling is exporting data from TiDB (less than 1 TB), if the TiDB version is later than or equal to v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. +> +> However, in either of the following scenarios, Dumpling cannot automatically adjust the GC time: +> +> - The data size is very large (more than 1 TB). +> - Dumpling cannot connect directly to PD, for example, the TiDB cluster is on TiDB Cloud or on Kubernetes that is separated from Dumpling. +> +> In such scenarios, you must manually extend the GC time in advance to avoid export failure due to GC during the export process. +> +> For more details, see [Manually set the TiDB GC time](/dumpling-overview.md#manually-set-the-tidb-gc-time). + + + + + +> **Tip:** +> +> Specifically, when Dumpling is exporting data from TiDB (less than 1 TB), if the TiDB version is later than or equal to v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. +> +> However, in either of the following scenarios, Dumpling cannot automatically adjust the GC time: +> +> - The data size is very large (more than 1 TB). +> - Dumpling cannot connect directly to PD, for example, the TiDB cluster is on TiDB Cloud or on Kubernetes that is separated from Dumpling. +> +> In such scenarios, you must manually extend the GC time in advance to avoid export failure due to GC during the export process. +> +> For more details, see [Manually set the TiDB GC time](https://docs.pingcap.com/tidb/stable/dumpling-overview#manually-set-the-tidb-gc-time). + + For more information about GC, see [GC Overview](/garbage-collection-overview.md). diff --git a/develop/dev-guide-use-common-table-expression.md b/develop/dev-guide-use-common-table-expression.md index f9b89a6da9f31..71854c515692f 100644 --- a/develop/dev-guide-use-common-table-expression.md +++ b/develop/dev-guide-use-common-table-expression.md @@ -15,7 +15,7 @@ Since TiDB v5.1, TiDB supports the CTE of the ANSI SQL99 standard and recursion. ## Basic use -A Common Table Expression (CTE) is a temporary result set that can be referred to multiple times within a SQL statement to improve the statement readability and execution efficiency. You can apply the `WITH` statement to use CTE. +A Common Table Expression (CTE) is a temporary result set that can be referred to multiple times within a SQL statement to improve the statement readability and execution efficiency. You can apply the [`WITH`](/sql-statements/sql-statement-with.md) statement to use CTE. Common Table Expressions can be classified into two types: non-recursive CTE and recursive CTE. diff --git a/dm/deploy-a-dm-cluster-using-tiup-offline.md b/dm/deploy-a-dm-cluster-using-tiup-offline.md index 5930e27ca9eea..16a4f464f1784 100644 --- a/dm/deploy-a-dm-cluster-using-tiup-offline.md +++ b/dm/deploy-a-dm-cluster-using-tiup-offline.md @@ -46,7 +46,7 @@ This document describes how to deploy a DM cluster offline using TiUP. tiup mirror clone tidb-dm-${version}-linux-amd64 --os=linux --arch=amd64 \ --dm-master=${version} --dm-worker=${version} --dmctl=${version} \ --alertmanager=v0.17.0 --grafana=v4.0.3 --prometheus=v4.0.3 \ - --tiup=v$(tiup --version|grep 'tiup'|awk -F ' ' '{print $1}') --dm=v$(tiup --version|grep 'tiup'|awk -F ' ' '{print $1}') + --dm=v$(tiup --version|grep 'tiup'|awk -F ' ' '{print $1}') ``` The command above creates a directory named `tidb-dm-${version}-linux-amd64` in the current directory, which contains the component package managed by TiUP. diff --git a/dm/dm-enable-tls.md b/dm/dm-enable-tls.md index 37dc390701cc9..292be260a1580 100644 --- a/dm/dm-enable-tls.md +++ b/dm/dm-enable-tls.md @@ -109,7 +109,7 @@ This section introduces how to enable encrypted data transmission between DM com ### Enable encrypted data transmission for downstream TiDB -1. Configure the downstream TiDB to use encrypted connections. For detailed operatons, refer to [Configure TiDB server to use secure connections](/enable-tls-between-clients-and-servers.md#configure-tidb-server-to-use-secure-connections). +1. Configure the downstream TiDB to use encrypted connections. For detailed operations, refer to [Configure TiDB server to use secure connections](/enable-tls-between-clients-and-servers.md#configure-tidb-server-to-use-secure-connections). 2. Set the TiDB client certificate in the task configuration file: diff --git a/dm/dm-faq.md b/dm/dm-faq.md index c6075787a505d..5ee968c2986e6 100644 --- a/dm/dm-faq.md +++ b/dm/dm-faq.md @@ -365,7 +365,7 @@ To solve this issue, you are recommended to maintain DM clusters using TiUP. In ## Why DM-master cannot be connected when I use dmctl to execute commands? -When using dmctl execute commands, you might find the connection to DM master fails (even if you have specified the parameter value of `--master-addr` in the command), and the error message is like `RawCause: context deadline exceeded, Workaround: please check your network connection.`. But afer checking the network connection using commands like `telnet `, no exception is found. +When using dmctl execute commands, you might find the connection to DM master fails (even if you have specified the parameter value of `--master-addr` in the command), and the error message is like `RawCause: context deadline exceeded, Workaround: please check your network connection.`. But after checking the network connection using commands like `telnet `, no exception is found. In this case, you can check the environment variable `https_proxy` (note that it is **https**). If this variable is configured, dmctl automatically connects the host and port specified by `https_proxy`. If the host does not have a corresponding `proxy` forwarding service, the connection fails. diff --git a/dm/dm-open-api.md b/dm/dm-open-api.md index c0fefce993896..ada86e1cbf718 100644 --- a/dm/dm-open-api.md +++ b/dm/dm-open-api.md @@ -1346,7 +1346,7 @@ curl -X 'GET' \ "name": "string", "source_name": "string", "worker_name": "string", - "stage": "runing", + "stage": "running", "unit": "sync", "unresolved_ddl_lock_id": "string", "load_status": { diff --git a/dm/dm-table-routing.md b/dm/dm-table-routing.md index 63f1089e56ef6..2625c6aaf1fb1 100644 --- a/dm/dm-table-routing.md +++ b/dm/dm-table-routing.md @@ -86,7 +86,7 @@ To migrate the upstream instances to the downstream `test`.`t`, you must create Assuming in the scenario of sharded schemas and tables, you want to migrate the `test_{1,2,3...}`.`t_{1,2,3...}` tables in two upstream MySQL instances to the `test`.`t` table in the downstream TiDB instance. At the same time, you want to extract the source information of the sharded tables and write it to the downstream merged table. -To migrate the upstream instances to the downstream `test`.`t`, you must create routing rules similar to the previous section [Merge sharded schemas and tables](#merge-sharded-schemas-and-tables). In addtion, you need to add the `extract-table`, `extract-schema`, and `extract-source` configurations: +To migrate the upstream instances to the downstream `test`.`t`, you must create routing rules similar to the previous section [Merge sharded schemas and tables](#merge-sharded-schemas-and-tables). In addition, you need to add the `extract-table`, `extract-schema`, and `extract-source` configurations: - `extract-table`: For a sharded table matching `schema-pattern` and `table-pattern`, DM extracts the sharded table name by using `table-regexp` and writes the name suffix without the `t_` part to `target-column` of the merged table, that is, the `c_table` column. - `extract-schema`: For a sharded schema matching `schema-pattern` and `table-pattern`, DM extracts the sharded schema name by using `schema-regexp` and writes the name suffix without the `test_` part to `target-column` of the merged table, that is, the `c_schema` column. diff --git a/dm/maintain-dm-using-tiup.md b/dm/maintain-dm-using-tiup.md index 9ae7cf5842eaa..6e7e53f581299 100644 --- a/dm/maintain-dm-using-tiup.md +++ b/dm/maintain-dm-using-tiup.md @@ -394,7 +394,7 @@ All operations above performed on the cluster machine use the SSH client embedde Then you can use the `--native-ssh` command-line flag to enable the system-native command-line tool: -- Deploy a cluster: `tiup dm deploy --native-ssh`. Fill in the name of your cluster for ``, the DM version to be deployed (such as `v8.0.0`) for `` , and the topology file name for ``. +- Deploy a cluster: `tiup dm deploy --native-ssh`. Fill in the name of your cluster for ``, the DM version to be deployed (such as `v8.1.0`) for `` , and the topology file name for ``. - Start a cluster: `tiup dm start --native-ssh`. - Upgrade a cluster: `tiup dm upgrade ... --native-ssh` diff --git a/dm/manually-handling-sharding-ddl-locks.md b/dm/manually-handling-sharding-ddl-locks.md index 9780b5b8172fa..663547ec5b3d1 100644 --- a/dm/manually-handling-sharding-ddl-locks.md +++ b/dm/manually-handling-sharding-ddl-locks.md @@ -55,7 +55,7 @@ You can use `shard-ddl-lock [task] [flags]` to view the DDL lock information on shard-ddl-lock test ``` -
+
Expected output ``` diff --git a/dm/monitor-a-dm-cluster.md b/dm/monitor-a-dm-cluster.md index f40a66dc3d709..2af6bb1ac852e 100644 --- a/dm/monitor-a-dm-cluster.md +++ b/dm/monitor-a-dm-cluster.md @@ -94,7 +94,7 @@ The following metrics show only when `task-mode` is in the `incremental` or `all | total sqls jobs | The number of newly added jobs per unit of time | N/A | N/A | | finished sqls jobs | The number of finished jobs per unit of time | N/A | N/A | | statement execution latency | The duration that the binlog replication unit executes the statement to the downstream (in seconds) | N/A | N/A | -| add job duration | The duration tht the binlog replication unit adds a job to the queue (in seconds) | N/A | N/A | +| add job duration | The duration that the binlog replication unit adds a job to the queue (in seconds) | N/A | N/A | | DML conflict detect duration | The duration that the binlog replication unit detects the conflict in DML (in seconds) | N/A | N/A | | skipped event duration | The duration that the binlog replication unit skips a binlog event (in seconds) | N/A | N/A | | unsynced tables | The number of tables that have not received the shard DDL statement in the current subtask | N/A | N/A | diff --git a/dm/quick-start-create-source.md b/dm/quick-start-create-source.md index a3af12981f8a6..d2c18e1675e2e 100644 --- a/dm/quick-start-create-source.md +++ b/dm/quick-start-create-source.md @@ -84,7 +84,7 @@ The returned results are as follows: After creating a data source, you can use the following command to query the data source: -- If you konw the `source-id` of the data source, you can use the `dmctl config source ` command to directly check the configuration of the data source: +- If you know the `source-id` of the data source, you can use the `dmctl config source ` command to directly check the configuration of the data source: {{< copyable "shell-regular" >}} diff --git a/dm/quick-start-create-task.md b/dm/quick-start-create-task.md index a42af559e0c2a..ec5af7512951f 100644 --- a/dm/quick-start-create-task.md +++ b/dm/quick-start-create-task.md @@ -74,7 +74,7 @@ To run a TiDB server, use the following command: {{< copyable "shell-regular" >}} ```bash -wget https://download.pingcap.org/tidb-community-server-v8.0.0-linux-amd64.tar.gz +wget https://download.pingcap.org/tidb-community-server-v8.1.0-linux-amd64.tar.gz tar -xzvf tidb-latest-linux-amd64.tar.gz mv tidb-latest-linux-amd64/bin/tidb-server ./ ./tidb-server diff --git a/download-ecosystem-tools.md b/download-ecosystem-tools.md index fe65a624f7cc1..7a28ae868bada 100644 --- a/download-ecosystem-tools.md +++ b/download-ecosystem-tools.md @@ -28,7 +28,7 @@ You can download TiDB Toolkit from the following link: https://download.pingcap.org/tidb-community-toolkit-{version}-linux-{arch}.tar.gz ``` -`{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v6.2.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v6.2.0-linux-amd64.tar.gz`. +`{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.1.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.1.0-linux-amd64.tar.gz`. > **Note:** > diff --git a/dumpling-overview.md b/dumpling-overview.md index df2d02a47f7d8..d376f157f9276 100644 --- a/dumpling-overview.md +++ b/dumpling-overview.md @@ -87,7 +87,7 @@ Dumpling exports data to SQL files by default. You can also export data to SQL f {{< copyable "shell-regular" >}} ```shell -dumpling -u root -P 4000 -h 127.0.0.1 --filetype sql -t 8 -o /tmp/test -r 200000 -F 256MiB +tiup dumpling -u root -P 4000 -h 127.0.0.1 --filetype sql -t 8 -o /tmp/test -r 200000 -F 256MiB ``` In the command above: @@ -121,7 +121,7 @@ When you export data to CSV files, you can use `--sql ` to filter the recor {{< copyable "shell-regular" >}} ```shell -./dumpling -u root -P 4000 -h 127.0.0.1 -o /tmp/test --filetype csv --sql 'select * from `test`.`sbtest1` where id < 100' -F 100MiB --output-filename-template 'test.sbtest1.{{.Index}}' +tiup dumpling -u root -P 4000 -h 127.0.0.1 -o /tmp/test --filetype csv --sql 'select * from `test`.`sbtest1` where id < 100' -F 100MiB --output-filename-template 'test.sbtest1.{{.Index}}' ``` In the command above: @@ -236,7 +236,7 @@ export AWS_SECRET_ACCESS_KEY=${SecretKey} Dumpling also supports reading credential files from `~/.aws/credentials`. For more information about URI parameter descriptions, see [URI Formats of External Storage Services](/external-storage-uri.md). ```shell -./dumpling -u root -P 4000 -h 127.0.0.1 -r 200000 -o "s3://${Bucket}/${Folder}" +tiup dumpling -u root -P 4000 -h 127.0.0.1 -r 200000 -o "s3://${Bucket}/${Folder}" ``` ### Filter the exported data @@ -246,7 +246,7 @@ Dumpling also supports reading credential files from `~/.aws/credentials`. For m By default, Dumpling exports all databases except system databases (including `mysql`, `sys`, `INFORMATION_SCHEMA`, `PERFORMANCE_SCHEMA`, `METRICS_SCHEMA`, and `INSPECTION_SCHEMA`). You can use `--where ` to select the records to be exported. ```shell -./dumpling -u root -P 4000 -h 127.0.0.1 -o /tmp/test --where "id < 100" +tiup dumpling -u root -P 4000 -h 127.0.0.1 -o /tmp/test --where "id < 100" ``` The above command exports the data that matches `id < 100` from each table. Note that you cannot use the `--where` parameter together with `--sql`. @@ -258,7 +258,7 @@ Dumpling can filter specific databases or tables by specifying the table filter {{< copyable "shell-regular" >}} ```shell -./dumpling -u root -P 4000 -h 127.0.0.1 -o /tmp/test -r 200000 --filter "employees.*" --filter "*.WorkOrder" +tiup dumpling -u root -P 4000 -h 127.0.0.1 -o /tmp/test -r 200000 --filter "employees.*" --filter "*.WorkOrder" ``` The above command exports all the tables in the `employees` database and the `WorkOrder` tables in all databases. @@ -329,8 +329,8 @@ The `--snapshot` option can be set to a TSO (the `Position` field output by the {{< copyable "shell-regular" >}} ```shell -./dumpling --snapshot 417773951312461825 -./dumpling --snapshot "2020-07-02 17:12:45" +tiup dumpling --snapshot 417773951312461825 +tiup dumpling --snapshot "2020-07-02 17:12:45" ``` The TiDB historical data snapshots when the TSO is `417773951312461825` and the time is `2020-07-02 17:12:45` are exported. diff --git a/dynamic-config.md b/dynamic-config.md index cff4e08482ead..9c6c40a8db132 100644 --- a/dynamic-config.md +++ b/dynamic-config.md @@ -343,10 +343,16 @@ select @@tidb_slow_log_threshold; The following TiDB configuration items can be modified dynamically: | Configuration item | SQL variable | Description | -| :--- | :--- | -| `instance.tidb_enable_slow_log` | `tidb_enable_slow_log` | Whether to enable slow log | -| `instance.tidb_slow_log_threshold` | `tidb_slow_log_threshold` | The threshold of slow log | -| `instance.tidb_expensive_query_time_threshold` | `tidb_expensive_query_time_threshold` | The threshold of a expensive query | +| --- | --- | --- | +| `instance.tidb_enable_slow_log` | `tidb_enable_slow_log` | Controls whether to enable slow log | +| `instance.tidb_slow_log_threshold` | `tidb_slow_log_threshold` | Specifies the threshold of slow log | +| `instance.tidb_expensive_query_time_threshold` | `tidb_expensive_query_time_threshold` | Specifies the threshold of an expensive query | +| `instance.tidb_enable_collect_execution_info` | `tidb_enable_collect_execution_info` | Controls whether to record the execution information of operators | +| `instance.tidb_record_plan_in_slow_log` | `tidb_record_plan_in_slow_log` | Controls whether to record execution plans in the slow log | +| `instance.tidb_force_priority` | `tidb_force_priority` | Specifies the priority of statements that are submitted from this TiDB instance | +| `instance.max_connections` | `max_connections` | Specifies the maximum number of concurrent connections permitted for this TiDB instance | +| `instance.tidb_enable_ddl` | `tidb_enable_ddl` | Controls whether this TiDB instance can become a DDL owner | +| `pessimistic-txn.constraint-check-in-place-pessimistic` | `tidb_constraint_check_in_place_pessimistic` | Controls whether to defer the unique constraint check of a unique index to the next time when this index requires a lock or to the time when the transaction is committed | ### Modify TiFlash configuration dynamically diff --git a/enable-tls-between-components.md b/enable-tls-between-components.md index af73a7887ddfc..3f48566c2da66 100644 --- a/enable-tls-between-components.md +++ b/enable-tls-between-components.md @@ -158,16 +158,17 @@ The Common Name is used for caller verification. In general, the callee needs to To verify component caller's identity, you need to mark the certificate user identity using `Common Name` when generating the certificate, and to check the caller's identity by configuring the `Common Name` list for the callee. +> **Note:** +> +> Currently the `cert-allowed-cn` configuration item of the PD can only be set to one value. Therefore, the `commonName` of all authentication objects must be set to the same value. + - TiDB Configure in the configuration file or command-line arguments: ```toml [security] - cluster-verify-cn = [ - "TiDB-Server", - "TiKV-Control", - ] + cluster-verify-cn = ["TiDB"] ``` - TiKV @@ -176,9 +177,7 @@ To verify component caller's identity, you need to mark the certificate user ide ```toml [security] - cert-allowed-cn = [ - "TiDB-Server", "PD-Server", "TiKV-Control", "RawKvClient1", - ] + cert-allowed-cn = ["TiDB"] ``` - PD @@ -187,7 +186,7 @@ To verify component caller's identity, you need to mark the certificate user ide ```toml [security] - cert-allowed-cn = ["TiKV-Server", "TiDB-Server", "PD-Control"] + cert-allowed-cn = ["TiDB"] ``` - TiFlash (New in v4.0.5) @@ -196,14 +195,14 @@ To verify component caller's identity, you need to mark the certificate user ide ```toml [security] - cert_allowed_cn = ["TiKV-Server", "TiDB-Server"] + cert_allowed_cn = ["TiDB"] ``` Configure in the `tiflash-learner.toml` file: ```toml [security] - cert-allowed-cn = ["PD-Server", "TiKV-Server", "TiFlash-Server"] + cert-allowed-cn = ["TiDB"] ``` ## Reload certificates diff --git a/encryption-at-rest.md b/encryption-at-rest.md index 8671963d3118f..9653ba186c16d 100644 --- a/encryption-at-rest.md +++ b/encryption-at-rest.md @@ -287,7 +287,7 @@ The encryption algorithm currently supported by TiFlash is consistent with that The same master key can be shared by multiple instances of TiFlash, and can also be shared among TiFlash and TiKV. The recommended way to provide a master key in production is via AWS KMS. Alternatively, if using custom key is desired, supplying the master key via file is also supported. The specific method to generate master key and the format of the master key are the same as TiKV. -TiFlash uses the current data key to encrypt all data placed on the disk, including data files, Schmea files, and temporary data files generated during calculations. Data keys are automatically rotated by TiFlash every week by default, and the period is configurable. On key rotation, TiFlash does not rewrite all existing files to replace the key, but background compaction tasks are expected to rewrite old data into new data files, with the most recent data key, if the cluster gets constant write workload. TiFlash keeps track of the key and encryption method used to encrypt each of the files and use the information to decrypt the content on reads. +TiFlash uses the current data key to encrypt all data placed on the disk, including data files, Schema files, and temporary data files generated during calculations. Data keys are automatically rotated by TiFlash every week by default, and the period is configurable. On key rotation, TiFlash does not rewrite all existing files to replace the key, but background compaction tasks are expected to rewrite old data into new data files, with the most recent data key, if the cluster gets constant write workload. TiFlash keeps track of the key and encryption method used to encrypt each of the files and use the information to decrypt the content on reads. ### Key creation @@ -402,19 +402,19 @@ TiFlash also optimizes encrypted metadata operations in v4.0.9, and its compatib To enable S3 server-side encryption when backup to S3 using BR, pass `--s3.sse` argument and set value to "aws:kms". S3 will use its own KMS key for encryption. Example: ``` -./br backup full --pd --storage "s3:///" --s3.sse aws:kms +tiup br backup full --pd --storage "s3:///" --s3.sse aws:kms ``` To use a custom AWS KMS CMK that you created and owned, pass `--s3.sse-kms-key-id` in addition. In this case, both the BR process and all the TiKV nodes in the cluster would need access to the KMS CMK (for example, via AWS IAM), and the KMS CMK needs to be in the same AWS region as the S3 bucket used to store the backup. It is advised to grant access to the KMS CMK to BR process and TiKV nodes via AWS IAM. Refer to AWS documentation for usage of [IAM](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html). For example: ``` -./br backup full --pd --storage "s3:///" --s3.sse aws:kms --s3.sse-kms-key-id 0987dcba-09fe-87dc-65ba-ab0987654321 +tiup br backup full --pd --storage "s3:///" --s3.sse aws:kms --s3.sse-kms-key-id 0987dcba-09fe-87dc-65ba-ab0987654321 ``` When restoring the backup, both `--s3.sse` and `--s3.sse-kms-key-id` should NOT be used. S3 will figure out encryption settings by itself. The BR process and TiKV nodes in the cluster to restore the backup to would also need access to the KMS CMK, or the restore will fail. Example: ``` -./br restore full --pd --storage "s3:///" +tiup br restore full --pd --storage "s3:///" ``` ## BR Azure Blob Storage server-side encryption @@ -428,13 +428,13 @@ To specify an encryption scope for the backup data, you can use one of the follo - Include the `--azblob.encryption-scope` option in the `backup` command and set it to the scope name: ```shell - ./br backup full --pd --storage "azure:///" --azblob.encryption-scope scope1 + tiup br backup full --pd --storage "azure:///" --azblob.encryption-scope scope1 ``` - Include `encryption-scope` in the URI and set it to the scope name: ```shell - ./br backup full --pd --storage "azure:///?encryption-scope=scope1" + tiup br backup full --pd --storage "azure:///?encryption-scope=scope1" ``` For more information, see the Azure documentation: [Upload a blob with an encryption scope](https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-scope-manage?tabs=powershell#upload-a-blob-with-an-encryption-scope). @@ -442,7 +442,7 @@ For more information, see the Azure documentation: [Upload a blob with an encryp When restoring the backup, you do not need to specify the encryption scope. Azure Blob Storage automatically decrypts the data. For example: ```shell -./br restore full --pd --storage "azure:///" +tiup br restore full --pd --storage "azure:///" ``` ### Method 2: use an encryption key @@ -452,20 +452,20 @@ To specify an encryption key for the backup data, you can use one of the followi - Include the `--azblob.encryption-key` option in the `backup` command and set it to an AES256 encryption key: ```shell - ./br backup full --pd --storage "azure:///" --azblob.encryption-key + tiup br backup full --pd --storage "azure:///" --azblob.encryption-key ``` - Include `encryption-key` in the URI and set it to an AES256 encryption key. If the key contains URI reserved characters such as `&` and `%`, you need to percent-encode it first: ```shell - ./br backup full --pd --storage "azure:///?encryption-key=" + tiup br backup full --pd --storage "azure:///?encryption-key=" ``` - Set the `AZURE_ENCRYPTION_KEY` environment variable to an AES256 encryption key. Before running, make sure that you remember the encryption key in the environment variable to avoid forgetting it. ```shell export AZURE_ENCRYPTION_KEY= - ./br backup full --pd --storage "azure:///" + tiup br backup full --pd --storage "azure:///" ``` For more information, see the Azure documentation: [Provide an encryption key on a request to Blob storage](https://learn.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys). @@ -475,18 +475,18 @@ When restoring the backup, you need to specify the encryption key. For example: - Include the `--azblob.encryption-key` option in the `restore` command: ```shell - ./br restore full --pd --storage "azure:///" --azblob.encryption-key + tiup br restore full --pd --storage "azure:///" --azblob.encryption-key ``` - Include `encryption-key` in the URI: ```shell - ./br restore full --pd --storage "azure:///?encryption-key=" + tiup br restore full --pd --storage "azure:///?encryption-key=" ``` - Set the `AZURE_ENCRYPTION_KEY` environment variable: ```shell export AZURE_ENCRYPTION_KEY= - ./br restore full --pd --storage "azure:///" + tiup br restore full --pd --storage "azure:///" ``` diff --git a/error-codes.md b/error-codes.md index 1bba32e0a314b..ff7dd32a95f9a 100644 --- a/error-codes.md +++ b/error-codes.md @@ -432,9 +432,9 @@ TiDB is compatible with the error codes in MySQL, and in most cases returns the * Error Number: 8216 - The usage of automatic random columns is incorrect. + The usage of the `AUTO_RANDOM` columns is incorrect. - See [auto random](/auto-random.md) to modify. + See [`AUTO_RANDOM`](/auto-random.md) to modify. * Error Number: 8223 @@ -464,7 +464,7 @@ TiDB is compatible with the error codes in MySQL, and in most cases returns the * Error Number: 8228 - Unsupported types are specified when using `setval` on Sequence. + Unsupported types are specified when using `SETVAL` on Sequence. See [Sequence documentation](/sql-statements/sql-statement-create-sequence.md#examples) to find the example of the function. diff --git a/explain-index-merge.md b/explain-index-merge.md index 0e68f80ffff58..4fd15aa1ce63e 100644 --- a/explain-index-merge.md +++ b/explain-index-merge.md @@ -92,8 +92,8 @@ When using the intersection-type index merge to access tables, the optimizer can > > - You can use the SQL hint [`USE_INDEX_MERGE`](/optimizer-hints.md#use_index_merget1_name-idx1_name--idx2_name-) to force the optimizer to apply Index Merge, regardless of the setting of `tidb_enable_index_merge`. To enable Index Merge when the filtering conditions contain expressions that cannot be pushed down, you must use the SQL hint [`USE_INDEX_MERGE`](/optimizer-hints.md#use_index_merget1_name-idx1_name--idx2_name-). > -> - If the optimizer can choose the single index scan method (other than full table scan) for a query plan, the optimizer will not automatically use index merge. For the optimizer to use index merge, you need to use the optimizer hint. +> - If the optimizer can choose the single index scan method (other than full table scan) for a query plan, the optimizer will not automatically use index merge. For the optimizer to use index merge, you need to use the optimizer hint. Starting from v8.1.0, you can remove this limitation by setting [Optimizer Fix Control 52869](/optimizer-fix-controls.md#52869-new-in-v810). Removing this limitation enables the optimizer to choose index merge automatically in more queries, but might cause the optimizer to ignore the optimal execution plans. Therefore, it is recommended to conduct sufficient tests on actual use cases before removing this limitation to make sure that it will not cause performance regressions. > -> - Index Merge is not supported in [tempoaray tables](/temporary-tables.md) for now. +> - Index Merge is not supported in [temporary tables](/temporary-tables.md) for now. > > - The intersection-type index merge will not automatically be selected by the optimizer. You must specify the **table name and index name** using the [`USE_INDEX_MERGE`](/optimizer-hints.md#use_index_merget1_name-idx1_name--idx2_name-) hint for it to be selected. diff --git a/expression-syntax.md b/expression-syntax.md index fe5fae943a5ba..8cc7e4aeadbd8 100644 --- a/expression-syntax.md +++ b/expression-syntax.md @@ -18,7 +18,7 @@ The expressions can be divided into the following types: - ParamMarker (`?`), system variables, user variables and CASE expressions. -The following rules are the expression syntax, which is based on the [`parser.y`](https://github.com/pingcap/tidb/blob/master/pkg/parser/parser.y) rules of TiDB parser. For the navigable version of the following syntax diagram, refer to [TiDB SQL Syntax Diagram](https://pingcap.github.io/sqlgram/#Expression). +The following rules are the expression syntax, which is based on the [`parser.y`](https://github.com/pingcap/tidb/blob/master/pkg/parser/parser.y) rules of TiDB parser. ```ebnf+diagram Expression ::= diff --git a/external-storage-uri.md b/external-storage-uri.md index 6f84b5a0f9a9e..090629e331479 100644 --- a/external-storage-uri.md +++ b/external-storage-uri.md @@ -21,7 +21,7 @@ The basic format of the URI is as follows: - `access-key`: Specifies the access key. - `secret-access-key`: Specifies the secret access key. - - `session-token`: Specifies the temporary session token. BR does not support this parameter yet. + - `session-token`: Specifies the temporary session token. BR supports this parameter starting from v7.6.0. - `use-accelerate-endpoint`: Specifies whether to use the accelerate endpoint on Amazon S3 (defaults to `false`). - `endpoint`: Specifies the URL of custom endpoint for S3-compatible services (for example, ``). - `force-path-style`: Use path style access rather than virtual hosted style access (defaults to `true`). @@ -29,7 +29,7 @@ The basic format of the URI is as follows: - `sse`: Specifies the server-side encryption algorithm used to encrypt the uploaded objects (value options: ``, `AES256`, or `aws:kms`). - `sse-kms-key-id`: Specifies the KMS ID if `sse` is set to `aws:kms`. - `acl`: Specifies the canned ACL of the uploaded objects (for example, `private` or `authenticated-read`). - - `role-arn`: When you need to access Amazon S3 data from a third party using a specified [IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html), you can specify the corresponding [Amazon Resource Name (ARN)](https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) of the IAM role with the `role-arn` URL query parameter, such as `arn:aws:iam::888888888888:role/my-role`. For more information about using an IAM role to access Amazon S3 data from a third party, see [AWS documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_third-party.html). + - `role-arn`: When you need to access Amazon S3 data from a third party using a specified [IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html), you can specify the corresponding [Amazon Resource Name (ARN)](https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) of the IAM role with the `role-arn` URL query parameter, such as `arn:aws:iam::888888888888:role/my-role`. For more information about using an IAM role to access Amazon S3 data from a third party, see [AWS documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_third-party.html). BR supports this parameter starting from v7.6.0. - `external-id`: When you access Amazon S3 data from a third party, you might need to specify a correct [external ID](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html) to assume [the IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html). In this case, you can use this `external-id` URL query parameter to specify the external ID and make sure that you can assume the IAM role. An external ID is an arbitrary string provided by the third party together with the IAM role ARN to access the Amazon S3 data. Providing an external ID is optional when assuming an IAM role, which means if the third party does not require an external ID for the IAM role, you can assume the IAM role and access the corresponding Amazon S3 data without providing this parameter. The following is an example of an Amazon S3 URI for TiDB Lightning and BR. In this example, you need to specify a specific file path `testfolder`. diff --git a/faq/backup-and-restore-faq.md b/faq/backup-and-restore-faq.md index de4efdfc1cda6..789607df5987b 100644 --- a/faq/backup-and-restore-faq.md +++ b/faq/backup-and-restore-faq.md @@ -276,7 +276,7 @@ Note that even if you configures [table filter](/table-filter.md#syntax), **BR d - Statistics tables (`mysql.stat_*`). But statistics can be restored. See [Back up statistics](/br/br-snapshot-manual.md#back-up-statistics). - System variable tables (`mysql.tidb`, `mysql.global_variables`) -- [Other system tables](https://github.com/pingcap/tidb/blob/master/br/pkg/restore/systable_restore.go#L31) +- [Other system tables](https://github.com/pingcap/tidb/blob/master/br/pkg/restore/snap_client/systable_restore.go#L31) ### How to deal with the error of `cannot find rewrite rule` during restoration? diff --git a/faq/manage-cluster-faq.md b/faq/manage-cluster-faq.md index e56982aa51da7..aa4147e8483f6 100644 --- a/faq/manage-cluster-faq.md +++ b/faq/manage-cluster-faq.md @@ -73,7 +73,7 @@ TiDB provides a few features and [tools](/ecosystem-tool-user-guide.md), with wh The TiDB community is highly active. The engineers have been keeping optimizing features and fixing bugs. Therefore, the TiDB version is updated quite fast. If you want to keep informed of the latest version, see [TiDB Release Timeline](/releases/release-timeline.md). -It is recommeneded to deploy TiDB [using TiUP](/production-deployment-using-tiup.md) or [using TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/stable). TiDB has a unified management of the version number. You can view the version number using one of the following methods: +It is recommended to deploy TiDB [using TiUP](/production-deployment-using-tiup.md) or [using TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/stable). TiDB has a unified management of the version number. You can view the version number using one of the following methods: - `select tidb_version()` - `tidb-server -V` @@ -415,7 +415,10 @@ The memory usage of TiKV mainly comes from the block-cache of RocksDB, which is ### Can both TiDB data and RawKV data be stored in the same TiKV cluster? -No. TiDB (or data created from the transactional API) relies on a specific key format. It is not compatible with data created from RawKV API (or data from other RawKV-based services). +It depends on your TiDB version and whether TiKV API V2 is enabled ([`storage.api-version = 2`](/tikv-configuration-file.md#api-version-new-in-v610)). + +- If your TiDB version is v6.1.0 or later and TiKV API V2 is enabled, TiDB data and RawKV data can be stored in the same TiKV cluster. +- Otherwise, the answer is no because the key format of TiDB data (or data created using the transactional API) is incompatible with data created using the RawKV API (or data from other RawKV-based services). ## TiDB testing diff --git a/faq/migration-tidb-faq.md b/faq/migration-tidb-faq.md index 190deb6e80886..477c9559bc390 100644 --- a/faq/migration-tidb-faq.md +++ b/faq/migration-tidb-faq.md @@ -93,7 +93,7 @@ To migrate all the data or migrate incrementally from DB2 or Oracle to TiDB, see Currently, it is recommended to use OGG. -### Error: `java.sql.BatchUpdateExecption:statement count 5001 exceeds the transaction limitation` while using Sqoop to write data into TiDB in `batches` +### Error: `java.sql.BatchUpdateException:statement count 5001 exceeds the transaction limitation` while using Sqoop to write data into TiDB in `batches` In Sqoop, `--batch` means committing 100 `statement`s in each batch, but by default each `statement` contains 100 SQL statements. So, 100 * 100 = 10000 SQL statements, which exceeds 5000, the maximum number of statements allowed in a single TiDB transaction. diff --git a/faq/monitor-faq.md b/faq/monitor-faq.md index 5cfa36a2781cf..8b015916823d8 100644 --- a/faq/monitor-faq.md +++ b/faq/monitor-faq.md @@ -18,7 +18,7 @@ The monitoring system of TiDB consists of Prometheus and Grafana. From the dashb Yes. Find the startup script on the machine where Prometheus is started, edit the startup parameter and restart Prometheus. -```config +``` --storage.tsdb.retention="60d" ``` diff --git a/faq/sql-faq.md b/faq/sql-faq.md index 967cea5adaea1..f28100b19b2a8 100644 --- a/faq/sql-faq.md +++ b/faq/sql-faq.md @@ -151,7 +151,7 @@ TiDB supports modifying the [`sql_mode`](/system-variables.md#sql_mode) system v - Changes to [`GLOBAL`](/sql-statements/sql-statement-set-variable.md) scoped variables propagate to the rest servers of the cluster and persist across restarts. This means that you do not need to change the `sql_mode` value on each TiDB server. - Changes to `SESSION` scoped variables only affect the current client session. After restarting a server, the changes are lost. -## Error: `java.sql.BatchUpdateExecption:statement count 5001 exceeds the transaction limitation` while using Sqoop to write data into TiDB in batches +## Error: `java.sql.BatchUpdateException:statement count 5001 exceeds the transaction limitation` while using Sqoop to write data into TiDB in batches In Sqoop, `--batch` means committing 100 statements in each batch, but by default each statement contains 100 SQL statements. So, 100 * 100 = 10000 SQL statements, which exceeds 5000, the maximum number of statements allowed in a single TiDB transaction. diff --git a/foreign-key.md b/foreign-key.md index 4ecb881ca4f19..377a39ce065c5 100644 --- a/foreign-key.md +++ b/foreign-key.md @@ -308,7 +308,7 @@ Create Table | CREATE TABLE `child` ( - [DM](/dm/dm-overview.md) does not support foreign keys. DM disables the [`foreign_key_checks`](/system-variables.md#foreign_key_checks) of the downstream TiDB when replicating data to TiDB. Therefore, the cascading operations caused by foreign keys are not replicated from the upstream to the downstream, which might cause data inconsistency. - [TiCDC](/ticdc/ticdc-overview.md) v6.6.0 is compatible with foreign keys. The previous versions of TiCDC might report an error when replicating tables with foreign keys. It is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster when using a TiCDC version earlier than v6.6.0. - [BR](/br/backup-and-restore-overview.md) v6.6.0 is compatible with foreign keys. The previous versions of BR might report an error when restoring tables with foreign keys to a v6.6.0 or later cluster. It is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster before restoring the cluster when using a BR earlier than v6.6.0. -- When you use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), it is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster before importing data. +- When you use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), if the target table uses a foreign key, it is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster before importing data. For versions earlier than v6.6.0, disabling this system variable does not take effect, and you need to grant the `REFERENCES` privilege for the downstream database user, or manually create the target table in the downstream database in advance to ensure smooth data import. diff --git a/functions-and-operators/bit-functions-and-operators.md b/functions-and-operators/bit-functions-and-operators.md index 1430d63bcdb6b..3760495c08e54 100644 --- a/functions-and-operators/bit-functions-and-operators.md +++ b/functions-and-operators/bit-functions-and-operators.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/functions-and-operators/bit-functions-and-operators/','/doc # Bit Functions and Operators -TiDB supports all of the [bit functions and operators](https://dev.mysql.com/doc/refman/5.7/en/bit-functions.html) available in MySQL 5.7. +TiDB supports all of the [bit functions and operators](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html) available in MySQL 8.0. **Bit functions and operators:** @@ -19,3 +19,14 @@ TiDB supports all of the [bit functions and operators](https://dev.mysql.com/doc | [^](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_bitwise-xor) | Bitwise XOR | | [<<](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_left-shift) | Left shift | | [>>](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_right-shift) | Right shift | + +## MySQL compatibility + +There are some differences between MySQL 8.0 and earlier versions of MySQL in handling bit functions and operators. TiDB aims to follow the behavior of MySQL 8.0. + +## Known issues + +In the following cases, the query results in TiDB are the same as MySQL 5.7 but different from MySQL 8.0. + +- Bitwise operations with binary arguments. For more information, see [#30637](https://github.com/pingcap/tidb/issues/30637). +- The result of the `BIT_COUNT()` function. For more information, see [#44621](https://github.com/pingcap/tidb/issues/44621). diff --git a/functions-and-operators/cast-functions-and-operators.md b/functions-and-operators/cast-functions-and-operators.md index 3eb512f074f69..4643760f7abd9 100644 --- a/functions-and-operators/cast-functions-and-operators.md +++ b/functions-and-operators/cast-functions-and-operators.md @@ -6,16 +6,121 @@ aliases: ['/docs/dev/functions-and-operators/cast-functions-and-operators/','/do # Cast Functions and Operators -Cast functions and operators enable conversion of values from one data type to another. TiDB supports all of the [cast functions and operators](https://dev.mysql.com/doc/refman/5.7/en/cast-functions.html) available in MySQL 5.7. - -## List of cast functions and operators +Cast functions and operators enable conversion of values from one data type to another. TiDB supports all of the [cast functions and operators](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html) available in MySQL 8.0. | Name | Description | | ---------------------------------------- | -------------------------------- | -| [`BINARY`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#operator_binary) | Cast a string to a binary string | -| [`CAST()`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast) | Cast a value as a certain type | -| [`CONVERT()`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_convert) | Cast a value as a certain type | +| [`BINARY`](#binary) | Cast a string to a binary string | +| [`CAST()`](#cast) | Cast a value as a certain type | +| [`CONVERT()`](#convert) | Cast a value as a certain type | > **Note:** > > TiDB and MySQL display inconsistent results for `SELECT CAST(MeN AS CHAR)` (or its equivalent form `SELECT CONVERT(MeM, CHAR)`), where `MeN` represents a double-precision floating-point number in scientific notation. MySQL displays the complete numeric value when `-15 <= N <= 14` and the scientific notation when `N < -15` or `N > 14`. However, TiDB always displays the complete numeric value. For example, MySQL displays the result of `SELECT CAST(3.1415e15 AS CHAR)` as `3.1415e15`, while TiDB displays the result as `3141500000000000`. + +## BINARY + +The [`BINARY`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#operator_binary) operator has been deprecated since MySQL 8.0.27. It is recommended to use `CAST(... AS BINARY)` instead both in TiDB and MySQL. + +## CAST + +The [`CAST( AS [ARRAY])`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast) function is used to cast an expression to a specific type. + +This function is also used to create [Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes). + +The following types are supported: + +| Type | Description | Whether it can be used with multi-valued indexes | +|----------------------|------------------|------------------------------------------------------------| +| `BINARY(n)` | Binary string | No | +| `CHAR(n)` | Character string | Yes, but only if a length is specified | +| `DATE` | Date | Yes | +| `DATETIME(fsp)` | Date/time, where `fsp` is optional | Yes | +| `DECIMAL(n, m)` | Decimal number, where `n` and `m` are optional and are `10` and `0` if not specified | No | +| `DOUBLE` | Double precision floating-point number | No | +| `FLOAT(n)` | Floating-point number, where `n` is optional and should be between `0` and `53` | No | +| `JSON` | JSON | No | +| `REAL` | Floating-point number | Yes | +| `SIGNED [INTEGER]` | Signed integer | Yes | +| `TIME(fsp)` | Time | Yes | +| `UNSIGNED [INTEGER]` | Unsigned integer | Yes | +| `YEAR` | Year | No | + +Examples: + +The following statement converts a binary string from a HEX literal to a `CHAR`. + +```sql +SELECT CAST(0x54694442 AS CHAR); +``` + +```sql ++--------------------------+ +| CAST(0x54694442 AS CHAR) | ++--------------------------+ +| TiDB | ++--------------------------+ +1 row in set (0.0002 sec) +``` + +The following statement casts the values of the `a` attribute extracted from the JSON column to an unsigned array. Note that casting to an array is only supported as part of an index definition for multi-valued indexes. + +```sql +CREATE TABLE t ( + id INT PRIMARY KEY, + j JSON, + INDEX idx_a ((CAST(j->'$.a' AS UNSIGNED ARRAY))) +); +INSERT INTO t VALUES (1, JSON_OBJECT('a',JSON_ARRAY(1,2,3))); +INSERT INTO t VALUES (2, JSON_OBJECT('a',JSON_ARRAY(4,5,6))); +INSERT INTO t VALUES (3, JSON_OBJECT('a',JSON_ARRAY(7,8,9))); +ANALYZE TABLE t; +``` + +```sql + EXPLAIN SELECT * FROM t WHERE 1 MEMBER OF(j->'$.a')\G +*************************** 1. row *************************** + id: IndexMerge_10 + estRows: 2.00 + task: root +access object: +operator info: type: union +*************************** 2. row *************************** + id: ├─IndexRangeScan_8(Build) + estRows: 2.00 + task: cop[tikv] +access object: table:t, index:idx_a(cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array)) +operator info: range:[1,1], keep order:false, stats:partial[j:unInitialized] +*************************** 3. row *************************** + id: └─TableRowIDScan_9(Probe) + estRows: 2.00 + task: cop[tikv] +access object: table:t +operator info: keep order:false, stats:partial[j:unInitialized] +3 rows in set (0.00 sec) +``` + +## CONVERT + +The [`CONVERT()`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_convert) function is used to convert between [character sets](/character-set-and-collation.md). + +Example: + +```sql +SELECT CONVERT(0x616263 USING utf8mb4); +``` + +```sql ++---------------------------------+ +| CONVERT(0x616263 USING utf8mb4) | ++---------------------------------+ +| abc | ++---------------------------------+ +1 row in set (0.0004 sec) +``` + +## MySQL compatibility + +- TiDB does not support cast operations on `SPATIAL` types. For more information, see [#6347](https://github.com/pingcap/tidb/issues/6347). +- TiDB does not support `AT TIME ZONE` for `CAST()`. For more information, see [#51742](https://github.com/pingcap/tidb/issues/51742). +- `CAST(24 AS YEAR)` returns 2 digits in TiDB and 4 digits in MySQL. For more information, see [#29629](https://github.com/pingcap/tidb/issues/29629). diff --git a/functions-and-operators/control-flow-functions.md b/functions-and-operators/control-flow-functions.md index 7499c88f4b97f..0d2fa9ce88a12 100644 --- a/functions-and-operators/control-flow-functions.md +++ b/functions-and-operators/control-flow-functions.md @@ -6,11 +6,138 @@ aliases: ['/docs/dev/functions-and-operators/control-flow-functions/','/docs/dev # Control Flow Functions -TiDB supports all of the [control flow functions](https://dev.mysql.com/doc/refman/5.7/en/flow-control-functions.html) available in MySQL 5.7. +TiDB supports all of the [control flow functions](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html) available in MySQL 8.0. | Name | Description | |:--------------------------------------------------------------------------------------------------|:----------------------------------| -| [`CASE`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#operator_case) | Case operator | -| [`IF()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_if) | If/else construct | -| [`IFNULL()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_ifnull) | Null if/else construct | -| [`NULLIF()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_nullif) | Return NULL if expr1 = expr2 | +| [`CASE`](#case) | Case operator | +| [`IF()`](#if) | If/else construct | +| [`IFNULL()`](#ifnull) | Null if/else construct | +| [`NULLIF()`](#nullif) | Return `NULL` if expr1 = expr2 | + +## CASE + +The [`CASE`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#operator_case) operator enables you to perform conditional logic and customize query results based on specified conditions. + +Syntax: + +```sql +CASE + WHEN condition1 THEN result1 + WHEN condition2 THEN result2 + ... + ELSE default_result +END +``` + +Example: + +```sql +WITH RECURSIVE d AS (SELECT 1 AS n UNION ALL SELECT n+1 FROM d WHERE n<10) +SELECT n, CASE WHEN n MOD 2 THEN "odd" ELSE "even" END FROM d; +``` + +``` ++----+----------------------------------------------+ +| n | CASE WHEN n MOD 2 THEN "odd" ELSE "even" END | ++----+----------------------------------------------+ +| 1 | odd | +| 2 | even | +| 3 | odd | +| 4 | even | +| 5 | odd | +| 6 | even | +| 7 | odd | +| 8 | even | +| 9 | odd | +| 10 | even | ++----+----------------------------------------------+ +10 rows in set (0.00 sec) +``` + +## IF() + +The [`IF()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_if) function enables you to perform different actions based on whether a value or expression is true or not. + +Syntax: + +```sql +IF(condition, value_if_true, value_if_false) +``` + +Example: + +```sql +WITH RECURSIVE d AS (SELECT 1 AS n UNION ALL SELECT n+1 FROM d WHERE n<10) +SELECT n, IF(n MOD 2, "odd", "even") FROM d; +``` + +``` ++----+----------------------------+ +| n | IF(n MOD 2, "odd", "even") | ++----+----------------------------+ +| 1 | odd | +| 2 | even | +| 3 | odd | +| 4 | even | +| 5 | odd | +| 6 | even | +| 7 | odd | +| 8 | even | +| 9 | odd | +| 10 | even | ++----+----------------------------+ +10 rows in set (0.00 sec) +``` + +## IFNULL() + +The [`IFNULL(expr1,expr2)`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_ifnull) function is used to handle NULL values in queries. If `expr1` is not `NULL`, it returns `expr1`; otherwise, it returns `expr2`. + +Example: + +```sql +WITH data AS (SELECT NULL AS x UNION ALL SELECT 1 ) +SELECT x, IFNULL(x,'x has no value') FROM data; +``` + +``` ++------+----------------------------+ +| x | IFNULL(x,'x has no value') | ++------+----------------------------+ +| NULL | x has no value | +| 1 | 1 | ++------+----------------------------+ +2 rows in set (0.0006 sec) +``` + +## NULLIF() + +The [`NULLIF(expr1,expr2)`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_nullif) function returns `NULL` if both arguments are the same or if the first argument is `NULL`. Otherwise, it returns the first argument. + +Example: + +```sql +WITH RECURSIVE d AS (SELECT 1 AS n UNION ALL SELECT n+1 FROM d WHERE n<10) +SELECT n, NULLIF(n+n, n+2) FROM d; +``` + +``` ++----+------------------+ +| n | NULLIF(n+n, n+2) | ++----+------------------+ +| 1 | 2 | +| 2 | NULL | +| 3 | 6 | +| 4 | 8 | +| 5 | 10 | +| 6 | 12 | +| 7 | 14 | +| 8 | 16 | +| 9 | 18 | +| 10 | 20 | ++----+------------------+ +10 rows in set (0.00 sec) +``` + +In this example, when `n` equals `2`, both `n+n` and `n+2` equal `4`, making both arguments the same and causing the function to return `NULL`. \ No newline at end of file diff --git a/functions-and-operators/date-and-time-functions.md b/functions-and-operators/date-and-time-functions.md index dd3568582c71c..17ac1032a2e8c 100644 --- a/functions-and-operators/date-and-time-functions.md +++ b/functions-and-operators/date-and-time-functions.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/functions-and-operators/date-and-time-functions/','/docs/de # Date and Time Functions -TiDB supports all of the [date and time functions](https://dev.mysql.com/doc/refman/5.7/en/numeric-functions.html) available in MySQL 5.7. +TiDB supports all of the [date and time functions](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html) available in MySQL 8.0. > **Note:** > @@ -82,7 +82,7 @@ For details, see [Date and Time Functions](https://dev.mysql.com/doc/refman/8.0/ ## MySQL compatibility -The function `str_to_date()` is supported by TiDB, but is unable to parse all date and time values. In addition, the following date and time formatting options are **not implemented**: +The function `STR_TO_DATE()` is supported by TiDB, but is unable to parse all date and time values. In addition, the following date and time formatting options are **not implemented**: | Format | Description | |--------|---------------------------------------------------------------------------------------| @@ -101,4 +101,4 @@ See [issue #30082](https://github.com/pingcap/tidb/issues/30082) for more detail ## Related system variables -The `default_week_format` variable affects the `WEEK()` function. \ No newline at end of file +The [`default_week_format`](/system-variables.md#default_week_format) variable affects the `WEEK()` function. \ No newline at end of file diff --git a/functions-and-operators/encryption-and-compression-functions.md b/functions-and-operators/encryption-and-compression-functions.md index 181300d313346..342a6f6ceff06 100644 --- a/functions-and-operators/encryption-and-compression-functions.md +++ b/functions-and-operators/encryption-and-compression-functions.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/functions-and-operators/encryption-and-compression-function # Encryption and Compression Functions -TiDB supports most of the [encryption and compression functions](https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html) available in MySQL 5.7. +TiDB supports most of the [encryption and compression functions](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html) available in MySQL 8.0. ## Supported functions @@ -27,9 +27,15 @@ TiDB supports most of the [encryption and compression functions](https://dev.mys ## Related system variables -The `block_encryption_mode` variable sets the encryption mode that is used for `AES_ENCRYPT()` and `AES_DECRYPT()`. +The [`block_encryption_mode`](/system-variables.md#block_encryption_mode) variable sets the encryption mode that is used for `AES_ENCRYPT()` and `AES_DECRYPT()`. + +The [`validate_password.*`](/system-variables.md) variables affect the `VALIDATE_PASSWORD_STRENGTH()` function. ## Unsupported functions * `DES_DECRYPT()`, `DES_ENCRYPT()`, `OLD_PASSWORD()`, `ENCRYPT()`: these functions were deprecated in MySQL 5.7 and removed in 8.0. * Functions only available in MySQL Enterprise [Issue #2632](https://github.com/pingcap/tidb/issues/2632). + +## MySQL compatibility + +* TiDB does not support the `STATEMENT_DIGEST()` and `STATEMENT_DIGEST_TEXT()` functions. diff --git a/functions-and-operators/information-functions.md b/functions-and-operators/information-functions.md index 2f88e11d70972..eb9484cce28cc 100644 --- a/functions-and-operators/information-functions.md +++ b/functions-and-operators/information-functions.md @@ -6,24 +6,305 @@ aliases: ['/docs/dev/functions-and-operators/information-functions/','/docs/dev/ # Information Functions -TiDB supports most of the [information functions](https://dev.mysql.com/doc/refman/5.7/en/information-functions.html) available in MySQL 5.7. +TiDB supports most of the [information functions](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html) available in MySQL 8.0. ## TiDB supported MySQL functions | Name | Description | |:-----|:------------| -| [`BENCHMARK()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_benchmark) | Execute an expression in a loop | -| [`CONNECTION_ID()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_connection-id) | Return the connection ID (thread ID) for the connection | -| [`CURRENT_USER()`, `CURRENT_USER`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_current-user) | Return the authenticated user name and host name | -| [`DATABASE()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_database) | Return the default (current) database name | -| [`FOUND_ROWS()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_found-rows) | For a `SELECT` with a `LIMIT` clause, the number of the rows that are returned if there is no `LIMIT` clause | -| [`LAST_INSERT_ID()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_last-insert-id) | Return the value of the `AUTOINCREMENT` column for the last `INSERT` | -| [`ROW_COUNT()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_row-count) | The number of rows affected | -| [`SCHEMA()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_schema) | Synonym for `DATABASE()` | -| [`SESSION_USER()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_session-user) | Synonym for `USER()` | -| [`SYSTEM_USER()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_system-user) | Synonym for `USER()` | -| [`USER()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_user) | Return the user name and host name provided by the client | -| [`VERSION()`](https://dev.mysql.com/doc/refman/8.0/en/information-functions.html#function_version) | Return a string that indicates the MySQL server version | +| [`BENCHMARK()`](#benchmark) | Execute an expression in a loop | +| [`CONNECTION_ID()`](#connection_id) | Return the connection ID (thread ID) for the connection | +| [`CURRENT_ROLE()`](#current_role) | Return the role that is in use by the connection | +| [`CURRENT_USER()`, `CURRENT_USER`](#current_user) | Return the authenticated user name and host name | +| [`DATABASE()`](#database) | Return the default (current) database name | +| [`FOUND_ROWS()`](#found_rows) | For a `SELECT` with a `LIMIT` clause, the number of the rows that are returned if there is no `LIMIT` clause | +| [`LAST_INSERT_ID()`](#last_insert_id) | Return the value of the `AUTOINCREMENT` column for the last `INSERT` | +| [`ROW_COUNT()`](#row_count) | The number of rows affected | +| [`SCHEMA()`](#schema) | Synonym for `DATABASE()` | +| [`SESSION_USER()`](#session_user) | Synonym for `USER()` | +| [`SYSTEM_USER()`](#system_user) | Synonym for `USER()` | +| [`USER()`](#user) | Return the user name and host name provided by the client | +| [`VERSION()`](#version) | Return a string that indicates the MySQL server version | + +### BENCHMARK() + +The `BENCHMARK()` function executes the given expression a specified number of times. + +Syntax: + +```sql +BENCHMARK(count, expression) +``` + +- `count`: the number of times the expression to be executed. +- `expression`: the expression to be executed repeatedly. + +Example: + +```sql +SELECT BENCHMARK(5, SLEEP(2)); +``` + +``` ++------------------------+ +| BENCHMARK(5, SLEEP(2)) | ++------------------------+ +| 0 | ++------------------------+ +1 row in set (10.00 sec) +``` + +### CONNECTION_ID() + + + +The `CONNECTION_ID()` function returns the ID of the connection. Based on the value of the [`enable-32bits-connection-id`](/tidb-configuration-file.md#enable-32bits-connection-id-new-in-v730) configuration item for TiDB, this function returns a 32-bit or 64-bit connection ID. + +If [`enable-global-kill`](/tidb-configuration-file.md#enable-global-kill-new-in-v610) is enabled, the connection ID can be used to kill queries across multiple TiDB instances of the same cluster. + + + + + +The `CONNECTION_ID()` function returns the ID of the connection. Based on the value of the [`enable-32bits-connection-id`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-32bits-connection-id-new-in-v730) configuration item for TiDB, this function returns a 32-bit or 64-bit connection ID. + +If [`enable-global-kill`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-global-kill-new-in-v610) is enabled, the connection ID can be used to kill queries across multiple TiDB instances of the same cluster. + + + +```sql +SELECT CONNECTION_ID(); +``` + +``` ++-----------------+ +| CONNECTION_ID() | ++-----------------+ +| 322961414 | ++-----------------+ +1 row in set (0.00 sec) +``` + +### CURRENT_ROLE() + + + +The `CURRENT_ROLE()` function returns the current [role](/role-based-access-control.md) for the current session. + + + + + +The `CURRENT_ROLE()` function returns the current [role](https://docs.pingcap.com/tidb/stable/role-based-access-control) for the current session. + + + +```sql +SELECT CURRENT_ROLE(); +``` + +``` ++----------------+ +| CURRENT_ROLE() | ++----------------+ +| NONE | ++----------------+ +1 row in set (0.00 sec) +``` + +### CURRENT_USER() + +The `CURRENT_USER()` function returns the account that is used in the current session. + +```sql +SELECT CURRENT_USER(); +``` + +``` ++----------------+ +| CURRENT_USER() | ++----------------+ +| root@% | ++----------------+ +1 row in set (0.00 sec) +``` + +### DATABASE() + +The `DATABASE()` function returns the database schema that the current session is using. + +```sql +SELECT DATABASE(); +``` + +``` ++------------+ +| DATABASE() | ++------------+ +| test | ++------------+ +1 row in set (0.00 sec) +``` + +### FOUND_ROWS() + +The `FOUND_ROWS()` function returns the number of rows in the result set of the last executed `SELECT` statement. + +```sql +SELECT 1 UNION ALL SELECT 2; +``` + +``` ++------+ +| 1 | ++------+ +| 2 | +| 1 | ++------+ +2 rows in set (0.01 sec) +``` + +```sql +SELECT FOUND_ROWS(); +``` + +``` ++--------------+ +| FOUND_ROWS() | ++--------------+ +| 2 | ++--------------+ +1 row in set (0.00 sec) +``` + +> **Note:** +> +> The `SQL_CALC_FOUND_ROWS` query modifier, which calculates the total number of rows in a result set without considering the `LIMIT` clause, is only accepted if [`tidb_enable_noop_functions`](/system-variables.md#tidb_enable_noop_functions-new-in-v40) is enabled. This query modifier is deprecated starting from MySQL 8.0.17. It is recommended to use `COUNT(*)` instead. + +### LAST_INSERT_ID() + +The `LAST_INSERT_ID()` function returns the ID of the last inserted row in a table that contains an [`AUTO_INCREMENT`](/auto-increment.md) or [`AUTO_RANDOM`](/auto-random.md) column. + +```sql +CREATE TABLE t1(id SERIAL); +Query OK, 0 rows affected (0.17 sec) + +INSERT INTO t1() VALUES(); +Query OK, 1 row affected (0.03 sec) + +INSERT INTO t1() VALUES(); +Query OK, 1 row affected (0.00 sec) + +SELECT LAST_INSERT_ID(); ++------------------+ +| LAST_INSERT_ID() | ++------------------+ +| 3 | ++------------------+ +1 row in set (0.00 sec) + +TABLE t1; ++----+ +| id | ++----+ +| 1 | +| 3 | ++----+ +2 rows in set (0.00 sec) +``` + +> **Note** +> +> - In TiDB, [`AUTO_ID_CACHE`](/auto-increment.md#auto_id_cache) might lead to results that differ from those returned by MySQL. This discrepancy arises because TiDB caches IDs on each node, potentially leading to IDs that are out of order or have gaps. If maintaining strict ID ordering is essential for your application, you can enable [MySQL compatible mode](/auto-increment.md#mysql-compatibility-mode). +> +> - In the preceding example, IDs increase by 2 while MySQL would generate IDs incrementing by 1 in the same scenario. For more compatibility information, see [Auto-increment ID](/mysql-compatibility.md#auto-increment-id). + +### ROW_COUNT() + +The `ROW_COUNT()` function returns the number of affected rows. + +```sql +CREATE TABLE t1(id BIGINT UNSIGNED PRIMARY KEY AUTO_RANDOM); +Query OK, 0 rows affected, 1 warning (0.16 sec) + +INSERT INTO t1() VALUES (),(),(); +Query OK, 3 rows affected (0.02 sec) +Records: 3 Duplicates: 0 Warnings: 0 + +SELECT ROW_COUNT(); ++-------------+ +| ROW_COUNT() | ++-------------+ +| 3 | ++-------------+ +1 row in set (0.00 sec) +``` + +### SCHEMA() + +The `SCHEMA()` function is a synonym for [`DATABASE()`](#database). + +### SESSION_USER() + +The `SESSION_USER()` function is a synonym for [`USER()`](#user). + +### SYSTEM_USER() + +The `SYSTEM_USER()` function is a synonym for [`USER()`](#user). + +### USER() + +The `USER()` function returns the user of the current connection. This might differ slightly from the output of `CURRENT_USER()`, as `USER()` displays the actual IP address instead of a wildcard. + +```sql +SELECT USER(), CURRENT_USER(); +``` + +``` ++----------------+----------------+ +| USER() | CURRENT_USER() | ++----------------+----------------+ +| root@127.0.0.1 | root@% | ++----------------+----------------+ +1 row in set (0.00 sec) +``` + +### VERSION() + +The `VERSION()` function returns the TiDB version in a format that is compatible with MySQL. To get a more detailed result, you can use the [`TIDB_VERSION()`](/functions-and-operators/tidb-functions.md#tidb_version) function. + +```sql +SELECT VERSION(); ++--------------------+ +| VERSION() | ++--------------------+ +| 8.0.11-TiDB-v7.5.1 | ++--------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT TIDB_VERSION()\G +*************************** 1. row *************************** +TIDB_VERSION(): Release Version: v7.5.1 +Edition: Community +Git Commit Hash: 7d16cc79e81bbf573124df3fd9351c26963f3e70 +Git Branch: heads/refs/tags/v7.5.1 +UTC Build Time: 2024-02-27 14:28:32 +GoVersion: go1.21.6 +Race Enabled: false +Check Table Before Drop: false +Store: tikv +1 row in set (0.00 sec) +``` + +The preceding example is from TiDB v7.5.1, which identifies itself as MySQL 8.0.11. + + + +If you want to change the returned version, you can modify the [`server-version`](/tidb-configuration-file.md#server-version) configuration item. + + ## TiDB specific functions @@ -38,3 +319,5 @@ The following function is only supported by TiDB, and there is no equivalent fun * `CHARSET()` * `COERCIBILITY()` * `COLLATION()` +* `ICU_VERSION()` +* `ROLES_GRAPHML()` diff --git a/functions-and-operators/json-functions.md b/functions-and-operators/json-functions.md index 6763335887590..5f53cf940ef1e 100644 --- a/functions-and-operators/json-functions.md +++ b/functions-and-operators/json-functions.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/functions-and-operators/json-functions/','/docs/dev/referen # JSON Functions -TiDB supports most of the JSON functions that shipped with the GA release of MySQL 5.7. +TiDB supports most of the [JSON functions](https://dev.mysql.com/doc/refman/8.0/en/json-functions.html) available in MySQL 8.0. ## Functions that create JSON values @@ -35,18 +35,16 @@ TiDB supports most of the JSON functions that shipped with the GA release of MyS | Function Name | Description | | --------------------------------- | ----------- | | [JSON_APPEND(json_doc, path, value)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-append) | An alias to `JSON_ARRAY_APPEND` | -| [JSON_ARRAY_APPEND(json_doc, path, value)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-append) | Appends a value to the end of a JSON array at a specified path | -| [JSON_ARRAY_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-insert) | Inserts an array into the json document and returns the modified document | +| [JSON_ARRAY_APPEND(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-append) | Appends values to the end of the indicated arrays within a JSON document and returns the result | +| [JSON_ARRAY_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-insert) | Insert values into the specified locations of a JSON document and returns the result | | [JSON_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-insert) | Inserts data into a JSON document and returns the result | -| [JSON_MERGE(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge) | A deprecated alias for `JSON_MERGE_PRESERVE` | | [JSON_MERGE_PATCH(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge-patch) | Merge JSON documents | | [JSON_MERGE_PRESERVE(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge-preserve) | Merges two or more JSON documents and returns the merged result | +| [JSON_MERGE(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge) | A deprecated alias for `JSON_MERGE_PRESERVE` | | [JSON_REMOVE(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-remove) | Removes data from a JSON document and returns the result | | [JSON_REPLACE(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-replace) | Replaces existing values in a JSON document and returns the result | | [JSON_SET(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-set) | Inserts or updates data in a JSON document and returns the result | | [JSON_UNQUOTE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-unquote) | Unquotes a JSON value and returns the result as a string | -| [JSON_ARRAY_APPEND(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-append) | Appends values to the end of the indicated arrays within a JSON document and returns the result | -| [JSON_ARRAY_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-insert) | Insert values into the specified location of a JSON document and returns the result | ## Functions that return JSON value attributes @@ -76,3 +74,12 @@ TiDB supports most of the JSON functions that shipped with the GA release of MyS * [JSON Function Reference](https://dev.mysql.com/doc/refman/8.0/en/json-function-reference.html) * [JSON Data Type](/data-type-json.md) + +## Unsupported functions + +- `JSON_SCHEMA_VALID()` +- `JSON_SCHEMA_VALIDATION_REPORT()` +- `JSON_TABLE()` +- `JSON_VALUE()` + +For more information, see [#14486](https://github.com/pingcap/tidb/issues/14486). \ No newline at end of file diff --git a/functions-and-operators/locking-functions.md b/functions-and-operators/locking-functions.md index a9829511825d4..08a261cfa241c 100644 --- a/functions-and-operators/locking-functions.md +++ b/functions-and-operators/locking-functions.md @@ -5,7 +5,7 @@ summary: Learn about user-level locking functions in TiDB. # Locking Functions -TiDB supports most of the user-level [locking functions](https://dev.mysql.com/doc/refman/5.7/en/locking-functions.html) available in MySQL 5.7. +TiDB supports most of the user-level [locking functions](https://dev.mysql.com/doc/refman/8.0/en/locking-functions.html) available in MySQL 8.0. ## Supported functions @@ -20,6 +20,6 @@ TiDB supports most of the user-level [locking functions](https://dev.mysql.com/d ## MySQL compatibility * The minimum timeout permitted by TiDB is 1 second, and the maximum timeout is 1 hour (3600 seconds). This differs from MySQL, where both 0 second and unlimited timeouts (`timeout=-1`) are permitted. TiDB will automatically convert out-of-range values to the nearest permitted value and convert `timeout=-1` to 3600 seconds. -* TiDB does not automatically detect deadlocks caused by user-level locks. Deadlocked sessions will timeout after a maximum of 1 hour, but can also be manually resolved by using `KILL` on one of the affected sessions. You can also prevent deadlocks by always acquiring user-level locks in the same order. +* TiDB does not automatically detect deadlocks caused by user-level locks. Deadlocked sessions will timeout after a maximum of 1 hour, but can also be manually resolved by using [`KILL`](/sql-statements/sql-statement-kill.md) on one of the affected sessions. You can also prevent deadlocks by always acquiring user-level locks in the same order. * Locks take effect on all TiDB servers in the cluster. This differs from MySQL Cluster and Group Replication where locks are local to a single server. * `IS_USED_LOCK()` returns `1` if it is called from another session and is unable to return the ID of the process that is holding the lock. diff --git a/functions-and-operators/miscellaneous-functions.md b/functions-and-operators/miscellaneous-functions.md index e0f3e73f0d9b4..c50284089cee2 100644 --- a/functions-and-operators/miscellaneous-functions.md +++ b/functions-and-operators/miscellaneous-functions.md @@ -6,32 +6,415 @@ aliases: ['/docs/dev/functions-and-operators/miscellaneous-functions/','/docs/de # Miscellaneous Functions -TiDB supports most of the [miscellaneous functions](https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html) available in MySQL 5.7. +TiDB supports most of the [miscellaneous functions](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html) available in MySQL 8.0. ## Supported functions | Name | Description | |:------------|:-----------------------------------------------------------------------------------------------| -| [`ANY_VALUE()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_any-value) | Suppress `ONLY_FULL_GROUP_BY` value rejection | -| [`BIN_TO_UUID()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_bin-to-uuid) | Convert UUID from binary format to text format | -| [`DEFAULT()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_default) | Returns the default value for a table column | -| [`INET_ATON()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_inet-aton) | Return the numeric value of an IP address | -| [`INET_NTOA()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_inet-ntoa) | Return the IP address from a numeric value | -| [`INET6_ATON()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_inet6-aton) | Return the numeric value of an IPv6 address | -| [`INET6_NTOA()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_inet6-ntoa) | Return the IPv6 address from a numeric value | -| [`IS_IPV4()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_is-ipv4) | Whether argument is an IPv4 address | -| [`IS_IPV4_COMPAT()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_is-ipv4-compat) | Whether argument is an IPv4-compatible address | -| [`IS_IPV4_MAPPED()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_is-ipv4-mapped) | Whether argument is an IPv4-mapped address | -| [`IS_IPV6()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_is-ipv6) | Whether argument is an IPv6 address | -| [`NAME_CONST()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_name-const) | Can be used to rename a column name | -| [`SLEEP()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_sleep) | Sleep for a number of seconds. Note that for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, the `SLEEP()` function has a limitation wherein it can only support a maximum sleep time of 300 seconds. | -| [`UUID()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_uuid) | Return a Universal Unique Identifier (UUID) | -| [`UUID_TO_BIN()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_uuid-to-bin) | Convert UUID from text format to binary format | -| [`VALUES()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_values) | Defines the values to be used during an INSERT | +| [`ANY_VALUE()`](#any_value) | Suppress `ONLY_FULL_GROUP_BY` value rejection | +| [`BIN_TO_UUID()`](#bin_to_uuid) | Convert UUID from binary format to text format | +| [`DEFAULT()`](#default) | Returns the default value for a table column | +| [`GROUPING()`](#grouping) | Modifier for `GROUP BY` operations | +| [`INET_ATON()`](#inet_aton) | Return the numeric value of an IP address | +| [`INET_NTOA()`](#inet_ntoa) | Return the IP address from a numeric value | +| [`INET6_ATON()`](#inet6_aton) | Return the numeric value of an IPv6 address | +| [`INET6_NTOA()`](#inet6_ntoa) | Return the IPv6 address from a numeric value | +| [`IS_IPV4()`](#is_ipv4) | Whether argument is an IPv4 address | +| [`IS_IPV4_COMPAT()`](#is_ipv4_compat) | Whether argument is an IPv4-compatible address | +| [`IS_IPV4_MAPPED()`](#is_ipv4_mapped) | Whether argument is an IPv4-mapped address | +| [`IS_IPV6()`](#is_ipv6) | Whether argument is an IPv6 address | +| [`IS_UUID()`](#is_uuid) | Whether argument is an UUID | +| [`NAME_CONST()`](#name_const) | Can be used to rename a column name | +| [`SLEEP()`](#sleep) | Sleep for a number of seconds. Note that for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, the `SLEEP()` function has a limitation wherein it can only support a maximum sleep time of 300 seconds. | +| [`UUID()`](#uuid) | Return a Universal Unique Identifier (UUID) | +| [`UUID_TO_BIN()`](#uuid_to_bin) | Convert UUID from text format to binary format | +| [`VALUES()`](#values) | Defines the values to be used during an INSERT | + +### ANY_VALUE() + +The `ANY_VALUE()` function returns any value from a group of values. Typically, it is used in scenarios where you need to include non-aggregated columns in your `SELECT` statement along with a `GROUP BY` clause. + +```sql +CREATE TABLE fruits (id INT PRIMARY KEY, name VARCHAR(255)); +Query OK, 0 rows affected (0.14 sec) + +INSERT INTO fruits VALUES (1,'apple'),(2,'apple'),(3,'pear'),(4,'banana'),(5, 'pineapple'); +Query OK, 5 rows affected (0.01 sec) +Records: 5 Duplicates: 0 Warnings: 0 + +SELECT id,name FROM fruits GROUP BY name; +ERROR 1055 (42000): Expression #1 of SELECT list is not in GROUP BY clause and contains nonaggregated column 'test.fruits.id' which is not functionally dependent on columns in GROUP BY clause; this is incompatible with sql_mode=only_full_group_by + +SELECT ANY_VALUE(id),GROUP_CONCAT(id),name FROM fruits GROUP BY name; ++---------------+------------------+-----------+ +| ANY_VALUE(id) | GROUP_CONCAT(id) | name | ++---------------+------------------+-----------+ +| 1 | 1,2 | apple | +| 3 | 3 | pear | +| 4 | 4 | banana | +| 5 | 5 | pineapple | ++---------------+------------------+-----------+ +4 rows in set (0.00 sec) +``` + +In the preceding example, TiDB returns an error for the first `SELECT` statement because the `id` column is non-aggregated and not included in the `GROUP BY` clause. To address the issue, the second `SELECT` query uses `ANY_VALUE()` to get any value from each group and uses `GROUP_CONCAT()` to concatenate all values of the `id` column within each group into a single string. This approach enables you to get one value from each group and all values of the group without changing the SQL mode for non-aggregated columns. + +### BIN_TO_UUID() + +`BIN_TO_UUID()` and `UUID_TO_BIN()` can be used to convert between a textual format UUID and a binary format. Both functions accept two arguments. + +- The first argument specifies the value to be converted. +- The second argument (optional) controls the ordering of the fields in the binary format. + +```sql +SET @a := UUID(); +Query OK, 0 rows affected (0.00 sec) + +SELECT @a; ++--------------------------------------+ +| @a | ++--------------------------------------+ +| 9a17b457-eb6d-11ee-bacf-5405db7aad56 | ++--------------------------------------+ +1 row in set (0.00 sec) + +SELECT UUID_TO_BIN(@a); ++------------------------------------+ +| UUID_TO_BIN(@a) | ++------------------------------------+ +| 0x9A17B457EB6D11EEBACF5405DB7AAD56 | ++------------------------------------+ +1 row in set (0.00 sec) + +SELECT BIN_TO_UUID(0x9A17B457EB6D11EEBACF5405DB7AAD56); ++-------------------------------------------------+ +| BIN_TO_UUID(0x9A17B457EB6D11EEBACF5405DB7AAD56) | ++-------------------------------------------------+ +| 9a17b457-eb6d-11ee-bacf-5405db7aad56 | ++-------------------------------------------------+ +1 row in set (0.00 sec) + +SELECT UUID_TO_BIN(@a, 1); ++----------------------------------------+ +| UUID_TO_BIN(@a, 1) | ++----------------------------------------+ +| 0x11EEEB6D9A17B457BACF5405DB7AAD56 | ++----------------------------------------+ +1 row in set (0.00 sec) + +SELECT BIN_TO_UUID(0x11EEEB6D9A17B457BACF5405DB7AAD56, 1); ++----------------------------------------------------+ +| BIN_TO_UUID(0x11EEEB6D9A17B457BACF5405DB7AAD56, 1) | ++----------------------------------------------------+ +| 9a17b457-eb6d-11ee-bacf-5405db7aad56 | ++----------------------------------------------------+ +1 row in set (0.00 sec) +``` + +See also [UUID()](#uuid) and [Best practices for UUID](/best-practices/uuid.md). + +### DEFAULT() + +The `DEFAULT()` function is used to get the default value of a column. + +```sql +CREATE TABLE t1 (id INT PRIMARY KEY, c1 INT DEFAULT 5); +Query OK, 0 rows affected (0.15 sec) + +INSERT INTO t1 VALUES (1, 1); +Query OK, 1 row affected (0.01 sec) + +UPDATE t1 SET c1=DEFAULT(c1)+3; +Query OK, 1 row affected (0.02 sec) +Rows matched: 1 Changed: 1 Warnings: 0 + +TABLE t1; ++----+------+ +| id | c1 | ++----+------+ +| 1 | 8 | ++----+------+ +1 row in set (0.00 sec) +``` + +In the preceding example, the `UPDATE` statement sets the value of the `c1` column to the default value of the column (which is `5`) plus `3`, resulting in a new value of `8`. + +### GROUPING() + +See [`GROUP BY` modifiers](/functions-and-operators/group-by-modifier.md). + +### INET_ATON() + +The `INET_ATON()` function converts an IPv4 address in dotted-quad notation into a binary version that can be stored efficiently. + +```sql +SELECT INET_ATON('127.0.0.1'); +``` + +``` ++------------------------+ +| INET_ATON('127.0.0.1') | ++------------------------+ +| 2130706433 | ++------------------------+ +1 row in set (0.00 sec) +``` + +### INET_NTOA() + +The `INET_NTOA()` function converts a binary IPv4 address into a dotted-quad notation. + +```sql +SELECT INET_NTOA(2130706433); +``` + +``` ++-----------------------+ +| INET_NTOA(2130706433) | ++-----------------------+ +| 127.0.0.1 | ++-----------------------+ +1 row in set (0.00 sec) +``` + +### INET6_ATON() + +The `INET6_ATON()` function is similar to [`INET_ATON()`](#inet_aton), but `INET6_ATON()` can also handle IPv6 addresses. + +```sql +SELECT INET6_ATON('::1'); +``` + +``` ++--------------------------------------+ +| INET6_ATON('::1') | ++--------------------------------------+ +| 0x00000000000000000000000000000001 | ++--------------------------------------+ +1 row in set (0.00 sec) +``` + +### INET6_NTOA() + +The `INET6_NTOA()` function is similar to [`INET_NTOA()`](#inet_ntoa), but `INET6_NTOA()` can also handle IPv6 addresses. + +```sql +SELECT INET6_NTOA(0x00000000000000000000000000000001); +``` + +``` ++------------------------------------------------+ +| INET6_NTOA(0x00000000000000000000000000000001) | ++------------------------------------------------+ +| ::1 | ++------------------------------------------------+ +1 row in set (0.00 sec) +``` + +### IS_IPV4() + +The `IS_IPV4()` function tests whether the given argument is an IPv4 address or not. + +```sql +SELECT IS_IPV4('127.0.0.1'); +``` + +``` ++----------------------+ +| IS_IPV4('127.0.0.1') | ++----------------------+ +| 1 | ++----------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT IS_IPV4('300.0.0.1'); +``` + +``` ++----------------------+ +| IS_IPV4('300.0.0.1') | ++----------------------+ +| 0 | ++----------------------+ +1 row in set (0.00 sec) +``` + +### IS_IPV4_COMPAT() + +The `IS_IPV4_COMPAT()` function tests whether the given argument is an IPv4-compatible address. + +```sql +SELECT IS_IPV4_COMPAT(INET6_ATON('::127.0.0.1')); +``` + +``` ++-------------------------------------------+ +| IS_IPV4_COMPAT(INET6_ATON('::127.0.0.1')) | ++-------------------------------------------+ +| 1 | ++-------------------------------------------+ +1 row in set (0.00 sec) +``` + +### IS_IPV4_MAPPED() + +The `IS_IPV4_MAPPED()` function tests whether the given argument is an IPv4-mapped address. + +```sql +SELECT IS_IPV4_MAPPED(INET6_ATON('::ffff:127.0.0.1')); +``` + +``` ++------------------------------------------------+ +| IS_IPV4_MAPPED(INET6_ATON('::ffff:127.0.0.1')) | ++------------------------------------------------+ +| 1 | ++------------------------------------------------+ +1 row in set (0.00 sec) +``` + +### IS_IPV6() + +The `IS_IPV6()` function tests whether the given argument is an IPv6 address. + +```sql +SELECT IS_IPV6('::1'); +``` + +``` ++----------------+ +| IS_IPV6('::1') | ++----------------+ +| 1 | ++----------------+ +1 row in set (0.00 sec) +``` + +### IS_UUID() + +The `IS_UUID()` function tests whether the given argument is a [UUID](/best-practices/uuid.md). + +```sql +SELECT IS_UUID('eb48c08c-eb71-11ee-bacf-5405db7aad56'); +``` + +``` ++-------------------------------------------------+ +| IS_UUID('eb48c08c-eb71-11ee-bacf-5405db7aad56') | ++-------------------------------------------------+ +| 1 | ++-------------------------------------------------+ +1 row in set (0.00 sec) +``` + +### NAME_CONST() + +The `NAME_CONST()` function is used to name columns. It is recommended to use column aliases instead. + +```sql +SELECT NAME_CONST('column name', 'value') UNION ALL SELECT 'another value'; +``` + +``` ++---------------+ +| column name | ++---------------+ +| another value | +| value | ++---------------+ +2 rows in set (0.00 sec) +``` + +The preceding statement uses `NAME_CONST()` and the following statement uses the recommended way of aliasing columns. + +```sql +SELECT 'value' AS 'column name' UNION ALL SELECT 'another value'; +``` + +``` ++---------------+ +| column name | ++---------------+ +| value | +| another value | ++---------------+ +2 rows in set (0.00 sec) +``` + +### SLEEP() + +The `SLEEP()` function is used to pause the execution of queries for a specified number of seconds. + +```sql +SELECT SLEEP(1.5); +``` + +``` ++------------+ +| SLEEP(1.5) | ++------------+ +| 0 | ++------------+ +1 row in set (1.50 sec) +``` + +### UUID() + +The `UUID()` function returns a universally unique identifier (UUID) version 1 as defined in [RFC 4122](https://datatracker.ietf.org/doc/html/rfc4122). + +```sql +SELECT UUID(); +``` + +``` ++--------------------------------------+ +| UUID() | ++--------------------------------------+ +| cb4d5ae6-eb6b-11ee-bacf-5405db7aad56 | ++--------------------------------------+ +1 row in set (0.00 sec) +``` + +See also [Best practices for UUID](/best-practices/uuid.md). + +### UUID_TO_BIN + +See [BIN_TO_UUID()](#bin_to_uuid). + +### VALUES() + +The `VALUES(col_name)` function is used to reference the value of a specific column in the `ON DUPLICATE KEY UPDATE` clause of an [`INSERT`](/sql-statements/sql-statement-insert.md) statement. + +```sql +CREATE TABLE t1 (id INT PRIMARY KEY, c1 INT); +Query OK, 0 rows affected (0.17 sec) + +INSERT INTO t1 VALUES (1,51),(2,52),(3,53),(4,54),(5,55); +Query OK, 5 rows affected (0.01 sec) +Records: 5 Duplicates: 0 Warnings: 0 + +INSERT INTO t1 VALUES(2,22),(4,44) ON DUPLICATE KEY UPDATE c1=VALUES(id)+100; +Query OK, 4 rows affected (0.01 sec) +Records: 2 Duplicates: 2 Warnings: 0 + +TABLE t1; ++----+------+ +| id | c1 | ++----+------+ +| 1 | 51 | +| 2 | 102 | +| 3 | 53 | +| 4 | 104 | +| 5 | 55 | ++----+------+ +5 rows in set (0.00 sec) +``` ## Unsupported functions | Name | Description | |:------------|:-----------------------------------------------------------------------------------------------| -| [`UUID_SHORT()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_uuid-short) | Provides a UUID that is unique given certain assumptions not present in TiDB [TiDB #4620](https://github.com/pingcap/tidb/issues/4620) | -| [`MASTER_WAIT_POS()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_master-pos-wait) | Relates to MySQL replication | +| [`UUID_SHORT()`](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_uuid-short) | Provides a UUID that is unique given certain assumptions not present in TiDB [TiDB #4620](https://github.com/pingcap/tidb/issues/4620) | \ No newline at end of file diff --git a/functions-and-operators/numeric-functions-and-operators.md b/functions-and-operators/numeric-functions-and-operators.md index ab228e0f1ea40..cf0b088ab1434 100644 --- a/functions-and-operators/numeric-functions-and-operators.md +++ b/functions-and-operators/numeric-functions-and-operators.md @@ -6,11 +6,11 @@ aliases: ['/docs/dev/functions-and-operators/numeric-functions-and-operators/',' # Numeric Functions and Operators -TiDB supports all of the [numeric functions and operators](https://dev.mysql.com/doc/refman/5.7/en/numeric-functions.html) available in MySQL 5.7. +TiDB supports all of the [numeric functions and operators](https://dev.mysql.com/doc/refman/8.0/en/numeric-functions.html) available in MySQL 8.0. ## Arithmetic operators -| Name | Description | +| Name | Description | |:----------------------------------------------------------------------------------------------|:----------------------------------| | [`+`](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_plus) | Addition operator | | [`-`](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_minus) | Minus operator | @@ -24,33 +24,37 @@ TiDB supports all of the [numeric functions and operators](https://dev.mysql.com | Name | Description | |:----------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------| -| [`POW()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_pow) | Return the argument raised to the specified power | -| [`POWER()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_power) | Return the argument raised to the specified power | +| [`ABS()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_abs) | Return the absolute value | +| [`ACOS()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_acos) | Return the arc cosine | +| [`ASIN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_asin) | Return the arc sine | +| [`ATAN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_atan) | Return the arc tangent | +| [`ATAN2(), ATAN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_atan2) | Return the arc tangent of the two arguments | +| [`CEIL()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ceil) | Return the smallest integer value not less than the argument | +| [`CEILING()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ceiling) | Return the smallest integer value not less than the argument | +| [`CONV()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_conv) | Convert numbers between different number bases | +| [`COS()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_cos) | Return the cosine | +| [`COT()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_cot) | Return the cotangent | +| [`CRC32()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_crc32) | Compute a cyclic redundancy check value | +| [`DEGREES()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_degrees) | Convert radians to degrees | | [`EXP()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_exp) | Raise to the power of | -| [`SQRT()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sqrt) | Return the square root of the argument | +| [`FLOOR()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_floor) | Return the largest integer value not greater than the argument | | [`LN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ln) | Return the natural logarithm of the argument | | [`LOG()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log) | Return the natural logarithm of the first argument | -| [`LOG2()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log2) | Return the base-2 logarithm of the argument | | [`LOG10()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log10) | Return the base-10 logarithm of the argument | +| [`LOG2()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log2) | Return the base-2 logarithm of the argument | +| [`MOD()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_mod) | Return the remainder | | [`PI()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_pi) | Return the value of pi | -| [`TAN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_tan) | Return the tangent of the argument | -| [`COT()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_cot) | Return the cotangent | -| [`SIN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sin) | Return the sine of the argument | -| [`COS()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_cos) | Return the cosine | -| [`ATAN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_atan) | Return the arc tangent | -| [`ATAN2(), ATAN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_atan2) | Return the arc tangent of the two arguments | -| [`ASIN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_asin) | Return the arc sine | -| [`ACOS()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_acos) | Return the arc cosine | +| [`POW()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_pow) | Return the argument raised to the specified power | +| [`POWER()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_power) | Return the argument raised to the specified power | | [`RADIANS()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_radians) | Return argument converted to radians | -| [`DEGREES()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_degrees) | Convert radians to degrees | -| [`MOD()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_mod) | Return the remainder | -| [`ABS()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_abs) | Return the absolute value | -| [`CEIL()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ceil) | Return the smallest integer value not less than the argument | -| [`CEILING()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ceiling) | Return the smallest integer value not less than the argument | -| [`FLOOR()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_floor) | Return the largest integer value not greater than the argument | -| [`ROUND()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_round) | Round the argument | | [`RAND()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_rand) | Return a random floating-point value | +| [`ROUND()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_round) | Round the argument | | [`SIGN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sign) | Return the sign of the argument | -| [`CONV()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_conv) | Convert numbers between different number bases | +| [`SIN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sin) | Return the sine of the argument | +| [`SQRT()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sqrt) | Return the square root of the argument | +| [`TAN()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_tan) | Return the tangent of the argument | | [`TRUNCATE()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_truncate) | Truncate to specified number of decimal places | -| [`CRC32()`](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_crc32) | Compute a cyclic redundancy check value | + +## Related system variables + +[`div_precision_increment`](/system-variables.md#div_precision_increment-new-in-v800) is used to set the precision for the `/` operator. \ No newline at end of file diff --git a/functions-and-operators/precision-math.md b/functions-and-operators/precision-math.md index fca0983e8ab41..e3916a1b75b92 100644 --- a/functions-and-operators/precision-math.md +++ b/functions-and-operators/precision-math.md @@ -51,7 +51,7 @@ DECIMAL columns do not store a leading `+` character or `-` character or leading DECIMAL columns do not permit values larger than the range implied by the column definition. For example, a `DECIMAL(3,0)` column supports a range of `-999` to `999`. A `DECIMAL(M,D)` column permits at most `M - D` digits to the left of the decimal point. -For more information about the internal format of the DECIMAL values, see [`mydecimal.go`](https://github.com/pingcap/tidb/blob/master/pkg/types/mydecimal.go) in TiDB souce code. +For more information about the internal format of the DECIMAL values, see [`mydecimal.go`](https://github.com/pingcap/tidb/blob/master/pkg/types/mydecimal.go) in TiDB source code. ## Expression handling diff --git a/functions-and-operators/sequence-functions.md b/functions-and-operators/sequence-functions.md new file mode 100644 index 0000000000000..bbf14b6b2e624 --- /dev/null +++ b/functions-and-operators/sequence-functions.md @@ -0,0 +1,18 @@ +--- +title: Sequence Functions +summary: This document introduces sequence functions supported in TiDB. +--- + +# Sequence Functions + +Sequence functions in TiDB are used to return or set values of sequence objects created using the [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) statement. + +| Function name | Feature description | +| :-------------- | :------------------------------------- | +| `NEXTVAL()` or `NEXT VALUE FOR` | Returns the next value of a sequence | +| `SETVAL()` | Sets the current value of a sequence | +| `LASTVAL()` | Returns the last used value of a sequence | + +## MySQL compatibility + +MySQL does not support the functions and statements for creating and manipulating sequences as defined in [ISO/IEC 9075-2](https://www.iso.org/standard/76584.html). diff --git a/functions-and-operators/set-operators.md b/functions-and-operators/set-operators.md index e5177dcf7012a..2c1f869e8e515 100644 --- a/functions-and-operators/set-operators.md +++ b/functions-and-operators/set-operators.md @@ -116,7 +116,7 @@ TiDB supports using parentheses to specify the precedence of set operations. Exp ## Use `ORDER BY` and `LIMIT` -TiDB supports using [`ORDER BY`](/media/sqlgram/OrderByOptional.png) or [`LIMIT`](/media/sqlgram/LimitClause.png) clause in set operations. These two clauses must be at the end of the entire statement. +TiDB supports using `ORDER BY` or `LIMIT` clause on the entire result of set operations. These two clauses must be at the end of the entire statement. ```sql (SELECT * FROM t1 UNION ALL SELECT * FROM t1 INTERSECT SELECT * FROM t2) ORDER BY a LIMIT 2; diff --git a/functions-and-operators/string-functions.md b/functions-and-operators/string-functions.md index 4f0c3751de455..746fb53c2b080 100644 --- a/functions-and-operators/string-functions.md +++ b/functions-and-operators/string-functions.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/functions-and-operators/string-functions/','/docs/dev/refer # String Functions -TiDB supports most of the [string functions](https://dev.mysql.com/doc/refman/5.7/en/string-functions.html) available in MySQL 5.7, some of the [string functions](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html) available in MySQL 8.0, and some of the [functions](https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlqr/SQL-Functions.html#GUID-93EC62F8-415D-4A7E-B050-5D5B2C127009) available in Oracle 21. +TiDB supports most of the [string functions](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html) available in MySQL 8.0, and some of the [functions](https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlqr/SQL-Functions.html#GUID-93EC62F8-415D-4A7E-B050-5D5B2C127009) available in Oracle 21. @@ -134,7 +134,7 @@ SELECT CustomerName, BIT_LENGTH(CustomerName) AS BitLengthOfName FROM Customers; ### [`CHAR()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char) -The `CHAR()` function is used to get the corresponding character of a specific ASCII value. It performs the opposite operation of `ASCII()`, which returns the ASCII value of a specific character. +The `CHAR()` function is used to get the corresponding character of a specific ASCII value. It performs the opposite operation of `ASCII()`, which returns the ASCII value of a specific character. If multiple arguments are supplied, the function works on all arguments and are then concaternated together. Examples: @@ -188,6 +188,19 @@ SELECT CHAR(50089); +--------------+ ``` +```sql +SELECT CHAR(65,66,67); +``` + +``` ++----------------+ +| CHAR(65,66,67) | ++----------------+ +| ABC | ++----------------+ +1 row in set (0.00 sec) +``` + ### [`CHAR_LENGTH()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char-length) The `CHAR_LENGTH()` function is used to get the total number of characters in a given argument as an integer. @@ -205,10 +218,10 @@ SELECT CHAR_LENGTH("TiDB") AS LengthOfString; ``` ```sql -SELECT CustomerName, CHAR_LENGTH(CustomerName) AS LenghtOfName FROM Customers; +SELECT CustomerName, CHAR_LENGTH(CustomerName) AS LengthOfName FROM Customers; +--------------------+--------------+ -| CustomerName | LenghtOfName | +| CustomerName | LengthOfName | +--------------------+--------------+ | Albert Einstein | 15 | | Robert Oppenheimer | 18 | @@ -223,7 +236,7 @@ SELECT CustomerName, CHAR_LENGTH(CustomerName) AS LenghtOfName FROM Customers; The `CHARACTER_LENGTH()` function is the same as the `CHAR_LENGTH()` function. Both functions can be used synonymously because they generate the same output. -## [`CONCAT()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_concat) +### [`CONCAT()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_concat) The `CONCAT()` function concatenates one or more arguments into a single string. @@ -406,11 +419,85 @@ Output: ### [`ELT()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_elt) -Return string at index number. +The `ELT()` function returns the element at the index number. + +```sql +SELECT ELT(3, 'This', 'is', 'TiDB'); +``` + +```sql ++------------------------------+ +| ELT(3, 'This', 'is', 'TiDB') | ++------------------------------+ +| TiDB | ++------------------------------+ +1 row in set (0.00 sec) +``` + +The preceding example returns the third element, which is `'TiDB'`. ### [`EXPORT_SET()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_export-set) -Return a string such that for every bit set in the value bits, you get an on string and for every unset bit, you get an off string. +The `EXPORT_SET()` function returns a string that consists of a specified number (`number_of_bits`) of `on`/`off` values, optionally separated by `separator`. These values are based on whether the corresponding bit in the `bits` argument is `1`, where the first value corresponds to the rightmost (lowest) bit of `bits`. + +Syntax: + +```sql +EXPORT_SET(bits, on, off, [separator[, number_of_bits]]) +``` + +- `bits`: an integer representing the bit value. +- `on`: the string to be returned if the corresponding bit is `1`. +- `off`: the string to be returned if the corresponding bit is `0`. +- `separator` (optional): the separator character in the result string. +- `number_of_bits` (optional): the number of bits to be processed. If it is not set, `64` (the max size of bits) is used by default, which means that `bits` is treated as an unsigned 64-bit integer. + +Examples: + +In the following example, `number_of_bits` is set to `5`, resulting in 5 values, separated by `|`. Because only 3 bits are given, the other bits are considered unset. Therefore, setting `number_of_bits` to either `101` or `00101` results in the same output. + +```sql +SELECT EXPORT_SET(b'101',"ON",'off','|',5); +``` + +```sql ++-------------------------------------+ +| EXPORT_SET(b'101',"ON",'off','|',5) | ++-------------------------------------+ +| ON|off|ON|off|off | ++-------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, `bits` is set to `00001111`, `on` is set to `x`, and `off` is set to `_`. This causes the function to return `____` for the `0` bits and `xxxx` for the `1` bits. Therefore, when processing with the bits in `00001111` from right to left, the function returns `xxxx____`. + +```sql +SELECT EXPORT_SET(b'00001111', 'x', '_', '', 8); +``` + +```sql ++------------------------------------------+ +| EXPORT_SET(b'00001111', 'x', '_', '', 8) | ++------------------------------------------+ +| xxxx____ | ++------------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, `bits` is set to `00001111`, `on` is set to `x`, and `off` is set to `_`. This causes the function to return `x` for each `1` bit and `_` for each `0` bit. Therefore, when processing with the bits in `01010101` from right to left, the function returns `x_x_x_x_`. + +```sql +SELECT EXPORT_SET(b'01010101', 'x', '_', '', 8); +``` + +```sql ++------------------------------------------+ +| EXPORT_SET(b'01010101', 'x', '_', '', 8) | ++------------------------------------------+ +| x_x_x_x_ | ++------------------------------------------+ +1 row in set (0.00 sec) +``` ### [`FIELD()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_field) @@ -1226,20 +1313,199 @@ SELECT LPAD('TiDB',-2,'>'); ### [`LTRIM()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_ltrim) -Remove leading spaces. +The `LTRIM()` function removes leading spaces from a given string. + +If the argument is `NULL`, this function returns `NULL`. + +> **Note:** +> +> This function only removes the space character (U+0020) and does not remove other space-like characters such as tab (U+0009) or non-breaking space (U+00A0). + +Examples: + +In the following example, the `LTRIM()` function removes the leading spaces from `' hello'` and returns `hello`. + +```sql +SELECT LTRIM(' hello'); +``` + +``` ++--------------------+ +| LTRIM(' hello') | ++--------------------+ +| hello | ++--------------------+ +1 row in set (0.00 sec) +``` + +In the following example, [`CONCAT()`](#concat) is used to enclose the result of `LTRIM(' hello')` with `«` and `»`. This formatting makes it a bit easier to see that all leading spaces are removed. + +```sql +SELECT CONCAT('«',LTRIM(' hello'),'»'); +``` + +``` ++------------------------------------+ +| CONCAT('«',LTRIM(' hello'),'»') | ++------------------------------------+ +| «hello» | ++------------------------------------+ +1 row in set (0.00 sec) +``` ### [`MAKE_SET()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_make-set) -Return a set of comma-separated strings that have the corresponding bit in bits set. +The `MAKE_SET()` function returns a set of comma-separated strings based on whether a corresponding bit in the `bits` argument is set to `1`. + +Syntax: + +```sql +MAKE_SET(bits, str1, str2, ...) +``` + +- `bits`: controls which subsequent string arguments to include in the result set. If `bits` is set to `NULL`, the function returns `NULL`. +- `str1, str2, ...`: a list of strings. Each string corresponds to a bit in the `bits` argument from right to left. `str1` corresponds to the first bit from the right, `str2` corresponds to the second bit from the right, and so on. If the corresponding bit is `1`, the string is included in the result; otherwise, it is not included. + +Examples: + +In the following example, because all bits are set to `0` in the `bits` argument, the function does not include any subsequent strings in the result and returns an empty string. + +```sql +SELECT MAKE_SET(b'000','foo','bar','baz'); +``` + +``` ++------------------------------------+ +| MAKE_SET(b'000','foo','bar','baz') | ++------------------------------------+ +| | ++------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, because only the first bit from the right is `1`, the function only returns the first string `foo`. + +```sql +SELECT MAKE_SET(b'001','foo','bar','baz'); +``` + +``` ++------------------------------------+ +| MAKE_SET(b'001','foo','bar','baz') | ++------------------------------------+ +| foo | ++------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, because only the second bit from the right is `1`, the function only returns the second string `bar`. + +```sql +SELECT MAKE_SET(b'010','foo','bar','baz'); +``` + +``` ++------------------------------------+ +| MAKE_SET(b'010','foo','bar','baz') | ++------------------------------------+ +| bar | ++------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, because only the third bit from the right is `1`, the function only returns the third string `baz`. + +```sql +SELECT MAKE_SET(b'100','foo','bar','baz'); +``` + +``` ++------------------------------------+ +| MAKE_SET(b'100','foo','bar','baz') | ++------------------------------------+ +| baz | ++------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, because all bits are `1`, the function returns all three strings in a comma-separated result set. + +```sql +SELECT MAKE_SET(b'111','foo','bar','baz'); +``` + +``` ++------------------------------------+ +| MAKE_SET(b'111','foo','bar','baz') | ++------------------------------------+ +| foo,bar,baz | ++------------------------------------+ +1 row in set (0.0002 sec) +``` ### [`MID()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_mid) -Return a substring starting from the specified position. +The `MID(str, pos, len)` function returns a substring starting from the specified `pos` position with the `len` length. + +If any of the arguments are `NULL`, the function returns `NULL`. + +TiDB does not support the two-argument variant of this function. For more information, see [#52420](https://github.com/pingcap/tidb/issues/52420). + +Examples: + +In the following example, `MID()` returns the substring of the input string starting from the second character (`b`) with a length of `3` characters. + +```sql +SELECT MID('abcdef',2,3); +``` + +``` ++-------------------+ +| MID('abcdef',2,3) | ++-------------------+ +| bcd | ++-------------------+ +1 row in set (0.00 sec) +``` ### [`NOT LIKE`](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_not-like) Negation of simple pattern matching. +This function performs the inverse operation of [`LIKE`](#like). + +Examples: + +In the following example, `NOT LIKE` returns `0` (False) because `aaa` matches the `a%` pattern. + +```sql +SELECT 'aaa' LIKE 'a%', 'aaa' NOT LIKE 'a%'; +``` + +``` ++-----------------+---------------------+ +| 'aaa' LIKE 'a%' | 'aaa' NOT LIKE 'a%' | ++-----------------+---------------------+ +| 1 | 0 | ++-----------------+---------------------+ +1 row in set (0.00 sec) +``` + +In the following example, `NOT LIKE` returns `1` (True) because `aaa` does not match the `b%` pattern. + +```sql +SELECT 'aaa' LIKE 'b%', 'aaa' NOT LIKE 'b%'; +``` + +``` ++-----------------+---------------------+ +| 'aaa' LIKE 'b%' | 'aaa' NOT LIKE 'b%' | ++-----------------+---------------------+ +| 0 | 1 | ++-----------------+---------------------+ +1 row in set (0.00 sec) +``` + ### [`NOT REGEXP`](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_not-regexp) Negation of [`REGEXP`](#regexp). @@ -1292,40 +1558,456 @@ SELECT n, OCT(n) FROM nr; ### [`OCTET_LENGTH()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_octet-length) -Synonym for `LENGTH()`. +Synonym for [`LENGTH()`](#length). ### [`ORD()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_ord) -Return character code for leftmost character of the argument. +Return the character code for the leftmost character of the given argument. + +This function is similar to [`CHAR()`](#char) but works the other way around. + +Examples: + +Taking `a` and `A` as an example, `ORD()` returns `97` for `a` and `65` for `A`. + +```sql +SELECT ORD('a'), ORD('A'); +``` + +``` ++----------+----------+ +| ORD('a') | ORD('A') | ++----------+----------+ +| 97 | 65 | ++----------+----------+ +1 row in set (0.00 sec) +``` + +If you take the character code obtained from `ORD()` as input, you can get the original characters back using the `CHAR()` function. Note that the output format might vary depending on whether the `binary-as-hex` option is enabled in your MySQL client. + +```sql +SELECT CHAR(97), CHAR(65); +``` + +``` ++----------+----------+ +| CHAR(97) | CHAR(65) | ++----------+----------+ +| a | A | ++----------+----------+ +1 row in set (0.01 sec) +``` + +The following example shows how `ORD()` handles multibyte characters. Here, both `101` and `0x65` are the UTF-8 encoded values for the `e` character, but in different formats. And both `50091` and `0xC3AB` represent the same values, but for the `ë` character. + +```sql +SELECT ORD('e'), ORD('ë'), HEX('e'), HEX('ë'); +``` + +``` ++----------+-----------+----------+-----------+ +| ORD('e') | ORD('ë') | HEX('e') | HEX('ë') | ++----------+-----------+----------+-----------+ +| 101 | 50091 | 65 | C3AB | ++----------+-----------+----------+-----------+ +1 row in set (0.00 sec) +``` ### [`POSITION()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_position) -Synonym for `LOCATE()`. +Synonym for [`LOCATE()`](#locate). ### [`QUOTE()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_quote) Escape the argument for use in an SQL statement. +If the argument is `NULL`, the function returns `NULL`. + +Example: + +To display the result directly instead of showing a hexadecimal-encoded value, you need to start the MySQL client with the [`--skip-binary-as-hex`](https://dev.mysql.com/doc/refman/8.0/en/mysql-command-options.html#option_mysql_binary-as-hex) option. + +The following example shows that the ASCII NULL character is escaped as `\0` and the single quote character `'` is escaped as `\'`: + +```sql +SELECT QUOTE(0x002774657374); +``` + +``` ++-----------------------+ +| QUOTE(0x002774657374) | ++-----------------------+ +| '\0\'test' | ++-----------------------+ +1 row in set (0.00 sec) +``` + ### [`REGEXP`](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_regexp) Pattern matching using regular expressions. +Examples: + +In this example a number of strings are matched against two regular expressions. + +```sql +WITH vals AS ( + SELECT 'TiDB' AS v + UNION ALL + SELECT 'Titanium' + UNION ALL + SELECT 'Tungsten' + UNION ALL + SELECT 'Rust' +) +SELECT + v, + v REGEXP '^Ti' AS 'starts with "Ti"', + v REGEXP '^.{4}$' AS 'Length is 4 characters' +FROM + vals; +``` + +``` ++----------+------------------+------------------------+ +| v | starts with "Ti" | Length is 4 characters | ++----------+------------------+------------------------+ +| TiDB | 1 | 1 | +| Titanium | 1 | 0 | +| Tungsten | 0 | 0 | +| Rust | 0 | 1 | ++----------+------------------+------------------------+ +4 rows in set (0.00 sec) +``` + +The following example demonstrates that `REGEXP` is not limited to the `SELECT` clause. For example, you can also use it in the `WHERE` clause of the query. + +```sql +SELECT + v +FROM ( + SELECT 'TiDB' AS v + ) AS vals +WHERE + v REGEXP 'DB$'; +``` + +``` ++------+ +| v | ++------+ +| TiDB | ++------+ +1 row in set (0.01 sec) +``` + ### [`REGEXP_INSTR()`](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-instr) Return the starting index of the substring that matches the regular expression (Partly compatible with MySQL. For more details, see [Regular expression compatibility with MySQL](#regular-expression-compatibility-with-mysql)). +The `REGEXP_INSTR(str, regexp, [start, [match, [ret, [match_type]]]])` function returns the position of the match if the regular expression (`regexp`) matches the string (`str`). + +If either the `str` or `regexp` is `NULL`, then the function returns `NULL`. + +Examples: + +In the example below you can see that the `^.b.$` matches `abc`. + +```sql +SELECT REGEXP_INSTR('abc','^.b.$'); +``` + +``` ++-----------------------------+ +| REGEXP_INSTR('abc','^.b.$') | ++-----------------------------+ +| 1 | ++-----------------------------+ +1 row in set (0.00 sec) +``` + +The following example uses the third argument to look for a match with a different start position in the string. + +```sql +SELECT REGEXP_INSTR('abcabc','a'); +``` + +``` ++----------------------------+ +| REGEXP_INSTR('abcabc','a') | ++----------------------------+ +| 1 | ++----------------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT REGEXP_INSTR('abcabc','a',2); +``` + +``` ++------------------------------+ +| REGEXP_INSTR('abcabc','a',2) | ++------------------------------+ +| 4 | ++------------------------------+ +1 row in set (0.00 sec) +``` + +The following example uses the 4th argument to look for the second match. + +```sql +SELECT REGEXP_INSTR('abcabc','a',1,2); +``` + +``` ++--------------------------------+ +| REGEXP_INSTR('abcabc','a',1,2) | ++--------------------------------+ +| 4 | ++--------------------------------+ +1 row in set (0.00 sec) +``` + +The following example uses the 5th argument to return the value _after_ the mach, instead of the value of the match. + +```sql +SELECT REGEXP_INSTR('abcabc','a',1,1,1); +``` + +``` ++----------------------------------+ +| REGEXP_INSTR('abcabc','a',1,1,1) | ++----------------------------------+ +| 2 | ++----------------------------------+ +1 row in set (0.00 sec) +``` + +The following example uses the 6th argument to add the `i` flag to get a case insensitive match. For more details about regular expression `match_type`, see [`match_type` compatibility](#match_type-compatibility). + +```sql +SELECT REGEXP_INSTR('abcabc','A',1,1,0,''); +``` + +``` ++-------------------------------------+ +| REGEXP_INSTR('abcabc','A',1,1,0,'') | ++-------------------------------------+ +| 0 | ++-------------------------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT REGEXP_INSTR('abcabc','A',1,1,0,'i'); +``` + +``` ++--------------------------------------+ +| REGEXP_INSTR('abcabc','A',1,1,0,'i') | ++--------------------------------------+ +| 1 | ++--------------------------------------+ +1 row in set (0.00 sec) +``` + +Besides `match_type`, the [collation](/character-set-and-collation.md) also influences the matching. The following example uses a case-sensitive and a case-insensitive collation to demonstrate this. + +```sql +SELECT REGEXP_INSTR('abcabc','A' COLLATE utf8mb4_general_ci); +``` + +``` ++-------------------------------------------------------+ +| REGEXP_INSTR('abcabc','A' COLLATE utf8mb4_general_ci) | ++-------------------------------------------------------+ +| 1 | ++-------------------------------------------------------+ +1 row in set (0.01 sec) +``` + +```sql +SELECT REGEXP_INSTR('abcabc','A' COLLATE utf8mb4_bin); +``` + +``` ++------------------------------------------------+ +| REGEXP_INSTR('abcabc','A' COLLATE utf8mb4_bin) | ++------------------------------------------------+ +| 0 | ++------------------------------------------------+ +1 row in set (0.00 sec) +``` + ### [`REGEXP_LIKE()`](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-like) Whether the string matches the regular expression (Partly compatible with MySQL. For more details, see [Regular expression compatibility with MySQL](#regular-expression-compatibility-with-mysql)). +The `REGEXP_LIKE(str, regex, [match_type])` function is used to test if a regular expression matches a string. Optionally the `match_type` can be used to change the matching behavior. + +Examples: + +The following example shows that `^a` matches `abc`. + +```sql +SELECT REGEXP_LIKE('abc','^a'); +``` + +``` ++-------------------------+ +| REGEXP_LIKE('abc','^a') | ++-------------------------+ +| 1 | ++-------------------------+ +1 row in set (0.00 sec) +``` + +This following example shows that `^A` does not match `abc`. + +```sql +SELECT REGEXP_LIKE('abc','^A'); +``` + +``` ++-------------------------+ +| REGEXP_LIKE('abc','^A') | ++-------------------------+ +| 0 | ++-------------------------+ +1 row in set (0.00 sec) +``` + +This example matches `^A` against `abc`, which now matches because of the `i` flag which enabled case insensitive matching. For more details about the regular expression `match_type`, see [`match_type` compatibility](#match_type-compatibility). + +```sql +SELECT REGEXP_LIKE('abc','^A','i'); +``` + +``` ++-----------------------------+ +| REGEXP_LIKE('abc','^A','i') | ++-----------------------------+ +| 1 | ++-----------------------------+ +1 row in set (0.00 sec) +``` + ### [`REGEXP_REPLACE()`](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-replace) Replace substrings that match the regular expression (Partly compatible with MySQL. For more details, see [Regular expression compatibility with MySQL](#regular-expression-compatibility-with-mysql)). +The `REGEXP_REPLACE(str, regexp, replace, [start, [match, [match_type]]])` function can be used to replace strings based on regular expressions. + +Examples: + +In the following example, two o's are replaced by `i`. + +```sql +SELECT REGEXP_REPLACE('TooDB', 'o{2}', 'i'); +``` + +``` ++--------------------------------------+ +| REGEXP_REPLACE('TooDB', 'o{2}', 'i') | ++--------------------------------------+ +| TiDB | ++--------------------------------------+ +1 row in set (0.00 sec) +``` + +The following example starts the match at the third character, causing the regular expression not to match and not do any replacement. + +```sql +SELECT REGEXP_REPLACE('TooDB', 'o{2}', 'i',3); +``` + +``` ++----------------------------------------+ +| REGEXP_REPLACE('TooDB', 'o{2}', 'i',3) | ++----------------------------------------+ +| TooDB | ++----------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, the 5th argument is used to set if the first or the second match is used for the replacement. + +```sql +SELECT REGEXP_REPLACE('TooDB', 'o', 'i',1,1); +``` + +``` ++---------------------------------------+ +| REGEXP_REPLACE('TooDB', 'o', 'i',1,1) | ++---------------------------------------+ +| TioDB | ++---------------------------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT REGEXP_REPLACE('TooDB', 'o', 'i',1,2); +``` + +``` ++---------------------------------------+ +| REGEXP_REPLACE('TooDB', 'o', 'i',1,2) | ++---------------------------------------+ +| ToiDB | ++---------------------------------------+ +1 row in set (0.00 sec) +``` + +The following example uses the 6th argument to set the `match_type` for case insensitive matching. For more details about the regular expression `match_type`, see [`match_type` compatibility](#match_type-compatibility). + +```sql +SELECT REGEXP_REPLACE('TooDB', 'O{2}','i',1,1); +``` + +``` ++-----------------------------------------+ +| REGEXP_REPLACE('TooDB', 'O{2}','i',1,1) | ++-----------------------------------------+ +| TooDB | ++-----------------------------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT REGEXP_REPLACE('TooDB', 'O{2}','i',1,1,'i'); +``` + +``` ++---------------------------------------------+ +| REGEXP_REPLACE('TooDB', 'O{2}','i',1,1,'i') | ++---------------------------------------------+ +| TiDB | ++---------------------------------------------+ +1 row in set (0.00 sec) +``` + ### [`REGEXP_SUBSTR()`](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-substr) Return the substring that matches the regular expression (Partly compatible with MySQL. For more details, see [Regular expression compatibility with MySQL](#regular-expression-compatibility-with-mysql)). +The `REGEXP_SUBSTR(str, regexp, [start, [match, [match_type]]])` function is used to get a substring based on a regular expression. + +The following example uses the `Ti.{2}` regular expression to get the `TiDB` substring of the `This is TiDB` string. + +```sql +SELECT REGEXP_SUBSTR('This is TiDB','Ti.{2}'); +``` + +``` ++----------------------------------------+ +| REGEXP_SUBSTR('This is TiDB','Ti.{2}') | ++----------------------------------------+ +| TiDB | ++----------------------------------------+ +1 row in set (0.00 sec) +``` + ### [`REPEAT()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_repeat) Repeat a string the specified number of times. @@ -1400,7 +2082,7 @@ Return the specified rightmost number of characters. ### [`RLIKE`](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_regexp) -Synonym for `REGEXP`. +Synonym for [`REGEXP`](#regexp). ### [`RPAD()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_rpad) @@ -1641,7 +2323,7 @@ Output: ## Regular expression compatibility with MySQL -The following sections describe the regular expression compatibility with MySQL. +The following sections describe the regular expression compatibility with MySQL, including `REGEXP_INSTR()`, `REGEXP_LIKE()`, `REGEXP_REPLACE()`, and `REGEXP_SUBSTR()`. ### Syntax compatibility @@ -1658,6 +2340,15 @@ The value options of `match_type` between TiDB and MySQL are: - TiDB does not support `"u"`, which means Unix-only line endings in MySQL. +| `match_type` | MySQL | TiDB | Description | +|:------------:|-------|------|----------------------------------------| +| c | Yes | Yes | Case-sensitive matching | +| i | Yes | Yes | Case-insensitive matching | +| m | Yes | Yes | Multi-line mode | +| s | No | Yes | Matches newlines, same as `n` in MySQL | +| n | Yes | No | Matches newlines, same as `s` in TiDB | +| u | Yes | No | UNIX™ line endings | + ### Data type compatibility The difference between TiDB and MySQL support for the binary string type: @@ -1679,3 +2370,7 @@ The difference between TiDB and MySQL support for the binary string type: ```sql SELECT REGEXP_REPLACE('abcd','(.*)(.{2})$','\\1') AS s; ``` + +### Known issues + +- [GitHub Issue #37981](https://github.com/pingcap/tidb/issues/37981) \ No newline at end of file diff --git a/functions-and-operators/tidb-functions.md b/functions-and-operators/tidb-functions.md index ee86bd482b449..790b3888821be 100644 --- a/functions-and-operators/tidb-functions.md +++ b/functions-and-operators/tidb-functions.md @@ -149,7 +149,7 @@ ORDER BY 4 rows in set (0.031 sec) ``` -`TIDB_DECODE_KEY` returns valid JSON on success and retuns the argument value if it fails to decode. +`TIDB_DECODE_KEY` returns valid JSON on success and returns the argument value if it fails to decode. ### TIDB_DECODE_PLAN @@ -333,12 +333,14 @@ To enable the checksum feature of single-row data in TiDB (controlled by the sys SET GLOBAL tidb_enable_row_level_checksum = ON; ``` +This configuration only takes effect for newly created sessions, so you need to reconnect to TiDB. + Create table `t` and insert data: ```sql USE test; -CREATE TABLE t (id INT PRIMARY KEY, k INT, c int); -INSERT INTO TABLE t values (1, 10, a); +CREATE TABLE t (id INT PRIMARY KEY, k INT, c CHAR(1)); +INSERT INTO t values (1, 10, 'a'); ``` The following statement shows how to query the checksum value of the row where `id = 1` in table `t`: diff --git a/get-started-with-tidb-lightning.md b/get-started-with-tidb-lightning.md index 9f929d33ab047..55605a5577bd6 100644 --- a/get-started-with-tidb-lightning.md +++ b/get-started-with-tidb-lightning.md @@ -1,6 +1,7 @@ --- title: Quick Start for TiDB Lightning aliases: ['/docs/dev/get-started-with-tidb-lightning/','/docs/dev/how-to/get-started/tidb-lightning/'] +summary: TiDB Lightning is a tool for importing MySQL data into a TiDB cluster. It is recommended for test and trial purposes only, not for production or development environments. The process involves preparing full backup data, deploying the TiDB cluster, installing TiDB Lightning, starting TiDB Lightning, and checking data integrity. For detailed features and usage, refer to the TiDB Lightning Overview. --- # Quick Start for TiDB Lightning diff --git a/grafana-pd-dashboard.md b/grafana-pd-dashboard.md index 13790d5aa6f41..048dc0c7b3d69 100644 --- a/grafana-pd-dashboard.md +++ b/grafana-pd-dashboard.md @@ -78,7 +78,7 @@ The following is the description of PD Dashboard metrics items: - Store Write rate keys: The total written keys on each TiKV instance - Hot cache write entry number: The number of peers on each TiKV instance that are in the write hotspot statistics module - Selector events: The event count of Selector in the hotspot scheduling module -- Direction of hotspot move leader: The direction of leader movement in the hotspot scheduling. The positive number means scheduling into the instance. The negtive number means scheduling out of the instance +- Direction of hotspot move leader: The direction of leader movement in the hotspot scheduling. The positive number means scheduling into the instance. The negative number means scheduling out of the instance - Direction of hotspot move peer: The direction of peer movement in the hotspot scheduling. The positive number means scheduling into the instance. The negative number means scheduling out of the instance ![PD Dashboard - Hot write metrics](/media/pd-dashboard-hotwrite-v4.png) diff --git a/grafana-tikv-dashboard.md b/grafana-tikv-dashboard.md index d39f714025f05..18579e8bd2fcf 100644 --- a/grafana-tikv-dashboard.md +++ b/grafana-tikv-dashboard.md @@ -459,6 +459,41 @@ This section provides a detailed description of these key metrics on the **TiKV- - Encrypt/decrypt data nanos: The histogram of duration on encrypting/decrypting data each time - Read/write encryption meta duration: The time consumed for reading/writing encryption meta files +### Log Backup + +- Handle Event Rate: The speed of handling write events +- Initial Scan Generate Event Throughput: Incremental scanning speed when generating a new listener stream +- Abnormal Checkpoint TS Lag: The lag of the current checkpoint TS to the present time for each task +- Memory Of Events: An estimated amount of memory occupied by temporary data generated by incremental scanning +- Observed Region Count: The number of Regions currently listened to +- Errors: The number and type of retryable and non-fatal errors +- Fatal Errors: The number and type of fatal errors. Usually, fatal errors cause the task to be paused. +- Checkpoint TS of Tasks: Checkpoint TS for each task +- Flush Duration: The heat map of how long it takes for moving cached data to external storage +- Initial Scanning Duration: The heat map of how long it takes for incremental scanning when creating a new listening stream +- Convert Raft Event Duration: The heat map of how long it takes to transform a Raft log entry into backup data after creating a listening stream +- Command Batch Size: The batch size (within a single Raft group) of the listening Raft command +- Save to Temp File Duration: The heat map of how long it takes to temporarily store a batch of backup data (spanning several tasks) into the temporary file area +- Write to Temp File Duration: The heat map of how long it takes to temporarily store a batch of backup data (from a particular task) into the temporary file area +- System Write Call Duration: The heat map of how long it takes to write a batch of backup data (from a Region) to a temporary file +- Internal Message Type: The type of messages received by the actor responsible for the log backup within TiKV +- Internal Message Handling Duration (P90|P99): The speed of consuming and processing each type of messages +- Initial Scan RocksDB Throughput: The read traffic generated by RocksDB internal logging during incremental scanning +- Initial Scan RocksDB Operation: The number of individual operations logged internally by RocksDB during incremental scanning +- Initial Scanning Trigger Reason: The reason for triggering incremental scanning +- Region Checkpoint Key Putting: The number of checkpoint operations logged to the PD + +> **Note:** +> +> The following monitoring metrics all use TiDB nodes as their data source, but they have some impact on the log backup process. Therefore, they are placed in the **TiKV Details** dashboard for ease of reference. TiKV actively pushes progress most of the time, but it is normal for some of the following monitoring metrics to occasionally not have sampled data. + +- Request Checkpoint Batch Size: The request batch size when the log backup coordinator requests checkpoint information for each TiKV +- Tick Duration \[P99|P90\]: The time taken by the tick inside the coordinator +- Region Checkpoint Failure Reason: The reason why a Region checkpoint cannot advance within the coordinator +- Request Result: The record of the coordinator's success or failure in advancing the Region checkpoint +- Get Region Operation Count: The number of times the coordinator requests Region information from the PD +- Try Advance Trigger Time: The time taken for the coordinator to attempt to advance the checkpoint + ### Explanation of Common Parameters #### gRPC Message Type diff --git a/hardware-and-software-requirements.md b/hardware-and-software-requirements.md index a77f808439766..944b719df7b68 100644 --- a/hardware-and-software-requirements.md +++ b/hardware-and-software-requirements.md @@ -38,6 +38,10 @@ As an open-source distributed SQL database with high performance, TiDB can be de Amazon Linux 2
  • x86_64
  • ARM 64
+ + Amazon Linux 2023 +
  • x86_64
  • ARM 64
+ Rocky Linux 9.1 or later
  • x86_64
  • ARM 64
@@ -51,7 +55,7 @@ As an open-source distributed SQL database with high performance, TiDB can be de
  • x86_64
  • ARM 64
- openEuler 22.03 LTS SP1 + openEuler 22.03 LTS SP1/SP3
  • x86_64
  • ARM 64
@@ -102,7 +106,7 @@ As an open-source distributed SQL database with high performance, TiDB can be de | Libraries required for compiling and running TiDB | Version | | :--- | :--- | | Golang | 1.21 or later | -| Rust | nightly-2022-07-31 or later | +| Rust | nightly-2023-12-28 or later | | GCC | 7.x | | LLVM | 13.0 or later | diff --git a/information-schema/information-schema-analyze-status.md b/information-schema/information-schema-analyze-status.md index b7c21f047f117..4a57f1433ed7d 100644 --- a/information-schema/information-schema-analyze-status.md +++ b/information-schema/information-schema-analyze-status.md @@ -11,8 +11,6 @@ Starting from TiDB v6.1.0, the `ANALYZE_STATUS` table supports showing cluster-l Starting from TiDB v6.1.0, you can view the history tasks within the last 7 days through the system table `mysql.analyze_jobs`. -{{< copyable "sql" >}} - ```sql USE information_schema; DESC analyze_status; @@ -77,3 +75,8 @@ Fields in the `ANALYZE_STATUS` table are described as follows: * `REMAINING_SECONDS`: The estimated time (in seconds) remaining for the task to complete. * `PROGRESS`: The progress of the task. * `ESTIMATED_TOTAL_ROWS`: The total rows that need to be analyzed by the task. + +## See also + +- [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) +- [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md) \ No newline at end of file diff --git a/information-schema/information-schema-character-sets.md b/information-schema/information-schema-character-sets.md index 82d195decba6c..b94170bb5f8ff 100644 --- a/information-schema/information-schema-character-sets.md +++ b/information-schema/information-schema-character-sets.md @@ -54,3 +54,10 @@ The description of columns in the `CHARACTER_SETS` table is as follows: * `DEFAULT_COLLATE_NAME` The default collation name of the character set. * `DESCRIPTION` The description of the character set. * `MAXLEN` The maximum length required to store a character in this character set. + +## See also + +- [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) +- [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) +- [`INFORMATION_SCHEMA.COLLATIONS`](/information-schema/information-schema-collations.md) +- [`INFORMATION_SCHEMA.COLLATION_CHARACTER_SET_APPLICABILITY`](/information-schema/information-schema-collation-character-set-applicability.md) \ No newline at end of file diff --git a/information-schema/information-schema-collation-character-set-applicability.md b/information-schema/information-schema-collation-character-set-applicability.md index 5650b4e35144a..4c2f1509ed2ec 100644 --- a/information-schema/information-schema-collation-character-set-applicability.md +++ b/information-schema/information-schema-collation-character-set-applicability.md @@ -47,3 +47,10 @@ The description of columns in the `COLLATION_CHARACTER_SET_APPLICABILITY` table * `COLLATION_NAME`: The name of the collation. * `CHARACTER_SET_NAME`: The name of the character set which the collation belongs to. + +## See also + +- [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) +- [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) +- [`INFORMATION_SCHEMA.CHARACTER_SETS`](/information-schema/information-schema-character-sets.md) +- [`INFORMATION_SCHEMA.COLLATIONS`](/information-schema/information-schema-collations.md) \ No newline at end of file diff --git a/information-schema/information-schema-collations.md b/information-schema/information-schema-collations.md index 4278891894494..cdb3f9c4a9af2 100644 --- a/information-schema/information-schema-collations.md +++ b/information-schema/information-schema-collations.md @@ -53,3 +53,10 @@ The description of columns in the `COLLATIONS` table is as follows: * `IS_DEFAULT`: Whether this collation is the default collation of the character set it belongs to. * `IS_COMPILED`: Whether the character set is compiled into the server. * `SORTLEN`: The minimum length of memory allocated when the collation sorts characters. + +## See also + +- [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) +- [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) +- [`INFORMATION_SCHEMA.CHARACTER_SETS`](/information-schema/information-schema-character-sets.md) +- [`INFORMATION_SCHEMA.COLLATION_CHARACTER_SET_APPLICABILITY`](/information-schema/information-schema-collation-character-set-applicability.md) \ No newline at end of file diff --git a/information-schema/information-schema-columns.md b/information-schema/information-schema-columns.md index 5c675bf4a56d8..fd564f293ee55 100644 --- a/information-schema/information-schema-columns.md +++ b/information-schema/information-schema-columns.md @@ -121,4 +121,8 @@ The output is as follows: | a | int(11) | YES | | NULL | | +-------+---------+------+------+---------+-------+ 1 row in set (0.00 sec) -``` \ No newline at end of file +``` + +## See also + +- [`SHOW COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) \ No newline at end of file diff --git a/information-schema/information-schema-ddl-jobs.md b/information-schema/information-schema-ddl-jobs.md index 21f4b69ae5304..c1ec1749cfe8c 100644 --- a/information-schema/information-schema-ddl-jobs.md +++ b/information-schema/information-schema-ddl-jobs.md @@ -7,8 +7,6 @@ summary: Learn the `DDL_JOBS` information_schema table. The `DDL_JOBS` table provides an `INFORMATION_SCHEMA` interface to the [`ADMIN SHOW DDL JOBS`](/sql-statements/sql-statement-admin-show-ddl.md) command. It provides information about DDL operations in the TiDB cluster, such as the current status, DDL statements, start time, end time, database names, and table names. -{{< copyable "sql" >}} - ```sql USE information_schema; DESC ddl_jobs; @@ -34,8 +32,6 @@ DESC ddl_jobs; 12 rows in set (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql SELECT * FROM ddl_jobs LIMIT 3\G ``` diff --git a/information-schema/information-schema-deadlocks.md b/information-schema/information-schema-deadlocks.md index e755d59cda0f9..debbeb65834c2 100644 --- a/information-schema/information-schema-deadlocks.md +++ b/information-schema/information-schema-deadlocks.md @@ -12,7 +12,7 @@ USE INFORMATION_SCHEMA; DESC deadlocks; ``` -Thhe output is as follows: +The output is as follows: ```sql +-------------------------+---------------------+------+------+---------+-------+ diff --git a/information-schema/information-schema-engines.md b/information-schema/information-schema-engines.md index cf185b60fe483..734bad64b44b6 100644 --- a/information-schema/information-schema-engines.md +++ b/information-schema/information-schema-engines.md @@ -7,8 +7,6 @@ summary: Learn the `ENGINES` information_schema table. The `ENGINES` table provides information about storage engines. For compatibility, TiDB will always describe InnoDB as the only supported engine. In addition, other column values in the `ENGINES` table are also fixed values. -{{< copyable "sql" >}} - ```sql USE information_schema; DESC engines; @@ -28,8 +26,6 @@ DESC engines; 6 rows in set (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql SELECT * FROM engines; ``` @@ -50,4 +46,8 @@ The description of columns in the `ENGINES` table is as follows: * `COMMENT`: The brief comment on the storage engine. * `TRANSACTIONS`: Whether the storage engine supports transactions. * `XA`: Whether the storage engine supports XA transactions. -* `SAVEPOINTS`: Whether the storage engine supports `savepoints`. \ No newline at end of file +* `SAVEPOINTS`: Whether the storage engine supports `savepoints`. + +## See also + +- [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) \ No newline at end of file diff --git a/information-schema/information-schema-memory-usage.md b/information-schema/information-schema-memory-usage.md index 63cb732f698d1..885d143881d1b 100644 --- a/information-schema/information-schema-memory-usage.md +++ b/information-schema/information-schema-memory-usage.md @@ -59,3 +59,19 @@ The columns in the `MEMORY_USAGE` table are described as follows: * GC_TOTAL: The number of times Golang GC is triggered by memory usage, from the time TiDB is started to the current time. * DISK_USAGE: The disk usage for the current data spill operation, in bytes. * QUERY_FORCE_DISK: The number of times data is spilled to disk, from the time TiDB is started to the current time. + +## See also + + + +- [TiDB memory control](/configure-memory-usage.md) +- [Tune TiKV memory parameter performance](/tune-tikv-memory-performance.md) + + + + + +- [TiDB memory control](https://docs.pingcap.com/tidb/stable/configure-memory-usage) +- [Tune TiKV memory parameter performance](https://docs.pingcap.com/tidb/stable/tune-tikv-memory-performance) + + \ No newline at end of file diff --git a/information-schema/information-schema-partitions.md b/information-schema/information-schema-partitions.md index e9c1023be0bd7..044474113732e 100644 --- a/information-schema/information-schema-partitions.md +++ b/information-schema/information-schema-partitions.md @@ -5,7 +5,7 @@ summary: Learn the `PARTITIONS` INFORMATION_SCHEMA table. # PARTITIONS -The `PARTITIONS` table provides information about partitioned tables. +The `PARTITIONS` table provides information about [partitioned tables](/partitioned-table.md). ```sql USE INFORMATION_SCHEMA; @@ -115,3 +115,7 @@ SUBPARTITION_ORDINAL_POSITION: NULL TIDB_PLACEMENT_POLICY_NAME: NULL 2 rows in set (0.00 sec) ``` + +## See also + +- [Explain statements using partitions](/explain-partitions.md) \ No newline at end of file diff --git a/information-schema/information-schema-processlist.md b/information-schema/information-schema-processlist.md index 919ac63b9da20..7d228fe385e2a 100644 --- a/information-schema/information-schema-processlist.md +++ b/information-schema/information-schema-processlist.md @@ -5,7 +5,7 @@ summary: Learn the `PROCESSLIST` information_schema table. # PROCESSLIST -`PROCESSLIST`, just like `SHOW PROCESSLIST`, is used to view the requests that are being handled. +`PROCESSLIST`, just like [`SHOW PROCESSLIST`](/sql-statements/sql-statement-show-processlist.md), is used to view the requests that are being handled. The `PROCESSLIST` table has additional columns not present in `SHOW PROCESSLIST`: @@ -15,8 +15,6 @@ The `PROCESSLIST` table has additional columns not present in `SHOW PROCESSLIST` * A `TxnStart` column to show the start time of the transaction. * A `RESOURCE_GROUP` column to show the resource group name. -{{< copyable "sql" >}} - ```sql USE information_schema; DESC processlist; @@ -43,8 +41,6 @@ DESC processlist; 13 rows in set (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql SELECT * FROM processlist\G ``` @@ -87,8 +83,6 @@ Fields in the `PROCESSLIST` table are described as follows: `CLUSTER_PROCESSLIST` is the cluster system table corresponding to `PROCESSLIST`. It is used to query the `PROCESSLIST` information of all TiDB nodes in the cluster. The table schema of `CLUSTER_PROCESSLIST` has one more column than `PROCESSLIST`, the `INSTANCE` column, which stores the address of the TiDB node this row of data is from. -{{< copyable "sql" >}} - ```sql SELECT * FROM information_schema.cluster_processlist; ``` diff --git a/information-schema/information-schema-referential-constraints.md b/information-schema/information-schema-referential-constraints.md index 5ca13f52f72ae..a24c4625f069a 100644 --- a/information-schema/information-schema-referential-constraints.md +++ b/information-schema/information-schema-referential-constraints.md @@ -5,7 +5,7 @@ summary: Learn the `REFERENTIAL_CONSTRAINTS` INFORMATION_SCHEMA table. # REFERENTIAL_CONSTRAINTS -The `REFERENTIAL_CONSTRAINTS` table provides information about `FOREIGN KEY` relationships between tables. +The `REFERENTIAL_CONSTRAINTS` table provides information about [`FOREIGN KEY`](/foreign-key.md) relationships between tables. ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-runaway-watches.md b/information-schema/information-schema-runaway-watches.md index d36711fb69f62..f65d557e574b5 100644 --- a/information-schema/information-schema-runaway-watches.md +++ b/information-schema/information-schema-runaway-watches.md @@ -32,16 +32,12 @@ DESC RUNAWAY_WATCHES; 8 rows in set (0.00 sec) ``` -> **Warning:** -> -> This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. - ## Examples Query the watch list of runaway queries: ```sql -SELECT * FROM INFORMATION_SCHEMA.RUNAWAY_WATCHES; +SELECT * FROM INFORMATION_SCHEMA.RUNAWAY_WATCHES\G ``` The output is as follows: @@ -77,7 +73,7 @@ QUERY WATCH ADD RESOURCE GROUP rg1 SQL TEXT EXACT TO 'select * from sbtest.sbtes Query the watch list of runaway queries again: ```sql -SELECT * FROM INFORMATION_SCHEMA.RUNAWAY_WATCHES\G; +SELECT * FROM INFORMATION_SCHEMA.RUNAWAY_WATCHES\G ``` The output is as follows: diff --git a/information-schema/information-schema-schemata.md b/information-schema/information-schema-schemata.md index 9e8305c35390c..b94c625ca16c6 100644 --- a/information-schema/information-schema-schemata.md +++ b/information-schema/information-schema-schemata.md @@ -5,9 +5,7 @@ summary: Learn the `SCHEMATA` information_schema table. # SCHEMATA -The `SCHEMATA` table provides information about databases. The table data is equivalent to the result of the `SHOW DATABASES` statement. - -{{< copyable "sql" >}} +The `SCHEMATA` table provides information about databases. The table data is equivalent to the result of the [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) statement. ```sql USE information_schema; @@ -27,8 +25,6 @@ desc SCHEMATA; 5 rows in set (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql SELECT * FROM SCHEMATA; ``` diff --git a/information-schema/information-schema-sequences.md b/information-schema/information-schema-sequences.md index 60a0cb408e2ff..5666651f86745 100644 --- a/information-schema/information-schema-sequences.md +++ b/information-schema/information-schema-sequences.md @@ -37,7 +37,7 @@ Create a sequence `test.seq` and query the next value of the sequence: ```sql CREATE SEQUENCE test.seq; -SELECT nextval(test.seq); +SELECT NEXTVAL(test.seq); SELECT * FROM sequences\G ``` @@ -45,7 +45,7 @@ The output is as follows: ```sql +-------------------+ -| nextval(test.seq) | +| NEXTVAL(test.seq) | +-------------------+ | 1 | +-------------------+ @@ -75,3 +75,11 @@ SEQUENCE_SCHEMA: test COMMENT: 1 row in set (0.00 sec) ``` + +## See also + +- [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) +- [`SHOW CREATE SEQUENCE`](/sql-statements/sql-statement-show-create-sequence.md) +- [`ALTER SEQUENCE`](/sql-statements/sql-statement-alter-sequence.md) +- [`DROP SEQUENCE`](/sql-statements/sql-statement-drop-sequence.md) +- [Sequence functions](/functions-and-operators/sequence-functions.md) \ No newline at end of file diff --git a/information-schema/information-schema-session-variables.md b/information-schema/information-schema-session-variables.md index df941a7a20846..22c604d7c04e5 100644 --- a/information-schema/information-schema-session-variables.md +++ b/information-schema/information-schema-session-variables.md @@ -5,7 +5,7 @@ summary: Learn the `SESSION_VARIABLES` INFORMATION_SCHEMA table. # SESSION_VARIABLES -The `SESSION_VARIABLES` table provides information about session variables. The table data is similar to the result of the `SHOW SESSION VARIABLES` statement. +The `SESSION_VARIABLES` table provides information about session variables. The table data is similar to the result of the [`SHOW SESSION VARIABLES`](/sql-statements/sql-statement-show-variables.md) statement. ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-slow-query.md b/information-schema/information-schema-slow-query.md index 0dc2c9dc6f3b8..b8fe57c66fe87 100644 --- a/information-schema/information-schema-slow-query.md +++ b/information-schema/information-schema-slow-query.md @@ -5,7 +5,17 @@ summary: Learn the `SLOW_QUERY` INFORMATION_SCHEMA table. # SLOW_QUERY -The `SLOW_QUERY` table provides the slow query information of the current node, which is the parsing result of the TiDB slow log file. The column names in the table are corresponding to the field names in the slow log. + + +The `SLOW_QUERY` table provides the slow query information of the current node, which is the parsing result of the TiDB [slow log file](/tidb-configuration-file.md#slow-query-file). The column names in the table are corresponding to the field names in the slow log. + + + + + +The `SLOW_QUERY` table provides the slow query information of the current node, which is the parsing result of the TiDB [slow log file](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#slow-query-file). The column names in the table are corresponding to the field names in the slow log. + + > **Note:** > @@ -24,7 +34,7 @@ DESC SLOW_QUERY; The output is as follows: -```sqlsql +``` +-------------------------------+---------------------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------------------------------+---------------------+------+------+---------+-------+ diff --git a/information-schema/information-schema-table-constraints.md b/information-schema/information-schema-table-constraints.md index 2b20328a69d50..5df7965a9b47b 100644 --- a/information-schema/information-schema-table-constraints.md +++ b/information-schema/information-schema-table-constraints.md @@ -5,7 +5,7 @@ summary: Learn the `TABLE_CONSTRAINTS` information_schema table. # TABLE_CONSTRAINTS -The `TABLE_CONSTRAINTS` table describes which tables have constraints. +The `TABLE_CONSTRAINTS` table describes which tables have [constraints](/constraints.md). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-tidb-servers-info.md b/information-schema/information-schema-tidb-servers-info.md index c51b6f8ff7806..dacdc9eecd392 100644 --- a/information-schema/information-schema-tidb-servers-info.md +++ b/information-schema/information-schema-tidb-servers-info.md @@ -50,7 +50,7 @@ The output is as follows: PORT: 4000 STATUS_PORT: 10080 LEASE: 45s - VERSION: 8.0.11-TiDB-v8.0.0 + VERSION: 8.0.11-TiDB-v8.1.0 GIT_HASH: 827d8ff2d22ac4c93ae1b841b79d468211e1d393 BINLOG_STATUS: Off LABELS: diff --git a/information-schema/information-schema-tikv-region-status.md b/information-schema/information-schema-tikv-region-status.md index f9d225ee66b5d..417ded9a76734 100644 --- a/information-schema/information-schema-tikv-region-status.md +++ b/information-schema/information-schema-tikv-region-status.md @@ -11,13 +11,13 @@ The `TIKV_REGION_STATUS` table shows some basic information of TiKV Regions via > > This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. -{{< copyable "sql" >}} - ```sql -USE information_schema; -DESC tikv_region_status; +USE INFORMATION_SCHEMA; +DESC TIKV_REGION_STATUS; ``` +The output is as follows: + ```sql +---------------------------+-------------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | @@ -31,6 +31,9 @@ DESC tikv_region_status; | IS_INDEX | tinyint(1) | NO | | 0 | | | INDEX_ID | bigint(21) | YES | | NULL | | | INDEX_NAME | varchar(64) | YES | | NULL | | +| IS_PARTITION | tinyint(1) | NO | | 0 | | +| PARTITION_ID | bigint(21) | YES | | NULL | | +| PARTITION_NAME | varchar(64) | YES | | NULL | | | EPOCH_CONF_VER | bigint(21) | YES | | NULL | | | EPOCH_VERSION | bigint(21) | YES | | NULL | | | WRITTEN_BYTES | bigint(21) | YES | | NULL | | @@ -40,7 +43,7 @@ DESC tikv_region_status; | REPLICATIONSTATUS_STATE | varchar(64) | YES | | NULL | | | REPLICATIONSTATUS_STATEID | bigint(21) | YES | | NULL | | +---------------------------+-------------+------+------+---------+-------+ -17 rows in set (0.00 sec) +20 rows in set (0.00 sec) ``` The descriptions of the columns in the `TIKV_REGION_STATUS` table are as follows: @@ -54,6 +57,9 @@ The descriptions of the columns in the `TIKV_REGION_STATUS` table are as follows * `IS_INDEX`: Whether the Region data is an index. 0 means that it is not an index, while 1 means that it is an index. If the current Region contains both table data and index data, there will be multiple rows of records, and `IS_INDEX` is 0 and 1 respectively. * `INDEX_ID`: The ID of the index to which the Region belongs. If `IS_INDEX` is 0, the value of this column is NULL. * `INDEX_NAME`: The name of the index to which the Region belongs. If `IS_INDEX` is 0, the value of this column is NULL. +* `IS_PARTITION`: Whether the table to which the Region belongs is partitioned. +* `PARTITION_ID`: If the table to which the Region belongs is partitioned, this column displays the ID of the partition to which the Region belongs. +* `PARTITION_NAME`: If the table to which the Region belongs is partitioned, this column displays the name of the partition to which the Region belongs. * `EPOCH_CONF_VER`: The version number of the Region configuration. The version number increases when a peer is added or removed. * `EPOCH_VERSION`: The current version number of the Region. The version number increases when the Region is split or merged. * `WRITTEN_BYTES`: The amount of data (bytes) written to the Region. diff --git a/information-schema/information-schema-user-privileges.md b/information-schema/information-schema-user-privileges.md index 06c8e1507a9d6..d65ca027078cf 100644 --- a/information-schema/information-schema-user-privileges.md +++ b/information-schema/information-schema-user-privileges.md @@ -126,3 +126,7 @@ Fields in the `USER_PRIVILEGES` table are described as follows: * `TABLE_CATALOG`: The name of the catalog to which the table belongs. This value is always `def`. * `PRIVILEGE_TYPE`: The privilege type to be granted. Only one privilege type is shown in each row. * `IS_GRANTABLE`: If you have the `GRANT OPTION` privilege, the value is `YES`; otherwise, the value is `NO`. + +## See also + +- [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) diff --git a/information-schema/information-schema-variables-info.md b/information-schema/information-schema-variables-info.md index 6ed4dc7bec579..ed5eb70ad42a3 100644 --- a/information-schema/information-schema-variables-info.md +++ b/information-schema/information-schema-variables-info.md @@ -5,7 +5,7 @@ summary: Learn the `VARIABLES_INFO` information_schema table. # VARIABLES_INFO -The `VARIABLES_INFO` table provides information about the default value, current value, and scope of system variables in the current TiDB instance or TiDB cluster. +The `VARIABLES_INFO` table provides information about the default value, current value, and scope of [system variables](/system-variables.md) in the current TiDB instance or TiDB cluster. ```sql USE information_schema; diff --git a/information-schema/information-schema-views.md b/information-schema/information-schema-views.md index da3e4b15bf9c3..a3e796d503e50 100644 --- a/information-schema/information-schema-views.md +++ b/information-schema/information-schema-views.md @@ -5,7 +5,7 @@ summary: Learn the `VIEWS` INFORMATION_SCHEMA table. # VIEWS -The `VIEWS` table provides information about SQL views. +The `VIEWS` table provides information about [SQL views](/views.md). ```sql USE INFORMATION_SCHEMA; @@ -67,4 +67,9 @@ Fields in the `VIEWS` table are described as follows: * `DEFINER`: The name of the user who creates the view, which is in the format of `'user_name'@'host_name'`. * `SECURITY_TYPE`: The value of `SQL SECURITY`. The value options are `DEFINER` and `INVOKER`. * `CHARACTER_SET_CLIENT`: The value of the `character_set_client` session variable when the view is created. -* `COLLATION_CONNECTION`: The value of the `collation_connection` session variable when the view is created. \ No newline at end of file +* `COLLATION_CONNECTION`: The value of the `collation_connection` session variable when the view is created. + +## See also + +- [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) +- [`DROP VIEW`](/sql-statements/sql-statement-drop-view.md) \ No newline at end of file diff --git a/information-schema/information-schema.md b/information-schema/information-schema.md index 43a5b47288636..16ce9f0e13812 100644 --- a/information-schema/information-schema.md +++ b/information-schema/information-schema.md @@ -8,7 +8,7 @@ aliases: ['/docs/dev/system-tables/system-table-information-schema/','/docs/dev/ Information Schema provides an ANSI-standard way of viewing system metadata. TiDB also provides a number of custom `INFORMATION_SCHEMA` tables, in addition to the tables included for MySQL compatibility. -Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefit of querying `INFORMATION_SCHEMA` is that it is possible to join between tables. +Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` statement. The benefit of querying `INFORMATION_SCHEMA` is that it is possible to join between tables. ## Tables for MySQL compatibility diff --git a/media/region-panel.png b/media/region-panel.png index 5b1fa4520c5ea..578a59cd87ef6 100644 Binary files a/media/region-panel.png and b/media/region-panel.png differ diff --git a/media/tidb-cloud/looker-studio-configure-connection.png b/media/tidb-cloud/looker-studio-configure-connection.png new file mode 100644 index 0000000000000..82f5055f897fa Binary files /dev/null and b/media/tidb-cloud/looker-studio-configure-connection.png differ diff --git a/media/tidb-cloud/looker-studio-custom-query.png b/media/tidb-cloud/looker-studio-custom-query.png new file mode 100644 index 0000000000000..8426e2da3c71a Binary files /dev/null and b/media/tidb-cloud/looker-studio-custom-query.png differ diff --git a/media/tidb-cloud/looker-studio-simple-chart.png b/media/tidb-cloud/looker-studio-simple-chart.png new file mode 100644 index 0000000000000..257f9b19203da Binary files /dev/null and b/media/tidb-cloud/looker-studio-simple-chart.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-failover.png b/media/tidb-cloud/recovery-group/recovery-group-failover.png new file mode 100644 index 0000000000000..ff2c15c37a520 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-failover.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-overview.png b/media/tidb-cloud/recovery-group/recovery-group-overview.png new file mode 100644 index 0000000000000..fe97b0067ce39 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-overview.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-protected.png b/media/tidb-cloud/recovery-group/recovery-group-protected.png new file mode 100644 index 0000000000000..5e44702cb732e Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-protected.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-reprotected.png b/media/tidb-cloud/recovery-group/recovery-group-reprotected.png new file mode 100644 index 0000000000000..7d1afe53ed384 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-reprotected.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-unprotected.png b/media/tidb-cloud/recovery-group/recovery-group-unprotected.png new file mode 100644 index 0000000000000..9597a81be24a5 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-unprotected.png differ diff --git a/migrate-from-csv-files-to-tidb.md b/migrate-from-csv-files-to-tidb.md index 38b8afa009f05..7cfdfdfee140f 100644 --- a/migrate-from-csv-files-to-tidb.md +++ b/migrate-from-csv-files-to-tidb.md @@ -63,6 +63,8 @@ data-source-dir = "${data-path}" # A local path or S3 path. For example, 's3://m separator = ',' # Delimiter. Can be zero or multiple characters. delimiter = '"' +# Line terminator. By default, \r, \n, and \r\n are all treated as line terminators. +# terminator = "\r\n" # Configures whether the CSV file has a table header. # If this item is set to true, TiDB Lightning uses the first line of the CSV file to parse the corresponding relationship of fields. header = true @@ -104,6 +106,8 @@ In a strict-format CSV file, each field only takes up one line. It must meet the - The delimiter is empty. - Each field does not contain CR (`\r`) or LF (`\n`). +You need to explicitly specify the line terminator `terminator` for a `strict-format` CSV file. + If your CSV file meets the above requirements, you can speed up the import by enabling the `strict-format` mode as follows: ```toml diff --git a/migrate-small-mysql-to-tidb.md b/migrate-small-mysql-to-tidb.md index 6846b8a792ab2..01c8731560ee2 100644 --- a/migrate-small-mysql-to-tidb.md +++ b/migrate-small-mysql-to-tidb.md @@ -137,8 +137,8 @@ To view the historical status of the migration task and other internal metrics, If you have deployed Prometheus, Alertmanager, and Grafana when deploying DM using TiUP, you can access Grafana using the IP address and port specified during the deployment. You can then select the DM dashboard to view DM-related monitoring metrics. -- The log directory of DM-master: specified by the DM-master process parameter `--log-file`. If you have deployd DM using TiUP, the log directory is `/dm-deploy/dm-master-8261/log/` by default. -- The log directory of DM-worker: specified by the DM-worker process parameter `--log-file`. If you have deployd DM using TiUP, the log directory is `/dm-deploy/dm-worker-8262/log/` by default. +- The log directory of DM-master: specified by the DM-master process parameter `--log-file`. If you have deployed DM using TiUP, the log directory is `/dm-deploy/dm-master-8261/log/` by default. +- The log directory of DM-worker: specified by the DM-worker process parameter `--log-file`. If you have deployed DM using TiUP, the log directory is `/dm-deploy/dm-worker-8262/log/` by default. ## What's next diff --git a/migrate-with-pt-ghost.md b/migrate-with-pt-ghost.md index 6b49e1c677523..4505e7a38c3f6 100644 --- a/migrate-with-pt-ghost.md +++ b/migrate-with-pt-ghost.md @@ -7,7 +7,7 @@ summary: Learn how to use DM to replicate incremental data from databases that u In production scenarios, table locking during DDL execution can block the reads from or writes to the database to a certain extent. Therefore, online DDL tools are often used to execute DDLs to minimize the impact on reads and writes. Common DDL tools are [gh-ost](https://github.com/github/gh-ost) and [pt-osc](https://www.percona.com/doc/percona-toolkit/3.0/pt-online-schema-change.html). -When using DM to migrate data from MySQL to TiDB, you can enbale `online-ddl` to allow collaboration of DM and gh-ost or pt-osc. +When using DM to migrate data from MySQL to TiDB, you can enable `online-ddl` to allow collaboration of DM and gh-ost or pt-osc. For the detailed replication instructions, refer to the following documents by scenarios: diff --git a/mysql-compatibility.md b/mysql-compatibility.md index 0851074746c9a..995aadb056742 100644 --- a/mysql-compatibility.md +++ b/mysql-compatibility.md @@ -56,7 +56,6 @@ You can try out TiDB features on [TiDB Playground](https://play.tidbcloud.com/?u + `FULLTEXT` syntax and indexes [#1793](https://github.com/pingcap/tidb/issues/1793) + `SPATIAL` (also known as `GIS`/`GEOMETRY`) functions, data types and indexes [#6347](https://github.com/pingcap/tidb/issues/6347) + Character sets other than `ascii`, `latin1`, `binary`, `utf8`, `utf8mb4`, and `gbk`. -+ SYS schema + Optimizer trace + XML Functions + X-Protocol [#1109](https://github.com/pingcap/tidb/issues/1109) @@ -158,9 +157,7 @@ For more information, refer to [Understand the Query Execution Plan](/explain-ov ### Built-in functions -TiDB supports most of the built-in functions in MySQL, but not all. You can use the statement `SHOW BUILTINS` to get a list of the available functions. - -For more information, refer to the [TiDB SQL Grammar](https://pingcap.github.io/sqlgram/#functioncallkeyword). +TiDB supports most of the built-in functions in MySQL, but not all. You can use the statement [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) to get a list of the available functions. ### DDL operations @@ -277,6 +274,10 @@ The following column types are supported by MySQL but **not** by TiDB: - `SQL_TSI_*` (includes SQL_TSI_MONTH, SQL_TSI_WEEK, SQL_TSI_DAY, SQL_TSI_HOUR, SQL_TSI_MINUTE, and SQL_TSI_SECOND, but excludes SQL_TSI_YEAR) +### Regular expressions + +For information about TiDB regular expression compatibility with MySQL, including `REGEXP_INSTR()`, `REGEXP_LIKE()`, `REGEXP_REPLACE()`, and `REGEXP_SUBSTR()`, see [Regular expression compatibility with MySQL](/functions-and-operators/string-functions.md#regular-expression-compatibility-with-mysql). + ### Incompatibility due to deprecated features TiDB does not implement specific features deprecated in MySQL, including: diff --git a/mysql-schema.md b/mysql-schema.md index 02d95816d31e5..95be0fc1bc10b 100644 --- a/mysql-schema.md +++ b/mysql-schema.md @@ -69,10 +69,6 @@ Currently, the `help_topic` is NULL. ## TTL related system tables -> **Note:** - -> The TTL related system tables are not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - * `tidb_ttl_table_status`: the previously executed TTL job and ongoing TTL job for all TTL tables * `tidb_ttl_task`: the current ongoing TTL subtasks * `tidb_ttl_job_history`: the execution history of TTL tasks in the last 90 days diff --git a/online-unsafe-recovery.md b/online-unsafe-recovery.md index 2310c2b546671..591dc9b36c019 100644 --- a/online-unsafe-recovery.md +++ b/online-unsafe-recovery.md @@ -38,7 +38,7 @@ Before using Online Unsafe Recovery, make sure that the following requirements a ### Step 1. Specify the stores that cannot be recovered -To trigger automatic recovery, use PD Control to execute [`unsafe remove-failed-stores [,,...]`](/pd-control.md#unsafe-remove-failed-stores-store-ids--show) and specify **all** the TiKV nodes that cannot be recovered, seperated by commas. +To trigger automatic recovery, use PD Control to execute [`unsafe remove-failed-stores [,,...]`](/pd-control.md#unsafe-remove-failed-stores-store-ids--show) and specify **all** the TiKV nodes that cannot be recovered, separated by commas. {{< copyable "shell-regular" >}} @@ -174,7 +174,7 @@ After the recovery is completed, the data and index might be inconsistent. Use t ADMIN CHECK TABLE table_name; ``` -If there are inconsistent indexes, you can fix the index inconsistency by renaming the old index, creating a new index, and then droping the old index. +If there are inconsistent indexes, you can fix the index inconsistency by renaming the old index, creating a new index, and then dropping the old index. 1. Rename the old index: diff --git a/optimizer-fix-controls.md b/optimizer-fix-controls.md index e79f2613bb480..0fc68bc3fce56 100644 --- a/optimizer-fix-controls.md +++ b/optimizer-fix-controls.md @@ -70,3 +70,10 @@ SET SESSION tidb_opt_fix_control = '44262:ON,44389:ON'; - Possible values: `[0, 2147483647]` - This variable sets the threshold for the optimizer's heuristic strategy to select access paths. If the estimated rows for an access path (such as `Index_A`) is much smaller than that of other access paths (default `1000` times), the optimizer skips the cost comparison and directly selects `Index_A`. - `0` means to disable this heuristic strategy. + +### [`52869`](https://github.com/pingcap/tidb/issues/52869) New in v8.1.0 + +- Default value: `OFF` +- Possible values: `ON`, `OFF` +- As stated in the **Note** of [Explain Statements Using Index Merge](/explain-index-merge.md#examples), if the optimizer can choose the single index scan method (other than full table scan) for a query plan, the optimizer will not automatically use index merge. +- You can remove this limitation by enabling this fix control. Removing this limitation enables the optimizer to choose index merge automatically in more queries, but might cause the optimizer to ignore the optimal execution plans. Therefore, it is recommended to conduct sufficient tests on actual use cases before removing this limitation to make sure that it will not cause performance regressions. diff --git a/optimizer-hints.md b/optimizer-hints.md index b23dd8c14f294..5aeff467a2601 100644 --- a/optimizer-hints.md +++ b/optimizer-hints.md @@ -139,6 +139,10 @@ SELECT /*+ NO_MERGE_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id = t2.id; ### INL_JOIN(t1_name [, tl_name ...]) +> **Note:** +> +> In some cases, the `INL_JOIN` hint might not take effect. For more information, see [`INL_JOIN` hint does not take effect](#inl_join-hint-does-not-take-effect). + The `INL_JOIN(t1_name [, tl_name ...])` hint tells the optimizer to use the index nested loop join algorithm for the given table(s). This algorithm might consume less system resources and take shorter processing time in some scenarios and might produce an opposite result in other scenarios. If the result set is less than 10,000 rows after the outer table is filtered by the `WHERE` condition, it is recommended to use this hint. For example: {{< copyable "sql" >}} @@ -930,7 +934,74 @@ The warning is as follows: In this case, you need to place the hint directly after the `SELECT` keyword. For more details, see the [Syntax](#syntax) section. -### INL_JOIN, INL_HASH_JOIN, and INL_MERGE_JOIN hints do not take effect due to collation incompatibility +### `INL_JOIN` hint does not take effect + +#### `INL_JOIN` hint does not take effect when built-in functions are used on columns for joining tables + +In some cases, if you use a built-in function on a column that joins tables, the optimizer might fail to choose the `IndexJoin` plan, resulting in the `INL_JOIN` hint not taking effect either. + +For example, the following query uses the built-in function `substr` on the column `tname` that joins tables: + +```sql +CREATE TABLE t1 (id varchar(10) primary key, tname varchar(10)); +CREATE TABLE t2 (id varchar(10) primary key, tname varchar(10)); +EXPLAIN SELECT /*+ INL_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id=t2.id and SUBSTR(t1.tname,1,2)=SUBSTR(t2.tname,1,2); +``` + +The execution plan is as follows: + +```sql ++------------------------------+----------+-----------+---------------+-----------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++------------------------------+----------+-----------+---------------+-----------------------------------------------------------------------+ +| HashJoin_12 | 12500.00 | root | | inner join, equal:[eq(test.t1.id, test.t2.id) eq(Column#5, Column#6)] | +| ├─Projection_17(Build) | 10000.00 | root | | test.t2.id, test.t2.tname, substr(test.t2.tname, 1, 2)->Column#6 | +| │ └─TableReader_19 | 10000.00 | root | | data:TableFullScan_18 | +| │ └─TableFullScan_18 | 10000.00 | cop[tikv] | table:t2 | keep order:false, stats:pseudo | +| └─Projection_14(Probe) | 10000.00 | root | | test.t1.id, test.t1.tname, substr(test.t1.tname, 1, 2)->Column#5 | +| └─TableReader_16 | 10000.00 | root | | data:TableFullScan_15 | +| └─TableFullScan_15 | 10000.00 | cop[tikv] | table:t1 | keep order:false, stats:pseudo | ++------------------------------+----------+-----------+---------------+-----------------------------------------------------------------------+ +7 rows in set, 1 warning (0.01 sec) +``` + +```sql +SHOW WARNINGS; +``` + +``` ++---------+------+------------------------------------------------------------------------------------+ +| Level | Code | Message | ++---------+------+------------------------------------------------------------------------------------+ +| Warning | 1815 | Optimizer Hint /*+ INL_JOIN(t1, t2) */ or /*+ TIDB_INLJ(t1, t2) */ is inapplicable | ++---------+------+------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +As you can see from the preceding example, the `INL_JOIN` hint does not take effect. This is due to a limitation of the optimizer that prevents using the `Projection` or `Selection` operator as the probe side of `IndexJoin`. + +Starting from TiDB v8.0.0, you can avoid this issue by setting [`tidb_enable_inl_join_inner_multi_pattern`](/system-variables.md#tidb_enable_inl_join_inner_multi_pattern-new-in-v700) to `ON`. + +```sql +SET @@tidb_enable_inl_join_inner_multi_pattern=ON; +Query OK, 0 rows affected (0.00 sec) + +EXPLAIN SELECT /*+ INL_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id=t2.id AND SUBSTR(t1.tname,1,2)=SUBSTR(t2.tname,1,2); ++------------------------------+--------------+-----------+---------------+--------------------------------------------------------------------------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++------------------------------+--------------+-----------+---------------+--------------------------------------------------------------------------------------------------------------------------------------------+ +| IndexJoin_18 | 12500.00 | root | | inner join, inner:Projection_14, outer key:test.t1.id, inner key:test.t2.id, equal cond:eq(Column#5, Column#6), eq(test.t1.id, test.t2.id) | +| ├─Projection_32(Build) | 10000.00 | root | | test.t1.id, test.t1.tname, substr(test.t1.tname, 1, 2)->Column#5 | +| │ └─TableReader_34 | 10000.00 | root | | data:TableFullScan_33 | +| │ └─TableFullScan_33 | 10000.00 | cop[tikv] | table:t1 | keep order:false, stats:pseudo | +| └─Projection_14(Probe) | 100000000.00 | root | | test.t2.id, test.t2.tname, substr(test.t2.tname, 1, 2)->Column#6 | +| └─TableReader_13 | 10000.00 | root | | data:TableRangeScan_12 | +| └─TableRangeScan_12 | 10000.00 | cop[tikv] | table:t2 | range: decided by [eq(test.t2.id, test.t1.id)], keep order:false, stats:pseudo | ++------------------------------+--------------+-----------+---------------+--------------------------------------------------------------------------------------------------------------------------------------------+ +7 rows in set (0.00 sec) +``` + +### `INL_JOIN`, `INL_HASH_JOIN`, and `INL_MERGE_JOIN` hints do not take effect due to collation incompatibility When the collation of the join key is incompatible between two tables, the `IndexJoin` operator cannot be utilized to execute the query. In this case, the [`INL_JOIN`](#inl_joint1_name--tl_name-), [`INL_HASH_JOIN`](#inl_hash_join), and [`INL_MERGE_JOIN`](#inl_merge_join) hints do not take effect. For example: @@ -967,7 +1038,7 @@ SHOW WARNINGS; 1 row in set (0.00 sec) ``` -### `INL_JOIN` hint does not take effect because of join order +#### `INL_JOIN` hint does not take effect due to join order The [`INL_JOIN(t1, t2)`](#inl_joint1_name--tl_name-) or `TIDB_INLJ(t1, t2)` hint semantically instructs `t1` and `t2` to act as inner tables in an `IndexJoin` operator to join with other tables, rather than directly joining them using an `IndexJoin` operator. For example: diff --git a/oracle-functions-to-tidb.md b/oracle-functions-to-tidb.md index 49d1f7c4b3380..44472b49c9640 100644 --- a/oracle-functions-to-tidb.md +++ b/oracle-functions-to-tidb.md @@ -65,13 +65,13 @@ TiDB distinguishes between `NULL` and an empty string `''`. Oracle supports reading and writing to the same table in an `INSERT` statement. For example: ```sql -INSERT INTO table1 VALUES (feild1,(SELECT feild2 FROM table1 WHERE...)) +INSERT INTO table1 VALUES (field1,(SELECT field2 FROM table1 WHERE...)) ``` TiDB does not support reading and writing to the same table in a `INSERT` statement. For example: ```sql -INSERT INTO table1 VALUES (feild1,(SELECT T.fields2 FROM table1 T WHERE...)) +INSERT INTO table1 VALUES (field1,(SELECT T.fields2 FROM table1 T WHERE...)) ``` ### Get the first n rows from a query diff --git a/partitioned-table.md b/partitioned-table.md index 1f614f01babaa..1a70299593a64 100644 --- a/partitioned-table.md +++ b/partitioned-table.md @@ -610,7 +610,7 @@ Starting from v7.0.0, TiDB supports Key partitioning. For TiDB versions earlier Both Key partitioning and Hash partitioning can evenly distribute data into a certain number of partitions. The difference is that Hash partitioning only supports distributing data based on a specified integer expression or an integer column, while Key partitioning supports distributing data based on a column list, and partitioning columns of Key partitioning are not limited to the integer type. The Hash algorithm of TiDB for Key partitioning is different from that of MySQL, so the table data distribution is also different. -To create a Key partitioned table, you need to append a `PARTITION BY KEY (columList)` clause to the `CREATE TABLE` statement. `columList` is a column list with one or more column names. The data type of each column in the list can be any type except `BLOB`, `JSON`, and `GEOMETRY` (Note that TiDB does not support `GEOMETRY`). In addition, you might also need to append `PARTITIONS num` (where `num` is a positive integer indicating how many partitions a table is divided into), or append the definition of the partition names. For example, adding `(PARTITION p0, PARTITION p1)` means dividing the table into two partitions named `p0` and `p1`. +To create a Key partitioned table, you need to append a `PARTITION BY KEY (columnList)` clause to the `CREATE TABLE` statement. `columnList` is a column list with one or more column names. The data type of each column in the list can be any type except `BLOB`, `JSON`, and `GEOMETRY` (Note that TiDB does not support `GEOMETRY`). In addition, you might also need to append `PARTITIONS num` (where `num` is a positive integer indicating how many partitions a table is divided into), or append the definition of the partition names. For example, adding `(PARTITION p0, PARTITION p1)` means dividing the table into two partitions named `p0` and `p1`. The following operation creates a Key partitioned table, which is divided into 4 partitions by `store_id`: diff --git a/pd-configuration-file.md b/pd-configuration-file.md index 5fec37f838d67..f727ce33ba9b5 100644 --- a/pd-configuration-file.md +++ b/pd-configuration-file.md @@ -481,7 +481,7 @@ Configuration items related to the [TiDB Dashboard](/dashboard/dashboard-intro.m Configuration items related to the replication mode of all Regions. See [Enable the DR Auto-Sync mode](/two-data-centers-in-one-city-deployment.md#enable-the-dr-auto-sync-mode) for details. -## Controllor +## Controller This section describes the configuration items that are built into PD for [Resource Control](/tidb-resource-control.md). diff --git a/pd-control.md b/pd-control.md index 6eba08566eb3a..c03a750fbe874 100644 --- a/pd-control.md +++ b/pd-control.md @@ -29,7 +29,7 @@ To obtain `pd-ctl` of the latest version, download the TiDB server installation > **Note:** > -> `{version}` in the link indicates the version number of TiDB. For example, the download link for `v8.0.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-server-v8.0.0-linux-amd64.tar.gz`. +> `{version}` in the link indicates the version number of TiDB. For example, the download link for `v8.1.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-server-v8.1.0-linux-amd64.tar.gz`. ### Compile from source code diff --git a/performance-tuning-methods.md b/performance-tuning-methods.md index 9d64deb6165c1..af4573457d918 100644 --- a/performance-tuning-methods.md +++ b/performance-tuning-methods.md @@ -187,7 +187,7 @@ The number of `StmtPrepare` commands per second is much greater than that of `St ![OLTP-Query](/media/performance/prepared_statement_leaking.png) -- In the QPS panel, the red bold line indicates the number of failed queries, and the Y axis on the right indicates the coordinate value of the number. In this example, the number of failed quries per second is 74.6. +- In the QPS panel, the red bold line indicates the number of failed queries, and the Y axis on the right indicates the coordinate value of the number. In this example, the number of failed queries per second is 74.6. - In the CPS By Type panel, the number of `StmtPrepare` commands per second is much greater than that of `StmtClose` per second, which indicates that an object leak occurs in the application for prepared statements. - In the Queries Using Plan Cache OPS panel, `avg-miss` is almost equal to `StmtExecute` in the CPS By Type panel, which indicates that almost all SQL executions miss the execution plan cache. @@ -256,7 +256,7 @@ The Duration panel contains the average and P99 latency of all statements, and t - in-txn: The interval between processing the previous SQL and receiving the next SQL statement when the connection is within a transaction. - not-in-txn: The interval between processing the previous SQL and receiving the next SQL statement when the connection is not within a transaction. -An applications perform transactions with the same database connction. By comparing the average query latency with the connection idle duration, you can determine if TiDB is the bottleneck for overall system, or if user response time jitter is caused by TiDB. +An applications perform transactions with the same database connection. By comparing the average query latency with the connection idle duration, you can determine if TiDB is the bottleneck for overall system, or if user response time jitter is caused by TiDB. - If the application workload is not read-only and contains transactions, by comparing the average query latency with `avg-in-txn`, you can determine the proportion in processing transactions inside and outside the database, and identify the bottleneck in user response time. - If the application workload is read-only or autocommit mode is on, you can compare the average query latency with `avg-not-in-txn`. diff --git a/performance-tuning-overview.md b/performance-tuning-overview.md index c4ba353e763f3..4332f9cc9b590 100644 --- a/performance-tuning-overview.md +++ b/performance-tuning-overview.md @@ -107,7 +107,7 @@ After identifying the bottleneck of a system through performance analysis, you c According to [Amdahl's Law](https://en.wikipedia.org/wiki/Amdahl%27s_law), the maximum gain from performance tuning depends on the percentage of the optimized part in the overall system. Therefore, you need to identify the system bottlenecks and the corresponding percentage based on the performance data, and then predict the gains after the bottleneck is resolved or optimized. -Note that even if a solution can bring the greatest potential benefits by tunning the largest bottleneck, you still need to evaluate the risks and costs of this solution. For example: +Note that even if a solution can bring the greatest potential benefits by tuning the largest bottleneck, you still need to evaluate the risks and costs of this solution. For example: - The most straightforward tuning objective solution for a resource-overloaded system is to expand its capacity, but in practice, the expansion solution might be too costly to be adopted. - When a slow query in a business module causes a slow response of the entire module, upgrading to a new version of the database can solve the slow query issue, but it might also affect modules that did not have this issue. Therefore, this solution might have a potentially high risk. A low-risk solution is to skip the database version upgrade and rewrite the existing slow queries for the current database version. diff --git a/post-installation-check.md b/post-installation-check.md index 176e7d8f25f02..e82481515361c 100644 --- a/post-installation-check.md +++ b/post-installation-check.md @@ -63,7 +63,7 @@ The following information indicates successful login: ```sql Welcome to the MySQL monitor. Commands end with ; or \g. Your MySQL connection id is 3 -Server version: 8.0.11-TiDB-v8.0.0 TiDB Server (Apache License 2.0) Community Edition, MySQL 8.0 compatible +Server version: 8.0.11-TiDB-v8.1.0 TiDB Server (Apache License 2.0) Community Edition, MySQL 8.0 compatible Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective diff --git a/privilege-management.md b/privilege-management.md index 39bf87ad5dc7b..4bdb158a18bf8 100644 --- a/privilege-management.md +++ b/privilege-management.md @@ -299,7 +299,7 @@ Requires the `CREATE VIEW` privilege. ### DROP DATABASE -Requires the `DROP` privilege for the table. +Requires the `DROP` privilege for the database. ### DROP INDEX diff --git a/production-deployment-using-tiup.md b/production-deployment-using-tiup.md index 5d5c8899b08bc..b334a0bc78d28 100644 --- a/production-deployment-using-tiup.md +++ b/production-deployment-using-tiup.md @@ -83,9 +83,21 @@ Perform the following steps in this section to deploy a TiDB cluster offline usi #### Prepare the TiUP offline component package -Method 1: On the [official download page](https://www.pingcap.com/download/), select the offline mirror package (TiUP offline package included) of the target TiDB version. Note that you need to download the server package and toolkit package at the same time. +**Method 1**: Download the offline binary packages (TiUP offline package included) of the target TiDB version using the following links. You need to download both the server and toolkit packages. Note that your downloading means you agree to the [Privacy Policy](https://www.pingcap.com/privacy-policy/). -Method 2: Manually pack an offline component package using `tiup mirror clone`. The detailed steps are as follows: +``` +https://download.pingcap.org/tidb-community-server-{version}-linux-{arch}.tar.gz +``` + +``` +https://download.pingcap.org/tidb-community-toolkit-{version}-linux-{arch}.tar.gz +``` + +> **Tip:** +> +> `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.1.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.1.0-linux-amd64.tar.gz`. + +**Method 2**: Manually pack an offline component package using `tiup mirror clone`. The detailed steps are as follows: 1. Install the TiUP package manager online. @@ -197,7 +209,7 @@ The `local_install.sh` script automatically runs the `tiup mirror set tidb-commu #### Merge offline packages -If you download the offline packages from the [official download page](https://www.pingcap.com/download/), you need to merge the server package and the toolkit package into an offline mirror. If you manually package the offline component packages using the `tiup mirror clone` command, you can skip this step. +If you download the offline packages via download links, you need to merge the server package and the toolkit package into an offline mirror. If you manually package the offline component packages using the `tiup mirror clone` command, you can skip this step. Run the following commands to merge the offline toolkit package into the server package directory: @@ -334,13 +346,13 @@ Before you run the `deploy` command, use the `check` and `check --apply` command {{< copyable "shell-regular" >}} ```shell - tiup cluster deploy tidb-test v8.0.0 ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] + tiup cluster deploy tidb-test v8.1.0 ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] ``` In the `tiup cluster deploy` command above: - `tidb-test` is the name of the TiDB cluster to be deployed. -- `v8.0.0` is the version of the TiDB cluster to be deployed. You can see the latest supported versions by running `tiup list tidb`. +- `v8.1.0` is the version of the TiDB cluster to be deployed. You can see the latest supported versions by running `tiup list tidb`. - `topology.yaml` is the initialization configuration file. - `--user root` indicates logging into the target machine as the `root` user to complete the cluster deployment. The `root` user is expected to have `ssh` and `sudo` privileges to the target machine. Alternatively, you can use other users with `ssh` and `sudo` privileges to complete the deployment. - `[-i]` and `[-p]` are optional. If you have configured login to the target machine without password, these parameters are not required. If not, choose one of the two parameters. `[-i]` is the private key of the root user (or other users specified by `--user`) that has access to the target machine. `[-p]` is used to input the user password interactively. diff --git a/quick-start-with-tidb.md b/quick-start-with-tidb.md index ab7d80d333bd3..aaff2420be923 100644 --- a/quick-start-with-tidb.md +++ b/quick-start-with-tidb.md @@ -81,10 +81,10 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in {{< copyable "shell-regular" >}} ```shell - tiup playground v8.0.0 --db 2 --pd 3 --kv 3 + tiup playground v8.1.0 --db 2 --pd 3 --kv 3 ``` - The command downloads a version cluster to the local machine and starts it, such as v8.0.0. To view the latest version, run `tiup list tidb`. + The command downloads a version cluster to the local machine and starts it, such as v8.1.0. To view the latest version, run `tiup list tidb`. This command returns the access methods of the cluster: @@ -202,10 +202,10 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in {{< copyable "shell-regular" >}} ```shell - tiup playground v8.0.0 --db 2 --pd 3 --kv 3 + tiup playground v8.1.0 --db 2 --pd 3 --kv 3 ``` - The command downloads a version cluster to the local machine and starts it, such as v8.0.0. To view the latest version, run `tiup list tidb`. + The command downloads a version cluster to the local machine and starts it, such as v8.1.0. To view the latest version, run `tiup list tidb`. This command returns the access methods of the cluster: @@ -437,7 +437,7 @@ Other requirements for the target machine include: ``` - ``: Set the cluster name - - ``: Set the TiDB cluster version, such as `v8.0.0`. You can see all the supported TiDB versions by running the `tiup list tidb` command + - ``: Set the TiDB cluster version, such as `v8.1.0`. You can see all the supported TiDB versions by running the `tiup list tidb` command - `-p`: Specify the password used to connect to the target machine. > **Note:** diff --git a/releases/release-1.0.1.md b/releases/release-1.0.1.md index eec2891e4bba0..7ff5bc2e67a72 100644 --- a/releases/release-1.0.1.md +++ b/releases/release-1.0.1.md @@ -1,6 +1,7 @@ --- title: TiDB 1.0.1 Release Notes aliases: ['/docs/dev/releases/release-1.0.1/','/docs/dev/releases/101/'] +summary: TiDB 1.0.1 was released on November 1, 2017. Updates include support for canceling DDL Job, optimizing the `IN` expression, correcting the result type of the `Show` statement, supporting log slow query into a separate log file, and fixing bugs. TiKV now supports flow control with write bytes, reduces Raft allocation, increases coprocessor stack size to 10MB, and removes the useless log from the coprocessor. --- # TiDB 1.0.1 Release Notes diff --git a/releases/release-1.0.3.md b/releases/release-1.0.3.md index ac57f7eea1f47..29396839835c2 100644 --- a/releases/release-1.0.3.md +++ b/releases/release-1.0.3.md @@ -1,6 +1,7 @@ --- title: TiDB 1.0.3 Release Notes aliases: ['/docs/dev/releases/release-1.0.3/','/docs/dev/releases/103/'] +summary: TiDB 1.0.3 was released on November 28, 2017. Updates include performance optimization, new configuration options, and bug fixes. PD now supports adding more schedulers using API, and TiKV has fixed deadlock and leader value issues. To upgrade from 1.0.2 to 1.0.3, follow the rolling upgrade order of PD, TiKV, and TiDB. --- # TiDB 1.0.3 Release Notes diff --git a/releases/release-1.0.7.md b/releases/release-1.0.7.md index c8b9362d3fb67..6ba4487c8404a 100644 --- a/releases/release-1.0.7.md +++ b/releases/release-1.0.7.md @@ -1,6 +1,7 @@ --- title: TiDB 1.0.7 Release Notes aliases: ['/docs/dev/releases/release-1.0.7/','/docs/dev/releases/107/'] +summary: TiDB 1.0.7 is released with various updates including optimization of commands, fixing data race and resource leak issues, adding session variable for log query control, and improving stability of test results. PD and TiKV also have updates to fix scheduling loss issues, compatibility issues, and add support for table scan and remote mode in tikv-ctl. To upgrade from 1.0.6 to 1.0.7, follow the rolling upgrade order of PD, TiKV, and TiDB. --- # TiDB 1.0.7 Release Notes diff --git a/releases/release-2.0-ga.md b/releases/release-2.0-ga.md index a48622c9b17f9..f56693e4185a1 100644 --- a/releases/release-2.0-ga.md +++ b/releases/release-2.0-ga.md @@ -68,7 +68,7 @@ On April 27, 2018, TiDB 2.0 GA is released! Compared with TiDB 1.0, this release - Optimize the scheduling policies to prevent the disks from becoming full when the space of TiKV nodes is insufficient - Improve the scheduling efficiency of the balance-leader scheduler - Reduce the scheduling overhead of the balance-region scheduler - - Optimize the execution efficiency of the the hot-region scheduler + - Optimize the execution efficiency of the hot-region scheduler - Operations interface and configuration - Support TLS - Support prioritizing the PD leaders diff --git a/releases/release-2.0-rc.1.md b/releases/release-2.0-rc.1.md index aaff6e7e006df..de346b63c53c0 100644 --- a/releases/release-2.0-rc.1.md +++ b/releases/release-2.0-rc.1.md @@ -1,6 +1,7 @@ --- title: TiDB 2.0 RC1 Release Notes aliases: ['/docs/dev/releases/release-2.0-rc.1/','/docs/dev/releases/2rc1/'] +summary: TiDB 2.0 RC1, released on March 9, 2018, brings improvements in MySQL compatibility, SQL optimization, and stability. Key updates include memory usage limitation for SQL statements, Stream Aggregate operator support, configuration file validation, and HTTP API for configuration information. TiDB also enhances MySQL syntax compatibility, optimizer, and Boolean field length. PD sees logic and performance optimizations, while TiKV fixes gRPC call and adds gRPC APIs for metrics. Additionally, TiKV checks SSD usage, optimizes read performance, and improves metrics usage. --- # TiDB 2.0 RC1 Release Notes diff --git a/releases/release-2.0.1.md b/releases/release-2.0.1.md index 4270f3609892e..b8242d7eaca04 100644 --- a/releases/release-2.0.1.md +++ b/releases/release-2.0.1.md @@ -1,6 +1,7 @@ --- title: TiDB 2.0.1 Release Notes aliases: ['/docs/dev/releases/release-2.0.1/','/docs/dev/releases/201/'] +summary: TiDB 2.0.1 was released on May 16, 2018, with improvements in MySQL compatibility and system stability. Updates include real-time progress for 'Add Index', a new session variable for automatic statistics update, bug fixes, compatibility improvements, and behavior changes. PD added a new scheduler, optimized region balancing, and fixed various issues. TiKV fixed issues related to reading, thread calls, raftstore blocking, and split causing dirty read. Overall, the release focuses on enhancing performance, stability, and compatibility. --- # TiDB 2.0.1 Release Notes diff --git a/releases/release-2.0.6.md b/releases/release-2.0.6.md index 1c389a9079f5f..9dbb73443f984 100644 --- a/releases/release-2.0.6.md +++ b/releases/release-2.0.6.md @@ -1,6 +1,7 @@ --- title: TiDB 2.0.6 Release Notes aliases: ['/docs/dev/releases/release-2.0.6/','/docs/dev/releases/206/'] +summary: TiDB 2.0.6 was released on August 6, 2018, with improvements in system compatibility and stability. The release includes various improvements and bug fixes for TiDB and TiKV. Some notable improvements include reducing transaction conflicts, improving row count estimation accuracy, and adding a recover mechanism for panics during the execution of `ANALYZE TABLE`. Bug fixes address issues such as incompatible `DROP USER` statement behavior, OOM errors for `INSERT`/`LOAD DATA` statements, and incorrect results for prefix index and `DECIMAL` operations. TiKV also sees improvements in scheduler slots, rollback transaction records, and RocksDB log file management, along with a fix for a crash issue during data type conversion. --- # TiDB 2.0.6 Release Notes diff --git a/releases/release-2.0.9.md b/releases/release-2.0.9.md index 4caed728aa23a..0ce6b2a3b30bb 100644 --- a/releases/release-2.0.9.md +++ b/releases/release-2.0.9.md @@ -1,6 +1,7 @@ --- title: TiDB 2.0.9 Release Notes aliases: ['/docs/dev/releases/release-2.0.9/','/docs/dev/releases/209/'] +summary: TiDB 2.0.9 was released on November 19, 2018, with significant improvements in system compatibility and stability. The release includes fixes for various issues, such as empty statistics histogram, panic issue with UNION ALL statement, stack overflow issue, and support for specifying utf8mb4 character set. PD and TiKV also received fixes for issues related to server startup failure and interface limits. --- # TiDB 2.0.9 Release Notes diff --git a/releases/release-2.1-ga.md b/releases/release-2.1-ga.md index 6b52b6ffd5718..db3b5629a9bf1 100644 --- a/releases/release-2.1-ga.md +++ b/releases/release-2.1-ga.md @@ -1,6 +1,7 @@ --- title: TiDB 2.1 GA Release Notes aliases: ['/docs/dev/releases/release-2.1-ga/','/docs/dev/releases/2.1ga/'] +summary: TiDB 2.1 GA was released on November 30, 2018, with significant improvements in stability, performance, compatibility, and usability. The release includes optimizations in SQL optimizer, SQL executor, statistics, expressions, server, DDL, compatibility, Placement Driver (PD), TiKV, and tools. It also introduces TiDB Lightning for fast full data import and supports new TiDB Binlog. However, TiDB 2.1 does not support downgrading to v2.0.x or earlier due to the adoption of the new storage engine. Additionally, parallel DDL is enabled in TiDB 2.1, so clusters with TiDB version earlier than 2.0.1 cannot upgrade to 2.1 using rolling update. If upgrading from TiDB 2.0.6 or earlier to TiDB 2.1, ongoing DDL operations may slow down the upgrading process. --- # TiDB 2.1 GA Release Notes @@ -89,7 +90,7 @@ On November 30, 2018, TiDB 2.1 GA is released. See the following updates in this - Check the TiDB cluster information - - [Add the `auto_analyze_ratio` system variables to contorl the ratio of Analyze](/faq/sql-faq.md#whats-the-trigger-strategy-for-auto-analyze-in-tidb) + - [Add the `auto_analyze_ratio` system variables to control the ratio of Analyze](/faq/sql-faq.md#whats-the-trigger-strategy-for-auto-analyze-in-tidb) - [Add the `tidb_retry_limit` system variable to control the automatic retry times of transactions](/system-variables.md#tidb_retry_limit) diff --git a/releases/release-2.1.17.md b/releases/release-2.1.17.md index ba9e51a26aa0f..c2ce2a99b1a6f 100644 --- a/releases/release-2.1.17.md +++ b/releases/release-2.1.17.md @@ -46,7 +46,7 @@ TiDB Ansible version: 2.1.17 - Change `start ts` recorded in slow query logs from the last retry time to the first execution time when retrying TiDB transactions [#11878](https://github.com/pingcap/tidb/pull/11878) - Add the number of keys of a transaction in `LockResolver` to avoid the scan operation on the whole Region and reduce costs of resolving locking when the number of keys is reduced [#11889](https://github.com/pingcap/tidb/pull/11889) - Fix the issue that the `succ` field value might be incorrect in slow query logs [#11886](https://github.com/pingcap/tidb/pull/11886) - - Replace the `Index_ids` filed in slow query logs with the `Index_names` field to improve the usability of slow query logs [#12063](https://github.com/pingcap/tidb/pull/12063) + - Replace the `Index_ids` field in slow query logs with the `Index_names` field to improve the usability of slow query logs [#12063](https://github.com/pingcap/tidb/pull/12063) - Fix the connection break issue caused by TiDB parsing `-` into EOF Error when `Duration` contains `-` (like `select time(‘--’)`) [#11910](https://github.com/pingcap/tidb/pull/11910) - Remove an invalid Region from `RegionCache` more quickly to reduce the number of requests sent to this Region [#11931](https://github.com/pingcap/tidb/pull/11931) - Fix the connection break issue caused by incorrectly handling the OOM panic issue when `oom-action = "cancel"` and OOM occurs in the `Insert Into … Select` syntax [#12126](https://github.com/pingcap/tidb/pull/12126) diff --git a/releases/release-2.1.2.md b/releases/release-2.1.2.md index 810feece1c73b..5c1555236e222 100644 --- a/releases/release-2.1.2.md +++ b/releases/release-2.1.2.md @@ -1,6 +1,7 @@ --- title: TiDB 2.1.2 Release Notes aliases: ['/docs/dev/releases/release-2.1.2/','/docs/dev/releases/2.1.2/'] +summary: TiDB 2.1.2 and TiDB Ansible 2.1.2 were released on December 22, 2018. The release includes improvements in system compatibility and stability. Key updates include compatibility with TiDB Binlog of the Kafka version, improved exit mechanism during rolling updates, and fixes for various issues. PD and TiKV also received updates, such as fixing Region merge issues and support for configuration format in the unit of 'DAY'. Additionally, TiDB Lightning and TiDB Binlog were updated to support new features and eliminate bottlenecks. --- # TiDB 2.1.2 Release Notes diff --git a/releases/release-3.0-beta.md b/releases/release-3.0-beta.md index f17d7cce3b4cb..2aa292008af59 100644 --- a/releases/release-3.0-beta.md +++ b/releases/release-3.0-beta.md @@ -1,6 +1,7 @@ --- title: TiDB 3.0 Beta Release Notes aliases: ['/docs/dev/releases/release-3.0-beta/','/docs/dev/releases/3.0beta/'] +summary: TiDB 3.0 Beta, released on January 19, 2019, focuses on stability, SQL optimizer, statistics, and execution engine. New features include support for views, window functions, range partitioning, and hash partitioning. The SQL optimizer has been enhanced with various optimizations, including support for index join in transactions, constant propagation optimization, and support for subqueries in the DO statement. The SQL executor has also been optimized for better performance. Privilege management, server, compatibility, and DDL have all been improved. TiDB Lightning now supports batch import for a single table, while PD and TiKV have also received various enhancements and new features. --- # TiDB 3.0 Beta Release Notes diff --git a/releases/release-3.0-ga.md b/releases/release-3.0-ga.md index 54edd52772eed..39cdf517dcb98 100644 --- a/releases/release-3.0-ga.md +++ b/releases/release-3.0-ga.md @@ -1,6 +1,7 @@ --- title: TiDB 3.0 GA Release Notes aliases: ['/docs/dev/releases/release-3.0-ga/','/docs/dev/releases/3.0-ga/'] +summary: TiDB 3.0 GA was released on June 28, 2019, with improved stability, usability, and performance. New features include Window Functions, Views, partitioned tables, and the plugin framework. The SQL Optimizer has been optimized for better performance, and DDL now supports fast recovery of mistakenly deleted tables. TiKV now supports distributed GC, multi-thread Raftstore, and batch receiving and sending Raft messages. Tools like TiDB Lightning and TiDB Binlog have also been enhanced with new features and performance improvements. The TiDB Ansible has been upgraded to support deployment and operations for TiDB Lightning, and to optimize monitoring components. --- # TiDB 3.0 GA Release Notes diff --git a/releases/release-3.0.0-rc.2.md b/releases/release-3.0.0-rc.2.md index d8aa5737cebc4..27ec05bad3ce4 100644 --- a/releases/release-3.0.0-rc.2.md +++ b/releases/release-3.0.0-rc.2.md @@ -1,6 +1,7 @@ --- title: TiDB 3.0.0-rc.2 Release Notes aliases: ['/docs/dev/releases/release-3.0.0-rc.2/','/docs/dev/releases/3.0.0-rc.2/'] +summary: TiDB 3.0.0-rc.2 was released on May 28, 2019, with improvements in stability, usability, features, SQL optimizer, statistics, and execution engine. The release includes enhancements to the SQL optimizer, execution engine, server, DDL, PD, TiKV, and tools like TiDB Binlog and TiDB Lightning. Some notable improvements include support for Index Join in more scenarios, handling virtual columns properly, and adding a metric to track data replication downstream. --- # TiDB 3.0.0-rc.2 Release Notes diff --git a/releases/release-3.0.10.md b/releases/release-3.0.10.md index 662986493769a..bfe5818569f0d 100644 --- a/releases/release-3.0.10.md +++ b/releases/release-3.0.10.md @@ -52,7 +52,7 @@ TiDB Ansible version: 3.0.10 + Raftstore - Fix the system panic issue #6460 or data loss issue #598 caused by Region merge failure [#6481](https://github.com/tikv/tikv/pull/6481) - - Support `yield` to optimize scheduling fairness, and support pre-transfering the leader to improve leader scheduling stability [#6563](https://github.com/tikv/tikv/pull/6563) + - Support `yield` to optimize scheduling fairness, and support pre-transferring the leader to improve leader scheduling stability [#6563](https://github.com/tikv/tikv/pull/6563) ## PD diff --git a/releases/release-3.0.15.md b/releases/release-3.0.15.md index 87501606431cf..7c4f2b1023d46 100644 --- a/releases/release-3.0.15.md +++ b/releases/release-3.0.15.md @@ -1,6 +1,7 @@ --- title: TiDB 3.0.15 Release Notes aliases: ['/docs/dev/releases/release-3.0.15/'] +summary: TiDB 3.0.15 was released on June 5, 2020. New features include support for admin recover index and admin check index statements on partitioned tables, as well as optimization of memory allocation mechanism. Bug fixes address issues such as incorrect results in PointGet and inconsistent results between TiDB and MySQL when XOR operates on a floating-point number. TiKV fixes issues related to memory defragmentation and gRPC disconnection. --- # TiDB 3.0.15 Release Notes diff --git a/releases/release-3.0.4.md b/releases/release-3.0.4.md index 6b98ff46ac5ec..93df3984ed0b0 100644 --- a/releases/release-3.0.4.md +++ b/releases/release-3.0.4.md @@ -43,7 +43,7 @@ TiDB Ansible version: 3.0.4 ## TiDB - SQL Optimizer - - Fix the issue that invalid query ranges might be resulted when splitted by feedback [#12170](https://github.com/pingcap/tidb/pull/12170) + - Fix the issue that invalid query ranges might be resulted when split by feedback [#12170](https://github.com/pingcap/tidb/pull/12170) - Display the returned error of the `SHOW STATS_BUCKETS` statement in hexadecimal rather than return errors when the result contains invalid Keys [#12094](https://github.com/pingcap/tidb/pull/12094) - Fix the issue that when a query contains the `SLEEP` function (for example, `select 1 from (select sleep(1)) t;)`), column pruning causes invalid `sleep(1)` during query [#11953](https://github.com/pingcap/tidb/pull/11953) - Use index scan to lower IO when a query only concerns the number of columns rather than the table data [#12112](https://github.com/pingcap/tidb/pull/12112) diff --git a/releases/release-3.0.5.md b/releases/release-3.0.5.md index 2a8474be3bf61..a20f90118e935 100644 --- a/releases/release-3.0.5.md +++ b/releases/release-3.0.5.md @@ -1,6 +1,7 @@ --- title: TiDB 3.0.5 Release Notes aliases: ['/docs/dev/releases/release-3.0.5/','/docs/dev/releases/3.0.5/'] +summary: TiDB 3.0.5 was released on October 25, 2019, with various improvements and bug fixes. The release includes enhancements to the SQL optimizer, SQL execution engine, server, DDL, monitor, TiKV, PD, TiDB Binlog, TiDB Lightning, and TiDB Ansible. Improvements include support for boundary checking on Window Functions, fixing issues with index join and outer join, and adding monitoring metrics for various operations. Additionally, TiKV received storage and performance optimizations, while PD saw improvements in storage precision and HTTP request handling. TiDB Ansible also received updates to monitoring metrics and configuration file simplification. --- # TiDB 3.0.5 Release Notes diff --git a/releases/release-4.0.0-beta.1.md b/releases/release-4.0.0-beta.1.md index 03ed69dc18fb2..18fbea5598b25 100644 --- a/releases/release-4.0.0-beta.1.md +++ b/releases/release-4.0.0-beta.1.md @@ -84,7 +84,7 @@ TiDB Ansible version: 4.0.0-beta.1 + Fix the incorrect results of `BatchPointGet` when `plan cache` is enabled [#14855](https://github.com/pingcap/tidb/pull/14855) + Fix the issue that data is inserted into the wrong partitioned table after the timezone is modified [#14370](https://github.com/pingcap/tidb/pull/14370) + Fix the panic occurred when rebuilding expression using the invalid name of the `IsTrue` function during the outer join simplification [#14515](https://github.com/pingcap/tidb/pull/14515) - + Fix the the incorrect privilege check for the`show binding` statement [#14443](https://github.com/pingcap/tidb/pull/14443) + + Fix the incorrect privilege check for the`show binding` statement [#14443](https://github.com/pingcap/tidb/pull/14443) * TiKV + Fix the inconsistent behaviors of the `CAST` function in TiDB and TiKV [#6463](https://github.com/tikv/tikv/pull/6463) [#6461](https://github.com/tikv/tikv/pull/6461) [#6459](https://github.com/tikv/tikv/pull/6459) [#6474](https://github.com/tikv/tikv/pull/6474) [#6492](https://github.com/tikv/tikv/pull/6492) [#6569](https://github.com/tikv/tikv/pull/6569) diff --git a/releases/release-4.0.0-beta.md b/releases/release-4.0.0-beta.md index faa45cc7ec4bc..2c2a10fcbc23e 100644 --- a/releases/release-4.0.0-beta.md +++ b/releases/release-4.0.0-beta.md @@ -1,6 +1,7 @@ --- title: TiDB 4.0 Beta Release Notes aliases: ['/docs/dev/releases/release-4.0.0-beta/','/docs/dev/releases/4.0.0-beta/'] +summary: TiDB version 4.0.0-beta and TiDB Ansible version 4.0.0-beta were released on January 17, 2020. The release includes various improvements such as increased accuracy in calculating the cost of Index Join, support for Table Locks, and optimization of the error code of SQL error messages. TiKV was also upgraded to RocksDB version 6.4.6 and now supports quick backup and restoration. PD now supports optimizing hotspot scheduling and adding Placement Rules feature. TiDB Lightning added a parameter to set the password of the downstream database, and TiDB Ansible now supports deploying and maintaining TiFlash. --- # TiDB 4.0 Beta Release Notes diff --git a/releases/release-4.0.0-rc.2.md b/releases/release-4.0.0-rc.2.md index 7daa263e87642..2bb086df641c0 100644 --- a/releases/release-4.0.0-rc.2.md +++ b/releases/release-4.0.0-rc.2.md @@ -1,6 +1,7 @@ --- title: TiDB 4.0 RC.2 Release Notes aliases: ['/docs/dev/releases/release-4.0.0-rc.2/'] +summary: TiDB 4.0 RC.2 was released on May 15, 2020. The release includes compatibility changes, important bug fixes, new features, and bug fixes for TiDB, TiKV, PD, TiFlash, and various tools. Some notable changes include the removal of the size limit for a single transaction when TiDB Binlog is enabled, support for the BACKUP and RESTORE commands, and the addition of encryption-related monitoring metrics in Grafana dashboard. Additionally, there are numerous bug fixes for issues such as wrong partition selection, incorrect index range building, and performance reduction. The release also introduces new features like support for the auto_random option in the CREATE TABLE statement and the ability to manage replication tasks using cdc cli. --- # TiDB 4.0 RC.2 Release Notes @@ -89,7 +90,7 @@ TiDB version: 4.0.0-rc.2 - Change the name of the Count graph of **Read Index** in Grafana to **Ops** - Optimize the data for opening file descriptors when the system load is low to reduce system resource consumption - - Add the capacity-related configuration parameter to limit the the data storage capacity + - Add the capacity-related configuration parameter to limit the data storage capacity + Tools @@ -160,7 +161,7 @@ TiDB version: 4.0.0-rc.2 - Fix the issue that the backup data cannot be restored from GCS [#7739](https://github.com/tikv/tikv/pull/7739) - Fix the issue that KMS key ID is not validated during encryption at rest [#7719](https://github.com/tikv/tikv/pull/7719) - Fix the underlying correctness issue of the Coprocessor in compilers of different architecture [#7714](https://github.com/tikv/tikv/pull/7714) [#7730](https://github.com/tikv/tikv/pull/7730) - - Fix the `snapshot ingestion` error when encrytion is enabled [#7815](https://github.com/tikv/tikv/pull/7815) + - Fix the `snapshot ingestion` error when encryption is enabled [#7815](https://github.com/tikv/tikv/pull/7815) - Fix the `Invalid cross-device link` error when rewriting the configuration file [#7817](https://github.com/tikv/tikv/pull/7817) - Fix the issue of wrong toml format when writing the configuration file to an empty file [#7817](https://github.com/tikv/tikv/pull/7817) - Fix the issue that a destroyed peer in Raftstore can still process requests [#7836](https://github.com/tikv/tikv/pull/7836) diff --git a/releases/release-4.0.0-rc.md b/releases/release-4.0.0-rc.md index 08f4c0fb10e30..8ed7c2b47e8f7 100644 --- a/releases/release-4.0.0-rc.md +++ b/releases/release-4.0.0-rc.md @@ -38,7 +38,7 @@ TiUP version: 0.0.3 + TiDB - Fix the issue that replication between the upstream and downstream might go wrong when the DDL job is executed using the `PREPARE` statement because of the incorrect job query in the internal records [#15435](https://github.com/pingcap/tidb/pull/15435) - - Fix the issue of incorrect subquery result in the `Read Commited` isolation level [#15471](https://github.com/pingcap/tidb/pull/15471) + - Fix the issue of incorrect subquery result in the `Read Committed` isolation level [#15471](https://github.com/pingcap/tidb/pull/15471) - Fix the issue of incorrect results caused by the Inline Projection optimization [#15411](https://github.com/pingcap/tidb/pull/15411) - Fix the issue that the SQL Hint `INL_MERGE_JOIN` is executed incorrectly in some cases [#15515](https://github.com/pingcap/tidb/pull/15515) - Fix the issue that columns with the `AutoRandom` attribute are rebased when the negative number is explicitly written to these columns [#15397](https://github.com/pingcap/tidb/pull/15397) @@ -49,7 +49,7 @@ TiUP version: 0.0.3 - Add the case-insensitive collation so that users can enable `utf8mb4_general_ci` and `utf8_general_ci` in a new cluster [#33](https://github.com/pingcap/tidb/projects/33) - Enhance the `RECOVER TABLE` syntax to support recovering truncated tables [#15398](https://github.com/pingcap/tidb/pull/15398) - - Refuse to get started instead of returning an alert log when the the tidb-server status port is occupied [#15177](https://github.com/pingcap/tidb/pull/15177) + - Refuse to get started instead of returning an alert log when the tidb-server status port is occupied [#15177](https://github.com/pingcap/tidb/pull/15177) - Optimize the write performance of using a sequence as the default column values [#15216](https://github.com/pingcap/tidb/pull/15216) - Add the `DDLJobs` system table to query the details of DDL jobs [#14837](https://github.com/pingcap/tidb/pull/14837) - Optimize the `aggFuncSum` performance [#14887](https://github.com/pingcap/tidb/pull/14887) @@ -80,7 +80,7 @@ TiUP version: 0.0.3 + TiDB - Fix the issue that replication between the upstream and downstream might go wrong when the DDL job is executed using the `PREPARE` statement because of the incorrect job query in the internal records [#15435](https://github.com/pingcap/tidb/pull/15435) - - Fix the issue of incorrect subquery result in the `Read Commited` isolation level [#15471](https://github.com/pingcap/tidb/pull/15471) + - Fix the issue of incorrect subquery result in the `Read Committed` isolation level [#15471](https://github.com/pingcap/tidb/pull/15471) - Fix the issue of possible wrong behavior when using `INSERT ... VALUES` to specify the `BIT(N)` data type [#15350](https://github.com/pingcap/tidb/pull/15350) - Fix the issue that the DDL Job internal retry does not fully achieve the expected outcomes because the values of `ErrorCount` fail to be summed correctly [#15373](https://github.com/pingcap/tidb/pull/15373) - Fix the issue that Garbage Collection might work abnormally when TiDB connects to TiFlash [#15505](https://github.com/pingcap/tidb/pull/15505) diff --git a/releases/release-4.0.16.md b/releases/release-4.0.16.md index a1751d6d91a60..c552efed45683 100644 --- a/releases/release-4.0.16.md +++ b/releases/release-4.0.16.md @@ -81,7 +81,7 @@ TiDB version: 4.0.16 + PD - Fix a panic issue that occurs after the TiKV node is removed [#4344](https://github.com/tikv/pd/issues/4344) - - Fix slow leader election caused by stucked region syncer [#3936](https://github.com/tikv/pd/issues/3936) + - Fix slow leader election caused by stuck region syncer [#3936](https://github.com/tikv/pd/issues/3936) - Support that the evict leader scheduler can schedule regions with unhealthy peers [#4093](https://github.com/tikv/pd/issues/4093) + TiFlash diff --git a/releases/release-4.0.6.md b/releases/release-4.0.6.md index 27fbc9c3bd858..fd46dac44a0d3 100644 --- a/releases/release-4.0.6.md +++ b/releases/release-4.0.6.md @@ -106,7 +106,7 @@ TiDB version: 4.0.6 - Fix a bug of converting the `enum` and `set` types [#19778](https://github.com/pingcap/tidb/pull/19778) - Add a privilege check for `SHOW STATS_META` and `SHOW STATS_BUCKET` [#19760](https://github.com/pingcap/tidb/pull/19760) - Fix the error of unmatched column lengths caused by `builtinGreatestStringSig` and `builtinLeastStringSig` [#19758](https://github.com/pingcap/tidb/pull/19758) - - If unnecessary errors or warnings occur, the vectorized control expresions fall back to their scalar execution [#19749](https://github.com/pingcap/tidb/pull/19749) + - If unnecessary errors or warnings occur, the vectorized control expression fall back to their scalar execution [#19749](https://github.com/pingcap/tidb/pull/19749) - Fix the error of the `Apply` operator when the type of the correlation column is `Bit` [#19692](https://github.com/pingcap/tidb/pull/19692) - Fix the issue that occurs when the user queries `processlist` and `cluster_log` in MySQL 8.0 client [#19690](https://github.com/pingcap/tidb/pull/19690) - Fix the issue that plans of the same type have different plan digests [#19684](https://github.com/pingcap/tidb/pull/19684) diff --git a/releases/release-4.0.9.md b/releases/release-4.0.9.md index 016dede57f686..9b62baa1d96a9 100644 --- a/releases/release-4.0.9.md +++ b/releases/release-4.0.9.md @@ -1,5 +1,6 @@ --- title: TiDB 4.0.9 Release Notes +summary: TiDB 4.0.9 was released on December 21, 2020. The release includes compatibility changes, new features, improvements, bug fixes, and updates to TiKV, TiDB Dashboard, PD, TiFlash, and various tools. Notable changes include the deprecation of the `enable-streaming` configuration item in TiDB, support for storing the latest data of the storage engine on multiple disks in TiFlash, and various bug fixes in TiDB and TiKV. --- # TiDB 4.0.9 Release Notes diff --git a/releases/release-5.0.0-rc.md b/releases/release-5.0.0-rc.md index befdfb08b4236..f8a7f8ff444a9 100644 --- a/releases/release-5.0.0-rc.md +++ b/releases/release-5.0.0-rc.md @@ -1,5 +1,6 @@ --- title: TiDB 5.0 RC Release Notes +summary: TiDB v5.0.0-rc is the predecessor version of TiDB v5.0. It includes new features like clustered index, async commit, reduced jitters, Raft Joint Consensus algorithm, optimized `EXPLAIN` features, invisible index, and improved reliability for enterprise data. It also supports desensitizing error messages and log files for security. Performance improvements include async commit, optimizer stability, and reduced performance jitter. It also enhances system availability during Region membership change. Additionally, it supports backup and restore to AWS S3 and Google Cloud GCS, data import/export, and optimized `EXPLAIN` features for troubleshooting SQL performance issues. Deployment and maintenance improvements include enhanced `mirror` command and easier installation process. --- # TiDB 5.0 RC Release Notes diff --git a/releases/release-5.0.0.md b/releases/release-5.0.0.md index b4e13c57cfd7a..fa4f630eaa1ff 100644 --- a/releases/release-5.0.0.md +++ b/releases/release-5.0.0.md @@ -334,7 +334,7 @@ You can view the manually or automatically bound execution plan information by r When upgrading TiDB, to avoid performance jitter, you can enable the baseline capturing feature to allow the system to automatically capture and bind the latest execution plan and store it in the system table. After TiDB is upgraded, you can export the bound execution plan by running the `SHOW GLOBAL BINDING` command and decide whether to delete these plans. -This feature is disbled by default. You can enable it by modifying the server or setting the `tidb_capture_plan_baselines` global system variable to `ON`. When this feature is enabled, the system fetches the SQL statements that appear at least twice from the Statement Summary every `bind-info-lease` (the default value is `3s`), and automatically captures and binds these SQL statements. +This feature is disabled by default. You can enable it by modifying the server or setting the `tidb_capture_plan_baselines` global system variable to `ON`. When this feature is enabled, the system fetches the SQL statements that appear at least twice from the Statement Summary every `bind-info-lease` (the default value is `3s`), and automatically captures and binds these SQL statements. ### Improve stability of TiFlash queries diff --git a/releases/release-5.0.3.md b/releases/release-5.0.3.md index eb29090c88965..038bd3dfc8bc4 100644 --- a/releases/release-5.0.3.md +++ b/releases/release-5.0.3.md @@ -1,5 +1,6 @@ --- title: TiDB 5.0.3 Release Notes +summary: TiDB 5.0.3 was released on July 2, 2021. The release includes compatibility changes, feature enhancements, improvements, bug fixes, and updates for TiDB, TiKV, PD, TiFlash, and Tools like TiCDC, Backup & Restore (BR), and TiDB Lightning. Some notable changes include support for pushing down operators and functions to TiFlash, memory consumption limits for TiCDC, and bug fixes for various issues in TiDB, TiKV, PD, and TiFlash. --- # TiDB 5.0.3 Release Notes diff --git a/releases/release-5.0.4.md b/releases/release-5.0.4.md index fc7ad9805dcb9..be37425e8d9c6 100644 --- a/releases/release-5.0.4.md +++ b/releases/release-5.0.4.md @@ -1,5 +1,6 @@ --- title: TiDB 5.0.4 Release Notes +summary: Compatibility changes include fixes for slow `SHOW VARIABLES` execution, default value change for `tidb_stmt_summary_max_stmt_count`, and bug fixes that may cause upgrade incompatibilities. Feature enhancements include support for setting `tidb_enforce_mpp=1` and dynamic TiCDC configurations. Improvements cover auto-analyze trigger, MPP query retry support, and stable result mode. Bug fixes address various issues in TiDB, TiKV, PD, TiFlash, and tools like Dumpling and TiCDC. --- # TiDB 5.0.4 Release Notes diff --git a/releases/release-5.0.6.md b/releases/release-5.0.6.md index 03c185b560291..5f780a20fae3a 100644 --- a/releases/release-5.0.6.md +++ b/releases/release-5.0.6.md @@ -1,6 +1,7 @@ --- title: TiDB 5.0.6 Release Notes category: Releases +summary: TiDB 5.0.6 was released on December 31, 2021. The release includes compatibility changes, improvements, bug fixes, and updates to various tools such as TiCDC, TiKV, PD, TiDB Lightning, TiFlash, Backup & Restore (BR), and Dumpling. The changes include enhancements to error handling, performance improvements, bug fixes related to SQL statements, and various optimizations for different tools. --- # TiDB 5.0.6 Release Notes @@ -76,7 +77,7 @@ TiDB version: 5.0.6 - Fix the `INDEX OUT OF RANGE` error for a MPP query after deleting an empty `dual table` [#28250](https://github.com/pingcap/tidb/issues/28250) - Fix the TiDB panic when inserting invalid date values concurrently [#25393](https://github.com/pingcap/tidb/issues/25393) - Fix the unexpected `can not found column in Schema column` error for queries in the MPP mode [#30980](https://github.com/pingcap/tidb/issues/30980) - - Fix the issue that TiDB might panic when TiFlash is shuting down [#28096](https://github.com/pingcap/tidb/issues/28096) + - Fix the issue that TiDB might panic when TiFlash is shutting down [#28096](https://github.com/pingcap/tidb/issues/28096) - Fix the unexpected `index out of range` error when the planner is doing join reorder [#24095](https://github.com/pingcap/tidb/issues/24095) - Fix wrong results of the control functions (such as `IF` and `CASE WHEN`) when using the `ENUM` type data as parameters of such functions [#23114](https://github.com/pingcap/tidb/issues/23114) - Fix the wrong result of `CONCAT(IFNULL(TIME(3))` [#29498](https://github.com/pingcap/tidb/issues/29498) @@ -114,7 +115,7 @@ TiDB version: 5.0.6 - Fix a panic issue that occurs after the TiKV node is removed [#4344](https://github.com/tikv/pd/issues/4344) - Fix the issue that operator can get blocked due to down store [#3353](https://github.com/tikv/pd/issues/3353) - - Fix slow leader election caused by stucked Region syncer [#3936](https://github.com/tikv/pd/issues/3936) + - Fix slow leader election caused by stuck Region syncer [#3936](https://github.com/tikv/pd/issues/3936) - Fix the issue that the speed of removing peers is limited when repairing the down nodes [#4090](https://github.com/tikv/pd/issues/4090) - Fix the issue that the hotspot cache cannot be cleared when the Region heartbeat is less than 60 seconds [#4390](https://github.com/tikv/pd/issues/4390) diff --git a/releases/release-5.1.0.md b/releases/release-5.1.0.md index 704cb272766f6..608bad96342f1 100644 --- a/releases/release-5.1.0.md +++ b/releases/release-5.1.0.md @@ -1,5 +1,6 @@ --- title: TiDB 5.1 Release Notes +summary: TiDB 5.1 introduces support for Common Table Expression, dynamic privilege feature, and Stale Read. It also includes new statistics type, Lock View feature, and TiKV write rate limiter. Compatibility changes include new system and configuration variables. Other improvements and bug fixes are also part of this release. --- # TiDB 5.1 Release Notes diff --git a/releases/release-5.1.4.md b/releases/release-5.1.4.md index a969762ad9554..1fc9c2ee415d5 100644 --- a/releases/release-5.1.4.md +++ b/releases/release-5.1.4.md @@ -110,7 +110,7 @@ TiDB version: 5.1.4 - Fix a bug that the schedule generated by the region scatterer might decrease the number of peers [#4565](https://github.com/tikv/pd/issues/4565) - Fix the issue that Region statistics are not affected by `flow-round-by-digit` [#4295](https://github.com/tikv/pd/issues/4295) - - Fix slow leader election caused by stucked region syncer [#3936](https://github.com/tikv/pd/issues/3936) + - Fix slow leader election caused by stuck region syncer [#3936](https://github.com/tikv/pd/issues/3936) - Support that the evict leader scheduler can schedule regions with unhealthy peers [#4093](https://github.com/tikv/pd/issues/4093) - Fix the issue that the cold hotspot data cannot be deleted from the hotspot statistics [#4390](https://github.com/tikv/pd/issues/4390) - Fix a panic issue that occurs after the TiKV node is removed [#4344](https://github.com/tikv/pd/issues/4344) diff --git a/releases/release-5.1.5.md b/releases/release-5.1.5.md index 3e8750bf31e8b..43c9837f220b7 100644 --- a/releases/release-5.1.5.md +++ b/releases/release-5.1.5.md @@ -9,7 +9,7 @@ Release date: December 28, 2022 TiDB version: 5.1.5 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v5.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v5.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v5.1.5#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v5.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v5.1/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-5.2.2.md b/releases/release-5.2.2.md index 4595e7bb8e039..7ba8c354199b9 100644 --- a/releases/release-5.2.2.md +++ b/releases/release-5.2.2.md @@ -63,7 +63,7 @@ TiDB version: 5.2.2 - Fix the `INDEX OUT OF RANGE` error for a MPP query after deleting an empty `dual table`. [#28250](https://github.com/pingcap/tidb/issues/28250) - Fix the issue of false positive error log `invalid cop task execution summaries length` for MPP queries [#1791](https://github.com/pingcap/tics/issues/1791) - Fix the issue of error log `cannot found column in Schema column` for MPP queries [#28149](https://github.com/pingcap/tidb/pull/28149) - - Fix the issue that TiDB might panic when TiFlash is shuting down [#28096](https://github.com/pingcap/tidb/issues/28096) + - Fix the issue that TiDB might panic when TiFlash is shutting down [#28096](https://github.com/pingcap/tidb/issues/28096) - Remove the support for insecure 3DES (Triple Data Encryption Algorithm) based TLS cipher suites [#27859](https://github.com/pingcap/tidb/pull/27859) - Fix the issue that Lightning connects to offline TiKV nodes during pre-check and causes import failures [#27826](https://github.com/pingcap/tidb/pull/27826) - Fix the issue that pre-check cost too much time when importing many files to tables [#27605](https://github.com/pingcap/tidb/issues/27605) diff --git a/releases/release-5.2.4.md b/releases/release-5.2.4.md index 4064c23c9a7fe..a93247a1088c8 100644 --- a/releases/release-5.2.4.md +++ b/releases/release-5.2.4.md @@ -1,6 +1,7 @@ --- title: TiDB 5.2.4 Release Notes category: Releases +summary: Learn about the new features, compatibility changes, improvements, and bug fixes in TiDB 5.2.4. --- # TiDB 5.2.4 Release Notes diff --git a/releases/release-5.3.0.md b/releases/release-5.3.0.md index a7c026b88a0b1..0284ae17d7cf9 100644 --- a/releases/release-5.3.0.md +++ b/releases/release-5.3.0.md @@ -1,5 +1,6 @@ --- title: TiDB 5.3 Release Notes +summary: TiDB 5.3.0 introduces temporary tables, table attributes, and user privileges on TiDB Dashboard for improved performance and security. It also enhances TiDB Data Migration, supports parallel import using multiple TiDB Lightning instances, and continuous profiling for better observability. Compatibility changes and configuration file parameters have been modified. The release also includes new SQL features, security enhancements, stability improvements, and diagnostic efficiency. Additionally, bug fixes and improvements have been made to TiDB, TiKV, PD, TiFlash, and TiCDC. The cyclic replication feature between TiDB clusters has been removed. Telemetry now includes information about the usage of the TEMPORARY TABLE feature. --- # TiDB 5.3 Release Notes @@ -337,7 +338,7 @@ Starting from TiCDC v5.3.0, the cyclic replication feature between TiDB clusters - Fix the `INDEX OUT OF RANGE` error for a MPP query after deleting an empty `dual table` [#28250](https://github.com/pingcap/tidb/issues/28250) - Fix the issue of false positive error log `invalid cop task execution summaries length` for MPP queries [#1791](https://github.com/pingcap/tics/issues/1791) - Fix the issue of error log `cannot found column in Schema column` for MPP queries [#28149](https://github.com/pingcap/tidb/pull/28149) - - Fix the issue that TiDB might panic when TiFlash is shuting down [#28096](https://github.com/pingcap/tidb/issues/28096) + - Fix the issue that TiDB might panic when TiFlash is shutting down [#28096](https://github.com/pingcap/tidb/issues/28096) - Remove the support for insecure 3DES (Triple Data Encryption Algorithm) based TLS cipher suites [#27859](https://github.com/pingcap/tidb/pull/27859) - Fix the issue that Lightning connects to offline TiKV nodes during pre-check and causes import failures [#27826](https://github.com/pingcap/tidb/pull/27826) - Fix the issue that pre-check cost too much time when importing many files to tables [#27605](https://github.com/pingcap/tidb/issues/27605) @@ -376,7 +377,7 @@ Starting from TiCDC v5.3.0, the cyclic replication feature between TiDB clusters - Fix the issue that the scatter range scheduler cannot schedule empty Regions [#4118](https://github.com/tikv/pd/pull/4118) - Fix the issue that the key manager cost too much CPU [#4071](https://github.com/tikv/pd/issues/4071) - Fix the data race issue that might occur when setting configurations of hot Region scheduler [#4159](https://github.com/tikv/pd/issues/4159) - - Fix slow leader election caused by stucked Region syncer [#3936](https://github.com/tikv/pd/issues/3936) + - Fix slow leader election caused by stuck Region syncer [#3936](https://github.com/tikv/pd/issues/3936) + TiFlash diff --git a/releases/release-5.3.1.md b/releases/release-5.3.1.md index 8f1a1da8e4974..849287628fb14 100644 --- a/releases/release-5.3.1.md +++ b/releases/release-5.3.1.md @@ -1,5 +1,6 @@ --- title: TiDB 5.3.1 Release Notes +summary: TiDB 5.3.1 was released on March 3, 2022. The release includes compatibility changes, improvements, and bug fixes for TiDB, TiKV, PD, TiCDC, TiFlash, Backup & Restore (BR), and TiDB Data Migration (DM). Some notable changes include optimizing user login mode mapping, reducing TiCDC recovery time, and fixing various bugs in TiDB, TiKV, PD, TiFlash, and tools like TiCDC and TiDB Lightning. These fixes address issues related to data import, user login, garbage collection, configuration parameters, and more. --- # TiDB 5.3.1 Release Notes diff --git a/releases/release-5.3.4.md b/releases/release-5.3.4.md index ca20e7ca1c7d3..5addade1af19b 100644 --- a/releases/release-5.3.4.md +++ b/releases/release-5.3.4.md @@ -1,5 +1,6 @@ --- title: TiDB 5.3.4 Release Notes +summary: TiDB 5.3.4 was released on November 24, 2022. The release includes improvements to TiKV and bug fixes for TiDB, PD, TiFlash, Dumpling, and TiCDC. Some of the key bug fixes include issues related to TLS certificate reloading, Region cache cleanup, wrong data writing, database-level privileges, and authentication failures. Other fixes address issues with logical operators, stream timeout, leader switchover, and data dumping. --- # TiDB 5.3.4 Release Notes diff --git a/releases/release-5.4.0.md b/releases/release-5.4.0.md index 396693e162c44..832ac4884d262 100644 --- a/releases/release-5.4.0.md +++ b/releases/release-5.4.0.md @@ -1,5 +1,6 @@ --- title: TiDB 5.4 Release Notes +summary: TiDB 5.4 introduces support for the GBK character set, Index Merge, reading stale data, persisting statistics configuration, and using Raft Engine as the log storage engine of TiKV. It also improves backup impact, supports Azure Blob storage, and enhances TiFlash and the MPP engine. Compatibility changes include new system variables and configuration file parameters. Other improvements cover SQL, security, performance, stability, high availability, data migration, diagnostic efficiency, and deployment. Bug fixes address issues in TiDB, TiKV, PD, TiFlash, BR, TiCDC, DM, TiDB Lightning, and TiDB Binlog. --- # TiDB 5.4 Release Notes diff --git a/releases/release-5.4.3.md b/releases/release-5.4.3.md index 01dd37d348833..f23687bd3ad19 100644 --- a/releases/release-5.4.3.md +++ b/releases/release-5.4.3.md @@ -1,5 +1,6 @@ --- title: TiDB 5.4.3 Release Notes +summary: TiDB 5.4.3 was released on October 13, 2022. The release includes various improvements and bug fixes for TiKV, Tools, TiCDC, TiFlash, PD, and other tools. Improvements include support for configuring RocksDB write stall settings, optimizing Scatter Region to batch mode, and reducing performance overhead in multi-Region scenarios. Bug fixes address issues such as incorrect output of `SHOW CREATE PLACEMENT POLICY`, DDL statements getting stuck after PD node replacement, and various issues causing incorrect results and errors in TiDB, TiKV, PD, TiFlash, and other tools. The release also provides workarounds and affected versions for specific issues. --- # TiDB 5.4.3 Release Notes diff --git a/releases/release-6.0.0-dmr.md b/releases/release-6.0.0-dmr.md index 826bed2523789..6417a8c20462e 100644 --- a/releases/release-6.0.0-dmr.md +++ b/releases/release-6.0.0-dmr.md @@ -1,5 +1,6 @@ --- title: TiDB 6.0.0 Release Notes +summary: Learn about the new features, compatibility changes, improvements, and bug fixes in TiDB 6.0.0. --- # TiDB 6.0.0 Release Notes diff --git a/releases/release-6.1.0.md b/releases/release-6.1.0.md index 7f58ad1021752..c612f4d0f0260 100644 --- a/releases/release-6.1.0.md +++ b/releases/release-6.1.0.md @@ -9,7 +9,7 @@ Release date: June 13, 2022 TiDB version: 6.1.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) In 6.1.0, the key new features or improvements are as follows: @@ -342,7 +342,7 @@ In 6.1.0, the key new features or improvements are as follows: - CDC supports RawKV [#11965](https://github.com/tikv/tikv/issues/11965) - Support splitting a large snapshot file into multiple files [#11595](https://github.com/tikv/tikv/issues/11595) - Move the snapshot garbage collection from Raftstore to background thread to prevent snapshot GC from blocking Raftstore message loops [#11966](https://github.com/tikv/tikv/issues/11966) - - Support dynamic setting of the the maximum message length (`max-grpc-send-msg-len`) and the maximum batch size of gPRC messages (`raft-msg-max-batch-size`) [#12334](https://github.com/tikv/tikv/issues/12334) + - Support dynamic setting of the maximum message length (`max-grpc-send-msg-len`) and the maximum batch size of gPRC messages (`raft-msg-max-batch-size`) [#12334](https://github.com/tikv/tikv/issues/12334) - Support executing online unsafe recovery plan through Raft [#10483](https://github.com/tikv/tikv/issues/10483) + PD diff --git a/releases/release-6.1.1.md b/releases/release-6.1.1.md index 3bf82c7c20952..47aadba78f361 100644 --- a/releases/release-6.1.1.md +++ b/releases/release-6.1.1.md @@ -9,7 +9,7 @@ Release date: September 1, 2022 TiDB version: 6.1.1 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.1#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes @@ -81,7 +81,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- - Fix the issue that the wrong join reorder in some right outer join scenarios causes wrong query result [#36912](https://github.com/pingcap/tidb/issues/36912) @[winoros](https://github.com/winoros) - Fix the issue of incorrectly inferred null flag of the TiFlash `firstrow` aggregate function in the EqualAll case [#34584](https://github.com/pingcap/tidb/issues/34584) @[fixdb](https://github.com/fixdb) - Fix the issue that Plan Cache does not work when a binding is created with the `IGNORE_PLAN_CACHE` hint [#34596](https://github.com/pingcap/tidb/issues/34596) @[fzzf678](https://github.com/fzzf678) - - Fix the issu that an `EXCHANGE` operator is missing between the hash-partition window and the single-partition window [#35990](https://github.com/pingcap/tidb/issues/35990) @[LittleFall](https://github.com/LittleFall) + - Fix the issue that an `EXCHANGE` operator is missing between the hash-partition window and the single-partition window [#35990](https://github.com/pingcap/tidb/issues/35990) @[LittleFall](https://github.com/LittleFall) - Fix the issue that partitioned tables cannot fully use indexes to scan data in some cases [#33966](https://github.com/pingcap/tidb/issues/33966) @[mjonss](https://github.com/mjonss) - Fix the issue of wrong query result when a wrong default value is set for partial aggregation after the aggregation is pushed down [#35295](https://github.com/pingcap/tidb/issues/35295) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the issue that querying partitioned tables might get the `index-out-of-range` error in some cases [#35181](https://github.com/pingcap/tidb/issues/35181) @[mjonss](https://github.com/mjonss) diff --git a/releases/release-6.1.2.md b/releases/release-6.1.2.md index 45b6fa42a46ed..40c9003a0a083 100644 --- a/releases/release-6.1.2.md +++ b/releases/release-6.1.2.md @@ -9,7 +9,7 @@ Release date: October 24, 2022 TiDB version: 6.1.2 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.2#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Improvements diff --git a/releases/release-6.1.3.md b/releases/release-6.1.3.md index 1418205d0c1bd..2fa7956f4be62 100644 --- a/releases/release-6.1.3.md +++ b/releases/release-6.1.3.md @@ -9,7 +9,7 @@ Release date: December 5, 2022 TiDB version: 6.1.3 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.3#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-6.1.4.md b/releases/release-6.1.4.md index 7dc2dc3ed32d5..4e452fbdbb06d 100644 --- a/releases/release-6.1.4.md +++ b/releases/release-6.1.4.md @@ -9,7 +9,7 @@ Release date: February 8, 2023 TiDB version: 6.1.4 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.4#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-6.1.5.md b/releases/release-6.1.5.md index 846ffe4e056db..7d72630a47681 100644 --- a/releases/release-6.1.5.md +++ b/releases/release-6.1.5.md @@ -9,7 +9,7 @@ Release date: February 28, 2023 TiDB version: 6.1.5 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.5#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-6.1.6.md b/releases/release-6.1.6.md index c21c534b24fd7..17725f58ad6e3 100644 --- a/releases/release-6.1.6.md +++ b/releases/release-6.1.6.md @@ -9,7 +9,7 @@ Release date: April 12, 2023 TiDB version: 6.1.6 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.6#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Compatibility changes @@ -92,7 +92,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiDB Lightning - - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[gozssky](https://github.com/gozssky) + - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[sleepymole](https://github.com/sleepymole) - Fix the issue that TiDB Lightning panics in the split-region phase [#40934](https://github.com/pingcap/tidb/issues/40934) @[lance6716](https://github.com/lance6716) - Fix the issue that when importing data in Local Backend mode, the target columns do not automatically generate data if the compound primary key of the imported target table has an `auto_random` column and no value for the column is specified in the source data [#41454](https://github.com/pingcap/tidb/issues/41454) @[D3Hunter](https://github.com/D3Hunter) - Fix the issue that TiDB Lightning might incorrectly skip conflict resolution when all but the last TiDB Lightning instance encounters a local duplicate record during a parallel import [#40923](https://github.com/pingcap/tidb/issues/40923) @[lichunzhu](https://github.com/lichunzhu) diff --git a/releases/release-6.1.7.md b/releases/release-6.1.7.md index 728ea0b6dab7d..2977b24b64f94 100644 --- a/releases/release-6.1.7.md +++ b/releases/release-6.1.7.md @@ -9,7 +9,7 @@ Release date: July 12, 2023 TiDB version: 6.1.7 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.1.7#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.1/production-deployment-using-tiup) ## Improvements diff --git a/releases/release-6.2.0.md b/releases/release-6.2.0.md index 37ca32dd9c3f4..1c7d9e57dc1b7 100644 --- a/releases/release-6.2.0.md +++ b/releases/release-6.2.0.md @@ -1,5 +1,6 @@ --- title: TiDB 6.2.0 Release Notes +summary: TiDB 6.2.0-DMR introduces new features like visual execution plans, monitoring page, and lock view. It also supports concurrent DDL operations and enhances the performance of aggregation operations. TiKV now supports automatic CPU usage tuning and detailed configuration information listing. TiFlash adds FastScan for data scanning and improves error handling. BR now supports continuous data validation and automatically identifies the region of Amazon S3 buckets. TiCDC supports filtering DDL and DML events. There are also compatibility changes, bug fixes, and improvements across various tools. --- # TiDB 6.2.0 Release Notes @@ -221,7 +222,7 @@ In v6.2.0-DMR, the key new features and improvements are as follows: This feature does not need manual configuration. If your TiDB cluster is v6.1.0 or later versions and TiDB Lightning is v6.2.0 or later versions, the new physical import mode takes effect automatically. - [User document](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#scope-of-pausing-scheduling-during-import) [#35148](https://github.com/pingcap/tidb/issues/35148) @[gozssky](https://github.com/gozssky) + [User document](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#scope-of-pausing-scheduling-during-import) [#35148](https://github.com/pingcap/tidb/issues/35148) @[sleepymole](https://github.com/sleepymole) * Refactor the [user documentation of TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) to make its structure more reasonable and clear. The terms for "backend" is also modified to lower the understanding barrier for new users: diff --git a/releases/release-6.3.0.md b/releases/release-6.3.0.md index 517ddc6b08c55..ad4b13bd9faf8 100644 --- a/releases/release-6.3.0.md +++ b/releases/release-6.3.0.md @@ -1,5 +1,6 @@ --- title: TiDB 6.3.0 Release Notes +summary: TiDB 6.3.0-DMR, released on September 30, 2022, introduces new features and improvements, including encryption at rest using the SM4 algorithm in TiKV, authentication using the SM3 algorithm in TiDB, and support for JSON data type and functions. It also provides execution time metrics at a finer granularity, enhances output for slow logs and `TRACE` statements, and supports deadlock history information in TiDB Dashboard. Additionally, TiDB v6.3.0 introduces new system variables and configuration file parameters, and fixes various bugs and issues. The release also includes improvements in TiKV, PD, TiFlash, Backup & Restore (BR), TiCDC, TiDB Binlog, TiDB Data Migration (DM), and TiDB Lightning. --- # TiDB 6.3.0 Release Notes @@ -12,7 +13,7 @@ TiDB version: 6.3.0-DMR > > The TiDB 6.3.0-DMR documentation has been [archived](https://docs-archive.pingcap.com/tidb/v6.3/). PingCAP encourages you to use [the latest LTS version](https://docs.pingcap.com/tidb/stable) of the TiDB database. -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.3/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v6.3.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.3/quick-start-with-tidb) In v6.3.0-DMR, the key new features and improvements are as follows: @@ -335,7 +336,7 @@ Since v6.3.0, TiCDC no longer supports configuring Pulsar sink. [kop](https://gi - Fix the issue that the privilege check is skipped for `PREPARE` statements [#35784](https://github.com/pingcap/tidb/issues/35784) @[lcwangchao](https://github.com/lcwangchao) - Fix the issue that the system variable `tidb_enable_noop_variable` can be set to `WARN` [#36647](https://github.com/pingcap/tidb/issues/36647) @[lcwangchao](https://github.com/lcwangchao) - - Fix the issue that when an expression index is defined, the `ORDINAL_POSITION` column of the `INFORMAITON_SCHEMA.COLUMNS` table might be incorrect [#31200](https://github.com/pingcap/tidb/issues/31200) @[bb7133](https://github.com/bb7133) + - Fix the issue that when an expression index is defined, the `ORDINAL_POSITION` column of the `INFORMATION_SCHEMA.COLUMNS` table might be incorrect [#31200](https://github.com/pingcap/tidb/issues/31200) @[bb7133](https://github.com/bb7133) - Fix the issue that TiDB does not report an error when the timestamp is larger than `MAXINT32` [#31585](https://github.com/pingcap/tidb/issues/31585) @[bb7133](https://github.com/bb7133) - Fix the issue that TiDB server cannot be started when the enterprise plugin is used [#37319](https://github.com/pingcap/tidb/issues/37319) @[xhebox](https://github.com/xhebox) - Fix the incorrect output of `SHOW CREATE PLACEMENT POLICY` [#37526](https://github.com/pingcap/tidb/issues/37526) @[xhebox](https://github.com/xhebox) @@ -353,7 +354,7 @@ Since v6.3.0, TiCDC no longer supports configuring Pulsar sink. [kop](https://gi - Fix the issue that the cast and comparison between binary strings and JSON in TiDB are incompatible with MySQL [#31918](https://github.com/pingcap/tidb/issues/31918) [#25053](https://github.com/pingcap/tidb/issues/25053) @[YangKeao](https://github.com/YangKeao) - Fix the issue that `JSON_OBJECTAGG` and `JSON_ARRAYAGG` in TiDB are not compatible with MySQL on binary values [#25053](https://github.com/pingcap/tidb/issues/25053) @[YangKeao](https://github.com/YangKeao) - Fix the issue that the comparison between JSON opaque values causes panic [#37315](https://github.com/pingcap/tidb/issues/37315) @[YangKeao](https://github.com/YangKeao) - - Fix the issue that the single precision float cannot be used in JSON aggregation funtions [#37287](https://github.com/pingcap/tidb/issues/37287) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the single precision float cannot be used in JSON aggregation functions [#37287](https://github.com/pingcap/tidb/issues/37287) @[YangKeao](https://github.com/YangKeao) - Fix the issue that the `UNION` operator might return unexpected empty result [#36903](https://github.com/pingcap/tidb/issues/36903) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the issue that the result of the `castRealAsTime` expression is inconsistent with MySQL [#37462](https://github.com/pingcap/tidb/issues/37462) @[mengxin9014](https://github.com/mengxin9014) - Fix the issue that pessimistic DML operations lock non-unique index keys [#36235](https://github.com/pingcap/tidb/issues/36235) @[ekexium](https://github.com/ekexium) diff --git a/releases/release-6.4.0.md b/releases/release-6.4.0.md index 63495548356d4..b9f6eb353a396 100644 --- a/releases/release-6.4.0.md +++ b/releases/release-6.4.0.md @@ -1,5 +1,6 @@ --- title: TiDB 6.4.0 Release Notes +summary: TiDB 6.4.0-DMR introduces new features and improvements, including support for restoring a cluster to a specific point in time, compatibility with Linear Hash partitioning syntax, and a high-performance `AUTO_INCREMENT` mode. It also enhances fault recovery, memory usage control, and statistics collection. TiFlash now supports the SM4 algorithm for encryption at rest, and TiCDC supports replicating data to Kafka. The release also includes bug fixes and improvements across various tools and components. --- # TiDB 6.4.0 Release Notes @@ -12,7 +13,7 @@ TiDB version: 6.4.0-DMR > > The TiDB 6.4.0-DMR documentation has been [archived](https://docs-archive.pingcap.com/tidb/v6.4/). PingCAP encourages you to use [the latest LTS version](https://docs.pingcap.com/tidb/stable) of the TiDB database. -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.4/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v6.4.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.4/quick-start-with-tidb) In v6.4.0-DMR, the key new features and improvements are as follows: @@ -168,7 +169,7 @@ In v6.4.0-DMR, the key new features and improvements are as follows: * Be compatible with the Linear Hash partitioning syntax [#38450](https://github.com/pingcap/tidb/issues/38450) @[mjonss](https://github.com/mjonss) - In the earlier version, TiDB has supported the Hash, Range, and List partitioning. Starting from v6.4.0, TiDB can also be compatible with the syntaxt of [MySQL Linear Hash partitioning](https://dev.mysql.com/doc/refman/5.7/en/partitioning-linear-hash.html). + In the earlier version, TiDB has supported the Hash, Range, and List partitioning. Starting from v6.4.0, TiDB can also be compatible with the syntax of [MySQL Linear Hash partitioning](https://dev.mysql.com/doc/refman/5.7/en/partitioning-linear-hash.html). In TiDB, you can execute the existing DDL statements of your MySQL Linear Hash partitions directly, and TiDB will create the corresponding Hash partition tables (note that there is no Linear Hash partition inside TiDB). You can also execute the existing DML statements of your MySQL Linear Hash partitions directly, and TiDB will return the query result of the corresponding TiDB Hash partitions normally. This feature ensures the TiDB syntax compatibility with MySQL Linear Hash partitions and facilitates seamless migration from MySQL-based applications to TiDB. @@ -340,7 +341,7 @@ In v6.4.0-DMR, the key new features and improvements are as follows: - Add a new configuration item `apply-yield-write-size` to control the maximum number of bytes that the Apply thread can write for one Finite-state Machine in one round of poll, and relieve Raftstore congestion when the Apply thread writes a large volume of data [#13313](https://github.com/tikv/tikv/issues/13313) @[glorv](https://github.com/glorv) - Warm up the entry cache before migrating the leader of Region to avoid QPS jitter during the leader transfer process [#13060](https://github.com/tikv/tikv/issues/13060) @[cosven](https://github.com/cosven) - - Support pushing down the `json_constains` operator to Coprocessor [#13592](https://github.com/tikv/tikv/issues/13592) @[lizhenhuan](https://github.com/lizhenhuan) + - Support pushing down the `json_constrains` operator to Coprocessor [#13592](https://github.com/tikv/tikv/issues/13592) @[lizhenhuan](https://github.com/lizhenhuan) - Add the asynchronous function for `CausalTsProvider` to improve the flush performance in some scenarios [#13428](https://github.com/tikv/tikv/issues/13428) @[zeminzhou](https://github.com/zeminzhou) + PD diff --git a/releases/release-6.5.0.md b/releases/release-6.5.0.md index 12770d715160f..ac8a160c35997 100644 --- a/releases/release-6.5.0.md +++ b/releases/release-6.5.0.md @@ -9,7 +9,7 @@ Release date: December 29, 2022 TiDB version: 6.5.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) TiDB 6.5.0 is a Long-Term Support Release (LTS). diff --git a/releases/release-6.5.1.md b/releases/release-6.5.1.md index d2be099af32e5..25ee8c978a65e 100644 --- a/releases/release-6.5.1.md +++ b/releases/release-6.5.1.md @@ -9,7 +9,7 @@ Release date: March 10, 2023 TiDB version: 6.5.1 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.1#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes @@ -50,7 +50,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Support starting TiKV on a CPU with less than 1 core [#13586](https://github.com/tikv/tikv/issues/13586) [#13752](https://github.com/tikv/tikv/issues/13752) [#14017](https://github.com/tikv/tikv/issues/14017) @[andreid-db](https://github.com/andreid-db) - Increase the thread limit of the Unified Read Pool (`readpool.unified.max-thread-count`) to 10 times the CPU quota, to better handle high-concurrency queries [#13690](https://github.com/tikv/tikv/issues/13690) @[v01dstar](https://github.com/v01dstar) - - Change the the default value of `resolved-ts.advance-ts-interval` from `"1s"` to `"20s"`, to reduce cross-region traffic [#14100](https://github.com/tikv/tikv/issues/14100) @[overvenus](https://github.com/overvenus) + - Change the default value of `resolved-ts.advance-ts-interval` from `"1s"` to `"20s"`, to reduce cross-region traffic [#14100](https://github.com/tikv/tikv/issues/14100) @[overvenus](https://github.com/overvenus) + TiFlash @@ -182,6 +182,6 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that TiDB Lightning prechecks cannot find dirty data left by previously failed imports [#39477](https://github.com/pingcap/tidb/issues/39477) @[dsdashun](https://github.com/dsdashun) - Fix the issue that TiDB Lightning panics in the split-region phase [#40934](https://github.com/pingcap/tidb/issues/40934) @[lance6716](https://github.com/lance6716) - - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[gozssky](https://github.com/gozssky) + - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[sleepymole](https://github.com/sleepymole) - Fix the issue that TiDB Lightning might incorrectly skip conflict resolution when all but the last TiDB Lightning instance encounters a local duplicate record during a parallel import [#40923](https://github.com/pingcap/tidb/issues/40923) @[lichunzhu](https://github.com/lichunzhu) - Fix the issue that when importing data in Local Backend mode, the target columns do not automatically generate data if the compound primary key of the imported target table has an `auto_random` column and no value for the column is specified in the source data [#41454](https://github.com/pingcap/tidb/issues/41454) @[D3Hunter](https://github.com/D3Hunter) diff --git a/releases/release-6.5.2.md b/releases/release-6.5.2.md index 9bab3dacafb18..86435e538c5c0 100644 --- a/releases/release-6.5.2.md +++ b/releases/release-6.5.2.md @@ -9,7 +9,7 @@ Release date: April 21, 2023 TiDB version: 6.5.2 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.2#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-6.5.3.md b/releases/release-6.5.3.md index 093aed31dac16..38e48b99e0b31 100644 --- a/releases/release-6.5.3.md +++ b/releases/release-6.5.3.md @@ -9,7 +9,7 @@ Release date: June 14, 2023 TiDB version: 6.5.3 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.3#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-6.5.4.md b/releases/release-6.5.4.md index a4f5b5100dc3f..0c2667d5eb67f 100644 --- a/releases/release-6.5.4.md +++ b/releases/release-6.5.4.md @@ -9,7 +9,7 @@ Release date: August 28, 2023 TiDB version: 6.5.4 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.4#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes @@ -112,7 +112,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that killing a connection might cause go coroutine leaks [#46034](https://github.com/pingcap/tidb/issues/46034) @[pingyu](https://github.com/pingyu) - Fix the issue that the `tmp-storage-quota` configuration does not take effect [#45161](https://github.com/pingcap/tidb/issues/45161) [#26806](https://github.com/pingcap/tidb/issues/26806) @[wshwsh12](https://github.com/wshwsh12) - Fix the issue that TiFlash replicas might be unavailable when a TiFlash node is down in the cluster [#38484](https://github.com/pingcap/tidb/issues/38484) @[hehechen](https://github.com/hehechen) - - Fix the issue that TiDB crashes due to possible data race when reading and writing `Config.Lables` concurrently [#45561](https://github.com/pingcap/tidb/issues/45561) @[genliqi](https://github.com/gengliqi) + - Fix the issue that TiDB crashes due to possible data race when reading and writing `Config.Labels` concurrently [#45561](https://github.com/pingcap/tidb/issues/45561) @[genliqi](https://github.com/gengliqi) - Fix the issue that the client-go regularly updating `min-resolved-ts` might cause PD OOM when the cluster is large [#46664](https://github.com/pingcap/tidb/issues/46664) @[HuSharp](https://github.com/HuSharp) + TiKV @@ -132,7 +132,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that when etcd is already started but the client has not yet connected to it, calling the client might cause PD to panic [#6860](https://github.com/tikv/pd/issues/6860) @[HuSharp](https://github.com/HuSharp) - Fix the issue that a leader cannot exit for a long time [#6918](https://github.com/tikv/pd/issues/6918) @[bufferflies](https://github.com/bufferflies) - - Fix the issue that when the placement rule uses `LOCATION_LABLES`, SQL and the Rule Checker are not compatible [#38605](https://github.com/pingcap/tidb/issues/38605) @[nolouch](https://github.com/nolouch) + - Fix the issue that when the placement rule uses `LOCATION_LABELS`, SQL and the Rule Checker are not compatible [#38605](https://github.com/pingcap/tidb/issues/38605) @[nolouch](https://github.com/nolouch) - Fix the issue that PD might unexpectedly add multiple Learners to a Region [#5786](https://github.com/tikv/pd/issues/5786) @[HunDunDM](https://github.com/HunDunDM) - Fix the issue that unhealthy peers cannot be removed when rule checker selects peers [#6559](https://github.com/tikv/pd/issues/6559) @[nolouch](https://github.com/nolouch) - Fix the issue that failed learner peers in `unsafe recovery` are ignored in `auto-detect` mode [#6690](https://github.com/tikv/pd/issues/6690) @[v01dstar](https://github.com/v01dstar) diff --git a/releases/release-6.5.5.md b/releases/release-6.5.5.md index 16246a2af6c76..3d653d2f92a75 100644 --- a/releases/release-6.5.5.md +++ b/releases/release-6.5.5.md @@ -9,7 +9,7 @@ Release date: September 21, 2023 TiDB version: 6.5.5 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.5#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Improvements diff --git a/releases/release-6.5.6.md b/releases/release-6.5.6.md index 12913b3af00ed..59f9a350c4d30 100644 --- a/releases/release-6.5.6.md +++ b/releases/release-6.5.6.md @@ -9,7 +9,7 @@ Release date: December 7, 2023 TiDB version: 6.5.6 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.6#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes @@ -17,7 +17,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Introduce the [`tidb_opt_enable_hash_join`](https://docs.pingcap.com/tidb/v6.5/system-variables#tidb_opt_enable_hash_join-new-in-v656) system variable to control whether the optimizer selects hash joins for tables [#46695](https://github.com/pingcap/tidb/issues/46695) @[coderplay](https://github.com/coderplay) - After further testing, the default value of the TiCDC Changefeed configuration item [`case-sensitive`](/ticdc/ticdc-changefeed-config.md) is changed from `true` to `false`. This means that by default, table and database names in the TiCDC configuration file are case-insensitive [#10047](https://github.com/pingcap/tiflow/issues/10047) @[sdojjy](https://github.com/sdojjy) - TiCDC Changefeed introduces the following new configuration items: - - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](/ticdc/ticdc-ddl.md#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) + - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](https://docs.pingcap.com/tidb/v6.5/ticdc-ddl#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) - [`encoding-worker-num`](/ticdc/ticdc-changefeed-config.md) and [`flush-worker-num`](/ticdc/ticdc-changefeed-config.md): enables you to set different concurrency parameters for the redo module based on specifications of different machines [#10048](https://github.com/pingcap/tiflow/issues/10048) @[CharlesCheung96](https://github.com/CharlesCheung96) - [`compression`](/ticdc/ticdc-changefeed-config.md): enables you to configure the compression behavior of redo log files [#10176](https://github.com/pingcap/tiflow/issues/10176) @[sdojjy](https://github.com/sdojjy) - [`sink.cloud-storage-config`](/ticdc/ticdc-changefeed-config.md): enables you to set the automatic cleanup of historical data when replicating data to object storage [#10109](https://github.com/pingcap/tiflow/issues/10109) @[CharlesCheung96](https://github.com/CharlesCheung96) diff --git a/releases/release-6.5.7.md b/releases/release-6.5.7.md index b477298b0df1c..5d63eb623d118 100644 --- a/releases/release-6.5.7.md +++ b/releases/release-6.5.7.md @@ -9,7 +9,7 @@ Release date: January 8, 2024 TiDB version: 6.5.7 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.7#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-6.5.8.md b/releases/release-6.5.8.md index 4724e94b804a1..742b7d4c8c46e 100644 --- a/releases/release-6.5.8.md +++ b/releases/release-6.5.8.md @@ -9,7 +9,7 @@ Release date: February 2, 2024 TiDB version: 6.5.8 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v6.5.8#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-6.5.9.md b/releases/release-6.5.9.md new file mode 100644 index 0000000000000..d4699e224d41b --- /dev/null +++ b/releases/release-6.5.9.md @@ -0,0 +1,138 @@ +--- +title: TiDB 6.5.9 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 6.5.9. +--- + +# TiDB 6.5.9 Release Notes + +Release date: April 12, 2024 + +TiDB version: 6.5.9 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +- Add a TiKV configuration item [`track-and-verify-wals-in-manifest`](https://docs.pingcap.com/tidb/v6.5/tikv-configuration-file#track-and-verify-wals-in-manifest-new-in-v659) for RocksDB, which helps you investigate possible corruption of Write Ahead Log (WAL) [#16549](https://github.com/tikv/tikv/issues/16549) @[v01dstar](https://github.com/v01dstar) +- DR Auto-Sync supports configuring [`wait-recover-timeout`](https://docs.pingcap.com/tidb/v6.5/two-data-centers-in-one-city-deployment#enable-the-dr-auto-sync-mode), which enables you to control the waiting time for switching back to the `sync-recover` status after the network recovers [#6295](https://github.com/tikv/pd/issues/6295) @[disksing](https://github.com/disksing) + +## Improvements + ++ TiDB + + - When `force-init-stats` is set to `true`, TiDB waits for statistics initialization to finish before providing services during TiDB startup. This setting no longer blocks the startup of HTTP servers, which enables users to continue monitoring [#50854](https://github.com/pingcap/tidb/issues/50854) @[hawkingrei](https://github.com/hawkingrei) + - Optimize the issue that the `ANALYZE` statement blocks the metadata lock [#47475](https://github.com/pingcap/tidb/issues/47475) @[wjhuang2016](https://github.com/wjhuang2016) + ++ TiKV + + - Remove unnecessary async blocks to reduce memory usage [#16540](https://github.com/tikv/tikv/issues/16540) @[overvenus](https://github.com/overvenus) + - Avoid performing IO operations on snapshot files in raftstore threads to improve TiKV stability [#16564](https://github.com/tikv/tikv/issues/16564) @[Connor1996](https://github.com/Connor1996) + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + ++ Tools + + + Backup & Restore (BR) + + - Optimize the Recovery Point Objective (RPO) for log backup during rolling restarts. Now, the checkpoint lag of log backup tasks will be smaller during rolling restarts [#15410](https://github.com/tikv/tikv/issues/15410) @[YuJuncen](https://github.com/YuJuncen) + - Enhance the tolerance of log backup to merge operations. When encountering a reasonably long merge operation, log backup tasks are less likely to enter the error state [#16554](https://github.com/tikv/tikv/issues/16554) @[YuJuncen](https://github.com/YuJuncen) + - Support automatically abandoning log backup tasks when encountering a large checkpoint lag, to avoid prolonged blocking GC and potential cluster issues [#50803](https://github.com/pingcap/tidb/issues/50803) @[RidRisR](https://github.com/RidRisR) + - Alleviate the issue that the latency of the PITR log backup progress increases when Region leadership migration occurs [#13638](https://github.com/tikv/tikv/issues/13638) @[YuJuncen](https://github.com/YuJuncen) + - Improve the speed of merging SST files during data restore by using a more efficient algorithm [#50613](https://github.com/pingcap/tidb/issues/50613) @[Leavrth](https://github.com/Leavrth) + - Support ingesting SST files in batch during data restore [#16267](https://github.com/tikv/tikv/issues/16267) @[3pointer](https://github.com/3pointer) + - Remove an outdated compatibility check when using Google Cloud Storage (GCS) as the external storage [#50533](https://github.com/pingcap/tidb/issues/50533) @[lance6716](https://github.com/lance6716) + - Print the information of the slowest Region that affects global checkpoint advancement in logs and metrics during log backups [#51046](https://github.com/pingcap/tidb/issues/51046) @[YuJuncen](https://github.com/YuJuncen) + - Refactor the BR exception handling mechanism to increase tolerance for unknown errors [#47656](https://github.com/pingcap/tidb/issues/47656) @[3pointer](https://github.com/3pointer) + +## Bug fixes + ++ TiDB + + - Fix the issue that after a large number of tables are created, the newly created tables might lack the `stats_meta` information, causing subsequent query estimation to fail to get accurate row count information [#36004](https://github.com/pingcap/tidb/issues/36004) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + - Fix the issue that dropped tables are still counted by the Grafana `Stats Healthy Distribution` panel [#39349](https://github.com/pingcap/tidb/issues/39349) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + - Fix the issue that TiDB does not handle the `WHERE ` filtering condition in a SQL statement when the query of that statement involves the `MemTableScan` operator [#40937](https://github.com/pingcap/tidb/issues/40937) @[zhongzc](https://github.com/zhongzc) + - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that query results might be incorrect when you use Common Table Expressions (CTE) to access partitioned tables with missing statistics [#51873](https://github.com/pingcap/tidb/issues/51873) @[qw4990](https://github.com/qw4990) + - Fix the issue that query execution using MPP might lead to incorrect query results when a SQL statement contains `JOIN` and the `SELECT` list in the statement contains only constants [#50358](https://github.com/pingcap/tidb/issues/50358) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `AUTO_INCREMENT` attribute causes non-consecutive IDs due to unnecessary transaction conflicts when assigning auto-increment IDs [#50819](https://github.com/pingcap/tidb/issues/50819) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the monitoring metric `tidb_statistics_auto_analyze_total` on Grafana is not displayed as an integer [#51051](https://github.com/pingcap/tidb/issues/51051) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that errors might be returned during the concurrent merging of global statistics for partitioned tables [#48713](https://github.com/pingcap/tidb/issues/48713) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that getting the default value of a column returns an error if the column default value is dropped [#50043](https://github.com/pingcap/tidb/issues/50043) [#51324](https://github.com/pingcap/tidb/issues/51324) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `INSERT ignore` statement cannot fill in default values when the column is write-only [#40192](https://github.com/pingcap/tidb/issues/40192) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that TiDB crashes when `shuffleExec` quits unexpectedly [#48230](https://github.com/pingcap/tidb/issues/48230) @[wshwsh12](https://github.com/wshwsh12) + - Fix the goroutine leak issue that might occur when the `HashJoin` operator fails to spill to disk [#50841](https://github.com/pingcap/tidb/issues/50841) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that renaming tables does not take effect when committing multiple statements in a transaction [#39664](https://github.com/pingcap/tidb/issues/39664) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the query result is incorrect when the `IN()` predicate contains `NULL` [#51560](https://github.com/pingcap/tidb/issues/51560) @[winoros](https://github.com/winoros) + - Fix the issue that querying JSON of `BINARY` type might cause an error in some cases [#51547](https://github.com/pingcap/tidb/issues/51547) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that parallel `Apply` might generate incorrect results when the table has a clustered index [#51372](https://github.com/pingcap/tidb/issues/51372) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `tidb_merge_partition_stats_concurrency` variable does not take effect when `auto analyze` is processing partitioned tables [#47594](https://github.com/pingcap/tidb/issues/47594) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that after the time window for automatic statistics updates is configured, statistics might still be updated outside that time window [#49552](https://github.com/pingcap/tidb/issues/49552) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `approx_percentile` function might cause TiDB panic [#40463](https://github.com/pingcap/tidb/issues/40463) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that `BIT` type columns might cause query errors due to decode failures when they are involved in calculations of some functions [#49566](https://github.com/pingcap/tidb/issues/49566) [#50850](https://github.com/pingcap/tidb/issues/50850) [#50855](https://github.com/pingcap/tidb/issues/50855) @[jiyfhust](https://github.com/jiyfhust) + - Fix the goroutine leak issue that occurs when the memory usage of CTE queries exceed limits [#50337](https://github.com/pingcap/tidb/issues/50337) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiDB does not listen to the corresponding port when `force-init-stats` is configured [#51473](https://github.com/pingcap/tidb/issues/51473) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `ALTER TABLE ... COMPACT TIFLASH REPLICA` might incorrectly end when the primary key type is `VARCHAR` [#51810](https://github.com/pingcap/tidb/issues/51810) @[breezewish](https://github.com/breezewish) + - Fix the issue that the `tidb_gogc_tuner_threshold` system variable is not adjusted accordingly after the `tidb_server_memory_limit` variable is modified [#48180](https://github.com/pingcap/tidb/issues/48180) @[hawkingrei](https://github.com/hawkingrei) + - Fix the `Can't find column ...` error that might occur when aggregate functions are used for group calculations [#50926](https://github.com/pingcap/tidb/issues/50926) @[qw4990](https://github.com/qw4990) + - Fix the issue that the `REVERSE` function reports an error when processing columns of the `BIT` type [#50850](https://github.com/pingcap/tidb/issues/50850) [#50855](https://github.com/pingcap/tidb/issues/50855) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that `INSERT IGNORE` reports an error when inserting data in bulk into a table undergoing a DDL operation [#50993](https://github.com/pingcap/tidb/issues/50993) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the TiDB server adds a label via the HTTP interface and returns success, but does not take effect [#51427](https://github.com/pingcap/tidb/issues/51427) @[you06](https://github.com/you06) + - Fix the issue that the type returned by the `IFNULL` function is inconsistent with MySQL [#51765](https://github.com/pingcap/tidb/issues/51765) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the TiDB server is marked as health before the initialization is complete [#51596](https://github.com/pingcap/tidb/issues/51596) @[shenqidebaozi](https://github.com/shenqidebaozi) + - Fix the issue that querying the `TIDB_HOT_REGIONS` table might incorrectly return `INFORMATION_SCHEMA` tables [#50810](https://github.com/pingcap/tidb/issues/50810) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that `EXCHANGE PARTITION` incorrectly processes foreign keys [#51807](https://github.com/pingcap/tidb/issues/51807) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that executing queries containing CTEs causes TiDB to panic [#41688](https://github.com/pingcap/tidb/issues/41688) @[srstack](https://github.com/srstack) + ++ TiKV + + - Fix the issue that after the process of destroying Peer is interrupted by applying snapshots, it does not resume even after applying snapshots is complete [#16561](https://github.com/tikv/tikv/issues/16561) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that inactive Write Ahead Logs (WALs) in RocksDB might corrupt data [#16705](https://github.com/tikv/tikv/issues/16705) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that the monitoring metric `tikv_unified_read_pool_thread_count` has no data in some cases [#16629](https://github.com/tikv/tikv/issues/16629) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that JSON integers greater than the maximum `INT64` value but less than the maximum `UINT64` value are parsed as `FLOAT64` by TiKV, resulting in inconsistency with TiDB [#16512](https://github.com/tikv/tikv/issues/16512) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that during the execution of an optimistic transaction, if other transactions initiate the resolving lock operation on it, there is a small chance that the atomicity of the transaction might be broken if the transaction's primary key has data that was previously committed in Async Commit or 1PC mode [#16620](https://github.com/tikv/tikv/issues/16620) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue that the scaling progress is not correctly displayed [#7726](https://github.com/tikv/pd/issues/7726) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that data race occurs when the `MergeLabels` function is called [#7535](https://github.com/tikv/pd/issues/7535) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that the PD monitoring item `learner-peer-count` does not synchronize the old value after a leader switch [#7728](https://github.com/tikv/pd/issues/7728) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that the query result of `SHOW CONFIG` includes the deprecated configuration item `trace-region-flow` [#7917](https://github.com/tikv/pd/issues/7917) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that TiFlash might panic due to unstable network connections with PD during replica migration [#8323](https://github.com/pingcap/tiflash/issues/8323) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might crash due to data race in case of remote reads [#8685](https://github.com/pingcap/tiflash/issues/8685) @[solotzg](https://github.com/solotzg) + - Fix the issue that the `ENUM` column might cause TiFlash to crash during chunk encoding [#8674](https://github.com/pingcap/tiflash/issues/8674) @[yibin87](https://github.com/yibin87) + - Fix the issue that TiFlash might panic when you insert data to columns with invalid default values in non-strict `sql_mode` [#8803](https://github.com/pingcap/tiflash/issues/8803) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that if Region migration, split, or merge occurs after the precision of a `TIME` column is modified, queries might fail [#8601](https://github.com/pingcap/tiflash/issues/8601) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that too many logs are printed when a full backup fails [#51572](https://github.com/pingcap/tidb/issues/51572) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that removing a log backup task after it is paused does not immediately restore the GC safepoint [#52082](https://github.com/pingcap/tidb/issues/52082) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR could not back up the `AUTO_RANDOM` ID allocation progress in a union clustered index that contains an `AUTO_RANDOM` column [#52255](https://github.com/pingcap/tidb/issues/52255) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that stopping a log backup task causes TiDB to crash [#50839](https://github.com/pingcap/tidb/issues/50839) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that TiKV panics when a full backup fails to find a peer in some extreme cases [#16394](https://github.com/tikv/tikv/issues/16394) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Fix the issue that `snapshot lost caused by GC` is not reported in time when resuming a changefeed and the `checkpoint-ts` of the changefeed is smaller than the GC safepoint of TiDB [#10463](https://github.com/pingcap/tiflow/issues/10463) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that data is written to a wrong CSV file due to wrong BarrierTS in scenarios where DDL statements are executed frequently [#10668](https://github.com/pingcap/tiflow/issues/10668) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that the Syncpoint table might be incorrectly replicated [#10576](https://github.com/pingcap/tiflow/issues/10576) @[asddongmen](https://github.com/asddongmen) + - Fix the issue TiCDC panics when scheduling table replication tasks [#10613](https://github.com/pingcap/tiflow/issues/10613) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that data race in the KV client causes TiCDC to panic [#10718](https://github.com/pingcap/tiflow/issues/10718) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the file sequence number generated by the storage service might not increment correctly when using the storage sink [#10352](https://github.com/pingcap/tiflow/issues/10352) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC cannot access Azure and GCS properly in storage sink scenarios [#10592](https://github.com/pingcap/tiflow/issues/10592) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the old value part of `open-protocol` incorrectly outputs the default value according to the `STRING` type instead of its actual type [#10803](https://github.com/pingcap/tiflow/issues/10803) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that a changefeed with eventual consistency enabled might fail when the object storage sink encounters a temporary failure [#10710](https://github.com/pingcap/tiflow/issues/10710) @[CharlesCheung96](https://github.com/CharlesCheung96) + + + TiDB Data Migration (DM) + + - Fix the issue that data is lost when the upstream primary key is of binary type [#10672](https://github.com/pingcap/tiflow/issues/10672) @[GMHDBJD](https://github.com/GMHDBJD) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning reports an error when encountering invalid symbolic link files during file scanning [#49423](https://github.com/pingcap/tidb/issues/49423) @[lance6716](https://github.com/lance6716) \ No newline at end of file diff --git a/releases/release-6.6.0.md b/releases/release-6.6.0.md index 3791470727703..1a234a695f5dd 100644 --- a/releases/release-6.6.0.md +++ b/releases/release-6.6.0.md @@ -13,7 +13,7 @@ TiDB version: 6.6.0-[DMR](/releases/versioning.md#development-milestone-releases > > The TiDB 6.6.0-DMR documentation has been [archived](https://docs-archive.pingcap.com/tidb/v6.6/). PingCAP encourages you to use [the latest LTS version](https://docs.pingcap.com/tidb/stable) of the TiDB database. -Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.6/quick-start-with-tidb) | [Installation package](https://www.pingcap.com/download/?version=v6.6.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.6/quick-start-with-tidb) In v6.6.0-DMR, the key new features and improvements are as follows: @@ -518,7 +518,7 @@ In v6.6.0-DMR, the key new features and improvements are as follows: - Fix the issue that resources are not released when disabling the resource management module [#40546](https://github.com/pingcap/tidb/issues/40546) @[zimulala](https://github.com/zimulala) - Fix the issue that TTL tasks cannot trigger statistics updates in time [#40109](https://github.com/pingcap/tidb/issues/40109) @[YangKeao](https://github.com/YangKeao) - Fix the issue that unexpected data is read because TiDB improperly handles `NULL` values when constructing key ranges [#40158](https://github.com/pingcap/tidb/issues/40158) @[tiancaiamao](https://github.com/tiancaiamao) - - Fix the issue that illegal values are written to a table when the `MODIFT COLUMN` statement also changes the default value of a column [#40164](https://github.com/pingcap/tidb/issues/40164) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that illegal values are written to a table when the `MODIFY COLUMN` statement also changes the default value of a column [#40164](https://github.com/pingcap/tidb/issues/40164) @[wjhuang2016](https://github.com/wjhuang2016) - Fix the issue that the adding index operation is inefficient due to invalid Region cache when there are many Regions in a table [#38436](https://github.com/pingcap/tidb/issues/38436) @[tangenta](https://github.com/tangenta) - Fix data race occurred in allocating auto-increment IDs [#40584](https://github.com/pingcap/tidb/issues/40584) @[Dousir9](https://github.com/Dousir9) - Fix the issue that the implementation of the not operator in JSON is incompatible with the implementation in MySQL [#40683](https://github.com/pingcap/tidb/issues/40683) @[YangKeao](https://github.com/YangKeao) diff --git a/releases/release-7.0.0.md b/releases/release-7.0.0.md index a420208f94541..ddf2e5eda74ac 100644 --- a/releases/release-7.0.0.md +++ b/releases/release-7.0.0.md @@ -9,7 +9,7 @@ Release date: March 30, 2023 TiDB version: 7.0.0-[DMR](/releases/versioning.md#development-milestone-releases) -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.0/quick-start-with-tidb) | [Installation package](https://www.pingcap.com/download/?version=v7.0.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.0/quick-start-with-tidb) In v7.0.0-DMR, the key new features and improvements are as follows: @@ -268,7 +268,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: For more information, see [documentation](/sql-statements/sql-statement-load-data.md). -* TiDB Lightning supports enabling compressed transfers when sending key-value pairs to TiKV (GA) [#41163](https://github.com/pingcap/tidb/issues/41163) @[gozssky](https://github.com/gozssky) +* TiDB Lightning supports enabling compressed transfers when sending key-value pairs to TiKV (GA) [#41163](https://github.com/pingcap/tidb/issues/41163) @[sleepymole](https://github.com/sleepymole) Starting from v6.6.0, TiDB Lightning supports compressing locally encoded and sorted key-value pairs for network transfer when sending them to TiKV, thus reducing the amount of data transferred over the network and lowering the network bandwidth overhead. In the earlier TiDB versions before this feature is supported, TiDB Lightning requires relatively high network bandwidth and incurs high traffic charges in case of large data volumes. @@ -419,7 +419,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: + TiDB Lightning - - TiDB Lightning Physical Import Mode supports separating data import and index import to improve import speed and stability [#42132](https://github.com/pingcap/tidb/issues/42132) @[gozssky](https://github.com/gozssky) + - TiDB Lightning Physical Import Mode supports separating data import and index import to improve import speed and stability [#42132](https://github.com/pingcap/tidb/issues/42132) @[sleepymole](https://github.com/sleepymole) Add the `add-index-by-sql` parameter. The default value is `false`, which means that TiDB Lightning encodes both row data and index data into KV pairs and import them into TiKV together. If you set it to `true`, it means that TiDB Lightning adds indexes via the `ADD INDEX` SQL statement after importing the row data to improve import speed and stability. @@ -504,7 +504,7 @@ We would like to thank the following contributors from the TiDB community: - [BornChanger](https://github.com/BornChanger) - [Dousir9](https://github.com/Dousir9) - [erwadba](https://github.com/erwadba) -- [HappyUncle](https://github.com/HappyUncle) +- [happy-v587](https://github.com/happy-v587) - [jiyfhust](https://github.com/jiyfhust) - [L-maple](https://github.com/L-maple) - [liumengya94](https://github.com/liumengya94) diff --git a/releases/release-7.1.0.md b/releases/release-7.1.0.md index d3c34c86061fc..57c38a1c77750 100644 --- a/releases/release-7.1.0.md +++ b/releases/release-7.1.0.md @@ -9,7 +9,7 @@ Release date: May 31, 2023 TiDB version: 7.1.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.1.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) TiDB 7.1.0 is a Long-Term Support Release (LTS). diff --git a/releases/release-7.1.1.md b/releases/release-7.1.1.md index 9aa331a081959..361f5e48a5879 100644 --- a/releases/release-7.1.1.md +++ b/releases/release-7.1.1.md @@ -9,7 +9,7 @@ Release date: July 24, 2023 TiDB version: 7.1.1 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.1.1#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-7.1.2.md b/releases/release-7.1.2.md index 6185034c45a85..7963f74bb2f5f 100644 --- a/releases/release-7.1.2.md +++ b/releases/release-7.1.2.md @@ -9,7 +9,7 @@ Release date: October 25, 2023 TiDB version: 7.1.2 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.1.2#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) ## Compatibility changes diff --git a/releases/release-7.1.3.md b/releases/release-7.1.3.md index 7feb53015cf64..85129eb2c3677 100644 --- a/releases/release-7.1.3.md +++ b/releases/release-7.1.3.md @@ -9,13 +9,13 @@ Release date: December 21, 2023 TiDB version: 7.1.3 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.1.3#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) ## Compatibility changes - After further testing, the default value of the TiCDC Changefeed configuration item [`case-sensitive`](/ticdc/ticdc-changefeed-config.md) is changed from `true` to `false`. This means that by default, table and database names in the TiCDC configuration file are case-insensitive [#10047](https://github.com/pingcap/tiflow/issues/10047) @[sdojjy](https://github.com/sdojjy) - TiCDC Changefeed introduces the following new configuration items: - - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](/ticdc/ticdc-ddl.md#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) + - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](https://docs.pingcap.com/tidb/v7.1/ticdc-ddl#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) - [`encoding-worker-num`](/ticdc/ticdc-changefeed-config.md) and [`flush-worker-num`](/ticdc/ticdc-changefeed-config.md): enables you to set different concurrency parameters for the redo module based on specifications of different machines [#10048](https://github.com/pingcap/tiflow/issues/10048) @[CharlesCheung96](https://github.com/CharlesCheung96) - [`compression`](/ticdc/ticdc-changefeed-config.md): enables you to configure the compression behavior of redo log files [#10176](https://github.com/pingcap/tiflow/issues/10176) @[sdojjy](https://github.com/sdojjy) - [`sink.cloud-storage-config`](/ticdc/ticdc-changefeed-config.md): enables you to set the automatic cleanup of historical data when replicating data to object storage [#10109](https://github.com/pingcap/tiflow/issues/10109) @[CharlesCheung96](https://github.com/CharlesCheung96) @@ -96,7 +96,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue that TiKV reports the `ServerIsBusy` error because it can not append the raft log [#15800](https://github.com/tikv/tikv/issues/15800) @[tonyxuqqi](https://github.com/tonyxuqqi) - Fix the issue that snapshot restore might get stuck when BR crashes [#15684](https://github.com/tikv/tikv/issues/15684) @[YuJuncen](https://github.com/YuJuncen) - Fix the issue that Resolved TS in stale read might cause TiKV OOM issues when tracking large transactions [#14864](https://github.com/tikv/tikv/issues/14864) @[overvenus](https://github.com/overvenus) - - Fix the issue that damaged SST files might be spreaded to other TiKV nodes [#15986](https://github.com/tikv/tikv/issues/15986) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that damaged SST files might be spread to other TiKV nodes [#15986](https://github.com/tikv/tikv/issues/15986) @[Connor1996](https://github.com/Connor1996) - Fix the issue that the joint state of DR Auto-Sync might time out when scaling out [#15817](https://github.com/tikv/tikv/issues/15817) @[Connor1996](https://github.com/Connor1996) - Fix the issue that the scheduler command variables are incorrect in Grafana on the cloud environment [#15832](https://github.com/tikv/tikv/issues/15832) @[Connor1996](https://github.com/Connor1996) - Fix the issue that stale peers are retained and block resolved-ts after Regions are merged [#15919](https://github.com/tikv/tikv/issues/15919) @[overvenus](https://github.com/overvenus) diff --git a/releases/release-7.1.4.md b/releases/release-7.1.4.md index 1189244d47691..04a5c88fe10cc 100644 --- a/releases/release-7.1.4.md +++ b/releases/release-7.1.4.md @@ -9,7 +9,7 @@ Release date: March 11, 2024 TiDB version: 7.1.4 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.1.4#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) ## Compatibility changes @@ -108,6 +108,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue that the `tidb_merge_partition_stats_concurrency` variable does not take effect when `auto analyze` is processing partitioned tables [#47594](https://github.com/pingcap/tidb/issues/47594) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that the `index out of range` error might occur when a query involves JOIN operations [#42588](https://github.com/pingcap/tidb/issues/42588) @[AilinKid](https://github.com/AilinKid) - Fix the issue that wrong results might be returned when TiFlash late materialization processes associated columns [#49241](https://github.com/pingcap/tidb/issues/49241) [#51204](https://github.com/pingcap/tidb/issues/51204) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that parallel `Apply` might generate incorrect results when the table has a clustered index [#51372](https://github.com/pingcap/tidb/issues/51372) @[guo-shaoge](https://github.com/guo-shaoge) + TiKV diff --git a/releases/release-7.1.5.md b/releases/release-7.1.5.md new file mode 100644 index 0000000000000..385347f612574 --- /dev/null +++ b/releases/release-7.1.5.md @@ -0,0 +1,119 @@ +--- +title: TiDB 7.1.5 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.1.5. +--- + +# TiDB 7.1.5 Release Notes + +Release date: April 26, 2024 + +TiDB version: 7.1.5 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) + +## Compatibility changes + +- Add a TiKV configuration item [`track-and-verify-wals-in-manifest`](https://docs.pingcap.com/tidb/v7.1/tikv-configuration-file#track-and-verify-wals-in-manifest-new-in-v659-and-v715) for RocksDB, which helps you investigate possible corruption of Write Ahead Log (WAL) [#16549](https://github.com/tikv/tikv/issues/16549) @[v01dstar](https://github.com/v01dstar) + +## Improvements + ++ TiDB + + - Support loading Regions in batch from PD to speed up the conversion process from the KV range to Regions when querying large tables [#51326](https://github.com/pingcap/tidb/issues/51326) @[SeaRise](https://github.com/SeaRise) + - Optimize the issue that the `ANALYZE` statement blocks the metadata lock [#47475](https://github.com/pingcap/tidb/issues/47475) @[wjhuang2016](https://github.com/wjhuang2016) + - Add a timeout mechanism for LDAP authentication to avoid the issue of resource lock (RLock) not being released in time [#51883](https://github.com/pingcap/tidb/issues/51883) @[YangKeao](https://github.com/YangKeao) + ++ TiKV + + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - Avoid performing IO operations on snapshot files in raftstore threads to improve TiKV stability [#16564](https://github.com/tikv/tikv/issues/16564) @[Connor1996](https://github.com/Connor1996) + ++ PD + + - Upgrade the etcd version to v3.4.30 [#7904](https://github.com/tikv/pd/issues/7904) @[JmPotato](https://github.com/JmPotato) + ++ Tools + + + Backup & Restore (BR) + + - Support automatically abandoning log backup tasks when encountering a large checkpoint lag, to avoid prolonged blocking GC and potential cluster issues [#50803](https://github.com/pingcap/tidb/issues/50803) @[RidRisR](https://github.com/RidRisR) + - Add PITR integration test cases to cover compatibility testing for log backup and adding index acceleration [#51987](https://github.com/pingcap/tidb/issues/51987) @[Leavrth](https://github.com/Leavrth) + - Remove the invalid verification for active DDL jobs when log backup starts [#52733](https://github.com/pingcap/tidb/issues/52733) @[Leavrth](https://github.com/Leavrth) + +## Bug fixes + ++ TiDB + + - Fix the issue that querying JSON of `BINARY` type might cause an error in some cases [#51547](https://github.com/pingcap/tidb/issues/51547) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that query execution using MPP might lead to incorrect query results when a SQL statement contains `JOIN` and the `SELECT` list in the statement contains only constants [#50358](https://github.com/pingcap/tidb/issues/50358) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the TiDB server is marked as health before the initialization is complete [#51596](https://github.com/pingcap/tidb/issues/51596) @[shenqidebaozi](https://github.com/shenqidebaozi) + - Fix the issue that `ALTER TABLE ... COMPACT TIFLASH REPLICA` might incorrectly end when the primary key type is `VARCHAR` [#51810](https://github.com/pingcap/tidb/issues/51810) @[breezewish](https://github.com/breezewish) + - Fix the issue that TiDB crashes when `shuffleExec` quits unexpectedly [#48230](https://github.com/pingcap/tidb/issues/48230) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the `SURVIVAL_PREFERENCES` attribute might not appear in the output of the `SHOW CREATE PLACEMENT POLICY` statement under certain conditions [#51699](https://github.com/pingcap/tidb/issues/51699) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that after the time window for automatic statistics updates is configured, statistics might still be updated outside that time window [#49552](https://github.com/pingcap/tidb/issues/49552) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `approx_percentile` function might cause TiDB panic [#40463](https://github.com/pingcap/tidb/issues/40463) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that the query result is incorrect when the `IN()` predicate contains `NULL` [#51560](https://github.com/pingcap/tidb/issues/51560) @[winoros](https://github.com/winoros) + - Fix the issue that the configuration file does not take effect when it contains an invalid configuration item [#51399](https://github.com/pingcap/tidb/issues/51399) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that `EXCHANGE PARTITION` incorrectly processes foreign keys [#51807](https://github.com/pingcap/tidb/issues/51807) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that querying the `TIDB_HOT_REGIONS` table might incorrectly return `INFORMATION_SCHEMA` tables [#50810](https://github.com/pingcap/tidb/issues/50810) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the type returned by the `IFNULL` function is inconsistent with MySQL [#51765](https://github.com/pingcap/tidb/issues/51765) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the TTL feature causes data hotspots due to incorrect data range splitting in some cases [#51527](https://github.com/pingcap/tidb/issues/51527) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that TiDB keeps sending probe requests to a TiFlash node that has been offline [#46602](https://github.com/pingcap/tidb/issues/46602) @[zyguan](https://github.com/zyguan) + - Fix the issue that AutoID Leader change might cause the value of the auto-increment column to decrease in the case of `AUTO_ID_CACHE=1` [#52600](https://github.com/pingcap/tidb/issues/52600) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that executing `INSERT IGNORE` might result in inconsistency between the unique index and the data [#51784](https://github.com/pingcap/tidb/issues/51784) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that adding a unique index might cause TiDB to panic [#52312](https://github.com/pingcap/tidb/issues/52312) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the Window function might panic when there is a related subquery in it [#42734](https://github.com/pingcap/tidb/issues/42734) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) + - Fix the performance regression issue caused by disabling predicate pushdown in TableDual [#50614](https://github.com/pingcap/tidb/issues/50614) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `EXPLAIN` statement might display incorrect column IDs in the result when statistics for certain columns are not fully loaded [#52207](https://github.com/pingcap/tidb/issues/52207) @[time-and-fate](https://github.com/time-and-fate) + ++ TiKV + + - Fix the issue that resolve-ts is blocked when a stale Region peer ignores the GC message [#16504](https://github.com/tikv/tikv/issues/16504) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that inactive Write Ahead Logs (WALs) in RocksDB might corrupt data [#16705](https://github.com/tikv/tikv/issues/16705) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that the monitoring metric `tikv_unified_read_pool_thread_count` has no data in some cases [#16629](https://github.com/tikv/tikv/issues/16629) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that during the execution of an optimistic transaction, if other transactions initiate the resolving lock operation on it, there is a small chance that the atomicity of the transaction might be broken if the transaction's primary key has data that was previously committed in Async Commit or 1PC mode [#16620](https://github.com/tikv/tikv/issues/16620) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue that the scheduling of write hotspots might break placement policy constraints [#7848](https://github.com/tikv/pd/issues/7848) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that the query result of `SHOW CONFIG` includes the deprecated configuration item `trace-region-flow` [#7917](https://github.com/tikv/pd/issues/7917) @[rleungx](https://github.com/rleungx) + - Fix the issue that the scaling progress is not correctly displayed [#7726](https://github.com/tikv/pd/issues/7726) @[CabinfeverB](https://github.com/CabinfeverB) + ++ TiFlash + + - Fix incorrect `local_region_num` values in logs [#8895](https://github.com/pingcap/tiflash/issues/8895) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that querying generated columns returns an error [#8787](https://github.com/pingcap/tiflash/issues/8787) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `ENUM` column might cause TiFlash to crash during chunk encoding [#8674](https://github.com/pingcap/tiflash/issues/8674) @[yibin87](https://github.com/yibin87) + - Fix the issue that TiFlash might panic when you insert data to columns with invalid default values in non-strict `sql_mode` [#8803](https://github.com/pingcap/tiflash/issues/8803) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that if Region migration, split, or merge occurs after the precision of a `TIME` column is modified, queries might fail [#8601](https://github.com/pingcap/tiflash/issues/8601) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that removing a log backup task after it is paused does not immediately restore the GC safepoint [#52082](https://github.com/pingcap/tidb/issues/52082) @[3pointer](https://github.com/3pointer) + - Fix the issue that too many logs are printed when a full backup fails [#51572](https://github.com/pingcap/tidb/issues/51572) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR could not back up the `AUTO_RANDOM` ID allocation progress if the `AUTO_RANDOM` column is in a composite clustered index [#52255](https://github.com/pingcap/tidb/issues/52255) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiKV panics when a full backup fails to find a peer in some extreme cases [#16394](https://github.com/tikv/tikv/issues/16394) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that a PD connection failure could cause the TiDB instance where the log backup advancer owner is located to panic [#52597](https://github.com/pingcap/tidb/issues/52597) @[YuJuncen](https://github.com/YuJuncen) + - Fix an unstable test case [#52547](https://github.com/pingcap/tidb/issues/52547) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the global checkpoint of log backup is advanced ahead of the actual backup file write point due to TiKV restart, which might cause a small amount of backup data loss [#16809](https://github.com/tikv/tikv/issues/16809) @[YuJuncen](https://github.com/YuJuncen) + - Fix a rare issue that special event timing might cause the data loss in log backup [#16739](https://github.com/tikv/tikv/issues/16739) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the issue that TiCDC fails to execute the `Exchange Partition ... With Validation` DDL downstream after it is written upstream, causing the changefeed to get stuck [#10859](https://github.com/pingcap/tiflow/issues/10859) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that `snapshot lost caused by GC` is not reported in time when resuming a changefeed and the `checkpoint-ts` of the changefeed is smaller than the GC safepoint of TiDB [#10463](https://github.com/pingcap/tiflow/issues/10463) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that TiCDC panics when scheduling table replication tasks [#10613](https://github.com/pingcap/tiflow/issues/10613) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that data is written to a wrong CSV file due to wrong BarrierTS in scenarios where DDL statements are executed frequently [#10668](https://github.com/pingcap/tiflow/issues/10668) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that a changefeed with eventual consistency enabled might fail when the object storage sink encounters a temporary failure [#10710](https://github.com/pingcap/tiflow/issues/10710) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that the old value part of `open-protocol` incorrectly outputs the default value according to the `STRING` type instead of its actual type [#10803](https://github.com/pingcap/tiflow/issues/10803) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning panics when importing an empty table of Parquet format [#52518](https://github.com/pingcap/tidb/issues/52518) @[kennytm](https://github.com/kennytm) + - Fix the issue that sensitive information in logs is printed in server mode [#36374](https://github.com/pingcap/tidb/issues/36374) @[kennytm](https://github.com/kennytm) diff --git a/releases/release-7.2.0.md b/releases/release-7.2.0.md index ffaef46deddfb..b5016238aa89d 100644 --- a/releases/release-7.2.0.md +++ b/releases/release-7.2.0.md @@ -9,7 +9,7 @@ Release date: June 29, 2023 TiDB version: 7.2.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v7.2.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with-tidb) 7.2.0 introduces the following key features and improvements: @@ -324,7 +324,7 @@ We would like to thank the following contributors from the TiDB community: - [darraes](https://github.com/darraes) - [demoManito](https://github.com/demoManito) - [dhysum](https://github.com/dhysum) -- [HappyUncle](https://github.com/HappyUncle) +- [happy-v587](https://github.com/happy-v587) - [jiyfhust](https://github.com/jiyfhust) - [L-maple](https://github.com/L-maple) - [nyurik](https://github.com/nyurik) diff --git a/releases/release-7.3.0.md b/releases/release-7.3.0.md index 269a4761bcb25..fa8b53a66cfb9 100644 --- a/releases/release-7.3.0.md +++ b/releases/release-7.3.0.md @@ -9,7 +9,7 @@ Release date: August 14, 2023 TiDB version: 7.3.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.3/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v7.3.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.3/quick-start-with-tidb) 7.3.0 introduces the following major features. In addition to that, 7.3.0 also includes a series of enhancements (described in the [Feature details](#feature-details) section) to query stability in TiDB server and TiFlash. These enhancements are more miscellaneous in nature and not user-facing so they are not included in the following table. @@ -155,6 +155,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.3/quick-start-with- ### Behavior changes +* TiDB + + - MPP is a distributed computing framework provided by the TiFlash engine, which allows data exchange between nodes and provides high-performance, high-throughput SQL algorithms. Compared with other protocols, the MPP protocol is more mature and can provide better task and resource management. Starting from v7.3.0, when TiDB pushes computation tasks to TiFlash, the optimizer only generates execution plans using the MPP protocol by default. If [`tidb_allow_mpp`](/system-variables.md#tidb_allow_mpp-new-in-v50) is set to `OFF`, queries might return errors after you upgrade TiDB. It is recommended that you check the value of `tidb_allow_mpp` and set it to `ON` before the upgrade. If you still need the optimizer to choose one of the Cop, BatchCop, and MPP protocols for generating execution plans based on cost estimates, you can set the [`tidb_allow_tiflash_cop`](/system-variables.md#tidb_allow_tiflash_cop-new-in-v730) variable to `ON`. + * Backup & Restore (BR) - BR adds an empty cluster check before performing a full data restoration. By default, restoring data to a non-empty cluster is not allowed. If you want to force the restoration, you can use the `--filter` option to specify the corresponding table name to restore data to. diff --git a/releases/release-7.4.0.md b/releases/release-7.4.0.md index 2d3cd93e69a4c..64faee8d149eb 100644 --- a/releases/release-7.4.0.md +++ b/releases/release-7.4.0.md @@ -9,7 +9,7 @@ Release date: October 12, 2023 TiDB version: 7.4.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v7.4.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with-tidb) 7.4.0 introduces the following key features and improvements: diff --git a/releases/release-7.5.0.md b/releases/release-7.5.0.md index 085a1ed5cd9a4..6952fef28e4bb 100644 --- a/releases/release-7.5.0.md +++ b/releases/release-7.5.0.md @@ -9,7 +9,7 @@ Release date: December 1, 2023 TiDB version: 7.5.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.5.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) TiDB 7.5.0 is a Long-Term Support Release (LTS). @@ -172,6 +172,7 @@ Compared with the previous LTS 7.1.0, 7.5.0 includes new features, improvements, | Configuration file | Configuration parameter | Change type | Description | | -------- | -------- | -------- | -------- | | TiDB | [`tikv-client.copr-req-timeout`](/tidb-configuration-file.md#copr-req-timeout-new-in-v750) | Newly added | Sets the timeout of a single Coprocessor request. | +| TiKV | [`raftstore.inspect-interval`](/tikv-configuration-file.md#inspect-interval) | Modified | Changes the default value from `500ms` to `100ms` after optimizing the algorithm to improve the sensitivity of slow node detection. | | TiKV | [`raftstore.region-compact-min-redundant-rows`](/tikv-configuration-file.md#region-compact-min-redundant-rows-new-in-v710) | Modified | Sets the number of redundant MVCC rows required to trigger RocksDB compaction. Starting from v7.5.0, this configuration item takes effect for the `"raft-kv"` storage engine. | | TiKV | [`raftstore.region-compact-redundant-rows-percent`](/tikv-configuration-file.md#region-compact-redundant-rows-percent-new-in-v710) | Modified | Sets the percentage of redundant MVCC rows required to trigger RocksDB compaction. Starting from v7.5.0, this configuration item takes effect for the `"raft-kv"` storage engine. | | TiKV | [`raftstore.evict-cache-on-memory-ratio`](/tikv-configuration-file.md#evict-cache-on-memory-ratio-new-in-v750) | Newly added | When the memory usage of TiKV exceeds 90% of the system available memory, and the memory occupied by Raft entry cache exceeds the `evict-cache-on-memory-ratio` of used memory, TiKV evicts the Raft entry cache. | @@ -180,6 +181,7 @@ Compared with the previous LTS 7.1.0, 7.5.0 includes new features, improvements, | BR | [`--ignore-stats`](/br/br-snapshot-manual.md#back-up-statistics) | Newly added | Controls whether to back up and restore database statistics. When you set this parameter to `false`, the br command-line tool supports backing up and restoring statistics of columns, indexes, and tables. | | TiCDC | [`case-sensitive`](/ticdc/ticdc-changefeed-config.md) | Modified | Changes the default value from `true` to `false` after further tests, which means that the table names and database names in the TiCDC configuration file are case-insensitive by default. | | TiCDC | [`sink.dispatchers.partition`](/ticdc/ticdc-changefeed-config.md) | Modified | Controls how TiCDC dispatches incremental data to Kafka partitions. v7.5.0 introduces a new value option `columns`, which uses the explicitly specified column values to calculate the partition number. | +| TiCDC | [`changefeed-error-stuck-duration`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the duration for which the changefeed is allowed to automatically retry when internal errors or exceptions occur. | | TiCDC | [`encoding-worker-num`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the number of encoding and decoding workers in the redo module. | | TiCDC | [`flush-worker-num`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the number of flushing workers in the redo module. | | TiCDC | [`sink.column-selectors`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls the specified columns of data change events that TiCDC sends to Kafka when dispatching incremental data. | diff --git a/releases/release-7.5.1.md b/releases/release-7.5.1.md index 6a975f4df1186..2be9f132a7589 100644 --- a/releases/release-7.5.1.md +++ b/releases/release-7.5.1.md @@ -9,7 +9,7 @@ Release date: February 29, 2024 TiDB version: 7.5.1 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) | [Installation packages](https://www.pingcap.com/download/?version=v7.5.1#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) ## Compatibility changes @@ -43,6 +43,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with- - Support multiple accelerated `ADD INDEX` DDL tasks to be queued for execution, instead of falling back to normal `ADD INDEX` tasks [#47758](https://github.com/pingcap/tidb/issues/47758) @[tangenta](https://github.com/tangenta) - Improve the speed of adding indexes to empty tables [#49682](https://github.com/pingcap/tidb/issues/49682) @[zimulala](https://github.com/zimulala) ++ TiKV + + - Enhance the slow store detection algorithm by improving its sensitivity and reducing the false-positive rate, especially in intensive read and write load scenarios [#15909](https://github.com/tikv/tikv/issues/15909) @[LykxSassinator](https://github.com/LykxSassinator) + + TiFlash - Improve the calculation method for [Request Unit (RU)](/tidb-resource-control.md#what-is-request-unit-ru) to make RU values more stable [#8391](https://github.com/pingcap/tiflash/issues/8391) @[guo-shaoge](https://github.com/guo-shaoge) @@ -159,6 +163,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with- - Fix the issue that the joint state of DR Auto-Sync might time out when scaling out [#15817](https://github.com/tikv/tikv/issues/15817) @[Connor1996](https://github.com/Connor1996) - Fix the issue that Resolved TS might be blocked for two hours [#11847](https://github.com/tikv/tikv/issues/11847) [#15520](https://github.com/tikv/tikv/issues/15520) [#39130](https://github.com/pingcap/tidb/issues/39130) @[overvenus](https://github.com/overvenus) - Fix the issue that `cast_duration_as_time` might return incorrect results [#16211](https://github.com/tikv/tikv/issues/16211) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that TiKV hangs in corner cases (for example, when disk I/O operations are blocked), which affects availability [#16368](https://github.com/tikv/tikv/issues/16368) @[LykxSassinator](https://github.com/LykxSassinator) + PD diff --git a/releases/release-7.6.0.md b/releases/release-7.6.0.md index b5bee4048eb99..ad6f0a18eba1c 100644 --- a/releases/release-7.6.0.md +++ b/releases/release-7.6.0.md @@ -9,7 +9,7 @@ Release date: January 25, 2024 TiDB version: 7.6.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.6/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v7.6.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.6/quick-start-with-tidb) 7.6.0 introduces the following key features and improvements: @@ -208,7 +208,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.6/quick-start-with- Starting from v5.1.0, when you back up snapshots, BR automatically backs up system tables in the `mysql` schema, but does not restore these system tables by default. In v6.2.0, BR adds the parameter `--with-sys-table` to support restoring data in some system tables, providing more flexibility in operations. - To further reduce your management overhead and provide more intuitive default behavior, starting from v7.6.0, BR enables the parameter `--with-sys-table` by default and supports restoring user data for the `cloud_admin` user. This means that BR restores some system tables by default during restoration, especially user account and table statistics data. This improvement makes backup and restore operations more intuitive, thereby reducing the burden of manual configuration and improving the overall operation experience. + To further reduce your management overhead and provide more intuitive default behavior, starting from v7.6.0, BR enables the parameter `--with-sys-table` by default. This means that BR restores some system tables by default during restoration, especially user account and table statistics data. This improvement makes backup and restore operations more intuitive, thereby reducing the burden of manual configuration and improving the overall operation experience. For more information, see [documentation](/br/br-snapshot-guide.md). @@ -353,6 +353,7 @@ Starting from v7.6.0, the `TiDB-community-server` [binary-package](/binary-packa + Backup & Restore (BR) + - Support authentication using Amazon S3 `session-token` and `assume-role` during full backup recovery phase [#39832](https://github.com/pingcap/tidb/issues/39832) @[3pointer](https://github.com/3pointer) - Introduce a new integration test for Point-In-Time Recovery (PITR) in the `delete range` scenario, enhancing PITR stability [#47738](https://github.com/pingcap/tidb/issues/47738) @[Leavrth](https://github.com/Leavrth) - Improve the table creation performance of the `RESTORE` statement in scenarios with large datasets [#48301](https://github.com/pingcap/tidb/issues/48301) @[Leavrth](https://github.com/Leavrth) - Refactor the BR exception handling mechanism to increase tolerance for unknown errors [#47656](https://github.com/pingcap/tidb/issues/47656) @[3pointer](https://github.com/3pointer) diff --git a/releases/release-8.0.0.md b/releases/release-8.0.0.md index 0afc08d207e5a..5c161a906af04 100644 --- a/releases/release-8.0.0.md +++ b/releases/release-8.0.0.md @@ -9,7 +9,7 @@ Release date: March 29, 2024 TiDB version: 8.0.0 -Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with-tidb) | [Installation packages](https://www.pingcap.com/download/?version=v8.0.0#version-list) +Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with-tidb) 8.0.0 introduces the following key features and improvements: @@ -392,6 +392,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- - MemoryTracker can track the memory usage of the `IndexLookup` operator [#45901](https://github.com/pingcap/tidb/issues/45901) @[solotzg](https://github.com/solotzg) - MemoryTracker can track the memory usage of the `MemTableReaderExec` operator [#51456](https://github.com/pingcap/tidb/issues/51456) @[wshwsh12](https://github.com/wshwsh12) - Support loading Regions in batch from PD to speed up the conversion process from the KV range to Regions when querying large tables [#51326](https://github.com/pingcap/tidb/issues/51326) @[SeaRise](https://github.com/SeaRise) + - Optimize the query performance of the system tables `INFORMATION_SCHEMA.TABLES`, `INFORMATION_SCHEMA.STATISTICS`, `INFORMATION_SCHEMA.KEY_COLUMN_USAGE`, and `INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS`. Compared with earlier versions, the performance has been improved by up to 100 times. [#50305](https://github.com/pingcap/tidb/issues/50305) @[ywqzzy](https://github.com/ywqzzy) + TiKV diff --git a/releases/release-8.1.0.md b/releases/release-8.1.0.md new file mode 100644 index 0000000000000..76a954360180e --- /dev/null +++ b/releases/release-8.1.0.md @@ -0,0 +1,365 @@ +--- +title: TiDB 8.1.0 Release Notes +summary: Learn about the new features, compatibility changes, improvements, and bug fixes in TiDB 8.1.0. +--- + +# TiDB 8.1.0 Release Notes + + + +Release date: May 24, 2024 + +TiDB version: 8.1.0 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v8.1/production-deployment-using-tiup) + +TiDB 8.1.0 is a Long-Term Support Release (LTS). + +Compared with the previous LTS 7.5.0, 8.1.0 includes new features, improvements, and bug fixes released in [7.6.0-DMR](/releases/release-7.6.0.md) and [8.0.0-DMR](/releases/release-8.0.0.md). When you upgrade from 7.5.x to 8.1.0, you can download the [TiDB Release Notes PDF](https://download.pingcap.org/tidb-v7.6-to-v8.1-en-release-notes.pdf) to view all release notes between the two LTS versions. The following table lists some highlights from 7.6.0 to 8.1.0: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryFeature/EnhancementDescription
Scalability and PerformanceAcceleration of cluster snapshot restore speed (GA in v8.0.0)With this feature, BR can fully leverage the scale advantage of a cluster, enabling all TiKV nodes in the cluster to participate in the preparation step of data restores. This feature can significantly improve the restore speed of large datasets in large-scale clusters. Real-world tests show that this feature can saturate the download bandwidth, with the download speed improving by 8 to 10 times, and the end-to-end restore speed improving by approximately 1.5 to 3 times.
Achieve up to 10 times faster for creating tables in batch (experimental, introduced in v7.6.0)With the implementation of the new DDL architecture in v7.6.0, the performance of batch table creation has witnessed a remarkable improvement, up to 10 times faster. This substantial enhancement drastically reduces the time needed for creating numerous tables. This acceleration is particularly noteworthy in SaaS scenarios, where the prevalence of high volumes of tables, ranging from tens to hundreds of thousands, is a common challenge.
Use Active PD Followers to enhance PD's Region information query service (experimental, introduced in v7.6.0)TiDB v7.6.0 introduces an experimental feature "Active PD Follower", which allows PD followers to provide Region information query services. This feature improves the capability of the PD cluster to handle GetRegion and ScanRegions requests in clusters with a large number of TiDB nodes and Regions, thereby reducing the CPU pressure on PD leaders.
Bulk DML for much larger transactions (experimental, introduced in v8.0.0)Large batch DML jobs, such as extensive cleanup jobs, joins, or aggregations, can consume a significant amount of memory and have previously been limited at very large scales. Bulk DML (tidb_dml_type = "bulk") is a new DML type for handling large batch DML tasks more efficiently while providing transaction guarantees and mitigating OOM issues. This feature differs from import, load, and restore operations when used for data loading.
Enhance the stability of caching the schema information when there is a massive number of tables (experimental, introduced in v8.0.0)SaaS companies using TiDB as the system of record for their multi-tenant applications often need to store a substantial number of tables. In previous versions, handling table counts in the order of a million or more was feasible, but it had the potential to degrade the overall user experience. TiDB v8.0.0 improves the situation with the following enhancements: +
    +
  • Introduce a new schema information caching system, which provides a cache strategy based on the Least Recently Used (LRU) algorithm for table metadata. It prioritizes storing the metadata of frequently accessed tables in the cache, thereby reducing memory usage in scenarios with a large number of tables.
    • +
  • Implement a priority queue for auto analyze, making the process less rigid and enhancing stability across a wider array of tables.
    • +
+
Reliability and availabilityGlobal Sort (GA in v8.0.0)The Global Sort feature aims to improve the stability and efficiency of IMPORT INTO and CREATE INDEX. By globally sorting the data to be processed, this feature improves the stability, controllability, and scalability of data writing to TiKV, consequently enhancing the user experience and service quality of data import and index creation. With global sorting enabled, each IMPORT INTO or CREATE INDEX statement now supports importing or adding indexes for up to 40 TiB of data.
Cross-database SQL binding (introduced in v7.6.0)When managing hundreds of databases with the same schema, it is often necessary to apply SQL bindings across these databases. For example, in SaaS or PaaS data platforms, each user typically operates separate databases with the same schema and runs similar SQL queries on them. In this case, it is impractical to bind SQL for each database one by one. TiDB v7.6.0 introduces cross-database SQL bindings that enable matching bindings across all schema-equivalent databases.
Support TiProxy (GA in v8.0.0)Full support for the TiProxy service, easily deployable via deployment tooling, to manage and maintain connections to TiDB so that they live through rolling restarts, upgrades, or scaling events.
Data Migration (DM) officially supports MySQL 8.0 (GA in v7.6.0)Previously, using DM to migrate data from MySQL 8.0 is an experimental feature and is not available for production environments. TiDB v7.6.0 enhances the stability and compatibility of this feature to help you smoothly and quickly migrate data from MySQL 8.0 to TiDB in production environments. In v7.6.0, this feature becomes generally available (GA).
TiDB resource control supports managing queries that consume more resources than expected (GA in v8.1.0)Through the rules of resource groups, TiDB can automatically identify queries that consume more resources than expected, and then limit or cancel these queries. Even if the queries are not identified by the rules, you can still manually add query characteristics and take corresponding measures to reduce the impact of the sudden query performance problem on the entire database.
DB Operations and ObservabilitySupport monitoring index usage statistics (introduced in v8.0.0)Proper index design is a crucial prerequisite to maintaining database performance. TiDB v8.0.0 introduces the INFORMATION_SCHEMA.TIDB_INDEX_USAGE table and the sys.schema_unused_indexes view to provide usage statistics of indexes. This feature helps you assess the efficiency of indexes in the database and optimize the index design.
Data MigrationTiCDC supports the Simple protocol (introduced in v8.0.0)TiCDC introduces a new protocol, the Simple protocol. This protocol provides in-band schema tracking capabilities by embedding table schema information in DDL and BOOTSTRAP events.
TiCDC supports the Debezium format protocol (introduced in v8.0.0)TiCDC introduces a new protocol, the Debezium protocol. TiCDC can now publish data change events to a Kafka sink using a protocol that generates Debezium style messages.
TiCDC supports client authentication (introduced in v8.1.0)TiCDC supports client authentication using mutual Transport Layer Security (mTLS) or TiDB username and password. This feature enables CLI or OpenAPI clients to authenticate their connections to TiCDC.
+ +## Feature details + +### Reliability + +* Support managing queries that consume more resources than expected (GA) [#43691](https://github.com/pingcap/tidb/issues/43691) @[nolouch](https://github.com/nolouch) + + Sudden SQL query performance problems can cause a decline in overall database performance, which is the most common challenge to database stability. The reasons for these problems are diverse, such as untested new SQL statements, drastic changes in data volume, and sudden changes in execution plans. These problems are difficult to avoid completely at the source. TiDB v7.2.0 has introduced the capability to manage queries that consume more resources than expected to quickly reduce the impact of sudden query performance problems. This feature becomes generally available in v8.1.0. + + You can set the maximum execution time for a query in a resource group. When the execution time of a query exceeds the set value, the priority of the query is automatically reduced or the query is canceled. You can also set immediately matching identified queries through text or execution plans within a period of time, to avoid excessive resource consumption during the identification phase when the concurrency of problematic queries is too high. + + TiDB also supports the manual marking of queries. By using the [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) command, you can mark queries based on the SQL text, SQL Digest, or execution plan. The queries that match the mark can be downgraded or canceled, achieving the purpose of adding a SQL blocklist. + + The automatic management capability of queries that consume more resources than expected provides users with an effective means to quickly mitigate the impact of query problems on overall performance before the root cause is identified, thereby improving the stability of the database. + + For more information, see [documentation](/tidb-resource-control.md#manage-queries-that-consume-more-resources-than-expected-runaway-queries). + +### SQL + +* Support using more expressions to set default column values when creating a table (GA) [#50936](https://github.com/pingcap/tidb/issues/50936) @[zimulala](https://github.com/zimulala) + + Before v8.0.0, when you create a table, the default value of a column is limited to strings, numbers, dates, and certain expressions. Starting from v8.0.0, you can use more expressions as the default column values. For example, you can set the default value of a column to `DATE_FORMAT`. This feature helps you meet more diverse requirements. In v8.1.0, this feature becomes GA. + + Starting from v8.1.0, you can use expressions as default values when adding columns by `ADD COLUMN`. + + For more information, see [documentation](/data-type-default-values.md#specify-expressions-as-default-values). + +### DB operations + +* Enable the TiDB Distributed eXecution Framework (DXF) by default to enhance the performance and stability of `ADD INDEX` or `IMPORT INTO` tasks in parallel execution [#52441](https://github.com/pingcap/tidb/issues/52441) @[D3Hunter](https://github.com/D3Hunter) + + The DXF becomes generally available (GA) in v7.5.0, but it is disabled by default. This means that an `ADD INDEX` or `IMPORT INTO` task is executed only by one TiDB node by default. + + Starting from v8.1.0, TiDB enables this feature by default ([`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) defaults to `ON`). When enabled, the DXF can schedule multiple TiDB nodes to execute the same `ADD INDEX` or `IMPORT INTO` task in parallel, fully utilizing the resources of the TiDB cluster and greatly improving the performance of these tasks. In addition, you can linearly improve the performance of `ADD INDEX` and `IMPORT INTO` tasks by adding TiDB nodes and configuring [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) for the newly added nodes. + + For more information, see [documentation](/tidb-distributed-execution-framework.md). + +### Security + +* Enhance TiDB log desensitization (GA) [#52364](https://github.com/pingcap/tidb/issues/52364) @[xhebox](https://github.com/xhebox) + + The enhanced TiDB log desensitization supports removing sensitive data when users view logs, implemented by marking SQL text information in log files. You can control whether to mark log information to enable secure use of TiDB logs in different scenarios, enhancing the security and flexibility of using log desensitization. To use this feature, set the system variable `tidb_redact_log` to `MARKER`, and then the SQL text in TiDB's runtime logs is marked. In addition, you can use the `collect-log` subcommand on the TiDB server to remove marked sensitive data from the logs and display the logs in a secure manner. You can also remove all markers and get the normal logs. This feature became generally available in v8.1.0. + + For more information, see [documentation](/system-variables.md#tidb_redact_log). + +### Data migration + +* Support the `IMPORT INTO ... FROM SELECT` syntax (GA) [#49883](https://github.com/pingcap/tidb/issues/49883) @[D3Hunter](https://github.com/D3Hunter) + + Before v8.0.0, importing query results into a target table could only be done using the `INSERT INTO ... SELECT` statement, which is relatively inefficient in some large dataset scenarios. In v8.0.0, TiDB introduces `IMPORT INTO ... FROM SELECT` as an experimental feature, which enables you to import the results of a `SELECT` query into an empty TiDB target table. It achieves up to 8 times the performance of `INSERT INTO ... SELECT` and significantly reduces the import time. In addition, you can use `IMPORT INTO ... FROM SELECT` to import historical data queried with [`AS OF TIMESTAMP`](/as-of-timestamp.md). + + In v8.1.0, the `IMPORT INTO ... FROM SELECT` syntax becomes generally available (GA), enriching the functionality scenarios of the `IMPORT INTO` statement. + + For more information, see [documentation](/sql-statements/sql-statement-import-into.md). + +* TiDB Lightning simplifies conflict resolution strategies and supports handling conflicting data using the `replace` strategy (GA) [#51036](https://github.com/pingcap/tidb/issues/51036) @[lyzx2001](https://github.com/lyzx2001) + + Before v8.0.0, TiDB Lightning has [one data conflict resolution strategy](/tidb-lightning/tidb-lightning-logical-import-mode-usage.md#conflict-detection) for the logical import mode and [two data conflict resolution strategies](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#conflict-detection) for the physical import mode, which are not easy to understand and configure. + + In v8.0.0, TiDB Lightning deprecates the [old version of conflict detection](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#the-old-version-of-conflict-detection-deprecated-in-v800) strategy for the physical import mode, enables you to control the conflict detection strategy for both logical and physical import modes via the [`conflict.strategy`](/tidb-lightning/tidb-lightning-configuration.md) parameter (experimental), and simplifies the configuration of this parameter. In addition, in the physical import mode, the `replace` strategy supports retaining the latest data and overwriting the old data when the import detects data with primary key or unique key conflicts. In v8.1.0, the capability to handle conflicting data with the `replace` strategy becomes generally available (GA). + + For more information, see [documentation](/tidb-lightning/tidb-lightning-configuration.md). + +* TiCDC supports client authentication [#10636](https://github.com/pingcap/tiflow/issues/10636) @[CharlesCheung96](https://github.com/CharlesCheung96) + + In v8.1.0, TiCDC supports client authentication when you are using the TiCDC CLI or OpenAPI. This feature enables you to configure TiCDC to require client authentication using client certificates, thereby establishing mutual Transport Layer Security (mTLS). Additionally, you can configure authentication based on TiDB username and password. + + For more information, see [documentation](/ticdc/ticdc-client-authentication.md). + +## Compatibility changes + +> **Note:** +> +> This section provides compatibility changes you need to know when you upgrade from v8.0.0 to the current version (v8.1.0). If you are upgrading from v7.6.0 or earlier versions to the current version, you might also need to check the compatibility changes introduced in intermediate versions. + +### Behavior changes + +* In earlier versions, the `tidb.tls` configuration item in TiDB Lightning treats values `"false"` and `""` the same, as well as treating the values `"preferred"` and `"skip-verify"` the same. Starting from v8.1.0, TiDB Lightning distinguishes the behavior of `"false"`, `""`, `"skip-verify"`, and `"preferred"` for `tidb.tls`. For more information, see [TiDB Lightning configuration](/tidb-lightning/tidb-lightning-configuration.md). +* For tables with `AUTO_ID_CACHE=1`, TiDB supports a [centralized auto-increment ID allocating service](/auto-increment.md#mysql-compatibility-mode). In earlier versions, the primary TiDB node of this service automatically performs a `forceRebase` operation when the TiDB process exits (for example, during the TiDB node restart) to keep auto-assigned IDs as consecutive as possible. However, when there are too many tables with `AUTO_ID_CACHE=1`, executing `forceRebase` becomes very time-consuming, preventing TiDB from restarting promptly and even blocking data writes, thus affecting system availability. To resolve this issue, starting from v8.1.0, TiDB removes the `forceRebase` behavior, but this change will cause some auto-assigned IDs to be non-consecutive during the failover. +* In earlier versions, when processing a transaction containing `UPDATE` changes, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. Starting from v8.1.0, when using the MySQL sink, TiCDC only splits an `UPDATE` event into `DELETE` and `INSERT` events if the primary key or non-null unique index value is modified in the `UPDATE` event and the transaction `commitTS` is less than TiCDC `thresholdTs` (which is the current timestamp that TiCDC fetches from PD at TiCDC startup). This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. For more information, see [documentation](/ticdc/ticdc-behavior-change.md#mysql-sink). + +### System variables + +| Variable name | Change type | Description | +|--------|------------------------------|------| +| [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio) | Modified | Changes the value range from `[0, 18446744073709551615]` to `(0, 1]`. | +| [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) | Modified | Changes the default value from `OFF` to `ON`. This means that Distributed eXecution Framework (DXF) is enabled by default, which fully utilizes the resources of the TiDB cluster and greatly improves the performance of `ADD INDEX` and `IMPORT INTO` tasks. If you want to upgrade a cluster with the DXF enabled to v8.1.0 or later, disable the DXF (by setting `tidb_enable_dist_task` to `OFF`) before the upgrade, which avoids `ADD INDEX` operations during the upgrade causing data index inconsistency. After the upgrade, you can manually enable the DXF. | +| [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) | Modified | Changes the optional value from `""` or `background` to a string with a length of up to 64 characters, which enables you to control the service scope of each TiDB node more flexibly. Valid characters include digits `0-9`, letters `a-zA-Z`, underscores `_`, and hyphens `-`. The Distributed eXecution Framework (DXF) determines which TiDB nodes can be scheduled to execute distributed tasks based on the value of this variable. For specific rules, see [Task scheduling](/tidb-distributed-execution-framework.md#task-scheduling). | + +### Configuration file parameters + +| Configuration file | Configuration parameter | Change type | Description | +| -------- | -------- | -------- | -------- | +| TiDB| [`concurrently-init-stats`](/tidb-configuration-file.md#concurrently-init-stats-new-in-v810-and-v752) | Newly added | Controls whether to initialize statistics concurrently during TiDB startup. The default value is `false`. | +| TiDB Lightning | [`conflict.max-record-rows`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-configuration) | Modified | Starting from v8.1.0, there is no need to configure `conflict.max-record-rows` manually, because TiDB Lightning automatically assigns the value of `conflict.max-record-rows` with the value of `conflict.threshold`, regardless of the user input. `conflict.max-record-rows` will be deprecated in a future release. | +| TiDB Lightning | [`conflict.threshold`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) | Modified | Changes the default value from `9223372036854775807` to `10000` to quickly interrupt abnormal tasks so that you can make corresponding adjustments as soon as possible. This saves time and computational resources by avoiding the scenario where a large amount of conflicting data is discovered after the import, caused by abnormal data sources or incorrect table schema definitions. | +| TiCDC | [`security.client-allowed-user`](/ticdc/ticdc-server-config.md#cdc-server-configuration-file-parameters) | Newly added | Lists the usernames that are allowed for client authentication. Authentication requests with usernames not in this list will be rejected. The default value is null. | +| TiCDC | [`security.client-user-required`](/ticdc/ticdc-server-config.md#cdc-server-configuration-file-parameters) | Newly added | Controls whether to use username and password for client authentication. The default value is `false`. | +| TiCDC | [`security.mtls`](/ticdc/ticdc-server-config.md#cdc-server-configuration-file-parameters) | Newly added | Controls whether to enable the TLS client authentication. The default value is `false`. | +| TiCDC | [`sink.debezium.output-old-value`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | Controls whether to output the value before the row data changes. The default value is `true`. When it is disabled, the `UPDATE` event does not output the "before" field. | +| TiCDC | [`sink.open.output-old-value`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | Controls whether to output the value before the row data changes. The default value is `true`. When it is disabled, the `UPDATE` event does not output the "p" field. | + +## Deprecated features + +* It is planned to redesign [the auto-evolution of execution plan bindings](/sql-plan-management.md#baseline-evolution) in subsequent releases, and the related variables and behavior will change. +* The TiDB Lightning parameter [`conflict.max-record-rows`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) is scheduled for deprecation in a future release and will be subsequently removed. This parameter will be replaced by `conflict.threshold`, which means that the maximum number of conflicting records is consistent with the maximum number of conflicting records that can be tolerated in a single import task. +* Starting from v8.0.0, TiDB Lightning deprecates the [old version of conflict detection](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#the-old-version-of-conflict-detection-deprecated-in-v800) strategy for the physical import mode, and enables you to control the conflict detection strategy for both logical and physical import modes via the [`conflict.strategy`](/tidb-lightning/tidb-lightning-configuration.md) parameter. The [`duplicate-resolution`](/tidb-lightning/tidb-lightning-configuration.md) parameter for the old version of conflict detection will be removed in a future release. + +## Improvements + ++ TiDB + + - Improve the MySQL compatibility of foreign keys displayed in the output of `SHOW CREATE TABLE` [#51837](https://github.com/pingcap/tidb/issues/51837) @[negachov](https://github.com/negachov) + - Improve the MySQL compatibility of expression default values displayed in the output of `SHOW CREATE TABLE` [#52939](https://github.com/pingcap/tidb/issues/52939) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Support adding multiple indexes concurrently in the ingest mode [#52596](https://github.com/pingcap/tidb/issues/52596) @[lance6716](https://github.com/lance6716) + - Support configuring the system variable `tidb_service_scope` with various values, enhancing the utilization of the Distributed eXecution Framework (DXF) [#52441](https://github.com/pingcap/tidb/issues/52441) @[ywqzzy](https://github.com/ywqzzy) + - Enhance the handling of DNF items that are always `false` by directly ignoring such filter conditions, thus avoiding unnecessary full table scans [#40997](https://github.com/pingcap/tidb/issues/40997) @[hi-rustin](https://github.com/hi-rustin) + - Support using Optimizer Fix Controls to remove the limitation that the optimizer does not automatically choose Index Merge for a query when the optimizer can choose the single index scan method (other than full table scan) for the query [#52869](https://github.com/pingcap/tidb/issues/52869) @[time-and-fate](https://github.com/time-and-fate) + - Add the `total_kv_read_wall_time` metric to the column `execution info` of Coprocessor operators [#28937](https://github.com/pingcap/tidb/issues/28937) @[cfzjywxk](https://github.com/cfzjywxk) + - Add the `RU (max)` metric on the Resource Control dashboard [#49318](https://github.com/pingcap/tidb/issues/49318) @[nolouch](https://github.com/nolouch) + - Add a timeout mechanism for LDAP authentication to avoid the issue of resource lock (RLock) not being released in time [#51883](https://github.com/pingcap/tidb/issues/51883) @[YangKeao](https://github.com/YangKeao) + ++ TiKV + + - Avoid performing IO operations on snapshot files in Raftstore threads to improve TiKV stability [#16564](https://github.com/tikv/tikv/issues/16564) @[Connor1996](https://github.com/Connor1996) + - Accelerate the shutdown speed of TiKV [#16680](https://github.com/tikv/tikv/issues/16680) @[LykxSassinator](https://github.com/LykxSassinator) + - Add metrics for memory usage per thread [#15927](https://github.com/tikv/tikv/issues/15927) @[Connor1996](https://github.com/Connor1996) + ++ PD + + - Optimize the logic for `OperatorController` to reduce the overhead of competition locks [#7897](https://github.com/tikv/pd/issues/7897) @[nolouch](https://github.com/nolouch) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - Add PITR integration test cases to cover compatibility testing for log backup and adding index acceleration [#51987](https://github.com/pingcap/tidb/issues/51987) @[Leavrth](https://github.com/Leavrth) + - Remove the invalid verification for active DDL jobs when log backup starts [#52733](https://github.com/pingcap/tidb/issues/52733) @[Leavrth](https://github.com/Leavrth) + - Add test cases to test compatibility between PITR and the acceleration of adding indexes feature [#51988](https://github.com/pingcap/tidb/issues/51988) @[Leavrth](https://github.com/Leavrth) + - BR cleans up empty SST files during data recovery [#16005](https://github.com/tikv/tikv/issues/16005) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Improve memory stability during data recovery using redo logs to reduce the probability of OOM [#10900](https://github.com/pingcap/tiflow/issues/10900) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Significantly improve the stability of data replication in transaction conflict scenarios, with up to 10 times performance improvement [#10896](https://github.com/pingcap/tiflow/issues/10896) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that executing SQL statements containing tables with multi-valued indexes might return the `Can't find a proper physical plan for this query` error [#49438](https://github.com/pingcap/tidb/issues/49438) @[qw4990](https://github.com/qw4990) + - Fix the issue that automatic statistics collection gets stuck after an OOM error occurs [#51993](https://github.com/pingcap/tidb/issues/51993) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that after using BR to restore a table that has no statistics, the statistics health of that table is still 100% [#29769](https://github.com/pingcap/tidb/issues/29769) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB creates statistics for system tables during upgrade [#52040](https://github.com/pingcap/tidb/issues/52040) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that automatic statistics collection is triggered before the initialization of statistics finishes [#52346](https://github.com/pingcap/tidb/issues/52346) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that TiDB might crash when `tidb_mem_quota_analyze` is enabled and the memory used by updating statistics exceeds the limit [#52601](https://github.com/pingcap/tidb/issues/52601) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the TiDB synchronously loading statistics mechanism retries to load empty statistics indefinitely and prints the `fail to get stats version for this histogram` log [#52657](https://github.com/pingcap/tidb/issues/52657) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that expressions containing different collations might cause the query to panic when the new framework for collations is disabled [#52772](https://github.com/pingcap/tidb/issues/52772) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the `CPS by type` metric displays incorrect values [#52605](https://github.com/pingcap/tidb/issues/52605) @[nolouch](https://github.com/nolouch) + - Fix the issue that the nil pointer error occurs when you query `INFORMATION_SCHEMA.TIKV_REGION_STATUS` [#52013](https://github.com/pingcap/tidb/issues/52013) @[JmPotato](https://github.com/JmPotato) + - Fix the incorrect error message displayed when an invalid default value is specified for a column [#51592](https://github.com/pingcap/tidb/issues/51592) @[danqixu](https://github.com/danqixu) + - Fix the issue that adding indexes in the ingest mode might cause inconsistent data index in some corner cases [#51954](https://github.com/pingcap/tidb/issues/51954) @[lance6716](https://github.com/lance6716) + - Fix the issue that DDL operations get stuck when restoring a table with the foreign key [#51838](https://github.com/pingcap/tidb/issues/51838) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that adding indexes fails when the TiDB network is isolated [#51846](https://github.com/pingcap/tidb/issues/51846) @[ywqzzy](https://github.com/ywqzzy) + - Fix the issue that adding an index with the same name after renaming an index causes an error [#51431](https://github.com/pingcap/tidb/issues/51431) @[lance6716](https://github.com/lance6716) + - Fix the issue of inconsistent data indexes caused by cluster upgrade during adding indexes [#52411](https://github.com/pingcap/tidb/issues/52411) @[tangenta](https://github.com/tangenta) + - Fix the issue that adding indexes to large tables fails after enabling the Distributed eXecution Framework (DXF) [#52640](https://github.com/pingcap/tidb/issues/52640) @[tangenta](https://github.com/tangenta) + - Fix the issue that adding indexes concurrently reports the error `no such file or directory` [#52475](https://github.com/pingcap/tidb/issues/52475) @[tangenta](https://github.com/tangenta) + - Fix the issue that the temporary data cannot be cleaned up after adding indexes fails [#52639](https://github.com/pingcap/tidb/issues/52639) @[lance6716](https://github.com/lance6716) + - Fix the issue that the metadata lock fails to prevent DDL operations from executing in the plan cache scenario [#51407](https://github.com/pingcap/tidb/issues/51407) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the `IMPORT INTO` operation gets stuck when importing a large amount of data [#52884](https://github.com/pingcap/tidb/issues/52884) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB unexpectedly restarts when logging gRPC errors [#51301](https://github.com/pingcap/tidb/issues/51301) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that IndexHashJoin outputs redundant data when calculating Anti Left Outer Semi Join [#52923](https://github.com/pingcap/tidb/issues/52923) @[yibin87](https://github.com/yibin87) + - Fix the incorrect result of the TopN operator in correlated subqueries [#52777](https://github.com/pingcap/tidb/issues/52777) @[yibin87](https://github.com/yibin87) + - Fix the inaccurate execution time statistics of HashJoin probe [#52222](https://github.com/pingcap/tidb/issues/52222) @[windtalker](https://github.com/windtalker) + - Fix the issue that using `TABLESAMPLE` returns incorrect results in static partition pruning mode (`tidb_partition_prune_mode='static'`) [#52282](https://github.com/pingcap/tidb/issues/52282) @[tangenta](https://github.com/tangenta) + - Fix the issue that TTL is deviated by 1 hour in daylight saving time [#51675](https://github.com/pingcap/tidb/issues/51675) @[lcwangchao](https://github.com/lcwangchao) + - Fix the incorrect calculation and display of the number of connections (Connection Count) on the TiDB Dashboard Monitoring page [#51889](https://github.com/pingcap/tidb/issues/51889) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the status gets stuck when rolling back the partition DDL tasks [#51090](https://github.com/pingcap/tidb/issues/51090) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that the value of `max_remote_stream` is incorrect when executing `EXPLAIN ANALYZE` [#52646](https://github.com/pingcap/tidb/issues/52646) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that querying the `TIDB_HOT_REGIONS` table might incorrectly return `INFORMATION_SCHEMA` tables [#50810](https://github.com/pingcap/tidb/issues/50810) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the `EXPLAIN` statement might display incorrect column IDs in the result when statistics for certain columns are not fully loaded [#52207](https://github.com/pingcap/tidb/issues/52207) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that the type returned by the `IFNULL` function is inconsistent with MySQL [#51765](https://github.com/pingcap/tidb/issues/51765) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that adding a unique index might cause TiDB to panic [#52312](https://github.com/pingcap/tidb/issues/52312) @[wjhuang2016](https://github.com/wjhuang2016) + ++ TiKV + + - Fix the issue that resolve-ts is blocked when a stale Region peer ignores the GC message [#16504](https://github.com/tikv/tikv/issues/16504) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that inactive Write Ahead Logs (WALs) in RocksDB might corrupt data [#16705](https://github.com/tikv/tikv/issues/16705) @[Connor1996](https://github.com/Connor1996) + ++ PD + + - Fix the issue that TSO might get stuck when toggling the PD microservice mode on and off [#7849](https://github.com/tikv/pd/issues/7849) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that the `State` monitoring metric for DR Auto-Sync does not show any data [#7974](https://github.com/tikv/pd/issues/7974) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that checking binary versions might cause PD panic [#7978](https://github.com/tikv/pd/issues/7978) @[JmPotato](https://github.com/JmPotato) + - Fix the type conversion error that occurs when TTL parameters are parsed [#7980](https://github.com/tikv/pd/issues/7980) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the Leader fails to transfer when you switch it between two deployed data centers [#7992](https://github.com/tikv/pd/issues/7992) @[TonsnakeLin](https://github.com/TonsnakeLin) + - Fix the issue that `PrintErrln` in pd-ctl fails to output error messages to `stderr` [#8022](https://github.com/tikv/pd/issues/8022) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that PD might panic when generating `Merge` schedules [#8049](https://github.com/tikv/pd/issues/8049) @[nolouch](https://github.com/nolouch) + - Fix the panic issue caused by `GetAdditionalInfo` [#8079](https://github.com/tikv/pd/issues/8079) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the `Filter target` monitoring metric for PD does not provide scatter range information [#8125](https://github.com/tikv/pd/issues/8125) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the query result of `SHOW CONFIG` includes the deprecated configuration item `trace-region-flow` [#7917](https://github.com/tikv/pd/issues/7917) @[rleungx](https://github.com/rleungx) + - Fix the issue that the scaling progress is not correctly displayed [#7726](https://github.com/tikv/pd/issues/7726) @[CabinfeverB](https://github.com/CabinfeverB) + ++ TiFlash + + - Fix the issue that TiFlash might panic when you insert data to columns with invalid default values in non-strict `sql_mode` [#8803](https://github.com/pingcap/tiflash/issues/8803) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that TiFlash might return transiently incorrect results in high-concurrency read scenarios [#8845](https://github.com/pingcap/tiflash/issues/8845) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that in the disaggregated storage and compute architecture, the disk `used_size` metric displayed in Grafana is incorrect after you modify the value of the `storage.remote.cache.capacity` configuration item for TiFlash compute nodes [#8920](https://github.com/pingcap/tiflash/issues/8920) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash metadata might become corrupted and cause the process to panic when upgrading a cluster from a version earlier than v6.5.0 to v6.5.0 or later [#9039](https://github.com/pingcap/tiflash/issues/9039) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that in the disaggregated storage and compute architecture, TiFlash might panic when the compute node process is stopped [#8860](https://github.com/pingcap/tiflash/issues/8860) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiFlash might return an error when executing queries containing virtual generated columns [#8787](https://github.com/pingcap/tiflash/issues/8787) @[guo-shaoge](https://github.com/guo-shaoge) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that BR cannot back up the `AUTO_RANDOM` ID allocation progress in a union clustered index that contains an `AUTO_RANDOM` column [#52255](https://github.com/pingcap/tidb/issues/52255) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that removing a log backup task after it is paused does not immediately restore the GC safepoint [#52082](https://github.com/pingcap/tidb/issues/52082) @[3pointer](https://github.com/3pointer) + - Fix a rare issue that special event timing might cause the data loss in log backup [#16739](https://github.com/tikv/tikv/issues/16739) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the global checkpoint of log backup is advanced ahead of the actual backup file write point due to TiKV restart, which might cause a small amount of backup data loss [#16809](https://github.com/tikv/tikv/issues/16809) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the confusing information related to `--concurrency` appears in the log during a full backup [#50837](https://github.com/pingcap/tidb/issues/50837) @[BornChanger](https://github.com/BornChanger) + - Fix the issue that the Region fetched from PD does not have a Leader when restoring data using BR or importing data using TiDB Lightning in physical import mode [#51124](https://github.com/pingcap/tidb/issues/51124) [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that after pausing, stopping, and rebuilding the log backup task, the task status is normal, but the checkpoint does not advance [#53047](https://github.com/pingcap/tidb/issues/53047) @[RidRisR](https://github.com/RidRisR) + - Fix the unstable test case `TestClearCache` [#51671](https://github.com/pingcap/tidb/issues/51671) @[zxc111](https://github.com/zxc111) + - Fix the unstable test case `TestGetMergeRegionSizeAndCount` [#52095](https://github.com/pingcap/tidb/issues/52095) @[3pointer](https://github.com/3pointer) + - Fix the unstable integration test `br_tikv_outage` [#52673](https://github.com/pingcap/tidb/issues/52673) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the test case `TestGetTSWithRetry` takes too long to execute [#52547](https://github.com/pingcap/tidb/issues/52547) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiKV might panic when resuming a paused log backup task with unstable network connections to PD [#17020](https://github.com/tikv/tikv/issues/17020) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the issue that calling the API (`/api/v2/owner/resign`) that evicts the TiCDC owner node causes the TiCDC task to restart unexpectedly [#10781](https://github.com/pingcap/tiflow/issues/10781) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that when the downstream Pulsar is stopped, removing the changefeed causes the normal TiCDC process to get stuck, which causes other changefeed processes to get stuck [#10629](https://github.com/pingcap/tiflow/issues/10629) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the **Ownership history** panel in Grafana is unstable [#10796](https://github.com/pingcap/tiflow/issues/10796) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that restarting PD might cause the TiCDC node to restart with an error [#10799](https://github.com/pingcap/tiflow/issues/10799) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that high latency in the PD disk I/O causes severe latency in data replication [#9054](https://github.com/pingcap/tiflow/issues/9054) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the default value of `TIMEZONE` type is not set according to the correct time zone [#10931](https://github.com/pingcap/tiflow/issues/10931) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that `DROP PRIMARY KEY` and `DROP UNIQUE KEY` statements are not replicated correctly [#10890](https://github.com/pingcap/tiflow/issues/10890) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that TiCDC fails to execute the `Exchange Partition ... With Validation` DDL downstream after it is written upstream, causing the changefeed to get stuck [#10859](https://github.com/pingcap/tiflow/issues/10859) @[hongyunyan](https://github.com/hongyunyan) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning reports `no database selected` during data import due to incompatible SQL statements in the source files [#51800](https://github.com/pingcap/tidb/issues/51800) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB Lightning might print sensitive information to logs in server mode [#36374](https://github.com/pingcap/tidb/issues/36374) @[kennytm](https://github.com/kennytm) + - Fix the issue that killing the PD Leader causes TiDB Lightning to report the `invalid store ID 0` error during data import [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiDB Lightning reports the `Unknown column in where clause` error when processing conflicting data using the `replace` strategy [#52886](https://github.com/pingcap/tidb/issues/52886) @[lyzx2001](https://github.com/lyzx2001) + - Fix the issue that TiDB Lightning panics when importing an empty table of Parquet format [#52518](https://github.com/pingcap/tidb/issues/52518) @[kennytm](https://github.com/kennytm) + +## Performance test + +To learn about the performance of TiDB v8.1.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v8.1-performance-benchmarking-with-tpcc) and [Sysbench performance test report](https://docs.pingcap.com/tidbcloud/v8.1-performance-benchmarking-with-sysbench) of the TiDB Dedicated cluster. + +## Contributors + +We would like to thank the following contributors from the TiDB community: + +- [arturmelanchyk](https://github.com/arturmelanchyk) (First-time contributor) +- [CabinfeverB](https://github.com/CabinfeverB) +- [danqixu](https://github.com/danqixu) (First-time contributor) +- [imalasong](https://github.com/imalasong) (First-time contributor) +- [jiyfhust](https://github.com/jiyfhust) +- [negachov](https://github.com/negachov) (First-time contributor) +- [testwill](https://github.com/testwill) +- [yzhan1](https://github.com/yzhan1) (First-time contributor) +- [zxc111](https://github.com/zxc111) (First-time contributor) diff --git a/releases/release-notes.md b/releases/release-notes.md index 309cb5c462dd4..b64bd0f9b9934 100644 --- a/releases/release-notes.md +++ b/releases/release-notes.md @@ -1,10 +1,17 @@ --- title: Release Notes aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] +summary: TiDB has released multiple versions, including 8.1.0, 8.0.0-DMR, 7.6.0-DMR, 7.5.1, 7.5.0, 7.4.0-DMR, 7.3.0-DMR, 7.2.0-DMR, 7.1.4, 7.1.3, 7.1.2, 7.1.1, 7.1.0, 7.0.0-DMR, 6.6.0-DMR, 6.5.9, 6.5.8, 6.5.7, 6.5.6, 6.5.5, 6.5.4, 6.5.3, 6.5.2, 6.5.1, 6.5.0, 6.4.0-DMR, 6.3.0-DMR, 6.2.0-DMR, 6.1.7, 6.1.6, 6.1.5, 6.1.4, 6.1.3, 6.1.2, 6.1.1, 6.1.0, 6.0.0-DMR, 5.4.3, 5.4.2, 5.4.1, 5.4.0, 5.3.4, 5.3.3, 5.3.2, 5.3.1, 5.3.0, 5.2.4, 5.2.3, 5.2.2, 5.2.1, 5.2.0, 5.1.5, 5.1.4, 5.1.3, 5.1.2, 5.1.1, 5.1.0, 5.0.6, 5.0.5, 5.0.4, 5.0.3, 5.0.2, 5.0.1, 5.0.0, 5.0.0-rc, 4.0.16, 4.0.15, 4.0.14, 4.0.13, 4.0.12, 4.0.11, 4.0.10, 4.0.9, 4.0.8, 4.0.7, 4.0.6, 4.0.5, 4.0.4, 4.0.3, 4.0.2, 4.0.1, 4.0.0, 4.0.0-rc.2, 4.0.0-rc.1, 4.0.0-rc, 4.0.0-beta.2, 4.0.0-beta.1, 4.0.0-beta, 3.1.2, 3.1.1, 3.1.0, 3.1.0-rc, 3.1.0-beta.2, 3.1.0-beta.1, 3.1.0-beta, 3.0.20, 3.0.19, 3.0.18, 3.0.17, 3.0.16, 3.0.15, 3.0.14, 3.0.13, 3.0.12, 3.0.11, 3.0.10, 3.0.9, 3.0.8, 3.0.7, 3.0.6, 3.0.5, 3.0.4, 3.0.3, 3.0.2, 3.0.1, 3.0.0, 3.0.0-rc.3, 3.0.0-rc.2, 3.0.0-rc.1, 3.0.0-beta.1, 3.0.0-beta, 2.1.19, 2.1.18, 2.1.17, 2.1.16, 2.1.15, 2.1.14, 2.1.13, 2.1.12, 2.1.11, 2.1.10, 2.1.9, 2.1.8, 2.1.7, 2.1.6, 2.1.5, 2.1.4, 2.1.3, 2.1.2, 2.1.1, 2.1.0, 2.1.0-rc.5, 2.1.0-rc.4, 2.1.0-rc.3, 2.1.0-rc.2, 2.1.0-rc.1, 2.1.0-beta, 2.0.11, 2.0.10, 2.0.9, 2.0.8, 2.0.7, 2.0.6, 2.0.5, 2.0.4, 2.0.3, 2.0.2, 2.0.1, 2.0.0, 2.0.0-rc.5, 2.0.0-rc.4, 2.0.0-rc.3, 2.0.0-rc.1, 1.1.0-beta, 1.1.0-alpha, 1.0.8, 1.0.7, 1.0.6, 1.0.5, 1.0.4, 1.0.3, 1.0.2, 1.0.1, 1.0.0, Pre-GA, rc4, rc3, rc2, rc1. --- # TiDB Release Notes + + +## 8.1 + +- [8.1.0](/releases/release-8.1.0.md): 2024-05-24 + ## 8.0 - [8.0.0-DMR](/releases/release-8.0.0.md): 2024-03-29 @@ -32,6 +39,7 @@ aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] ## 7.1 +- [7.1.5](/releases/release-7.1.5.md): 2024-04-26 - [7.1.4](/releases/release-7.1.4.md): 2024-03-11 - [7.1.3](/releases/release-7.1.3.md): 2023-12-21 - [7.1.2](/releases/release-7.1.2.md): 2023-10-25 @@ -48,6 +56,7 @@ aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] ## 6.5 +- [6.5.9](/releases/release-6.5.9.md): 2024-04-12 - [6.5.8](/releases/release-6.5.8.md): 2024-02-02 - [6.5.7](/releases/release-6.5.7.md): 2024-01-08 - [6.5.6](/releases/release-6.5.6.md): 2023-12-07 diff --git a/releases/release-pre-ga.md b/releases/release-pre-ga.md index effafb4181013..c41a06b28855e 100644 --- a/releases/release-pre-ga.md +++ b/releases/release-pre-ga.md @@ -1,6 +1,7 @@ --- title: Pre-GA release notes aliases: ['/docs/dev/releases/release-pre-ga/','/docs/dev/releases/prega/'] +summary: TiDB Pre-GA release on August 30, 2017, focuses on MySQL compatibility, SQL optimization, stability, and performance. TiDB introduces SQL query optimizer enhancements, MySQL compatibility, JSON type support, and memory consumption reduction. Placement Driver (PD) now supports manual leader change, while TiKV uses dedicated Rocksdb for Raft log storage and improves performance. TiDB Connector for Spark Beta Release implements predicates pushdown, aggregation pushdown, and range pruning, capable of running TPC+H queries. --- # Pre-GA Release Notes diff --git a/releases/release-rc.3.md b/releases/release-rc.3.md index c63980ff53e5d..f8d49e8efb291 100644 --- a/releases/release-rc.3.md +++ b/releases/release-rc.3.md @@ -1,6 +1,7 @@ --- title: TiDB RC3 Release Notes aliases: ['/docs/dev/releases/release-rc.3/','/docs/dev/releases/rc3/'] +summary: TiDB RC3, released on June 16, 2017, focuses on MySQL compatibility, SQL optimization, stability, and performance. Highlights include refined privilege management, accelerated DDL, optimized load balancing, and open-sourced TiDB Ansible for easy cluster management. Detailed updates for TiDB, Placement Driver (PD), and TiKV include improved SQL query optimization, complete privilege management, support for HTTP API, system variables for query concurrency control, and more efficient data balance. PD supports gRPC, disaster recovery toolkit, and hot Region scheduling. TiKV supports gRPC, SST format snapshot, memory leak detection, and improved data importing speed. Overall, the release enhances performance, stability, and management capabilities. --- # TiDB RC3 Release Notes diff --git a/releases/release-timeline.md b/releases/release-timeline.md index 2354b85157e5c..79c60e96d574a 100644 --- a/releases/release-timeline.md +++ b/releases/release-timeline.md @@ -5,11 +5,16 @@ summary: Learn about the TiDB release timeline. # TiDB Release Timeline + + This document shows all the released TiDB versions in reverse chronological order. | Version | Release Date | | :--- | :--- | -| [8.0.0](/releases/release-8.0.0.md) | 2024-03-29 | +| [8.1.0](/releases/release-8.1.0.md) | 2024-05-24 | +| [7.1.5](/releases/release-7.1.5.md) | 2024-04-26 | +| [6.5.9](/releases/release-6.5.9.md) | 2024-04-12 | +| [8.0.0-DMR](/releases/release-8.0.0.md) | 2024-03-29 | | [7.1.4](/releases/release-7.1.4.md) | 2024-03-11 | | [7.5.1](/releases/release-7.5.1.md) | 2024-02-29 | | [6.5.8](/releases/release-6.5.8.md) | 2024-02-02 | diff --git a/replicate-data-to-kafka.md b/replicate-data-to-kafka.md index 7dd7bf3d3a722..1cf94d1c41252 100644 --- a/replicate-data-to-kafka.md +++ b/replicate-data-to-kafka.md @@ -31,7 +31,7 @@ The preceding steps are performed in a lab environment. You can also deploy a cl 2. Create a Kafka cluster. - - Lab environment: refer to [Apache Kakfa Quickstart](https://kafka.apache.org/quickstart) to start a Kafka cluster. + - Lab environment: refer to [Apache Kafka Quickstart](https://kafka.apache.org/quickstart) to start a Kafka cluster. - Production environment: refer to [Running Kafka in Production](https://docs.confluent.io/platform/current/kafka/deployment.html) to deploy a Kafka production cluster. 3. (Optional) Create a Flink cluster. diff --git a/resources/doc-templates/patch_release_note_template_zh.md b/resources/doc-templates/patch_release_note_template_zh.md index 6ead5f552bd7c..400d5628aea1f 100644 --- a/resources/doc-templates/patch_release_note_template_zh.md +++ b/resources/doc-templates/patch_release_note_template_zh.md @@ -5,7 +5,7 @@ summary: 了解 TiDB x.y.z 版本的兼容性变更、改进提升，以及错 # TiDB x.y.z Release Notes -发版日期：2023 年 x 月 x 日 +发版日期：2024 年 x 月 x 日 TiDB 版本：x.y.z diff --git a/runtime-filter.md b/runtime-filter.md index c4d01012a35d2..1d2031322929f 100644 --- a/runtime-filter.md +++ b/runtime-filter.md @@ -67,7 +67,7 @@ The execution process of Runtime Filter is as follows: | | filter data | | | | +-----+----v------+ +-------+--------+ - | TableFullScan | | TabelFullScan | + | TableFullScan | | TableFullScan | | store_sales | | date_dim | +-----------------+ +----------------+ ``` diff --git a/scale-tidb-using-tiup.md b/scale-tidb-using-tiup.md index 56c2540c4d887..cf9363accdd4f 100644 --- a/scale-tidb-using-tiup.md +++ b/scale-tidb-using-tiup.md @@ -127,7 +127,29 @@ This section exemplifies how to add a TiDB node to the `10.0.1.5` host. If you see `Scaled cluster out successfully`, the scale-out operation succeeds. -3. Check the cluster status: +3. Refresh the cluster configuration. + + > **Note:** + > + > This operation is only required after you add PD nodes. If you only add TiDB or TiKV nodes, this operation is unnecessary. + + 1. Refresh the cluster configuration: + + ```shell + tiup cluster reload --skip-restart + ``` + + 2. Refresh the Prometheus configuration and restart Prometheus: + + > **Note:** + > + > If you are using TiUP v1.15.0 or a later version, skip this step. If you are using a TiUP version earlier than v1.15.0, execute the following command to update the Prometheus configuration and restart Prometheus. + + ```shell + tiup cluster reload -R prometheus + ``` + +4. Check the cluster status: {{< copyable "shell-regular" >}} @@ -276,7 +298,7 @@ This section exemplifies how to remove a TiKV node from the `10.0.1.5` host. ``` Starting /root/.tiup/components/cluster/v1.12.3/cluster display TiDB Cluster: - TiDB Version: v8.0.0 + TiDB Version: v8.1.0 ID Role Host Ports Status Data Dir Deploy Dir -- ---- ---- ----- ------ -------- ---------- 10.0.1.3:8300 cdc 10.0.1.3 8300 Up data/cdc-8300 deploy/cdc-8300 @@ -307,7 +329,29 @@ This section exemplifies how to remove a TiKV node from the `10.0.1.5` host. If you see `Scaled cluster in successfully`, the scale-in operation succeeds. -3. Check the cluster status: +3. Refresh the cluster configuration. + + > **Note:** + > + > This operation is only required after you remove PD nodes. If you only remove TiDB or TiKV nodes, this operation is unnecessary. + + 1. Refresh the cluster configuration: + + ```shell + tiup cluster reload --skip-restart + ``` + + 2. Refresh the Prometheus configuration and restart Prometheus: + + > **Note:** + > + > If you are using TiUP v1.15.0 or a later version, skip this step. If you are using a TiUP version earlier than v1.15.0, execute the following command to update the Prometheus configuration and restart Prometheus. + + ```shell + tiup cluster reload -R prometheus + ``` + +4. Check the cluster status: The scale-in process takes some time. You can run the following command to check the scale-in status: diff --git a/schedule-replicas-by-topology-labels.md b/schedule-replicas-by-topology-labels.md index 9604dd74f371d..36dfab793fa12 100644 --- a/schedule-replicas-by-topology-labels.md +++ b/schedule-replicas-by-topology-labels.md @@ -14,9 +14,156 @@ To improve the high availability and disaster recovery capability of TiDB cluste To make this mechanism effective, you need to properly configure TiKV and PD so that the topology information of the cluster, especially the TiKV location information, is reported to PD during deployment. Before you begin, see [Deploy TiDB Using TiUP](/production-deployment-using-tiup.md) first. -## Configure `labels` based on the cluster topology +## Configure `labels` for TiKV, TiFlash, and TiDB -### Configure `labels` for TiKV and TiFlash +You can configure `labels` for TiKV, TiFlash, and TiDB based on the cluster topology. + +### Configure a cluster using TiUP (recommended) + +When using TiUP to deploy a cluster, you can configure the TiKV location in the [initialization configuration file](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file). TiUP will generate the corresponding configuration files for TiDB, TiKV, PD, and TiFlash during deployment. + +In the following example, a two-layer topology of `zone/host` is defined. The TiDB nodes, TiKV nodes, and TiFlash nodes of the cluster are distributed among three zones, z1, z2, and z3. + +- In each zone, there are two hosts that have TiDB instances deployed, and each host has a separate TiDB instance deployed. +- In each zone, there are two hosts that have TiKV instances deployed. In z1, each host has two TiKV instances deployed. In z2 and z3, each host has a separate TiKV instance deployed. +- In each zone, there are two hosts that have TiFlash instances deployed, and each host has a separate TiFlash instance deployed. + +In the following example, `tidb-host-machine-n` represents the IP address of the `n`th TiDB node, `tikv-host-machine-n` represents the IP address of the `n`th TiKV node, and `tiflash-host-machine-n` represents the IP address of the `n`th TiFlash node. + +``` +server_configs: + pd: + replication.location-labels: ["zone", "host"] +tidb_servers: +# z1 + - host: tidb-host-machine-1 + config: + labels: + zone: z1 + host: tidb-host-machine-1 + - host: tidb-host-machine-2 + config: + labels: + zone: z1 + host: tidb-host-machine-2 +# z2 + - host: tidb-host-machine-3 + config: + labels: + zone: z2 + host: tidb-host-machine-3 + - host: tikv-host-machine-4 + config: + labels: + zone: z2 + host: tidb-host-machine-4 +# z3 + - host: tidb-host-machine-5 + config: + labels: + zone: z3 + host: tidb-host-machine-5 + - host: tidb-host-machine-6 + config: + labels: + zone: z3 + host: tidb-host-machine-6 +tikv_servers: +# z1 + # machine-1 on z1 + - host: tikv-host-machine-1 + port：20160 + config: + server.labels: + zone: z1 + host: tikv-host-machine-1 + - host: tikv-host-machine-1 + port：20161 + config: + server.labels: + zone: z1 + host: tikv-host-machine-1 + # machine-2 on z1 + - host: tikv-host-machine-2 + port：20160 + config: + server.labels: + zone: z1 + host: tikv-host-machine-2 + - host: tikv-host-machine-2 + port：20161 + config: + server.labels: + zone: z1 + host: tikv-host-machine-2 +# z2 + - host: tikv-host-machine-3 + config: + server.labels: + zone: z2 + host: tikv-host-machine-3 + - host: tikv-host-machine-4 + config: + server.labels: + zone: z2 + host: tikv-host-machine-4 +# z3 + - host: tikv-host-machine-5 + config: + server.labels: + zone: z3 + host: tikv-host-machine-5 + - host: tikv-host-machine-6 + config: + server.labels: + zone: z3 + host: tikv-host-machine-6 + +tiflash_servers: +# z1 + - host: tiflash-host-machine-1 + learner_config: + server.labels: + zone: z1 + host: tiflash-host-machine-1 + - host: tiflash-host-machine-2 + learner_config: + server.labels: + zone: z1 + host: tiflash-host-machine-2 +# z2 + - host: tiflash-host-machine-3 + learner_config: + server.labels: + zone: z2 + host: tiflash-host-machine-3 + - host: tiflash-host-machine-4 + learner_config: + server.labels: + zone: z2 + host: tiflash-host-machine-4 +# z3 + - host: tiflash-host-machine-5 + learner_config: + server.labels: + zone: z3 + host: tiflash-host-machine-5 + - host: tiflash-host-machine-6 + learner_config: + server.labels: + zone: z3 + host: tiflash-host-machine-6 +``` + +For details, see [Geo-distributed Deployment topology](/geo-distributed-deployment-topology.md). + +> **Note:** +> +> If you have not configured `replication.location-labels` in the configuration file, when you deploy a cluster using this topology file, an error might occur. It is recommended that you confirm `replication.location-labels` is configured in the configuration file before deploying a cluster. + +### Configure a cluster using command lines or configuration files + +#### Configure `labels` for TiKV and TiFlash You can use the command-line flag or set the TiKV or TiFlash configuration file to bind some attributes in the form of key-value pairs. These attributes are called `labels`. After TiKV and TiFlash are started, they report their `labels` to PD so users can identify the location of TiKV and TiFlash nodes. @@ -50,7 +197,7 @@ rack = "" host = "" ``` -### (Optional) Configure `labels` for TiDB +#### (Optional) Configure `labels` for TiDB When [Follower read](/follower-read.md) is enabled, if you want TiDB to prefer to read data from the same region, you need to configure `labels` for TiDB nodes. @@ -68,7 +215,7 @@ host = "" > > Currently, TiDB depends on the `zone` label to match and select replicas that are in the same region. To use this feature, you need to include `zone` when [configuring `location-labels` for PD](#configure-location-labels-for-pd), and configure `zone` when configuring `labels` for TiDB, TiKV, and TiFlash. For more details, see [Configure `labels` for TiKV and TiFlash](#configure-labels-for-tikv-and-tiflash). -### Configure `location-labels` for PD +## Configure `location-labels` for PD According to the description above, the label can be any key-value pair used to describe TiKV attributes. But PD cannot identify the location-related labels and the layer relationship of these labels. Therefore, you need to make the following configuration for PD to understand the TiKV node topology. @@ -100,7 +247,7 @@ To configure `location-labels`, choose one of the following methods according to pd-ctl config set location-labels zone,rack,host ``` -### Configure `isolation-level` for PD +## Configure `isolation-level` for PD If `location-labels` has been configured, you can further enhance the topological isolation requirements on TiKV clusters by configuring `isolation-level` in the PD configuration file. @@ -127,115 +274,6 @@ The `location-level` configuration is an array of strings, which needs to corres > > `isolation-level` is empty by default, which means there is no mandatory restriction on the isolation level. To set it, you need to configure `location-labels` for PD and ensure that the value of `isolation-level` is one of `location-labels` names. -### Configure a cluster using TiUP (recommended) - -When using TiUP to deploy a cluster, you can configure the TiKV location in the [initialization configuration file](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file). TiUP will generate the corresponding configuration files for TiKV, PD, and TiFlash during deployment. - -In the following example, a two-layer topology of `zone/host` is defined. The TiKV nodes and TiFlash nodes of the cluster are distributed among three zones, z1, z2, and z3. - -- In each zone, there are two hosts that have TiKV instances deployed. In z1, each host has two TiKV instances deployed. In z2 and z3, each host has a separate TiKV instance deployed. -- In each zone, there are two hosts that have TiFlash instances deployed, and each host has a separate TiFlash instance deployed. - -In the following example, `tikv-host-machine-n` represents the IP address of the `n`th TiKV node, and `tiflash-host-machine-n` represents the IP address of the `n`th TiFlash node. - -``` -server_configs: - pd: - replication.location-labels: ["zone", "host"] - -tikv_servers: -# z1 - # machine-1 on z1 - - host: tikv-host-machine-1 - port：20160 - config: - server.labels: - zone: z1 - host: tikv-host-machine-1 - - host: tikv-host-machine-1 - port：20161 - config: - server.labels: - zone: z1 - host: tikv-host-machine-1 - # machine-2 on z1 - - host: tikv-host-machine-2 - port：20160 - config: - server.labels: - zone: z1 - host: tikv-host-machine-2 - - host: tikv-host-machine-2 - port：20161 - config: - server.labels: - zone: z1 - host: tikv-host-machine-2 -# z2 - - host: tikv-host-machine-3 - config: - server.labels: - zone: z2 - host: tikv-host-machine-3 - - host: tikv-host-machine-4 - config: - server.labels: - zone: z2 - host: tikv-host-machine-4 -# z3 - - host: tikv-host-machine-5 - config: - server.labels: - zone: z3 - host: tikv-host-machine-5 - - host: tikv-host-machine-6 - config: - server.labels: - zone: z3 - host: tikv-host-machine-6 - -tiflash_servers: -# z1 - - host: tiflash-host-machine-1 - learner_config: - server.labels: - zone: z1 - host: tiflash-host-machine-1 - - host: tiflash-host-machine-2 - learner_config: - server.labels: - zone: z1 - host: tiflash-host-machine-2 -# z2 - - host: tiflash-host-machine-3 - learner_config: - server.labels: - zone: z2 - host: tiflash-host-machine-3 - - host: tiflash-host-machine-4 - learner_config: - server.labels: - zone: z2 - host: tiflash-host-machine-4 -# z3 - - host: tiflash-host-machine-5 - learner_config: - server.labels: - zone: z3 - host: tiflash-host-machine-5 - - host: tiflash-host-machine-6 - learner_config: - server.labels: - zone: z3 - host: tiflash-host-machine-6 -``` - -For details, see [Geo-distributed Deployment topology](/geo-distributed-deployment-topology.md). - -> **Note:** -> -> If you have not configured `replication.location-labels` in the configuration file, when you deploy a cluster using this topology file, an error might occur. It is recommended that you confirm `replication.location-labels` is configured in the configuration file before deploying a cluster. - ## PD schedules based on topology label PD schedules replicas according to the label layer to make sure that different replicas of the same data are scattered as much as possible. diff --git a/scripts/get-en-duplicated-notes.py b/scripts/get-en-duplicated-notes.py index 83e0bc4da6cbf..a930392b16d3a 100644 --- a/scripts/get-en-duplicated-notes.py +++ b/scripts/get-en-duplicated-notes.py @@ -6,8 +6,8 @@ from shutil import move from os import remove -ext_path = r'/Users/userid/Documents/GitHub/mygithubid/docs/releases' # Specify the directory of the English release notes folder -main_path = r'/Users/userid/Documents/GitHub/mygithubid/docs/releases/release-7.1.2.md' # Specify the directory of the English release note file that you are preparing +ext_path = r'/Users/hfxsd/Documents/docs/releases' # Specify the directory of the English release notes folder +main_path = r'/Users/hfxsd/Documents/docs/releases/release-7.1.5.md' # Specify the directory of the English release note file that you are preparing # Get existing release notes from the English release notes folder def store_exst_rn(ext_path,main_path): diff --git a/scripts/release_notes_update_pr_author_info_add_dup.py b/scripts/release_notes_update_pr_author_info_add_dup.py index 92039bf89380d..e5b72fb43513f 100644 --- a/scripts/release_notes_update_pr_author_info_add_dup.py +++ b/scripts/release_notes_update_pr_author_info_add_dup.py @@ -12,6 +12,7 @@ import openpyxl import os import shutil +import requests version = '6.5.3' # Specifies the target TiDB version release_note_excel = r'/Users/userid/Downloads/download_tirelease_tmp_patch_6.5.3_release_note_2023-06-06.xlsx' # Specifies the path of release note table with PR links and issue links @@ -131,7 +132,8 @@ def update_pr_author_and_release_notes(excel_path): # If pr_author is ti-chi-bot or ti-srebot current_pr_author = row[pr_author_index] current_formated_rn= row[pr_formated_rn_index] - if current_pr_author in ['ti-chi-bot', 'ti-srebot']: + pr_response = requests.get(row[pr_link_index]) + if (current_pr_author in ['ti-chi-bot', 'ti-srebot']) and (pr_response.status_code == 200): print ("Replacing the author info for row " + str(row_index) + ".") actual_pr_author = get_pr_info_from_github(row[pr_link_index], row[pr_title_index], current_pr_author) # Get the PR author according to the cherry-pick PR pr_author_cell = sheet.cell(row=row_index, column=pr_author_index+1, value = actual_pr_author)#Fill in the pr_author_cell @@ -174,7 +176,7 @@ def create_release_file(version, dup_notes_levels, dup_notes): release_file = os.path.join(ext_path, f'release-{version}.md') shutil.copyfile(template_file, release_file) # Replace the file content - with open(release_file, 'r+') as file: + with open(release_file, 'r+', encoding='utf-8') as file: content = file.read() content = content.replace('x.y.z', version) version_parts = version.split('.') diff --git a/security-compatibility-with-mysql.md b/security-compatibility-with-mysql.md index ba6a5b0972cc8..c120a20188f6d 100644 --- a/security-compatibility-with-mysql.md +++ b/security-compatibility-with-mysql.md @@ -143,7 +143,7 @@ The support for TLS authentication is configured differently. For detailed infor ### `tidb_auth_token` -`tidb_auth_token` is a passwordless authentication method based on [JSON Web Token (JWT)](https://datatracker.ietf.org/doc/html/rfc7519). In v6.4.0, `tidb_auth_token` is only used for user authentication in TiDB Cloud. Starting from v6.5.0, you can also configure `tidb_auth_token` as a user authentication method for TiDB Self-Hosted. Different from password-based authentication methods such as `mysql_native_passsword` and `caching_sha2_password`, when you create users using `tidb_auth_token`, there is no need to set or store custom passwords. To log into TiDB, users only need to use a signed token instead of a password, which simplifies the authentication process and improves security. +`tidb_auth_token` is a passwordless authentication method based on [JSON Web Token (JWT)](https://datatracker.ietf.org/doc/html/rfc7519). In v6.4.0, `tidb_auth_token` is only used for user authentication in TiDB Cloud. Starting from v6.5.0, you can also configure `tidb_auth_token` as a user authentication method for TiDB Self-Hosted. Different from password-based authentication methods such as `mysql_native_password` and `caching_sha2_password`, when you create users using `tidb_auth_token`, there is no need to set or store custom passwords. To log into TiDB, users only need to use a signed token instead of a password, which simplifies the authentication process and improves security. #### JWT diff --git a/smooth-upgrade-tidb.md b/smooth-upgrade-tidb.md index c6fb1facc6559..e429cfc69088e 100644 --- a/smooth-upgrade-tidb.md +++ b/smooth-upgrade-tidb.md @@ -78,7 +78,7 @@ When using the smooth upgrade feature, note the following limitations. * Before the upgrade, consider the following restrictions: * If there is a canceling DDL job in the cluster, that is, an ongoing DDL job is being canceled by a user, because the job in the canceling state cannot be paused, TiDB will retry canceling the job. If the retry fails, an error is reported and the upgrade is exited. - * If the TiDB Distributed eXecution Framework (DXF) is enabled, disable it by setting [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `OFF` (its default value). Make sure that all ongoing distributed `ADD INDEX` and `IMPORT INTO` tasks are completed. Alternatively, you can cancel these tasks and wait until the upgrade is complete to restart them. Otherwise, the `ADD INDEX` operations during the upgrade might cause data index inconsistency. + * If your current TiDB version is earlier than v8.1.0 and the TiDB Distributed eXecution Framework (DXF) is enabled, disable it by setting [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `OFF`. Make sure that all ongoing distributed `ADD INDEX` and `IMPORT INTO` tasks are completed. Alternatively, you can cancel these tasks and wait until the upgrade is complete to restart them. Otherwise, the `ADD INDEX` operations during the upgrade might cause data index inconsistency. * In scenarios of using TiUP to upgrade TiDB, because TiUP upgrade has a timeout period, if the cluster has a large number of DDL jobs (more than 300) waiting in queues before the upgrade, the upgrade might fail. diff --git a/sql-mode.md b/sql-mode.md index bb8e0f697436d..9cf4d2190c858 100644 --- a/sql-mode.md +++ b/sql-mode.md @@ -29,13 +29,13 @@ In this statement, `modes` is a set of modes separated by commas (','). You can | `PIPES_AS_CONCAT` | Treats "\|\|" as a string concatenation operator (`+`) (the same as `CONCAT()`), not as an `OR` (full support) | | `ANSI_QUOTES` | Treats `"` as an identifier. If `ANSI_QUOTES` is enabled, only single quotes are treated as string literals, and double quotes are treated as identifiers. Therefore, double quotes cannot be used to quote strings. (full support)| | `IGNORE_SPACE` | If this mode is enabled, the system ignores space. For example: "user" and "user " are the same. (full support)| -| `ONLY_FULL_GROUP_BY` | A SQL statement is invalid if it refers to a column in `SELECT`, `HAVING`, or `ORDER BY` that is neither aggregated nor included in the `GROUP BY` clause. This is because displaying such a column in query result is abnormal. (full support)| +| `ONLY_FULL_GROUP_BY` | A SQL statement is invalid if it refers to a column in `SELECT`, `HAVING`, or `ORDER BY` that is neither aggregated nor included in the `GROUP BY` clause. This is because displaying such a column in query results is abnormal. This setting is affected by the [`tidb_enable_new_only_full_group_by_check`](/system-variables.md#tidb_enable_new_only_full_group_by_check-new-in-v610) system variable. (full support)| | `NO_UNSIGNED_SUBTRACTION` | Does not mark the result as `UNSIGNED` if an operand has no symbol in subtraction. (full support)| | `NO_DIR_IN_CREATE` | Ignores all `INDEX DIRECTORY` and `DATA DIRECTORY` directives when a table is created. This option is only useful for secondary replication servers (syntax support only) | | `NO_KEY_OPTIONS` | When you use the `SHOW CREATE TABLE` statement, MySQL-specific syntaxes such as `ENGINE` are not exported. Consider this option when migrating across DB types using mysqldump. (syntax support only)| | `NO_FIELD_OPTIONS` | When you use the `SHOW CREATE TABLE` statement, MySQL-specific syntaxes such as `ENGINE` are not exported. Consider this option when migrating across DB types using mysqldump. (syntax support only) | | `NO_TABLE_OPTIONS` | When you use the `SHOW CREATE TABLE` statement, MySQL-specific syntaxes such as `ENGINE` are not exported. Consider this option when migrating across DB types using mysqldump. (syntax support only)| -| `NO_AUTO_VALUE_ON_ZERO` | If this mode is enabled, when the value passed in the `AUTO_INCREMENT` column is `0` or a specific value, the system directly writes this value to this column. When `NULL` is passed, the system automatically generates the next serial number. (full support)| +| `NO_AUTO_VALUE_ON_ZERO` | If this mode is enabled, when the value passed in the [`AUTO_INCREMENT`](/auto-increment.md) column is `0` or a specific value, the system directly writes this value to this column. When `NULL` is passed, the system automatically generates the next serial number. (full support)| | `NO_BACKSLASH_ESCAPES` | If this mode is enabled, the `\` backslash symbol only stands for itself. (full support)| | `STRICT_TRANS_TABLES` | Enables the strict mode for the transaction storage engine and rolls back the entire statement after an illegal value is inserted. (full support) | | `STRICT_ALL_TABLES` | For transactional tables, rolls back the entire transaction statement after an illegal value is inserted. (full support) | diff --git a/sql-plan-management.md b/sql-plan-management.md index e402c92b4d7c6..956c283f91bf6 100644 --- a/sql-plan-management.md +++ b/sql-plan-management.md @@ -256,7 +256,7 @@ To use this binding method, you need to first get the `plan_digest` correspondin ```sql CREATE TABLE t(id INT PRIMARY KEY , a INT, KEY(a)); SELECT /*+ IGNORE_INDEX(t, a) */ * FROM t WHERE a = 1; - SELECT * FROM INFORMATION_SCHEMA.STATEMENTS_SUMMARY WHERE QUERY_SAMPLE_TEXT = 'SELECT /*+ IGNORE_INDEX(t, a) */ * FROM t WHERE a = 1'\G; + SELECT * FROM INFORMATION_SCHEMA.STATEMENTS_SUMMARY WHERE QUERY_SAMPLE_TEXT = 'SELECT /*+ IGNORE_INDEX(t, a) */ * FROM t WHERE a = 1'\G ``` The following is a part of the example query result of `statements_summary`: @@ -285,7 +285,7 @@ To use this binding method, you need to first get the `plan_digest` correspondin To verify whether the created binding takes effect, you can [view bindings](#view-bindings): ```sql -SHOW BINDINGS\G; +SHOW BINDINGS\G ``` ``` diff --git a/sql-plan-replayer.md b/sql-plan-replayer.md index d88f39f120708..e25630605a04a 100644 --- a/sql-plan-replayer.md +++ b/sql-plan-replayer.md @@ -31,7 +31,7 @@ Based on `sql-statement`, TiDB sorts out and exports the following on-site infor - The table schema in `sql-statement` - The statistics of the table in `sql-statement` - The result of `EXPLAIN [ANALYZE] sql-statement` -- Some internal procudures of query optimization +- Some internal procedures of query optimization If historical statistics are [enabled](/system-variables.md#tidb_enable_historical_stats), you can specify a time in the `PLAN REPLAYER` statement to get the historical statistics for the corresponding time. You can directly specify a time and date or specify a timestamp. TiDB looks for the historical statistics before the specified time and exports the latest one among them. @@ -269,6 +269,29 @@ The method of downloading the file of `PLAN REPLAYER CAPTURE` is the same as tha > > The result file of `PLAN REPLAYER CAPTURE` is kept in the TiDB cluster for up to one week. After one week, TiDB deletes the file. +### Remove the capture tasks + +If a capture task is no longer needed, you can remove it using the `PLAN REPLAYER CAPTURE REMOVE` statement. For example: + +```sql +mysql> PLAN REPLAYER CAPTURE '077a87a576e42360c95530ccdac7a1771c4efba17619e26be50a4cfd967204a0' '4838af52c1e07fc8694761ad193d16a689b2128bc5ced9d13beb31ae27b370ce'; +Query OK, 0 rows affected (0.01 sec) + +mysql> SELECT * FROM mysql.plan_replayer_task; ++------------------------------------------------------------------+------------------------------------------------------------------+---------------------+ +| sql_digest | plan_digest | update_time | ++------------------------------------------------------------------+------------------------------------------------------------------+---------------------+ +| 077a87a576e42360c95530ccdac7a1771c4efba17619e26be50a4cfd967204a0 | 4838af52c1e07fc8694761ad193d16a689b2128bc5ced9d13beb31ae27b370ce | 2024-05-21 11:26:10 | ++------------------------------------------------------------------+------------------------------------------------------------------+---------------------+ +1 row in set (0.01 sec) + +mysql> PLAN REPLAYER CAPTURE REMOVE '077a87a576e42360c95530ccdac7a1771c4efba17619e26be50a4cfd967204a0' '4838af52c1e07fc8694761ad193d16a689b2128bc5ced9d13beb31ae27b370ce'; +Query OK, 0 rows affected (0.01 sec) + +mysql> SELECT * FROM mysql.plan_replayer_task; +Empty set (0.01 sec) +``` + ## Use `PLAN REPLAYER CONTINUOUS CAPTURE` After `PLAN REPLAYER CONTINUOUS CAPTURE` is enabled, TiDB asynchronously records the applications' SQL statements with the `PLAN REPLAYER` method according to their `SQL DIGEST` and `PLAN DIGEST`. For SQL statements and execution plans that share the same DIGEST, `PLAN REPLAYER CONTINUOUS CAPTURE` does not record them repeatedly. diff --git a/sql-prepared-plan-cache.md b/sql-prepared-plan-cache.md index e70495ca1db9c..1b9194527dd1b 100644 --- a/sql-prepared-plan-cache.md +++ b/sql-prepared-plan-cache.md @@ -6,13 +6,6 @@ aliases: ['/tidb/dev/sql-prepare-plan-cache'] # SQL Prepared Execution Plan Cache -> **Warning:** -> -> If a cached `UPDATE` or `DELETE` statement encounters a DDL operation that modifies the relevant schema during execution, it might cause data inconsistency between tables and indexes. For more information, see [Issue #51407](https://github.com/pingcap/tidb/issues/51407). It is recommended to monitor the status of this issue and upgrade to [the latest LTS version](https://docs.pingcap.com/tidb/stable) to resolve this issue. Before upgrading, you can try the following workarounds: -> -> - Before executing DDL, temporarily [disable the prepared plan cache](/system-variables.md#tidb_enable_prepared_plan_cache-new-in-v610), and re-enable it after the DDL is complete. -> - Avoid executing DDL during business peak hours. After executing DDL, immediately run [`ADMIN CHECK TABLE`](/sql-statements/sql-statement-admin-check-table-index.md) to check the consistency between tables and indexes. If errors are found, rebuild the relevant indexes. - TiDB supports execution plan caching for `Prepare` and `Execute` queries. This includes both forms of prepared statements: - Using the `COM_STMT_PREPARE` and `COM_STMT_EXECUTE` protocol features. @@ -70,7 +63,7 @@ Starting from v6.1.0, the execution plan cache is enabled by default. You can co > **Note:** > -> The execution plan cache feature applies only to `Prepare`/`Execute` queries and does not take effect for normal queries. +> The [`tidb_enable_prepared_plan_cache`](/system-variables.md#tidb_enable_prepared_plan_cache-new-in-v610) system variable controls the execution plan cache only for `Prepare`/`Execute` queries, not for normal queries. For the execution plan cache for normal queries, see [SQL Non-Prepared Execution Plan Cache](/sql-non-prepared-plan-cache.md). After the execution plan cache feature is enabled, you can use the session-level system variable [`last_plan_from_cache`](/system-variables.md#last_plan_from_cache-new-in-v40) to see whether the previous `Execute` statement used the cached execution plan, for example: diff --git a/sql-statements/sql-statement-add-index.md b/sql-statements/sql-statement-add-index.md index c85653df8f98f..9d4ff05c11276 100644 --- a/sql-statements/sql-statement-add-index.md +++ b/sql-statements/sql-statement-add-index.md @@ -8,6 +8,10 @@ aliases: ['/docs/dev/sql-statements/sql-statement-add-index/','/docs/dev/referen The `ALTER TABLE.. ADD INDEX` statement adds an index to an existing table. This operation is online in TiDB, which means that neither reads or writes to the table are blocked by adding an index. +> **Tip:** +> +> The [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) can be used to speed up the operation of this statement. + > **Warning:** @@ -96,3 +100,4 @@ mysql> EXPLAIN SELECT * FROM t1 WHERE c1 = 3; * [ADD COLUMN](/sql-statements/sql-statement-add-column.md) * [CREATE TABLE](/sql-statements/sql-statement-create-table.md) * [EXPLAIN](/sql-statements/sql-statement-explain.md) +* [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) diff --git a/sql-statements/sql-statement-admin-show-ddl.md b/sql-statements/sql-statement-admin-show-ddl.md index 9da7821eac9b2..efdb46707bfbe 100644 --- a/sql-statements/sql-statement-admin-show-ddl.md +++ b/sql-statements/sql-statement-admin-show-ddl.md @@ -11,7 +11,12 @@ The `ADMIN SHOW DDL [JOBS|JOB QUERIES]` statement shows information about runnin ```ebnf+diagram AdminStmt ::= - 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList | 'JOB' 'QUERIES' 'LIMIT' m 'OFFSET' n )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) + 'ADMIN' 'SHOW' 'DDL' + ( + 'JOBS' Int64Num? WhereClauseOptional + | 'JOB' 'QUERIES' NumList + | 'JOB' 'QUERIES' 'LIMIT' m ( ('OFFSET' | ',') n )? + ) NumList ::= Int64Num ( ',' Int64Num )* @@ -24,21 +29,27 @@ WhereClauseOptional ::= ### `ADMIN SHOW DDL` -To view the status of the currently running DDL jobs, use `ADMIN SHOW DDL`. The output includes the current schema version, the DDL ID and address of the owner, the running DDL jobs and SQL statements, and the DDL ID of the current TiDB instance. +To view the status of the currently running DDL jobs, use `ADMIN SHOW DDL`. The output includes the current schema version, the DDL ID and address of the owner, the running DDL jobs and SQL statements, and the DDL ID of the current TiDB instance. The returned result fields are described as follows: -{{< copyable "sql" >}} +- `SCHEMA_VER`: a number indicating the version of the schema. +- `OWNER_ID`: the UUID of the DDL owner. See also [`TIDB_IS_DDL_OWNER()`](/functions-and-operators/tidb-functions.md). +- `OWNER_ADDRESS`: the IP address of the DDL owner. +- `RUNNING_JOBS`: details about the running DDL job. +- `SELF_ID`: the UUID of the TiDB node to which you are currently connected. If `SELF_ID` is the same as the `OWNER_ID`, it means that you are connected to the DDL owner. +- `QUERY`: the statement of the query. ```sql -ADMIN SHOW DDL; +ADMIN SHOW DDL\G; ``` ```sql -mysql> ADMIN SHOW DDL; -+------------+--------------------------------------+---------------+--------------+--------------------------------------+-------+ -| SCHEMA_VER | OWNER_ID | OWNER_ADDRESS | RUNNING_JOBS | SELF_ID | QUERY | -+------------+--------------------------------------+---------------+--------------+--------------------------------------+-------+ -| 26 | 2d1982af-fa63-43ad-a3d5-73710683cc63 | 0.0.0.0:4000 | | 2d1982af-fa63-43ad-a3d5-73710683cc63 | | -+------------+--------------------------------------+---------------+--------------+--------------------------------------+-------+ +*************************** 1. row *************************** + SCHEMA_VER: 26 + OWNER_ID: 2d1982af-fa63-43ad-a3d5-73710683cc63 +OWNER_ADDRESS: 0.0.0.0:4000 + RUNNING_JOBS: + SELF_ID: 2d1982af-fa63-43ad-a3d5-73710683cc63 + QUERY: 1 row in set (0.00 sec) ``` @@ -52,9 +63,12 @@ The `ADMIN SHOW DDL JOBS` statement is used to view all the results in the curre - `DB_NAME`: the name of the database where the DDL operation is performed. - `TABLE_NAME`: the name of the table where the DDL operation is performed. - `JOB_TYPE`: the type of DDL operation. Common job types include the following: - - `ingest`: Ingestion with accelerated index backfilling as configured by [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630). - - `txn`: Basic transactional backfill. - - `txn-merge`: Transactional backfill with a temporary index that gets merged with the original index when the backfill is finished. + - `create schema`: for [`CREATE SCHEMA`](/sql-statements/sql-statement-create-database.md) operations. + - `create table`: for [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) operations. + - `create view`: for [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) operations. + - `ingest`: ingestion with accelerated index backfilling as configured by [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630). + - `txn`: basic transactional backfilling. + - `add index /* txn-merge */`: transactional backfilling with a temporary index that gets merged with the original index when the backfilling is finished. - `SCHEMA_STATE`: the current state of the schema object that the DDL operates on. If `JOB_TYPE` is `ADD INDEX`, it is the state of the index; if `JOB_TYPE` is `ADD COLUMN`, it is the state of the column; if `JOB_TYPE` is `CREATE TABLE`, it is the state of the table. Common states include the following: - `none`: indicates that it does not exist. Generally, after the `DROP` operation or after the `CREATE` operation fails and rolls back, it will become the `none` state. - `delete only`, `write only`, `delete reorganization`, `write reorganization`: these four states are intermediate states. For their specific meanings, see [How the Online DDL Asynchronous Change Works in TiDB](/ddl-introduction.md#how-the-online-ddl-asynchronous-change-works-in-tidb). As the intermediate state conversion is fast, these states are generally not visible during operation. Only when performing `ADD INDEX` operation can the `write reorganization` state be seen, indicating that index data is being added. @@ -62,15 +76,21 @@ The `ADMIN SHOW DDL JOBS` statement is used to view all the results in the curre - `SCHEMA_ID`: the ID of the database where the DDL operation is performed. - `TABLE_ID`: the ID of the table where the DDL operation is performed. - `ROW_COUNT`: when performing the `ADD INDEX` operation, it is the number of data rows that have been added. +- `CREATE_TIME`: the creation time of the DDL operation. - `START_TIME`: the start time of the DDL operation. +- `END_TIME`: the end time of the DDL operation. - `STATE`: the state of the DDL operation. Common states include the following: - - `queueing`: indicates that the operation job has entered the DDL job queue but has not been executed because it is still waiting for an earlier DDL job to complete. Another reason might be that after executing the `DROP` operation, it will become the `none` state, but it will soon be updated to the `synced` state, indicating that all TiDB instances have been synchronized to that state. + - `none`: indicates that the operation has not started yet. + - `queueing`: indicates that the operation job has entered the DDL job queue but has not been executed because it is still waiting for an earlier DDL job to complete. Another reason might be that after executing the `DROP` operation, the `queueing` state will become the `done` state, but it will soon be updated to the `synced` state, indicating that all TiDB instances have been synchronized to that state. - `running`: indicates that the operation is being executed. - `synced`: indicates that the operation has been executed successfully and all TiDB instances have been synchronized to this state. - `rollback done`: indicates that the operation has failed and the rollback has been completed. - `rollingback`: indicates that the operation has failed and is rolling back. - `cancelling`: indicates that the operation is being canceled. This state only appears when you use the [`ADMIN CANCEL DDL JOBS`](/sql-statements/sql-statement-admin-cancel-ddl.md) command to cancel the DDL job. + - `cancelled`: indicates that the operation has been canceled. + - `pausing`: indicates that the operation is being paused. - `paused`: indicates that the operation has been paused. This state only appears when you use the [`ADMIN PAUSED DDL JOBS`](/sql-statements/sql-statement-admin-pause-ddl.md) command to pause the DDL job. You can use the [`ADMIN RESUME DDL JOBS`](/sql-statements/sql-statement-admin-resume-ddl.md) command to resume the DDL job. + - `done`: indicates that the operation has been successfully executed on the TiDB owner node, but other TiDB nodes have not yet synchronized the changes performed by this DDL job. @@ -101,8 +121,6 @@ The `ADMIN SHOW DDL JOBS` statement is used to view all the results in the curre The following example shows the results of `ADMIN SHOW DDL JOBS`: -{{< copyable "sql" >}} - ```sql ADMIN SHOW DDL JOBS; ``` @@ -149,8 +167,6 @@ ADMIN SHOW DDL JOBS [NUM] [WHERE where_condition]; To view the original SQL statements of the DDL job corresponding to `job_id`, use `ADMIN SHOW DDL JOB QUERIES`: -{{< copyable "sql" >}} - ```sql ADMIN SHOW DDL JOBS; ADMIN SHOW DDL JOB QUERIES 51; @@ -172,8 +188,6 @@ You can only search the running DDL job corresponding to `job_id` within the las To view the original SQL statements of the DDL job within a specified range `[n+1, n+m]` corresponding to `job_id`, use `ADMIN SHOW DDL JOB QUERIES LIMIT m OFFSET n`: - {{< copyable "sql" >}} - ```sql ADMIN SHOW DDL JOB QUERIES LIMIT m; # Retrieve first m rows ADMIN SHOW DDL JOB QUERIES LIMIT n, m; # Retrieve rows [n+1, n+m] @@ -225,6 +239,8 @@ This statement is a TiDB extension to MySQL syntax. ## See also +* [DDL introduction](/ddl-introduction.md) * [ADMIN CANCEL DDL](/sql-statements/sql-statement-admin-cancel-ddl.md) * [ADMIN PAUSE DDL](/sql-statements/sql-statement-admin-pause-ddl.md) * [ADMIN RESUME DDL](/sql-statements/sql-statement-admin-resume-ddl.md) +* [INFORMATION_SCHEMA.DDL_JOBS](/information-schema/information-schema-ddl-jobs.md) diff --git a/sql-statements/sql-statement-alter-index.md b/sql-statements/sql-statement-alter-index.md index f729fb3ef24d7..d060bc1a31a1c 100644 --- a/sql-statements/sql-statement-alter-index.md +++ b/sql-statements/sql-statement-alter-index.md @@ -118,7 +118,7 @@ Query OK, 0 rows affected (0.02 sec) ## MySQL compatibility * Invisible indexes in TiDB are modeled on the equivalent feature from MySQL 8.0. -* Similiar to MySQL, TiDB does not permit `PRIMARY KEY` indexes to be made invisible. +* Similar to MySQL, TiDB does not permit `PRIMARY KEY` indexes to be made invisible. ## See also diff --git a/sql-statements/sql-statement-alter-placement-policy.md b/sql-statements/sql-statement-alter-placement-policy.md index e56848ff57e30..2f140911c5bf6 100644 --- a/sql-statements/sql-statement-alter-placement-policy.md +++ b/sql-statements/sql-statement-alter-placement-policy.md @@ -68,7 +68,7 @@ AdvancedPlacementOption ::= CREATE PLACEMENT POLICY p1 PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1"; CREATE TABLE t1 (i INT) PLACEMENT POLICY=p1; -- Assign policy p1 to table t1 ALTER PLACEMENT POLICY p1 PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1,us-west-2" FOLLOWERS=4; -- The rules of t1 will be updated automatically. -SHOW CREATE PLACEMENT POLICY p1\G; +SHOW CREATE PLACEMENT POLICY p1\G ``` ``` diff --git a/sql-statements/sql-statement-alter-sequence.md b/sql-statements/sql-statement-alter-sequence.md new file mode 100644 index 0000000000000..72f896d467876 --- /dev/null +++ b/sql-statements/sql-statement-alter-sequence.md @@ -0,0 +1,215 @@ +--- +title: ALTER SEQUENCE +summary: An overview of the usage of ALTER SEQUENCE for the TiDB database. +--- + +# ALTER SEQUENCE + +The `ALTER SEQUENCE` statement alters sequence objects in TiDB. The sequence is a database object that is on a par with the `Table` and the `View` object. The sequence is used to generate serialized IDs in a customized way. + +## Synopsis + +```ebnf+diagram +CreateSequenceStmt ::= + 'ALTER' 'SEQUENCE' TableName CreateSequenceOptionListOpt + +TableName ::= + Identifier ('.' Identifier)? + +CreateSequenceOptionListOpt ::= + SequenceOption* + +SequenceOptionList ::= + SequenceOption + +SequenceOption ::= + ( 'INCREMENT' ( '='? | 'BY' ) | 'START' ( '='? | 'WITH' ) | ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' ) '='? ) SignedNum +| 'COMMENT' '='? stringLit +| 'NOMINVALUE' +| 'NO' ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' | 'CYCLE' ) +| 'NOMAXVALUE' +| 'NOCACHE' +| 'CYCLE' +| 'NOCYCLE' +| 'RESTART' ( ( '='? | 'WITH' ) SignedNum )? +``` + +## Syntax + +```sql +ALTER SEQUENCE sequence_name + [ INCREMENT [ BY | = ] increment ] + [ MINVALUE [=] minvalue | NO MINVALUE | NOMINVALUE ] + [ MAXVALUE [=] maxvalue | NO MAXVALUE | NOMAXVALUE ] + [ START [ WITH | = ] start ] + [ CACHE [=] cache | NOCACHE | NO CACHE] + [ CYCLE | NOCYCLE | NO CYCLE] + [table_options] +``` + +## Parameters + +|Parameters | Default value | Description | +| :-- | :-- | :--| +| `INCREMENT` | `1` | Specifies the increment of a sequence. Its positive or negative values can control the growth direction of the sequence. | +| `MINVALUE` | `1` or `-9223372036854775807` | Specifies the minimum value of a sequence. When `INCREMENT` > `0`, the default value is `1`. When `INCREMENT` < `0`, the default value is `-9223372036854775807`. | +| `MAXVALUE` | `9223372036854775806` or `-1` | Specifies the maximum value of a sequence. When `INCREMENT` > `0`, the default value is `9223372036854775806`. When `INCREMENT` < `0`, the default value is `-1`. | +| `START` | `MINVALUE` or `MAXVALUE`| Specifies the initial value of a sequence. When `INCREMENT` > `0`, the default value is `MINVALUE`. When `INCREMENT` < `0`, the default value is `MAXVALUE`. | +| `CACHE` | `1000` | Specifies the local cache size of a sequence in TiDB. | +| `CYCLE` | `NO CYCLE` | Specifies whether a sequence restarts from the minimum value (or maximum for the descending sequence). When `INCREMENT` > `0`, the default value is `MINVALUE`. When `INCREMENT` < `0`, the default value is `MAXVALUE`. | + +> **Note:** +> +> Changing the `START` value does not affect the generated values until you execute `ALTER SEQUENCE ... RESTART`. + +## `SEQUENCE` function + +You can control a sequence through the following expression functions: + ++ `NEXTVAL` or `NEXT VALUE FOR` + + Essentially, both are the `NEXTVAL()` function that gets the next valid value of a sequence object. The parameter of the `NEXTVAL()` function is the `identifier` of the sequence. + ++ `LASTVAL` + + This function gets the last used value of this session. If the value does not exist, `NULL` is used. The parameter of this function is the `identifier` of the sequence. + ++ `SETVAL` + + This function sets the progression of the current value for a sequence. The first parameter of this function is the `identifier` of the sequence; the second parameter is `num`. + +> **Note:** +> +> In the implementation of a sequence in TiDB, the `SETVAL` function cannot change the initial progression or cycle progression of this sequence. This function only returns the next valid value based on this progression. + +## Examples + +Create a sequence named `s1`: + +```sql +CREATE SEQUENCE s1; +``` + +``` +Query OK, 0 rows affected (0.15 sec) +``` + +Get the next two values from the sequence by executing the following SQL statement twice: + +```sql +SELECT NEXTVAL(s1); +``` + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 1 | ++-------------+ +1 row in set (0.01 sec) +``` + +```sql +SELECT NEXTVAL(s1); +``` + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 2 | ++-------------+ +1 row in set (0.00 sec) +``` + +Change the increment of the sequence to `2`: + +```sql +ALTER SEQUENCE s1 INCREMENT=2; +``` + +``` +Query OK, 0 rows affected (0.18 sec) +``` + +Now, get the next two values from the sequence again: + +```sql +SELECT NEXTVAL(s1); +``` + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 1001 | ++-------------+ +1 row in set (0.02 sec) +``` + +```sql +SELECT NEXTVAL(s1); +``` + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 1003 | ++-------------+ +1 row in set (0.00 sec) +``` + +As you can see from the output, the values now increase by two, following the `ALTER SEQUENCE` statement. + +You can also change other parameters of the sequence. For example, you can change the `MAXVALUE` of the sequence as follows: + +```sql +CREATE SEQUENCE s2 MAXVALUE=10; +``` + +``` +Query OK, 0 rows affected (0.17 sec) +``` + +```sql +ALTER SEQUENCE s2 MAXVALUE=100; +``` + +``` +Query OK, 0 rows affected (0.15 sec) +``` + +```sql +SHOW CREATE SEQUENCE s2\G +``` + +``` +*************************** 1. row *************************** + Sequence: s2 +Create Sequence: CREATE SEQUENCE `s2` start with 1 minvalue 1 maxvalue 100 increment by 1 cache 1000 nocycle ENGINE=InnoDB +1 row in set (0.00 sec) +``` + +## MySQL compatibility + +This statement is a TiDB extension. The implementation is modeled on sequences available in MariaDB. + +Except for the `SETVAL` function, all other functions have the same _progressions_ as MariaDB. Here "progression" means that the numbers in a sequence follow a certain arithmetic progression rule defined by the sequence. Although you can use `SETVAL` to set the current value of a sequence, the subsequent values of the sequence still follow the original progression rule. + +For example: + +``` +1, 3, 5, ... // The sequence starts from 1 and increments by 2. +SELECT SETVAL(seq, 6) // Sets the current value of a sequence to 6. +7, 9, 11, ... // Subsequent values still follow the progression rule. +``` + +In the `CYCLE` mode, the initial value of a sequence in the first round is the value of the `START` parameter, and the initial value in the subsequent rounds is the value of `MinValue` (`INCREMENT` > 0) or `MaxValue` (`INCREMENT` < 0). + +## See also + +* [CREATE SEQUENCE](/sql-statements/sql-statement-create-sequence.md) +* [DROP SEQUENCE](/sql-statements/sql-statement-drop-sequence.md) +* [SHOW CREATE SEQUENCE](/sql-statements/sql-statement-show-create-sequence.md) +* [Sequence Functions](/functions-and-operators/sequence-functions.md) diff --git a/sql-statements/sql-statement-backup.md b/sql-statements/sql-statement-backup.md index 064adca71db63..612b4a3d57b1b 100644 --- a/sql-statements/sql-statement-backup.md +++ b/sql-statements/sql-statement-backup.md @@ -138,9 +138,11 @@ BACKUP DATABASE `test` TO 's3://example-bucket-2020/backup-05/' Use `RATE_LIMIT` to limit the average upload speed per TiKV node to reduce network bandwidth. -By default, every TiKV node would run 4 backup threads. This value can be adjusted with the `CONCURRENCY` option. +Before the backup is completed, `BACKUP` would perform a checksum against the data on the cluster to verify correctness. If you are confident that this verification is unnecessary, you can disable the check by setting the `CHECKSUM` parameter to `FALSE`. -Before backup is completed, `BACKUP` would perform a checksum against the data on the cluster to verify correctness. This step can be disabled with the `CHECKSUM` option if you are confident that this is unnecessary. +To specify the number of concurrent tasks that BR can execute for backing up tables and indices, use the `CONCURRENCY` parameter. This parameter controls the thread pool size within BR, optimizing the performance and efficiency of backup operations. + +One task represents one table range or one index range, according to the backup schemas. For one table with one index, two tasks are used to back up this table. The default value of `CONCURRENCY` is `4`. If you need to back up lots of tables or indices, increase its value. {{< copyable "sql" >}} diff --git a/sql-statements/sql-statement-create-index.md b/sql-statements/sql-statement-create-index.md index 2211011be0ddd..57852a4a81ce2 100644 --- a/sql-statements/sql-statement-create-index.md +++ b/sql-statements/sql-statement-create-index.md @@ -6,7 +6,7 @@ aliases: ['/docs/dev/sql-statements/sql-statement-create-index/','/docs/dev/refe # CREATE INDEX -This statement adds a new index to an existing table. It is an alternative syntax to `ALTER TABLE .. ADD INDEX`, and included for MySQL compatibility. +This statement adds a new index to an existing table. It is an alternative syntax to [`ALTER TABLE .. ADD INDEX`](/sql-statements/sql-statement-alter-table.md), and included for MySQL compatibility. ## Synopsis @@ -106,28 +106,26 @@ Query OK, 0 rows affected (0.31 sec) In some scenarios, the filtering condition of a query is based on a certain expression. In these scenarios, the query performance is relatively poor because ordinary indexes cannot take effect, the query can only be executed by scanning the entire table. The expression index is a type of special index that can be created on an expression. Once an expression index is created, TiDB can use the index for the expression-based query, which significantly improves the query performance. -For example, if you want to create an index based on `lower(col1)`, execute the following SQL statement: - -{{< copyable "sql" >}} +For example, if you want to create an index based on `LOWER(col1)`, execute the following SQL statement: ```sql -CREATE INDEX idx1 ON t1 ((lower(col1))); +CREATE INDEX idx1 ON t1 ((LOWER(col1))); ``` Or you can execute the following equivalent statement: -{{< copyable "sql" >}} - ```sql -ALTER TABLE t1 ADD INDEX idx1((lower(col1))); +ALTER TABLE t1 ADD INDEX idx1((LOWER(col1))); ``` You can also specify the expression index when you create the table: -{{< copyable "sql" >}} - ```sql -CREATE TABLE t1(col1 char(10), col2 char(10), index((lower(col1)))); +CREATE TABLE t1 ( + col1 CHAR(10), + col2 CHAR(10), + INDEX ((LOWER(col1))) +); ``` > **Note:** @@ -136,8 +134,6 @@ CREATE TABLE t1(col1 char(10), col2 char(10), index((lower(col1)))); You can drop an expression index in the same way as dropping an ordinary index: -{{< copyable "sql" >}} - ```sql DROP INDEX idx1 ON t1; ``` @@ -145,17 +141,15 @@ DROP INDEX idx1 ON t1; Expression index involves various kinds of expressions. To ensure correctness, only some fully tested functions are allowed for creating an expression index. This means that only these functions are allowed in expressions in a production environment. You can get these functions by querying the `tidb_allow_function_for_expression_index` variable. Currently, the allowed functions are as follows: ``` -json_array, json_array_append, json_array_insert, json_contains, json_contains_path, json_depth, json_extract, json_insert, json_keys, json_length, json_merge_patch, json_merge_preserve, json_object, json_pretty, json_quote, json_remove, json_replace, json_search, json_set, json_storage_size, json_type, json_unquote, json_valid, lower, md5, reverse, tidb_shard, upper, vitess_hash +JSON_ARRAY, JSON_ARRAY_APPEND, JSON_ARRAY_INSERT, JSON_CONTAINS, JSON_CONTAINS_PATH, JSON_DEPTH, JSON_EXTRACT, JSON_INSERT, JSON_KEYS, JSON_LENGTH, JSON_MERGE_PATCH, JSON_MERGE_PRESERVE, JSON_OBJECT, JSON_PRETTY, JSON_QUOTE, JSON_REMOVE, JSON_REPLACE, JSON_SEARCH, JSON_SET, JSON_STORAGE_SIZE, JSON_TYPE, JSON_UNQUOTE, JSON_VALID, LOWER, MD5, REVERSE, TIDB_SHARD, UPPER, VITESS_HASH ``` -For the functions that are not included in the above list, those functions are not fully tested and not recommended for a production environment, which can be seen as experimental. Other expressions such as operators, `cast`, and `case when` are also seen as experimental and not recommended for production. +For the functions that are not included in the above list, those functions are not fully tested and not recommended for a production environment, which can be seen as experimental. Other expressions such as operators, `CAST`, and `CASE WHEN` are also seen as experimental and not recommended for production. If you still want to use those expressions, you can make the following configuration in the [TiDB configuration file](/tidb-configuration-file.md#allow-expression-index-new-in-v400): -{{< copyable "sql" >}} - ```sql allow-expression-index = true ``` @@ -168,63 +162,55 @@ allow-expression-index = true > > The expression in an expression index cannot contain the following content: > -> - Volatile functions, such as `rand()` and `now()`. -> - System variables and user variables. +> - Volatile functions, such as `RAND()` and `NOW()`. +> - [System variables](/system-variables.md) and [user variables](/user-defined-variables.md). > - Subqueries. -> - `AUTO_INCREMENT` column. You can remove this restriction by setting the value of `tidb_enable_auto_increment_in_generated` (system variable) to `true`. -> - Window functions. -> - ROW functions, such as `create table t (j json, key k (((j,j))));`. -> - Aggregate functions. +> - [`AUTO_INCREMENT`](/auto-increment.md) columns. You can remove this restriction by setting the value of [`tidb_enable_auto_increment_in_generated`](/system-variables.md#tidb_enable_auto_increment_in_generated) (system variable) to `true`. +> - [Window functions](/functions-and-operators/window-functions.md). +> - ROW functions, such as `CREATE TABLE t (j JSON, INDEX k (((j,j))));`. +> - [Aggregate functions](/functions-and-operators/aggregate-group-by-functions.md). > > An expression index implicitly takes up a name (for example, `_V$_{index_name}_{index_offset}`). If you try to create a new expression index with the name that a column has already had, an error occurs. In addition, if you add a new column with the same name, an error also occurs. > > Make sure that the number of function parameters in the expression of an expression index is correct. > -> When the expression of an index contains a string-related function, affected by the returned type and the length, creating the expression index might fail. In this situation, you can use the `cast()` function to explicitly specify the returned type and the length. For example, to create an expression index based on the `repeat(a, 3)` expression, you need to modify this expression to `cast(repeat(a, 3) as char(20))`. +> When the expression of an index contains a string-related function, affected by the returned type and the length, creating the expression index might fail. In this situation, you can use the `CAST()` function to explicitly specify the returned type and the length. For example, to create an expression index based on the `REPEAT(a, 3)` expression, you need to modify this expression to `CAST(REPEAT(a, 3) AS CHAR(20))`. When the expression in a query statement matches the expression in an expression index, the optimizer can choose the expression index for the query. In some cases, the optimizer might not choose an expression index depending on statistics. In this situation, you can force the optimizer to select an expression index by using optimizer hints. -In the following examples, suppose that you create the expression index `idx` on the expression `lower(col1)`: +In the following examples, suppose that you create the expression index `idx` on the expression `LOWER(col1)`: If the results of the query statement are the same expressions, the expression index applies. Take the following statement as an example: -{{< copyable "sql" >}} - ```sql -SELECT lower(col1) FROM t; +SELECT LOWER(col1) FROM t; ``` If the same expression is included in the filtering conditions, the expression index applies. Take the following statements as an example: -{{< copyable "sql" >}} - ```sql -SELECT * FROM t WHERE lower(col1) = "a"; -SELECT * FROM t WHERE lower(col1) > "a"; -SELECT * FROM t WHERE lower(col1) BETWEEN "a" AND "b"; -SELECT * FROM t WHERE lower(col1) in ("a", "b"); -SELECT * FROM t WHERE lower(col1) > "a" AND lower(col1) < "b"; -SELECT * FROM t WHERE lower(col1) > "b" OR lower(col1) < "a"; +SELECT * FROM t WHERE LOWER(col1) = "a"; +SELECT * FROM t WHERE LOWER(col1) > "a"; +SELECT * FROM t WHERE LOWER(col1) BETWEEN "a" AND "b"; +SELECT * FROM t WHERE LOWER(col1) IN ("a", "b"); +SELECT * FROM t WHERE LOWER(col1) > "a" AND LOWER(col1) < "b"; +SELECT * FROM t WHERE LOWER(col1) > "b" OR LOWER(col1) < "a"; ``` When the queries are sorted by the same expression, the expression index applies. Take the following statement as an example: -{{< copyable "sql" >}} - ```sql -SELECT * FROM t ORDER BY lower(col1); +SELECT * FROM t ORDER BY LOWER(col1); ``` If the same expression is included in the aggregate (`GROUP BY`) functions, the expression index applies. Take the following statements as an example: -{{< copyable "sql" >}} - ```sql -SELECT max(lower(col1)) FROM t; -SELECT min(col1) FROM t GROUP BY lower(col1); +SELECT MAX(LOWER(col1)) FROM t; +SELECT MIN(col1) FROM t GROUP BY LOWER(col1); ``` -To see the expression corresponding to the expression index, execute `show index`, or check the system tables `information_schema.tidb_indexes` and the table `information_schema.STATISTICS`. The `Expression` column in the output indicates the corresponded expression. For the non-expression indexes, the column shows `NULL`. +To see the expression corresponding to the expression index, execute [`SHOW INDEX`](/sql-statements/sql-statement-show-indexes.md), or check the system tables [`information_schema.tidb_indexes`](/information-schema/information-schema-tidb-indexes.md) and the table [`information_schema.STATISTICS`](/information-schema/information-schema-statistics.md). The `Expression` column in the output indicates the corresponded expression. For the non-expression indexes, the column shows `NULL`. The cost of maintaining an expression index is higher than that of maintaining other indexes, because the value of the expression needs to be calculated whenever a row is inserted or updated. The value of the expression is already stored in the index, so this value does not require recalculation when the optimizer selects the expression index. @@ -246,7 +232,7 @@ Multi-valued indexes are a kind of secondary index defined on an array column. I ### Create multi-valued indexes -You can create multi-valued indexes by using the `CAST(... AS ... ARRAY)` expression in the index definition, as creating an expression index. +You can create multi-valued indexes by using the [`CAST(... AS ... ARRAY)`](/functions-and-operators/cast-functions-and-operators.md#cast) function in the index definition, as creating an expression index. ```sql mysql> CREATE TABLE customers ( diff --git a/sql-statements/sql-statement-create-sequence.md b/sql-statements/sql-statement-create-sequence.md index 01946a387eaba..f73f273435cd7 100644 --- a/sql-statements/sql-statement-create-sequence.md +++ b/sql-statements/sql-statement-create-sequence.md @@ -12,7 +12,7 @@ The `CREATE SEQUENCE` statement creates sequence objects in TiDB. The sequence i ```ebnf+diagram CreateSequenceStmt ::= - 'CREATE' 'SEQUENCE' IfNotExists TableName CreateSequenceOptionListOpt CreateTableOptionListOpt + 'CREATE' 'SEQUENCE' IfNotExists TableName CreateSequenceOptionListOpt IfNotExists ::= ('IF' 'NOT' 'EXISTS')? @@ -28,6 +28,7 @@ SequenceOptionList ::= SequenceOption ::= ( 'INCREMENT' ( '='? | 'BY' ) | 'START' ( '='? | 'WITH' ) | ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' ) '='? ) SignedNum +| 'COMMENT' '='? stringLit | 'NOMINVALUE' | 'NO' ( 'MINVALUE' | 'MAXVALUE' | 'CACHE' | 'CYCLE' ) | 'NOMAXVALUE' @@ -69,7 +70,7 @@ You can control a sequence through the following expression functions: + `NEXTVAL` or `NEXT VALUE FOR` - Essentially, both are the `nextval()` function that gets the next valid value of a sequence object. The parameter of the `nextval()` function is the `identifier` of the sequence. + Essentially, both are the `NEXTVAL()` function that gets the next valid value of a sequence object. The parameter of the `NEXTVAL()` function is the `identifier` of the sequence. + `LASTVAL` @@ -97,51 +98,51 @@ You can control a sequence through the following expression functions: Query OK, 0 rows affected (0.06 sec) ``` -+ Use the `nextval()` function to get the next value of the sequence object: ++ Use the `NEXTVAL()` function to get the next value of the sequence object: {{< copyable "sql" >}} ```sql - SELECT nextval(seq); + SELECT NEXTVAL(seq); ``` ``` +--------------+ - | nextval(seq) | + | NEXTVAL(seq) | +--------------+ | 1 | +--------------+ 1 row in set (0.02 sec) ``` -+ Use the `lastval()` function to get the value generated by the last call to a sequence object in this session: ++ Use the `LASTVAL()` function to get the value generated by the last call to a sequence object in this session: {{< copyable "sql" >}} ```sql - SELECT lastval(seq); + SELECT LASTVAL(seq); ``` ``` +--------------+ - | lastval(seq) | + | LASTVAL(seq) | +--------------+ | 1 | +--------------+ 1 row in set (0.02 sec) ``` -+ Use the `setval()` function to set the current value (or the current position) of the sequence object: ++ Use the `SETVAL()` function to set the current value (or the current position) of the sequence object: {{< copyable "sql" >}} ```sql - SELECT setval(seq, 10); + SELECT SETVAL(seq, 10); ``` ``` +-----------------+ - | setval(seq, 10) | + | SETVAL(seq, 10) | +-----------------+ | 10 | +-----------------+ @@ -177,58 +178,58 @@ You can control a sequence through the following expression functions: Query OK, 0 rows affected (0.01 sec) ``` -+ When the sequence object has not been used in this session, the `lastval()` function returns a `NULL` value. ++ When the sequence object has not been used in this session, the `LASTVAL()` function returns a `NULL` value. {{< copyable "sql" >}} ```sql - SELECT lastval(seq2); + SELECT LASTVAL(seq2); ``` ``` +---------------+ - | lastval(seq2) | + | LASTVAL(seq2) | +---------------+ | NULL | +---------------+ 1 row in set (0.01 sec) ``` -+ The first valid value of the `nextval()` function for the sequence object is the value of `START` parameter. ++ The first valid value of the `NEXTVAL()` function for the sequence object is the value of `START` parameter. {{< copyable "sql" >}} ```sql - SELECT nextval(seq2); + SELECT NEXTVAL(seq2); ``` ``` +---------------+ - | nextval(seq2) | + | NEXTVAL(seq2) | +---------------+ | 3 | +---------------+ 1 row in set (0.00 sec) ``` -+ Although the `setval()` function can change the current value of the sequence object, it cannot change the arithmetic progression rule for the next value. ++ Although the `SETVAL()` function can change the current value of the sequence object, it cannot change the arithmetic progression rule for the next value. {{< copyable "sql" >}} ```sql - SELECT setval(seq2, 6); + SELECT SETVAL(seq2, 6); ``` ``` +-----------------+ - | setval(seq2, 6) | + | SETVAL(seq2, 6) | +-----------------+ | 6 | +-----------------+ 1 row in set (0.00 sec) ``` -+ When you use `nextval()` to get the next value, the next value will follow the arithmetic progression rule defined by the sequence. ++ When you use `NEXTVAL()` to get the next value, the next value will follow the arithmetic progression rule defined by the sequence. {{< copyable "sql" >}} @@ -306,7 +307,7 @@ For example: ``` 1, 3, 5, ... // The sequence starts from 1 and increments by 2. -select setval(seq, 6) // Sets the current value of a sequence to 6. +select SETVAL(seq, 6) // Sets the current value of a sequence to 6. 7, 9, 11, ... // Subsequent values still follow the progression rule. ``` @@ -314,5 +315,7 @@ In the `CYCLE` mode, the initial value of a sequence in the first round is the v ## See also +* [ALTER SEQUENCE](/sql-statements/sql-statement-alter-sequence.md) * [DROP SEQUENCE](/sql-statements/sql-statement-drop-sequence.md) * [SHOW CREATE SEQUENCE](/sql-statements/sql-statement-show-create-sequence.md) +* [Sequence Functions](/functions-and-operators/sequence-functions.md) diff --git a/sql-statements/sql-statement-drop-binding.md b/sql-statements/sql-statement-drop-binding.md index b240e6293ea3a..6413ccf5fc4d2 100644 --- a/sql-statements/sql-statement-drop-binding.md +++ b/sql-statements/sql-statement-drop-binding.md @@ -166,7 +166,7 @@ mysql> SELECT @@LAST_PLAN_FROM_BINDING; +--------------------------+ 1 row in set (0.01 sec) -mysql> SHOW BINDINGS\G; +mysql> SHOW BINDINGS\G *************************** 1. row *************************** Original_sql: select * from `test` . `t` where `a` = ? Bind_sql: SELECT /*+ use_index(@`sel_1` `test`.`t` ) ignore_index(`t` `a`)*/ * FROM `test`.`t` WHERE `a` = 1 @@ -187,7 +187,7 @@ No query specified mysql> DROP BINDING FOR SQL DIGEST '6909a1bbce5f64ade0a532d7058dd77b6ad5d5068aee22a531304280de48349f'; Query OK, 0 rows affected (0.00 sec) -mysql> SHOW BINDINGS\G; +mysql> SHOW BINDINGS\G Empty set (0.01 sec) ERROR: diff --git a/sql-statements/sql-statement-drop-sequence.md b/sql-statements/sql-statement-drop-sequence.md index ca346c168859f..fbc3cdec4f9f3 100644 --- a/sql-statements/sql-statement-drop-sequence.md +++ b/sql-statements/sql-statement-drop-sequence.md @@ -52,4 +52,5 @@ This statement is a TiDB extension. The implementation is modeled on sequences a ## See also * [CREATE SEQUENCE](/sql-statements/sql-statement-create-sequence.md) +* [ALTER SEQUENCE](/sql-statements/sql-statement-alter-sequence.md) * [SHOW CREATE SEQUENCE](/sql-statements/sql-statement-show-create-sequence.md) diff --git a/sql-statements/sql-statement-drop-stats.md b/sql-statements/sql-statement-drop-stats.md index 86ddcfd01bf93..8aeed832f72e6 100644 --- a/sql-statements/sql-statement-drop-stats.md +++ b/sql-statements/sql-statement-drop-stats.md @@ -12,34 +12,59 @@ The `DROP STATS` statement is used to delete the statistics of the selected tabl ```ebnf+diagram DropStatsStmt ::= - 'DROP' 'STATS' TableNameList - -TableNameList ::= - TableName ( ',' TableName )* + 'DROP' 'STATS' TableName ("PARTITION" partition | "GLOBAL")? ( ',' TableName )* TableName ::= Identifier ('.' Identifier)? ``` -## Examples +## Usage -{{< copyable "sql" >}} +The following statement deletes all statistics of `TableName`. If a partitioned table is specified, this statement deletes statistics of all partitions in this table as well as [GlobalStats generated in dynamic pruning mode](/statistics.md#collect-statistics-of-partitioned-tables-in-dynamic-pruning-mode). ```sql -CREATE TABLE t(a INT); +DROP STATS TableName +``` + ``` +Query OK, 0 rows affected (0.00 sec) +``` + +The following statement only deletes statistics of the specified partitions in `PartitionNameList`. ```sql -Query OK, 0 rows affected (0.01 sec) +DROP STATS TableName PARTITION PartitionNameList; ``` -{{< copyable "sql" >}} +``` +Query OK, 0 rows affected (0.00 sec) +``` + +The following statement only deletes GlobalStats generated in dynamic pruning mode of the specified table. ```sql -SHOW STATS_META WHERE db_name='test' and table_name='t'; +DROP STATS TableName GLOBAL; +``` + +``` +Query OK, 0 rows affected (0.00 sec) +``` + +## Examples + +```sql +CREATE TABLE t(a INT); +``` + +``` +Query OK, 0 rows affected (0.01 sec) ``` ```sql +SHOW STATS_META WHERE db_name='test' and table_name='t'; +``` + +``` +---------+------------+----------------+---------------------+--------------+-----------+ | Db_name | Table_name | Partition_name | Update_time | Modify_count | Row_count | +---------+------------+----------------+---------------------+--------------+-----------+ @@ -48,23 +73,19 @@ SHOW STATS_META WHERE db_name='test' and table_name='t'; 1 row in set (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql DROP STATS t; ``` -```sql +``` Query OK, 0 rows affected (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql SHOW STATS_META WHERE db_name='test' and table_name='t'; ``` -```sql +``` Empty set (0.00 sec) ``` diff --git a/sql-statements/sql-statement-explain-analyze.md b/sql-statements/sql-statement-explain-analyze.md index 462b0c8d84ec9..fc455434d9c09 100644 --- a/sql-statements/sql-statement-explain-analyze.md +++ b/sql-statements/sql-statement-explain-analyze.md @@ -34,7 +34,7 @@ ExplainableStmt ::= ## EXPLAIN ANALYZE output format -Different from `EXPLAIN`, `EXPLAIN ANALYZE` executes the corresponding SQL statement, records its runtime information, and returns the information together with the execution plan. Therefore, you can regard `EXPLAIN ANALYZE` as an extension of the `EXPLAIN` statement. Compared to `EXPLAIN` (for debugging query exeuction), the return results of `EXPLAIN ANALYZE` also include columns of information such as `actRows`, `execution info`, `memory`, and `disk`. The details of these columns are shown as follows: +Different from `EXPLAIN`, `EXPLAIN ANALYZE` executes the corresponding SQL statement, records its runtime information, and returns the information together with the execution plan. Therefore, you can regard `EXPLAIN ANALYZE` as an extension of the `EXPLAIN` statement. Compared to `EXPLAIN` (for debugging query execution), the return results of `EXPLAIN ANALYZE` also include columns of information such as `actRows`, `execution info`, `memory`, and `disk`. The details of these columns are shown as follows: | attribute name | description | |:----------------|:---------------------------------| diff --git a/sql-statements/sql-statement-explain.md b/sql-statements/sql-statement-explain.md index e99e5b1d12598..5a872c6ffc28a 100644 --- a/sql-statements/sql-statement-explain.md +++ b/sql-statements/sql-statement-explain.md @@ -336,7 +336,7 @@ In the output, `id`, `estRows`, `taskType`, `accessObject`, and `operatorInfo` h ## MySQL compatibility -* Both the format of `EXPLAIN` and the potential execution plans in TiDB differ substaintially from MySQL. +* Both the format of `EXPLAIN` and the potential execution plans in TiDB differ substantially from MySQL. * TiDB does not support the `FORMAT=JSON` or `FORMAT=TREE` options. * `FORMAT=tidb_json` in TiDB is the JSON format output of the default `EXPLAIN` result. The format and fields are different from the `FORMAT=JSON` output in MySQL. diff --git a/sql-statements/sql-statement-import-into.md b/sql-statements/sql-statement-import-into.md index db0fc465d20c5..c5a31416d1273 100644 --- a/sql-statements/sql-statement-import-into.md +++ b/sql-statements/sql-statement-import-into.md @@ -10,15 +10,19 @@ The `IMPORT INTO` statement lets you import data to TiDB via the [Physical Impor - `IMPORT INTO ... FROM FILE`: imports data files in formats such as `CSV`, `SQL`, and `PARQUET` into an empty table in TiDB. - `IMPORT INTO ... FROM SELECT`: imports the query result of a `SELECT` statement into an empty table in TiDB. You can also use it to import historical data queried with [`AS OF TIMESTAMP`](/as-of-timestamp.md). -> **Warning:** + + +> **Note:** > -> Currently, `IMPORT INTO ... FROM SELECT` is experimental. It is not recommended that you use it in production environments. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. +> Compared with [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), `IMPORT INTO` can be directly executed on TiDB nodes, supports automated distributed task scheduling and [TiDB Global Sort](/tidb-global-sort.md), and offers significant improvements in deployment, resource utilization, task configuration convenience, ease of invocation and integration, high availability, and scalability. It is recommended that you consider using `IMPORT INTO` instead of TiDB Lightning in appropriate scenarios. + + ## Restrictions - `IMPORT INTO` only supports importing data into existing empty tables in the database. - `IMPORT INTO` does not support transactions or rollback. Executing `IMPORT INTO` within an explicit transaction (`BEGIN`/`END`) will return an error. -- `IMPORT INTO` does not support working simultaneously with features such as [Backup & Restore](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview), [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md), [acceleration of adding indexes](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630), data import using TiDB Lightning, data replication using TiCDC, or [Point-in-Time Recovery (PITR)](https://docs.pingcap.com/tidb/stable/br-log-architecture). +- `IMPORT INTO` does not support working simultaneously with features such as [Backup & Restore](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview), [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md), [acceleration of adding indexes](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630), data import using TiDB Lightning, data replication using TiCDC, or [Point-in-Time Recovery (PITR)](https://docs.pingcap.com/tidb/stable/br-log-architecture). For more compatibility information, see [Compatibility of TiDB Lightning and `IMPORT INTO` with TiCDC and Log Backup](https://docs.pingcap.com/tidb/stable/tidb-lightning-compatibility-and-scenarios). - During the data import process, do not perform DDL or DML operations on the target table, and do not execute [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) for the target database. These operations can lead to import failures or data inconsistencies. In addition, it is **NOT** recommended to perform read operations during the import process, as the data being read might be inconsistent. Perform read and write operations only after the import is completed. - The import process consumes system resources significantly. For TiDB Self-Hosted, to get better performance, it is recommended to use TiDB nodes with at least 32 cores and 64 GiB of memory. TiDB writes sorted data to the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) during import, so it is recommended to configure high-performance storage media for TiDB Self-Hosted, such as flash memory. For more information, see [Physical Import Mode limitations](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode#requirements-and-restrictions). - For TiDB Self-Hosted, the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) is expected to have at least 90 GiB of available space. It is recommended to allocate storage space that is equal to or greater than the volume of data to be imported. @@ -142,7 +146,7 @@ The supported options are described as follows: | `FIELDS_DEFINED_NULL_BY=''` | CSV | Specifies the value that represents `NULL` in the fields. The default value is `\N`. | | `LINES_TERMINATED_BY=''` | CSV | Specifies the line terminator. By default, `IMPORT INTO` automatically identifies `\n`, `\r`, or `\r\n` as line terminators. If the line terminator is one of these three, you do not need to explicitly specify this option. | | `SKIP_ROWS=` | CSV | Specifies the number of rows to skip. The default value is `0`. You can use this option to skip the header in a CSV file. If you use a wildcard to specify the source files for import, this option applies to all source files that are matched by the wildcard in `fileLocation`. | -| `SPLIT_FILE` | CSV | Splits a single CSV file into multiple smaller chunks of around 256 MiB for parallel processing to improve import efficiency. This parameter only works for **non-compressed** CSV files and has the same usage restrictions as that of TiDB Lightning [`strict-format`](https://docs.pingcap.com/tidb/stable/tidb-lightning-data-source#strict-format). | +| `SPLIT_FILE` | CSV | Splits a single CSV file into multiple smaller chunks of around 256 MiB for parallel processing to improve import efficiency. This parameter only works for **non-compressed** CSV files and has the same usage restrictions as that of TiDB Lightning [`strict-format`](https://docs.pingcap.com/tidb/stable/tidb-lightning-data-source#strict-format). Note that you need to explicitly specify `LINES_TERMINATED_BY` for this option. | | `DISK_QUOTA=''` | All file formats | Specifies the disk space threshold that can be used during data sorting. The default value is 80% of the disk space in the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630). If the total disk size cannot be obtained, the default value is 50 GiB. When specifying `DISK_QUOTA` explicitly, make sure that the value does not exceed 80% of the disk space in the TiDB temporary directory. | | `DISABLE_TIKV_IMPORT_MODE` | All file formats | Specifies whether to disable switching TiKV to import mode during the import process. By default, switching TiKV to import mode is not disabled. If there are ongoing read-write operations in the cluster, you can enable this option to avoid impact from the import process. | | `THREAD=` | All file formats and query results of `SELECT` | Specifies the concurrency for import. For `IMPORT INTO ... FROM FILE`, the default value of `THREAD` is 50% of the number of CPU cores on the TiDB node, the minimum value is `1`, and the maximum value is the number of CPU cores. For `IMPORT INTO ... FROM SELECT`, the default value of `THREAD` is `2`, the minimum value is `1`, and the maximum value is two times the number of CPU cores on the TiDB node. To import data into a new cluster without any data, it is recommended to increase this concurrency appropriately to improve import performance. If the target cluster is already used in a production environment, it is recommended to adjust this concurrency according to your application requirements. | @@ -150,7 +154,7 @@ The supported options are described as follows: | `CHECKSUM_TABLE=''` | All file formats | Configures whether to perform a checksum check on the target table after the import to validate the import integrity. The supported values include `"required"` (default), `"optional"`, and `"off"`. `"required"` means performing a checksum check after the import. If the checksum check fails, TiDB will return an error and the import will exit. `"optional"` means performing a checksum check after the import. If an error occurs, TiDB will return a warning and ignore the error. `"off"` means not performing a checksum check after the import. | | `DETACHED` | All file formats | Controls whether to execute `IMPORT INTO` asynchronously. When this option is enabled, executing `IMPORT INTO` immediately returns the information of the import job (such as the `Job_ID`), and the job is executed asynchronously in the backend. | | `CLOUD_STORAGE_URI` | All file formats | Specifies the target address where encoded KV data for [Global Sort](/tidb-global-sort.md) is stored. When `CLOUD_STORAGE_URI` is not specified, `IMPORT INTO` determines whether to use Global Sort based on the value of the system variable [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740). If this system variable specifies a target storage address, `IMPORT INTO` uses this address for Global Sort. When `CLOUD_STORAGE_URI` is specified with a non-empty value, `IMPORT INTO` uses that value as the target storage address. When `CLOUD_STORAGE_URI` is specified with an empty value, local sorting is enforced. Currently, the target storage address only supports S3. For details about the URI configuration, see [Amazon S3 URI format](/external-storage-uri.md#amazon-s3-uri-format). When this feature is used, all TiDB nodes must have read and write access for the target S3 bucket, including at least these permissions: `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, `s3:PutObject`, `s3: AbortMultipartUpload`. | -| `DISABLE_PRECHECK` | All file formats and query results of `SELECT` | Setting this option disables pre-checks of non-critical itemes, such as checking whether there are CDC or PITR tasks. | +| `DISABLE_PRECHECK` | All file formats and query results of `SELECT` | Setting this option disables pre-checks of non-critical items, such as checking whether there are CDC or PITR tasks. | ## `IMPORT INTO ... FROM FILE` usage @@ -356,3 +360,4 @@ This statement is a TiDB extension to MySQL syntax. * [`SHOW IMPORT JOB(s)`](/sql-statements/sql-statement-show-import-job.md) * [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) +* [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) diff --git a/sql-statements/sql-statement-modify-column.md b/sql-statements/sql-statement-modify-column.md index 12748bfe00de6..55bb096c52065 100644 --- a/sql-statements/sql-statement-modify-column.md +++ b/sql-statements/sql-statement-modify-column.md @@ -88,7 +88,7 @@ Query OK, 0 rows affected (0.09 sec) {{< copyable "sql" >}} ```sql -SHOW CREATE TABLE t1\G; +SHOW CREATE TABLE t1\G ``` ```sql @@ -138,7 +138,7 @@ Query OK, 0 rows affected (2.52 sec) {{< copyable "sql" >}} ```sql -SHOW CREATE TABLE t1\G; +SHOW CREATE TABLE t1\G ``` ```sql diff --git a/sql-statements/sql-statement-overview.md b/sql-statements/sql-statement-overview.md new file mode 100644 index 0000000000000..40b5b6112bfc5 --- /dev/null +++ b/sql-statements/sql-statement-overview.md @@ -0,0 +1,335 @@ +--- +title: SQL Statement Overview +summary: Learn about supported SQL statements in TiDB. +--- + +# SQL Statement Overview + +TiDB uses SQL statements that aim to follow ISO/IEC SQL standards, with extensions for MySQL and TiDB-specific statements where necessary. + +## Schema management / Data definition statements (DDL) + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) | Modifies a database. | +| [`ALTER SEQUENCE`](/sql-statements/sql-statement-alter-sequence.md) | Modifies a sequence. | +| [`ALTER TABLE ... ADD COLUMN`](/sql-statements/sql-statement-add-column.md) | Adds a column to an existing table. | +| [`ALTER TABLE ... ADD INDEX`](/sql-statements/sql-statement-add-index.md) | Adds an index to an existing table. | +| [`ALTER TABLE ... ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) | Changes an index definition. | +| [`ALTER TABLE ... CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) | Changes a column definition. | +| [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) | Compacts a table. | +| [`ALTER TABLE ... DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) | Drops a column from a table. | +| [`ALTER TABLE ... MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) | Modifies a column definition. | +| [`ALTER TABLE ... RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) | Renames an index. | +| [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) | Changes a table definition. | +| [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) | Creates a new database. | +| [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md) | Creates a new index on a table. | +| [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) | Creates a new sequence object. | +| [`CREATE TABLE LIKE`](/sql-statements/sql-statement-create-table-like.md) | Copies the definition of an existing table, without copying any data. | +| [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) | Creates a new table. | +| [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) | Creates a new view. | +| [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) | Drops an existing database. | +| [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) | Drops an index from a table. | +| [`DROP SEQUENCE`](/sql-statements/sql-statement-drop-sequence.md) | Drops a sequence object. | +| [`DROP TABLE`](/sql-statements/sql-statement-drop-table.md) | Drops an existing table. | +| [`DROP VIEW`](/sql-statements/sql-statement-drop-view.md) | Drops an existing view. | +| [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) | Renames a table. | +| [`SHOW COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) | Shows the columns from a table. | +| [`SHOW CREATE DATABASE`](/sql-statements/sql-statement-show-create-database.md) | Shows the CREATE statement for a database. | +| [`SHOW CREATE SEQUENCE`](/sql-statements/sql-statement-show-create-sequence.md) | Shows the CREATE statement for a sequence. | +| [`SHOW CREATE TABLE`](/sql-statements/sql-statement-show-create-table.md) | Shows the CREATE statement for a table. | +| [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) | Shows a list of databases that the current user has privileges to. | +| [`SHOW FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) | Shows columns of a table. | +| [`SHOW INDEXES`](/sql-statements/sql-statement-show-indexes.md) | Shows indexes of a table. | +| [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) | An alias to `SHOW DATABASES`, which shows a list of databases that the current user has privileges to. | +| [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) | Shows the next row ID for a table. | +| [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) | Shows the Region information of a table in TiDB. | +| [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) | Shows various statistics about tables in TiDB. | +| [`SHOW TABLES`](/sql-statements/sql-statement-show-tables.md) | Shows tables in a database. | +| [`TRUNCATE`](/sql-statements/sql-statement-truncate.md) | Truncates all data from a table. | + +## Data manipulation statements (DML) + +| SQL Statement | Description | +|---------------|-------------| +| [`BATCH`](/sql-statements/sql-statement-batch.md) | Splits a DML statement into multiple statements in TiDB for execution. | +| [`DELETE`](/sql-statements/sql-statement-delete.md) | Deletes rows from a table. | +| [`INSERT`](/sql-statements/sql-statement-insert.md) | Inserts new rows into a table. | +| [`REPLACE`](/sql-statements/sql-statement-replace.md) | Replaces existing rows or inserts new rows. | +| [`SELECT`](/sql-statements/sql-statement-select.md) | Reads data from a table. | +| [`TABLE`](/sql-statements/sql-statement-table.md) | Retrieves rows from a table. | +| [`UPDATE`](/sql-statements/sql-statement-update.md) | Updates existing rows in a table. | +| [`WITH`](/sql-statements/sql-statement-with.md) | Defines common table expressions. | + +## Transaction statements + +| SQL Statement | Description | +|---------------|-------------| +| [`BEGIN`](/sql-statements/sql-statement-begin.md) | Begins a new transaction. | +| [`COMMIT`](/sql-statements/sql-statement-commit.md) | Commits the current transaction. | +| [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) | Rolls back the current transaction. | +| [`SAVEPOINT`](/sql-statements/sql-statement-savepoint.md) | Sets a savepoint within a transaction. | +| [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) | Changes the current isolation level on a `GLOBAL` or `SESSION` basis. | +| [`START TRANSACTION`](/sql-statements/sql-statement-start-transaction.md) | Starts a new transaction. | + +## Prepared statements + +| SQL Statement | Description | +|---------------|-------------| +| [`DEALLOCATE`](/sql-statements/sql-statement-deallocate.md) | Deallocates a prepared statement, freeing associated resources. | +| [`EXECUTE`](/sql-statements/sql-statement-execute.md) | Executes a prepared statement with specific parameter values. | +| [`PREPARE`](/sql-statements/sql-statement-prepare.md) | Creates a prepared statement with placeholders. | + +## Administrative statements + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) | Cancels a DDL job. | +| [`ADMIN CHECK [TABLE\|INDEX]`](/sql-statements/sql-statement-admin-check-table-index.md) | Checks the integrity of a table or index. | +| [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) | Computes the checksum of a table. | +| [`ADMIN CLEANUP INDEX`](/sql-statements/sql-statement-admin-cleanup.md) | Cleans up indexes from a table. | +| [`ADMIN PAUSE DDL`](/sql-statements/sql-statement-admin-pause-ddl.md) | Pauses DDL operations. | +| [`ADMIN RESUME DDL`](/sql-statements/sql-statement-admin-resume-ddl.md) | Resumes DDL operations. | +| [`ADMIN SHOW DDL [JOBS\|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) | Shows DDL jobs or job queries. | +| [`ADMIN SHOW TELEMETRY`](/sql-statements/sql-statement-admin-show-telemetry.md) | Shows telemetry data. | +| [`ADMIN`](/sql-statements/sql-statement-admin.md) | Performs various administrative tasks. | +| [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) | Included for [MySQL compatibility](/mysql-compatibility.md). It has no effective usage in TiDB. | +| [`SET `](/sql-statements/sql-statement-set-variable.md) | Modifies a system variable or user variable. | +| [`SET [NAMES\|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) | Set a character set and collation. | +| [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) | Splits a Region into smaller Regions. | + + + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) | Cancels a DDL job. | +| [`ADMIN CHECK [TABLE\|INDEX]`](/sql-statements/sql-statement-admin-check-table-index.md) | Checks the integrity of a table or index. | +| [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) | Computes the checksum of a table. | +| [`ADMIN CLEANUP INDEX`](/sql-statements/sql-statement-admin-cleanup.md) | Cleans up indexes from a table. | +| [`ADMIN PAUSE DDL`](/sql-statements/sql-statement-admin-pause-ddl.md) | Pauses DDL operations. | +| [`ADMIN RECOVER INDEX`](/sql-statements/sql-statement-admin-recover.md) | Recovers the consistency based on the redundant indexes. | +| [`ADMIN RESUME DDL`](/sql-statements/sql-statement-admin-resume-ddl.md) | Resumes DDL operations. | +| [`ADMIN SHOW DDL [JOBS\|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) | Shows DDL jobs or job queries. | +| [`ADMIN`](/sql-statements/sql-statement-admin.md) | Performs various administrative tasks. | +| [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) | Included for [MySQL compatibility](/mysql-compatibility.md). It has no effective usage in TiDB. | +| [`SET `](/sql-statements/sql-statement-set-variable.md) | Modifies a system variable or user variable. | +| [`SET [NAMES\|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) | Set a character set and collation. | +| [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) | Splits a Region into smaller Regions. | + + + +## Data import and export + +| SQL Statement | Description | +|---------------|-------------| +| [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) | Cancels an ongoing import job. | +| [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) | Imports data into a table via the [Physical Import Mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode) of TiDB Lightning. | +| [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) | Loads data into a table from Amazon S3 or Google Cloud Storage. | +| [`SHOW IMPORT JOB`](/sql-statements/sql-statement-show-import-job.md) | Shows the status of an import job. | + +## Backup & restore + +| SQL Statement | Description | +|---------------|-------------| +| [`BACKUP`](/sql-statements/sql-statement-backup.md) | Performs a distributed backup of the TiDB cluster. | +| [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) | Restores the cluster to a specific snapshot. | +| [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) | Restores a database and its data deleted by the `DROP` statement. | +| [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) | Restore the tables and data dropped by the `DROP` or `TRUNCATE` operation. | +| [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) | Recovers a deleted table and the data on it. | +| [`RESTORE`](/sql-statements/sql-statement-restore.md) | Restores a database from a backup. | +| [`SHOW BACKUPS`](/sql-statements/sql-statement-show-backups.md) | Shows backup tasks. | +| [`SHOW RESTORES`](/sql-statements/sql-statement-show-backups.md) | Shows restore tasks. | + +## Placement policy + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md) | Modifies a placement policy. | +| [`ALTER RANGE`](/sql-statements/sql-statement-alter-range.md) | Modifies the range of a placement policy. | +| [`CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-create-placement-policy.md) | Creates a new placement policy. | +| [`DROP PLACEMENT POLICY`](/sql-statements/sql-statement-drop-placement-policy.md) | Drops an existing placement policy. | +| [`SHOW CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-show-create-placement-policy.md) | Shows the `CREATE` statement for a placement policy. | +| [`SHOW PLACEMENT FOR`](/sql-statements/sql-statement-show-placement-for.md) | Shows placement rules for a specific table. | +| [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md) | Shows available placement labels. | +| [`SHOW PLACEMENT`](/sql-statements/sql-statement-show-placement.md) | Shows placement rules. | + +## Resource groups + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) | Modifies a resource group. | +| [`CALIBRATE RESOURCE`](/sql-statements/sql-statement-calibrate-resource.md) | Estimates and outputs the [Request Unit (RU)](/tidb-resource-control.md#what-is-request-unit-ru) capacity of the current cluster. | +| [`CREATE RESOURCE GROUP`](/sql-statements/sql-statement-create-resource-group.md) | Creates a new resource group. | +| [`DROP RESOURCE GROUP`](/sql-statements/sql-statement-drop-resource-group.md) | Drops a resource group. | +| [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) | Manages the runaway query watch list. | +| [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md) | Sets a resource group. | +| [`SHOW CREATE RESOURCE GROUP`](/sql-statements/sql-statement-show-create-resource-group.md) | Shows the `CREATE` statement for a resource group. | + + + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) | Modifies a resource group. | +| [`CREATE RESOURCE GROUP`](/sql-statements/sql-statement-create-resource-group.md) | Creates a new resource group. | +| [`DROP RESOURCE GROUP`](/sql-statements/sql-statement-drop-resource-group.md) | Drops a resource group. | +| [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) | Manages the runaway query watch list. | +| [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md) | Sets a resource group. | +| [`SHOW CREATE RESOURCE GROUP`](/sql-statements/sql-statement-show-create-resource-group.md) | Shows the `CREATE` statement for a resource group. | + + + +## Utility statements + +| SQL Statement | Description | +|---------------|-------------| +| [`DESC`](/sql-statements/sql-statement-desc.md) | An alias to `DESCRIBE`, which shows the structure of a table. | +| [`DESCRIBE`](/sql-statements/sql-statement-describe.md) | Shows the structure of a table. | +| [`DO`](/sql-statements/sql-statement-do.md) | Executes an expression but does not return any results. | +| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Shows the execution plan of a query. | +| [`TRACE`](/sql-statements/sql-statement-trace.md) | Provides detailed information about query execution. | +| [`USE`](/sql-statements/sql-statement-use.md) | Sets the current database. | + +## Show statements + + + +| SQL Statement | Description | +|---------------|-------------| +| [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) | Lists builtin functions. | +| [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) | Lists character sets. | +| [`SHOW COLLATIONS`](/sql-statements/sql-statement-show-collation.md) | Lists collations. | +| [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) | Shows errors from previously executed statements. | +| [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) | Included for [compatibility with MySQL](/mysql-compatibility.md). TiDB uses [Prometheus and Grafana](/tidb-monitoring-framework.md) for centralized metrics collection instead of `SHOW STATUS` for most metrics. | +| [`SHOW VARIABLES`](/sql-statements/sql-statement-show-variables.md) | Shows system variables. | +| [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) | Shows warnings and notes from previously executed statements. | + + + + + +| SQL Statement | Description | +|---------------|-------------| +| [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) | Lists builtin functions. | +| [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) | Lists character sets. | +| [`SHOW COLLATIONS`](/sql-statements/sql-statement-show-collation.md) | Lists collations. | +| [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) | Shows errors from previously executed statements. | +| [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) | Included for [compatibility with MySQL](/mysql-compatibility.md). TiDB Cloud provides [Monitoring](/tidb-cloud/monitor-tidb-cluster.md) for centralized metrics collection instead of `SHOW STATUS` for most metrics. | +| [`SHOW VARIABLES`](/sql-statements/sql-statement-show-variables.md) | Shows system variables. | +| [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) | Shows warnings and notes from previously executed statements. | + + + +## Instance management + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) | Modifies an instance. | +| [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) | Included for [compatibility with MySQL](/mysql-compatibility.md). TiDB uses [Prometheus and Grafana](/tidb-monitoring-framework.md) for centralized metrics collection instead of `SHOW STATUS` for most metrics. | +| [`KILL`](/sql-statements/sql-statement-kill.md) | Kills a connection in any TiDB instance in the current TiDB cluster. | +| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Shows the configuration of various components of TiDB. | +| [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) | Shows available storage engines. | +| [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) | Shows installed plugins. | +| [`SHOW PROCESSLIST`](/sql-statements/sql-statement-show-processlist.md) | Shows the current sessions connected to the same TiDB server. | +| [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) | Shows query profiles. Included for [compatibility with MySQL](/mysql-compatibility.md). Currently, it only returns an empty result. | +| [`SHUTDOWN`](/sql-statements/sql-statement-shutdown.md) | Stops the client-connected TiDB instance, not the entire TiDB cluster. | + + + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) | Modifies an instance. | +| [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) | Included for [compatibility with MySQL](/mysql-compatibility.md). TiDB Cloud provides [Monitoring](/tidb-cloud/monitor-tidb-cluster.md) for centralized metrics collection instead of `SHOW STATUS` for most metrics. | +| [`KILL`](/sql-statements/sql-statement-kill.md) | Kills a connection in any TiDB instance in the current TiDB cluster. | +| [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) | Shows available storage engines. | +| [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) | Shows installed plugins. | +| [`SHOW PROCESSLIST`](/sql-statements/sql-statement-show-processlist.md) | Shows the current sessions connected to the same TiDB server. | +| [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) | Shows query profiles. Included for [compatibility with MySQL](/mysql-compatibility.md). Currently only returns an empty result. | + + + +## Locking statements + +| SQL Statement | Description | +|---------------|-------------| +| [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md) | Locks statistics of tables or partitions. | +| [`LOCK TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | Locks tables for the current session. | +| [`UNLOCK STATS`](/sql-statements/sql-statement-unlock-stats.md) | Unlocks statistics of tables or partitions. | +| [`UNLOCK TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | Unlocks tables. | + +## Account management / Data Control Language + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) | Modifies a user. | +| [`CREATE ROLE`](/sql-statements/sql-statement-create-role.md) | Creates a role. | +| [`CREATE USER`](/sql-statements/sql-statement-create-user.md) | Creates a new user. | +| [`DROP ROLE`](/sql-statements/sql-statement-drop-role.md) | Drops an existing role. | +| [`DROP USER`](/sql-statements/sql-statement-drop-user.md) | Drops an existing user. | +| [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) | Reloads the in-memory copy of privileges from the privilege tables. | +| [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) | Grants privileges. | +| [`GRANT `](/sql-statements/sql-statement-grant-role.md) | Grants a role. | +| [`RENAME USER`](/sql-statements/sql-statement-rename-user.md) | Renames an existing user. | +| [`REVOKE `](/sql-statements/sql-statement-revoke-privileges.md) | Revokes privileges. | +| [`REVOKE `](/sql-statements/sql-statement-revoke-role.md) | Revokes a role. | +| [`SET DEFAULT ROLE`](/sql-statements/sql-statement-set-default-role.md) | Sets a default role. | +| [`SET PASSWORD`](/sql-statements/sql-statement-set-password.md) | Changes a password. | +| [`SET ROLE`](/sql-statements/sql-statement-set-role.md) | Enables roles in the current session. | +| [`SHOW CREATE USER`](/sql-statements/sql-statement-show-create-user.md) | Shows the `CREATE` statement for a user. | +| [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) | Shows privileges associated with a user. | +| [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) | Shows available privileges. | + +## TiCDC & TiDB Binlog + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ADMIN [SET\|SHOW\|UNSET] BDR ROLE`](/sql-statements/sql-statement-admin-bdr-role.md) | Manages BDR roles. | +| [`CHANGE DRAINER`](/sql-statements/sql-statement-change-drainer.md) | Modifies the status information for Drainer in the cluster. | +| [`CHANGE PUMP`](/sql-statements/sql-statement-change-pump.md) | Modifies the status information for Pump in the cluster. | +| [`SHOW DRAINER STATUS`](/sql-statements/sql-statement-show-drainer-status.md) | Shows the status for all Drainer nodes in the cluster. | +| [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) | Shows the latest TSO in the cluster. | +| [`SHOW PUMP STATUS`](/sql-statements/sql-statement-show-pump-status.md) | Shows the status information for all Pump nodes in the cluster. | + + + + + +> **Note:** +> +> [TiCDC](https://docs.pingcap.com/tidb/stable/ticdc-overview) & [TiDB Binlog](https://docs.pingcap.com/tidb/stable/tidb-binlog-overview) are tools for replicating TiDB data to the upstream for TiDB Self-Hosted. Most SQL statements for TiCDC and TiDB Binlog are not applicable to TiDB Cloud. For TiDB Cloud, you can use the [Changefeed](/tidb-cloud/changefeed-overview.md) feature in the [TiDB Cloud console](https://tidbcloud.com) instead to stream data. + +| SQL Statement | Description | +|---------------|-------------| +| [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) | Shows the latest TSO in the cluster. | + + + +## Statistics and plan management + +| SQL Statement | Description | +|---------------|-------------| +| [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) | Collects statistics about a table. | +| [`CREATE BINDING`](/sql-statements/sql-statement-create-binding.md) | Creates an execution plan binding for a SQL statement. | +| [`DROP BINDING`](/sql-statements/sql-statement-drop-binding.md) | Drops an execution plan binding from a SQL statement. | +| [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) | Drops statistics from a table. | +| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Works similar to `EXPLAIN`, with the major difference that it will execute the statement. | +| [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) | Loads statistics into TiDB. | +| [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md) | Shows statistics collection tasks. | +| [`SHOW BINDINGS`](/sql-statements/sql-statement-show-bindings.md) | Shows created SQL bindings. | +| [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) | Shows an estimation of how accurate statistics are believed to be. | +| [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-stats-histograms.md) | Shows the histogram information in statistics. | +| [`SHOW STATS_LOCKED`](/sql-statements/sql-statement-show-stats-locked.md) | Shows the tables whose statistics are locked. | +| [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) | Shows how many rows are in a table and how many rows are changed in that table. | diff --git a/sql-statements/sql-statement-prepare.md b/sql-statements/sql-statement-prepare.md index d13088ae7907e..7e45973d7bfc6 100644 --- a/sql-statements/sql-statement-prepare.md +++ b/sql-statements/sql-statement-prepare.md @@ -19,6 +19,12 @@ PrepareSQL ::= | UserVariable ``` +> **Note:** +> +> For each `PREPARE` statement, the maximum number of placeholders is 65535. + +To limit the number of `PREPARE` statements in the current TiDB instance, you can use the [`max_prepared_stmt_count`](/system-variables.md#max_prepared_stmt_count) system variable. + ## Examples ```sql diff --git a/sql-statements/sql-statement-query-watch.md b/sql-statements/sql-statement-query-watch.md index 8390a957f6d06..be7d49b00fc58 100644 --- a/sql-statements/sql-statement-query-watch.md +++ b/sql-statements/sql-statement-query-watch.md @@ -7,10 +7,6 @@ summary: An overview of the usage of QUERY WATCH for the TiDB database. The `QUERY WATCH` statement is used to manually manage the watch list of runaway queries in a resource group. -> **Warning:** -> -> This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. - > **Note:** > > This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. diff --git a/sql-statements/sql-statement-restore.md b/sql-statements/sql-statement-restore.md index 6a64b9187ff68..76562c7f8914e 100644 --- a/sql-statements/sql-statement-restore.md +++ b/sql-statements/sql-statement-restore.md @@ -129,9 +129,7 @@ RESTORE DATABASE * FROM 's3://example-bucket-2020/backup-05/' Use `RATE_LIMIT` to limit the average download speed per TiKV node to reduce network bandwidth. -By default, TiDB node would run 128 restore threads. This value can be adjusted with the `CONCURRENCY` option. - -Before restore is completed, `RESTORE` would perform a checksum against the data from the archive to verify correctness. This step can be disabled with the `CHECKSUM` option if you are confident that this is unnecessary. +Before the restore is completed, `RESTORE` would perform a checksum against the data in the backup files to verify correctness. If you are confident that this verification is unnecessary, you can disable the check by setting the `CHECKSUM` parameter to `FALSE`. {{< copyable "sql" >}} diff --git a/sql-statements/sql-statement-savepoint.md b/sql-statements/sql-statement-savepoint.md index 055eb6661499e..735ef8ccf352b 100644 --- a/sql-statements/sql-statement-savepoint.md +++ b/sql-statements/sql-statement-savepoint.md @@ -36,6 +36,19 @@ RELEASE SAVEPOINT identifier After the transaction is committed or rolled back, all savepoints in the transaction will be deleted. +## Synopsis + +```ebnf+diagram +SavepointStmt ::= + "SAVEPOINT" Identifier + +RollbackToStmt ::= + "ROLLBACK" "TO" "SAVEPOINT"? Identifier + +ReleaseSavepointStmt ::= + "RELEASE" "SAVEPOINT" Identifier +``` + ## Examples Create a table `t1`: @@ -141,6 +154,8 @@ SELECT * FROM t1; When `ROLLBACK TO SAVEPOINT` is used to roll back a transaction to a specified savepoint, MySQL releases the locks held only after the specified savepoint, while in TiDB pessimistic transaction, TiDB does not immediately release the locks held after the specified savepoint. Instead, TiDB releases all locks when the transaction is committed or rolled back. +TiDB does not support the MySQL syntax `ROLLBACK WORK TO SAVEPOINT ...`. + ## See also * [COMMIT](/sql-statements/sql-statement-commit.md) diff --git a/sql-statements/sql-statement-select.md b/sql-statements/sql-statement-select.md index 8077af19dc5ae..1e101c27e9a3a 100644 --- a/sql-statements/sql-statement-select.md +++ b/sql-statements/sql-statement-select.md @@ -10,84 +10,48 @@ The `SELECT` statement is used to read data from TiDB. ## Synopsis -**SelectStmt:** - -![SelectStmt](/media/sqlgram/SelectStmt.png) - -**FromDual:** - -![FromDual](/media/sqlgram/FromDual.png) - -**SelectStmtOpts:** +```ebnf+diagram +SelectStmt ::= + ( SelectStmtBasic | SelectStmtFromDualTable | SelectStmtFromTable ) + OrderBy? SelectStmtLimit? SelectLockOpt? SelectStmtIntoOption -![SelectStmtOpts](/media/sqlgram/SelectStmtOpts.png) +SelectStmtBasic ::= + "SELECT" SelectStmtOpts Field ("," Field)* ( "HAVING" Expression)? -**SelectStmtFieldList:** +SelectStmtFromDualTable ::= + "SELECT" SelectStmtOpts Field ("," Field)* "FROM" "DUAL" WhereClause? -![SelectStmtFieldList](/media/sqlgram/SelectStmtFieldList.png) +SelectStmtFromTable ::= + "SELECT" SelectStmtOpts Field ("," Field)* "FROM" TableRefsClause + WhereClause? GroupByClause? ( "HAVING" Expression)? WindowClause? -**TableRefsClause:** +SelectStmtOpts ::= + TableOptimizerHints DefaultFalseDistictOpt PriorityOpt SelectStmtSQLSmallResult + SelectStmtSQLBigResult SelectStmtSQLBufferResult SelectStmtSQLCache SelectStmtCalcFoundRows + SelectStmtStraightJoin -```ebnf+diagram TableRefsClause ::= TableRef AsOfClause? ( ',' TableRef AsOfClause? )* AsOfClause ::= 'AS' 'OF' 'TIMESTAMP' Expression -``` - -**WhereClauseOptional:** - -![WhereClauseOptional](/media/sqlgram/WhereClauseOptional.png) - -**SelectStmtGroup:** - -![SelectStmtGroup](/media/sqlgram/SelectStmtGroup.png) - -**HavingClause:** - -![HavingClause](/media/sqlgram/HavingClause.png) - -**OrderByOptional:** -![OrderByOptional](/media/sqlgram/OrderByOptional.png) +SelectStmtLimit ::= + ("LIMIT" LimitOption ( ("," | "OFFSET") LimitOption )? +| "FETCH" ("FIRST" | "NEXT") LimitOption? ("ROW" | "ROWS") "ONLY" ) -**SelectStmtLimit:** - -![SelectStmtLimit](/media/sqlgram/SelectStmtLimit.png) - -**FirstOrNext:** - -![FirstOrNext](/media/sqlgram/FirstOrNext.png) - -**FetchFirstOpt:** - -![FetchFirstOpt](/media/sqlgram/FetchFirstOpt.png) - -**RowOrRows:** - -![RowOrRows](/media/sqlgram/RowOrRows.png) - -**SelectLockOpt:** - -```ebnf+diagram SelectLockOpt ::= - ( ( 'FOR' 'UPDATE' ( 'OF' TableList )? 'NOWAIT'? ) -| ( 'LOCK' 'IN' 'SHARE' 'MODE' ) )? + ( 'FOR' 'UPDATE' ( 'OF' TableList )? 'NOWAIT'? +| 'LOCK' 'IN' 'SHARE' 'MODE' ) TableList ::= TableName ( ',' TableName )* -``` -**WindowClauseOptional** +WindowClause ::= + "WINDOW" WindowDefinition ("," WindowDefinition)* -![WindowClauseOptional](/media/sqlgram/WindowClauseOptional.png) - -**TableSampleOpt** - -```ebnf+diagram TableSampleOpt ::= - 'TABLESAMPLE' 'REGIONS()' + 'TABLESAMPLE' 'REGIONS' '(' ')' ``` ## Description of the syntax elements diff --git a/sql-statements/sql-statement-set-default-role.md b/sql-statements/sql-statement-set-default-role.md index 6686fcc5bf066..aa90b5cdcd0b1 100644 --- a/sql-statements/sql-statement-set-default-role.md +++ b/sql-statements/sql-statement-set-default-role.md @@ -9,21 +9,10 @@ This statement sets a specific role to be applied to a user by default. Thus, th ## Synopsis -**SetDefaultRoleStmt:** - -![SetDefaultRoleStmt](/media/sqlgram/SetDefaultRoleStmt.png) - -**SetDefaultRoleOpt:** - -![SetDefaultRoleOpt](/media/sqlgram/SetDefaultRoleOpt.png) - -**RolenameList:** - -![RolenameList](/media/sqlgram/RolenameList.png) - -**UsernameList:** - -![UsernameList](/media/sqlgram/UsernameList.png) +```ebnf+diagram +SetDefaultRoleStmt ::= + "SET" "DEFAULT" "ROLE" ( "NONE" | "ALL" | Rolename ("," Rolename)* ) "TO" Username ("," Username)* +``` ## Examples diff --git a/sql-statements/sql-statement-set-names.md b/sql-statements/sql-statement-set-names.md index 5c99b307cb9bc..dd76df13ff224 100644 --- a/sql-statements/sql-statement-set-names.md +++ b/sql-statements/sql-statement-set-names.md @@ -10,33 +10,10 @@ The statements `SET NAMES`, `SET CHARACTER SET` and `SET CHARSET` modify the var ## Synopsis -**SetNamesStmt:** - -![SetNamesStmt](/media/sqlgram/SetNamesStmt.png) - -**VariableAssignmentList:** - -![VariableAssignmentList](/media/sqlgram/VariableAssignmentList.png) - -**VariableAssignment:** - -![VariableAssignment](/media/sqlgram/VariableAssignment.png) - -**CharsetName:** - -![CharsetName](/media/sqlgram/CharsetName.png) - -**StringName:** - -![StringName](/media/sqlgram/StringName.png) - -**CharsetKw:** - -![CharsetKw](/media/sqlgram/CharsetKw.png) - -**CharsetNameOrDefault:** - -![CharsetNameOrDefault](/media/sqlgram/CharsetNameOrDefault.png) +```ebnf+diagram +SetNamesStmt ::= + "SET" ("NAMES" ("DEFAULT" | CharsetName ("COLLATE" ("DEFAULT" | CollationName))?) | ("CHARSET" | ("CHAR" | "CHARACTER") "SET") ("DEFAULT" | CharsetName)) +``` ## Examples diff --git a/sql-statements/sql-statement-set-password.md b/sql-statements/sql-statement-set-password.md index 069aff17737e7..2d3ae1e3b6e2e 100644 --- a/sql-statements/sql-statement-set-password.md +++ b/sql-statements/sql-statement-set-password.md @@ -10,7 +10,7 @@ This statement changes the user password for a user account in the TiDB system d ## Synopsis -```enbf+diagram +```ebnf+diagram SetPasswordStmt ::= "SET" "PASSWORD" ( "FOR" Username )? "=" ( stringLit | "PASSWORD" "(" stringLit ")" ) ``` diff --git a/sql-statements/sql-statement-set-role.md b/sql-statements/sql-statement-set-role.md index 703b8e62db0a1..242677164cd0a 100644 --- a/sql-statements/sql-statement-set-role.md +++ b/sql-statements/sql-statement-set-role.md @@ -10,24 +10,15 @@ The `SET ROLE` statement is used to enable roles in the current session. After e ## Synopsis -**SetRoleStmt:** - -![SetRoleStmt](/media/sqlgram/SetRoleStmt.png) - -**SetRoleOpt:** - -![SetRoleOpt](/media/sqlgram/SetRoleOpt.png) - -**SetDefaultRoleOpt:** - -![SetDefaultRoleOpt](/media/sqlgram/SetDefaultRoleOpt.png) +```ebnf+diagram +SetRoleStmt ::= + "SET" "ROLE" ( "DEFAULT" | "ALL" ( "EXCEPT" Rolename ("," Rolename)* )? | "NONE" | Rolename ("," Rolename)* )? +``` ## Examples Create a user `'u1'@'%'` and three roles: `'r1'@'%'`, `'r2'@'%'` and `'r3'@'%'`. Grant these roles to `'u1'@'%'` and set `'r1'@'%'` as the default role of `'u1'@'%'`. -{{< copyable "sql" >}} - ```sql CREATE USER 'u1'@'%'; CREATE ROLE 'r1', 'r2', 'r3'; @@ -37,8 +28,6 @@ SET DEFAULT ROLE 'r1' TO 'u1'@'%'; Log in as `'u1'@'%'` and execute the following `SET ROLE` statement to enable all roles. -{{< copyable "sql" >}} - ```sql SET ROLE ALL; SELECT CURRENT_ROLE(); @@ -55,8 +44,6 @@ SELECT CURRENT_ROLE(); Execute the following `SET ROLE` statement to enable `'r2'` and `'r3'`. -{{< copyable "sql" >}} - ```sql SET ROLE 'r2', 'r3'; SELECT CURRENT_ROLE(); @@ -73,8 +60,6 @@ SELECT CURRENT_ROLE(); Execute the following `SET ROLE` statement to enable the default role(s). -{{< copyable "sql" >}} - ```sql SET ROLE DEFAULT; SELECT CURRENT_ROLE(); @@ -91,8 +76,6 @@ SELECT CURRENT_ROLE(); Execute the following `SET ROLE` statement to cancel all enabled role(s). -{{< copyable "sql" >}} - ```sql SET ROLE NONE; SELECT CURRENT_ROLE(); diff --git a/sql-statements/sql-statement-set-variable.md b/sql-statements/sql-statement-set-variable.md index 6fb9326754b5d..72f443fe160d0 100644 --- a/sql-statements/sql-statement-set-variable.md +++ b/sql-statements/sql-statement-set-variable.md @@ -6,7 +6,11 @@ aliases: ['/docs/dev/sql-statements/sql-statement-set-variable/','/docs/dev/refe # `SET [GLOBAL|SESSION] ` -The statement `SET [GLOBAL|SESSION]` modifies one of TiDB's built in variables, of either `SESSION` or `GLOBAL` scope. +The statement `SET [GLOBAL|SESSION]` modifies one of TiDB's built in variables. These variables can be [system variables](/system-variables.md) of either `SESSION` or `GLOBAL` scope or [user variables](/user-defined-variables.md). + +> **Warning:** +> +> User-defined variables are still an experimental feature. It is **NOT** recommended that you use them in the production environment. > **Note:** > @@ -14,13 +18,14 @@ The statement `SET [GLOBAL|SESSION]` modifies one of TiDB's built in variables, ## Synopsis -**SetStmt:** - -![SetStmt](/media/sqlgram/SetStmt.png) +```ebnf+diagram +SetVariableStmt ::= + "SET" Variable "=" Expression ("," Variable "=" Expression )* -**VariableAssignment:** - -![VariableAssignment](/media/sqlgram/VariableAssignment.png) +Variable ::= + ("GLOBAL" | "SESSION") SystemVariable +| UserVariable +``` ## Examples @@ -82,6 +87,21 @@ mysql> SHOW SESSION VARIABLES LIKE 'sql_mode'; 1 row in set (0.00 sec) ``` +User variables start with a `@`. + +```sql +SET @myvar := 5; +Query OK, 0 rows affected (0.00 sec) + +SELECT @myvar, @myvar + 1; ++--------+------------+ +| @myvar | @myvar + 1 | ++--------+------------+ +| 5 | 6 | ++--------+------------+ +1 row in set (0.00 sec) +``` + ## MySQL compatibility The following behavior differences apply: @@ -89,6 +109,7 @@ The following behavior differences apply: * Changes made with `SET GLOBAL` will be propagated to all TiDB instances in the cluster. This differs from MySQL, where changes do not propagate to replicas. * TiDB presents several variables as both readable and settable. This is required for MySQL compatibility, because it is common for both applications and connectors to read MySQL variables. For example: JDBC connectors both read and set query cache settings, despite not relying on the behavior. * Changes made with `SET GLOBAL` will persist through TiDB server restarts. This means that `SET GLOBAL` in TiDB behaves more similar to `SET PERSIST` as available in MySQL 8.0 and above. +* TiDB does not support `SET PERSIST` and `SET PERSIST_ONLY`, because TiDB persists global variables. ## See also diff --git a/sql-statements/sql-statement-show-analyze-status.md b/sql-statements/sql-statement-show-analyze-status.md index 087a6e8819176..ea82ae75fcbd2 100644 --- a/sql-statements/sql-statement-show-analyze-status.md +++ b/sql-statements/sql-statement-show-analyze-status.md @@ -14,6 +14,21 @@ Starting from TiDB v6.1.0, you can view the history tasks within the last 7 days Starting from TiDB v7.3.0, you can view the progress of the current `ANALYZE` task through the system table `mysql.analyze_jobs` or `SHOW ANALYZE STATUS`. +Currently, the `SHOW ANALYZE STATUS` statement returns the following columns: + +| Column name | Description | +| :--------------- | :------------- | +| `Table_schema` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Job_info` | The task information. If an index is analyzed, this information will include the index name. When `tidb_analyze_version =2`, this information will include configuration items such as sample rate. | +| `Processed_rows` | The number of rows that have been analyzed | +| `Start_time` | The time at which the task starts | +| `State` | The state of a task, including `pending`, `running`, `finished`, and `failed` | +| `Fail_reason` | The reason why the task fails. If the execution is successful, the value is `NULL`. | +| `Instance` | The TiDB instance that executes the task | +| `Process_id` | The process ID that executes the task | + ## Synopsis ```ebnf+diagram @@ -72,4 +87,4 @@ This statement is a TiDB extension to MySQL syntax. ## See also -* [ANALYZE_STATUS table](/information-schema/information-schema-analyze-status.md) \ No newline at end of file +* [ANALYZE_STATUS table](/information-schema/information-schema-analyze-status.md) diff --git a/sql-statements/sql-statement-show-bindings.md b/sql-statements/sql-statement-show-bindings.md index 53d2dcb001e76..e7b3866bc8ad1 100644 --- a/sql-statements/sql-statement-show-bindings.md +++ b/sql-statements/sql-statement-show-bindings.md @@ -10,30 +10,17 @@ The `SHOW BINDINGS` statement is used to display information about created SQL b ## Synopsis -**ShowStmt:** +```ebnf+diagram +ShowBindingsStmt ::= + "SHOW" ("GLOBAL" | "SESSION")? "BINDINGS" ShowLikeOrWhere? -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFilterable:** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) - -**GlobalScope:** - -![GlobalScope](/media/sqlgram/GlobalScope.png) - -**ShowLikeOrWhereOpt** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Syntax description -{{< copyable "sql" >}} - -```sql -SHOW [GLOBAL | SESSION] BINDINGS [ShowLikeOrWhereOpt]; -``` - This statement outputs the execution plan bindings at the GLOBAL or SESSION level. The default scope is SESSION. Currently `SHOW BINDINGS` outputs eight columns, as shown below: | Column Name | Description | @@ -50,8 +37,6 @@ This statement outputs the execution plan bindings at the GLOBAL or SESSION leve ## Examples -{{< copyable "sql" >}} - ```sql mysql> CREATE TABLE t1 ( id INT NOT NULL PRIMARY KEY auto_increment, diff --git a/sql-statements/sql-statement-show-builtins.md b/sql-statements/sql-statement-show-builtins.md index df6186adb3c42..ea5a934582067 100644 --- a/sql-statements/sql-statement-show-builtins.md +++ b/sql-statements/sql-statement-show-builtins.md @@ -10,14 +10,13 @@ aliases: ['/docs/dev/sql-statements/sql-statement-show-builtins/'] ## Synopsis -**ShowBuiltinsStmt:** - -![ShowBuiltinsStmt](/media/sqlgram/ShowBuiltinsStmt.png) +```ebnf+diagram +ShowBuiltinsStmt ::= + "SHOW" "BUILTINS" +``` ## Examples -{{< copyable "sql" >}} - ```sql SHOW BUILTINS; ``` diff --git a/sql-statements/sql-statement-show-character-set.md b/sql-statements/sql-statement-show-character-set.md index 478ed1fa5b98f..f6b192afaca3c 100644 --- a/sql-statements/sql-statement-show-character-set.md +++ b/sql-statements/sql-statement-show-character-set.md @@ -10,18 +10,22 @@ This statement provides a static list of available character sets in TiDB. The o ## Synopsis -**ShowCharsetStmt:** +```ebnf+diagram +ShowCharsetStmt ::= + "SHOW" ( ("CHARACTER" | "CHAR") "SET" | "CHARSET" ) ShowLikeOrWhere? -![ShowCharsetStmt](/media/sqlgram/ShowCharsetStmt.png) - -**CharsetKw:** - -![CharsetKw](/media/sqlgram/CharsetKw.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples ```sql -mysql> SHOW CHARACTER SET; +SHOW CHARACTER SET; +``` + +``` +---------+---------------+-------------------+--------+ | Charset | Description | Default collation | Maxlen | +---------+---------------+-------------------+--------+ @@ -34,6 +38,34 @@ mysql> SHOW CHARACTER SET; 5 rows in set (0.00 sec) ``` +```sql +SHOW CHARACTER SET LIKE 'utf8%'; +``` + +``` ++---------+---------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+---------------+-------------------+--------+ +| utf8 | UTF-8 Unicode | utf8_bin | 3 | +| utf8mb4 | UTF-8 Unicode | utf8mb4_bin | 4 | ++---------+---------------+-------------------+--------+ +2 rows in set (0.00 sec) +``` + +```sql +SHOW CHARACTER SET WHERE Description='UTF-8 Unicode'; +``` + +``` ++---------+---------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+---------------+-------------------+--------+ +| utf8 | UTF-8 Unicode | utf8_bin | 3 | +| utf8mb4 | UTF-8 Unicode | utf8mb4_bin | 4 | ++---------+---------------+-------------------+--------+ +2 rows in set (0.00 sec) +``` + ## MySQL compatibility The usage of `SHOW CHARACTER SET` statement in TiDB is fully compatible with MySQL. However, charsets in TiDB might have different default collations compared with MySQL. For details, refer to [Compatibility with MySQL](/mysql-compatibility.md). If you find any compatibility differences, [report a bug](https://docs.pingcap.com/tidb/stable/support). diff --git a/sql-statements/sql-statement-show-column-stats-usage.md b/sql-statements/sql-statement-show-column-stats-usage.md new file mode 100644 index 0000000000000..b7d28adc4d04e --- /dev/null +++ b/sql-statements/sql-statement-show-column-stats-usage.md @@ -0,0 +1,58 @@ +--- +title: SHOW COLUMN_STATS_USAGE +summary: An overview of the usage of SHOW COLUMN_STATS_USAGE for TiDB database. +--- + +# SHOW COLUMN_STATS_USAGE + +The `SHOW COLUMN_STATS_USAGE` statement shows the last usage time and collection time of column statistics. You can also use it to locate `PREDICATE COLUMNS` and columns on which statistics have been collected. + +Currently, the `SHOW COLUMN_STATS_USAGE` statement returns the following columns: + +| Column name | Description | +| -------- | ------------- | +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Column_name` | The column name | +| `Last_used_at` | The last time when the column statistics were used in the query optimization | +| `Last_analyzed_at` | The last time when the column statistics were collected | + +## Synopsis + +```ebnf+diagram +ShowColumnStatsUsageStmt ::= + "SHOW" "COLUMN_STATS_USAGE" ShowLikeOrWhere? + +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` + +## Examples + +```sql +SHOW COLUMN_STATS_USAGE; +``` + +``` ++---------+------------+----------------+-------------+--------------+---------------------+ +| Db_name | Table_name | Partition_name | Column_name | Last_used_at | Last_analyzed_at | ++---------+------------+----------------+-------------+--------------+---------------------+ +| test | t1 | | id | NULL | 2024-05-10 11:04:23 | +| test | t1 | | b | NULL | 2024-05-10 11:04:23 | +| test | t1 | | pad | NULL | 2024-05-10 11:04:23 | +| test | t | | a | NULL | 2024-05-10 11:37:06 | +| test | t | | b | NULL | 2024-05-10 11:37:06 | ++---------+------------+----------------+-------------+--------------+---------------------+ +5 rows in set (0.00 sec) +``` + +## MySQL compatibility + +This statement is a TiDB extension to MySQL syntax. + +## See also + +* [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) +* [Introduction to Statistics](/statistics.md) \ No newline at end of file diff --git a/sql-statements/sql-statement-show-columns-from.md b/sql-statements/sql-statement-show-columns-from.md index eef35f39bf7db..41de5fb623561 100644 --- a/sql-statements/sql-statement-show-columns-from.md +++ b/sql-statements/sql-statement-show-columns-from.md @@ -16,53 +16,25 @@ The statements `SHOW [FULL] FIELDS FROM `, `DESC `, `DES ## Synopsis -**ShowStmt:** +```ebnf+diagram +ShowColumnsFromStmt ::= + "SHOW" "FULL"? ("COLUMNS" | "FIELDS") ("FROM" | "IN") TableName ( ("FROM" | "IN") SchemaName)? ShowLikeOrWhere? -![ShowStmt](/media/sqlgram/ShowStmt.png) +TableName ::= + (Identifier ".")? Identifier -**ShowColumnsFilterable:** - -![ShowColumnsFilterable](/media/sqlgram/ShowColumnsFilterable.png) - -**OptFull:** - -![OptFull](/media/sqlgram/OptFull.png) - -**FieldsOrColumns:** - -![FieldsOrColumns](/media/sqlgram/FieldsOrColumns.png) - -**ShowTableAliasOpt:** - -![ShowTableAliasOpt](/media/sqlgram/ShowTableAliasOpt.png) - -**FromOrIn:** - -![FromOrIn](/media/sqlgram/FromOrIn.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) - -**ShowDatabaseNameOpt:** - -![ShowDatabaseNameOpt](/media/sqlgram/ShowDatabaseNameOpt.png) - -**DBName:** - -![DBName](/media/sqlgram/DBName.png) - -**ShowLikeOrWhereOpt:** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples ```sql -mysql> create view v1 as select 1; +mysql> CREATE VIEW v1 AS SELECT 1; Query OK, 0 rows affected (0.11 sec) -mysql> show columns from v1; +mysql> SHOW COLUMNS FROM v1; +-------+-----------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------+-----------+------+------+---------+-------+ @@ -70,7 +42,7 @@ mysql> show columns from v1; +-------+-----------+------+------+---------+-------+ 1 row in set (0.00 sec) -mysql> desc v1; +mysql> DESC v1; +-------+-----------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------+-----------+------+------+---------+-------+ @@ -78,7 +50,7 @@ mysql> desc v1; +-------+-----------+------+------+---------+-------+ 1 row in set (0.00 sec) -mysql> describe v1; +mysql> DESCRIBE v1; +-------+-----------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------+-----------+------+------+---------+-------+ @@ -86,7 +58,7 @@ mysql> describe v1; +-------+-----------+------+------+---------+-------+ 1 row in set (0.00 sec) -mysql> explain v1; +mysql> EXPLAIN v1; +-------+-----------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------+-----------+------+------+---------+-------+ @@ -94,7 +66,7 @@ mysql> explain v1; +-------+-----------+------+------+---------+-------+ 1 row in set (0.00 sec) -mysql> show fields from v1; +mysql> SHOW FIELDS FROM v1; +-------+-----------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | +-------+-----------+------+------+---------+-------+ @@ -102,7 +74,7 @@ mysql> show fields from v1; +-------+-----------+------+------+---------+-------+ 1 row in set (0.00 sec) -mysql> show full columns from v1; +mysql> SHOW FULL COLUMNS FROM v1; +-------+-----------+-----------+------+------+---------+-------+---------------------------------+---------+ | Field | Type | Collation | Null | Key | Default | Extra | Privileges | Comment | +-------+-----------+-----------+------+------+---------+-------+---------------------------------+---------+ @@ -110,7 +82,7 @@ mysql> show full columns from v1; +-------+-----------+-----------+------+------+---------+-------+---------------------------------+---------+ 1 row in set (0.00 sec) -mysql> show full columns from mysql.user; +mysql> SHOW FULL COLUMNS FROM mysql.user; +------------------------+---------------+-------------+------+------+---------+-------+---------------------------------+---------+ | Field | Type | Collation | Null | Key | Default | Extra | Privileges | Comment | +------------------------+---------------+-------------+------+------+---------+-------+---------------------------------+---------+ diff --git a/sql-statements/sql-statement-show-config.md b/sql-statements/sql-statement-show-config.md index bca81011141d4..bb23df9b5f8c5 100644 --- a/sql-statements/sql-statement-show-config.md +++ b/sql-statements/sql-statement-show-config.md @@ -14,20 +14,19 @@ The `SHOW CONFIG` statement is used to show the current configuration of various ## Synopsis -**ShowStmt:** +```ebnf+diagram +ShowConfigStmt ::= + "SHOW" "CONFIG" ShowLikeOrWhere? -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFilterable:** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples Show all configurations: -{{< copyable "sql" >}} - ```sql SHOW CONFIG; ``` @@ -45,8 +44,6 @@ SHOW CONFIG; Show the configuration where the `type` is `tidb`: -{{< copyable "sql" >}} - ```sql SHOW CONFIG WHERE type = 'tidb' AND name = 'advertise-address'; ``` @@ -62,8 +59,6 @@ SHOW CONFIG WHERE type = 'tidb' AND name = 'advertise-address'; You can also use the `LIKE` clause to show the configuration where the `type` is `tidb`: -{{< copyable "sql" >}} - ```sql SHOW CONFIG LIKE 'tidb'; ``` diff --git a/sql-statements/sql-statement-show-create-database.md b/sql-statements/sql-statement-show-create-database.md index 2231295f8393a..f8202c1f0a235 100644 --- a/sql-statements/sql-statement-show-create-database.md +++ b/sql-statements/sql-statement-show-create-database.md @@ -13,7 +13,7 @@ summary: An overview of the use of SHOW CREATE DATABASE in the TiDB database. ```ebnf+diagram ShowCreateDatabaseStmt ::= - "SHOW" "CREATE" "DATABASE" | "SCHEMA" ("IF" "NOT" "EXISTS")? DBName + "SHOW" "CREATE" ("DATABASE" | "SCHEMA") ("IF" "NOT" "EXISTS")? DBName ``` ## Examples diff --git a/sql-statements/sql-statement-show-create-placement-policy.md b/sql-statements/sql-statement-show-create-placement-policy.md index cdf84b2f4802f..eae33c3db8111 100644 --- a/sql-statements/sql-statement-show-create-placement-policy.md +++ b/sql-statements/sql-statement-show-create-placement-policy.md @@ -28,7 +28,7 @@ PolicyName ::= ```sql CREATE PLACEMENT POLICY p1 PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1" FOLLOWERS=4; CREATE TABLE t1 (a INT) PLACEMENT POLICY=p1; -SHOW CREATE PLACEMENT POLICY p1\G; +SHOW CREATE PLACEMENT POLICY p1\G ``` ``` diff --git a/sql-statements/sql-statement-show-create-sequence.md b/sql-statements/sql-statement-show-create-sequence.md index b8b9b5f69007d..1d9d56a840714 100644 --- a/sql-statements/sql-statement-show-create-sequence.md +++ b/sql-statements/sql-statement-show-create-sequence.md @@ -10,18 +10,13 @@ The `SHOW CREATE SEQUENCE` shows the detailed information of a sequence, which i ## Synopsis -**ShowCreateSequenceStmt:** - -![ShowCreateSequenceStmt](/media/sqlgram/ShowCreateSequenceStmt.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) +```ebnf+diagram +ShowCreateSequenceStmt ::= + "SHOW" "CREATE" "SEQUENCE" ( SchemaName "." )? TableName +``` ## Examples -{{< copyable "sql" >}} - ```sql CREATE SEQUENCE seq; ``` @@ -30,8 +25,6 @@ CREATE SEQUENCE seq; Query OK, 0 rows affected (0.03 sec) ``` -{{< copyable "sql" >}} - ```sql SHOW CREATE SEQUENCE seq; ``` @@ -52,4 +45,5 @@ This statement is a TiDB extension. The implementation is modeled on sequences a ## See also * [CREATE SEQUENCE](/sql-statements/sql-statement-create-sequence.md) +* [ALTER SEQUENCE](/sql-statements/sql-statement-alter-sequence.md) * [DROP SEQUENCE](/sql-statements/sql-statement-drop-sequence.md) diff --git a/sql-statements/sql-statement-show-create-table.md b/sql-statements/sql-statement-show-create-table.md index b4d58e4c74425..418eb63ba7321 100644 --- a/sql-statements/sql-statement-show-create-table.md +++ b/sql-statements/sql-statement-show-create-table.md @@ -10,13 +10,10 @@ This statement shows the exact statement to recreate an existing table using SQL ## Synopsis -**ShowCreateTableStmt:** - -![ShowCreateTableStmt](/media/sqlgram/ShowCreateTableStmt.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) +```ebnf+diagram +ShowCreateTableStmt ::= + "SHOW" "CREATE" "TABLE" (SchemaName ".")? TableName +``` ## Examples @@ -24,14 +21,12 @@ This statement shows the exact statement to recreate an existing table using SQL mysql> CREATE TABLE t1 (a INT); Query OK, 0 rows affected (0.12 sec) -mysql> SHOW CREATE TABLE t1; -+-------+------------------------------------------------------------------------------------------------------------+ -| Table | Create Table | -+-------+------------------------------------------------------------------------------------------------------------+ -| t1 | CREATE TABLE `t1` ( +mysql> SHOW CREATE TABLE t1\G +*************************** 1. row *************************** + Table: t1 +Create Table: CREATE TABLE `t1` ( `a` int(11) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin | -+-------+------------------------------------------------------------------------------------------------------------+ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin 1 row in set (0.00 sec) ``` diff --git a/sql-statements/sql-statement-show-create-user.md b/sql-statements/sql-statement-show-create-user.md index d43ffbce8cad7..b952c2dca3da3 100644 --- a/sql-statements/sql-statement-show-create-user.md +++ b/sql-statements/sql-statement-show-create-user.md @@ -10,13 +10,10 @@ This statement shows how to re-create a user using the `CREATE USER` syntax. ## Synopsis -**ShowCreateUserStmt:** - -![ShowCreateUserStmt](/media/sqlgram/ShowCreateUserStmt.png) - -**Username:** - -![Username](/media/sqlgram/Username.png) +```ebnf+diagram +ShowCreateUserStmt ::= + "SHOW" "CREATE" "USER" (Username ("@" Hostname)? | "CURRENT_USER" ( "(" ")" )? ) +``` ## Examples @@ -40,7 +37,17 @@ mysql> SHOW GRANTS FOR 'root'; ## MySQL compatibility -* The output of `SHOW CREATE USER` is designed to match MySQL, but several of the `CREATE` options are not yet supported by TiDB. Not yet supported options will be parsed but ignored. See [security compatibility] for more details. + + +* The output of `SHOW CREATE USER` is designed to match MySQL, but several of the `CREATE` options are not yet supported by TiDB. Not yet supported options will be parsed but ignored. See [Security compatibility](/security-compatibility-with-mysql.md) for more details. + + + + + +* The output of `SHOW CREATE USER` is designed to match MySQL, but several of the `CREATE` options are not yet supported by TiDB. Not yet supported options will be parsed but ignored. See [Security compatibility](https://docs.pingcap.com/tidb/stable/security-compatibility-with-mysql/) for more details. + + ## See also diff --git a/sql-statements/sql-statement-show-databases.md b/sql-statements/sql-statement-show-databases.md index 7b6a762582bd9..ef28478c9af19 100644 --- a/sql-statements/sql-statement-show-databases.md +++ b/sql-statements/sql-statement-show-databases.md @@ -12,13 +12,14 @@ This statement shows a list of databases that the current user has privileges to ## Synopsis -**ShowDatabasesStmt:** +```ebnf+diagram +ShowDatabasesStmt ::= + "SHOW" "DATABASES" ShowLikeOrWhere? -![ShowDatabasesStmt](/media/sqlgram/ShowDatabasesStmt.png) - -**ShowLikeOrWhereOpt:** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples @@ -59,3 +60,4 @@ The `SHOW DATABASES` statement in TiDB is fully compatible with MySQL. If you fi * [SHOW SCHEMAS](/sql-statements/sql-statement-show-schemas.md) * [DROP DATABASE](/sql-statements/sql-statement-drop-database.md) * [CREATE DATABASE](/sql-statements/sql-statement-create-database.md) +* [`INFORMATION_SCHEMA.SCHEMATA`](/information-schema/information-schema-schemata.md) diff --git a/sql-statements/sql-statement-show-engines.md b/sql-statements/sql-statement-show-engines.md index e0234219a503b..19ea33560cecb 100644 --- a/sql-statements/sql-statement-show-engines.md +++ b/sql-statements/sql-statement-show-engines.md @@ -10,12 +10,13 @@ This statement is used to list all supported storage engines. The syntax is incl ## Synopsis -**ShowEnginesStmt:** +```ebnf+diagram +ShowEnginesStmt ::= + "SHOW" "ENGINES" ShowLikeOrWhere? -![ShowEnginesStmt](/media/sqlgram/ShowEnginesStmt.png) - -```sql -SHOW ENGINES; +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression ``` ## Examples @@ -32,4 +33,4 @@ mysql> SHOW ENGINES; ## MySQL compatibility -* This statement will always only return InnoDB as the supported engine. Internally, TiDB will typically use TiKV as the storage engine. +* This statement will always only return InnoDB as the supported engine. Internally, TiDB will typically use [TiKV](/tikv-overview.md) as the storage engine. diff --git a/sql-statements/sql-statement-show-grants.md b/sql-statements/sql-statement-show-grants.md index 2fc346f6b9118..301d48b8056df 100644 --- a/sql-statements/sql-statement-show-grants.md +++ b/sql-statements/sql-statement-show-grants.md @@ -10,25 +10,17 @@ This statement shows a list of privileges associated with a user. As in MySQL, t ## Synopsis -**ShowGrantsStmt:** +```ebnf+diagram +ShowGrantsStmt ::= + "SHOW" "GRANTS" ("FOR" Username ("USING" RolenameList)?)? -![ShowGrantsStmt](/media/sqlgram/ShowGrantsStmt.png) +Username ::= + "CURRENT_USER" ( "(" ")" )? +| Username ("@" Hostname)? -**Username:** - -![Username](/media/sqlgram/Username.png) - -**UsingRoles:** - -![UsingRoles](/media/sqlgram/UsingRoles.png) - -**RolenameList:** - -![RolenameList](/media/sqlgram/RolenameList.png) - -**Rolename:** - -![Rolename](/media/sqlgram/Rolename.png) +RolenameList ::= + Rolename ("@" Hostname)? ("," Rolename ("@" Hostname)? )* +``` ## Examples diff --git a/sql-statements/sql-statement-show-indexes.md b/sql-statements/sql-statement-show-indexes.md index 30e97557594cb..9b1e8beef2c76 100644 --- a/sql-statements/sql-statement-show-indexes.md +++ b/sql-statements/sql-statement-show-indexes.md @@ -10,25 +10,14 @@ The statement `SHOW INDEXES [FROM|IN]` lists the indexes on a specified table. T ## Synopsis -**ShowIndexStmt:** +```ebnf+diagram +ShowIndexStmt ::= + "SHOW" ( "INDEX" | "INDEXES" | "KEYS" ) ("FROM" | "IN" ) TableName (("FROM" | "IN") SchemaName )? ShowLikeOrWhere? -![ShowIndexStmt](/media/sqlgram/ShowIndexStmt.png) - -**ShowIndexKwd:** - -![ShowIndexKwd](/media/sqlgram/ShowIndexKwd.png) - -**FromOrIn:** - -![FromOrIn](/media/sqlgram/FromOrIn.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) - -**ShowLikeOrWhereOpt:** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples @@ -73,3 +62,7 @@ The `SHOW INDEXES [FROM|IN]` statement in TiDB is fully compatible with MySQL. I * [SHOW CREATE TABLE](/sql-statements/sql-statement-show-create-table.md) * [DROP INDEX](/sql-statements/sql-statement-drop-index.md) * [CREATE INDEX](/sql-statements/sql-statement-create-index.md) +* [`information_schema.TIDB_INDEXES`](/information-schema/information-schema-tidb-indexes.md) +* [`information_schema.TIDB_INDEX_USAGE`](/information-schema/information-schema-tidb-index-usage.md) +* [`information_schema.KEY_COLUMN_USAGE`](/information-schema/information-schema-key-column-usage.md) +* [`sys.schema_unused_indexes`](/sys-schema/sys-schema-unused-indexes.md) diff --git a/sql-statements/sql-statement-show-placement-for.md b/sql-statements/sql-statement-show-placement-for.md index 31b7cd1f94665..7833feee80639 100644 --- a/sql-statements/sql-statement-show-placement-for.md +++ b/sql-statements/sql-statement-show-placement-for.md @@ -21,7 +21,7 @@ The statement returns a result set in which the `Scheduling_State` field indicat ```ebnf+diagram ShowStmt ::= - "PLACEMENT" "FOR" ShowPlacementTarget + "SHOW" "PLACEMENT" "FOR" ShowPlacementTarget ShowPlacementTarget ::= DatabaseSym DBName @@ -31,17 +31,15 @@ ShowPlacementTarget ::= ## Examples -{{< copyable "sql" >}} - ```sql CREATE PLACEMENT POLICY p1 PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1" FOLLOWERS=4; ALTER DATABASE test PLACEMENT POLICY=p1; CREATE TABLE t1 (a INT); SHOW PLACEMENT FOR DATABASE test; SHOW PLACEMENT FOR TABLE t1; -SHOW CREATE TABLE t1\G; +SHOW CREATE TABLE t1\G CREATE TABLE t3 (a INT) PARTITION BY RANGE (a) (PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (20)); -SHOW PLACEMENT FOR TABLE t3 PARTITION p1\G; +SHOW PLACEMENT FOR TABLE t3 PARTITION p1\G ``` ``` diff --git a/sql-statements/sql-statement-show-placement-labels.md b/sql-statements/sql-statement-show-placement-labels.md index 4e7c81e363a91..4e8fd3ab5f87b 100644 --- a/sql-statements/sql-statement-show-placement-labels.md +++ b/sql-statements/sql-statement-show-placement-labels.md @@ -15,13 +15,11 @@ summary: The usage of SHOW PLACEMENT LABELS in TiDB. ```ebnf+diagram ShowStmt ::= - "PLACEMENT" "LABELS" + "SHOW" "PLACEMENT" "LABELS" ShowLikeOrWhere? ``` ## Examples -{{< copyable "sql" >}} - ```sql SHOW PLACEMENT LABELS; ``` diff --git a/sql-statements/sql-statement-show-placement.md b/sql-statements/sql-statement-show-placement.md index de27500e62869..94f21423fc143 100644 --- a/sql-statements/sql-statement-show-placement.md +++ b/sql-statements/sql-statement-show-placement.md @@ -21,13 +21,11 @@ The statement returns a result set in which the `Scheduling_State` field indicat ```ebnf+diagram ShowStmt ::= - "PLACEMENT" + "SHOW" "PLACEMENT" ShowLikeOrWhere? ``` ## Examples -{{< copyable "sql" >}} - ```sql CREATE PLACEMENT POLICY p1 PRIMARY_REGION="us-east-1" REGIONS="us-east-1,us-west-1" FOLLOWERS=4; CREATE TABLE t1 (a INT) PLACEMENT POLICY=p1; diff --git a/sql-statements/sql-statement-show-plugins.md b/sql-statements/sql-statement-show-plugins.md index 660b9d83cd02e..37025e3759504 100644 --- a/sql-statements/sql-statement-show-plugins.md +++ b/sql-statements/sql-statement-show-plugins.md @@ -14,18 +14,13 @@ aliases: ['/docs/dev/sql-statements/sql-statement-show-plugins/'] ## Synopsis -**ShowStmt:** - -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFilterable:** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) +```ebnf+diagram +ShowPluginsStmt ::= + "SHOW" "PLUGINS" ShowLikeOrWhere? +``` ## Examples -{{< copyable "sql" >}} - ```sql SHOW PLUGINS; ``` @@ -39,8 +34,6 @@ SHOW PLUGINS; 1 row in set (0.000 sec) ``` -{{< copyable "sql" >}} - ```sql SHOW PLUGINS LIKE 'a%'; ``` @@ -57,3 +50,7 @@ SHOW PLUGINS LIKE 'a%'; ## MySQL compatibility The `SHOW PLUGINS` statement in TiDB is fully compatible with MySQL. If you find any compatibility differences, [report a bug](https://docs.pingcap.com/tidb/stable/support). + +## See also + +- [`ADMIN PLUGINS`](/sql-statements/sql-statement-admin.md#admin-plugins-related-statement) \ No newline at end of file diff --git a/sql-statements/sql-statement-show-privileges.md b/sql-statements/sql-statement-show-privileges.md index 65ac4722ab61e..8e1c394185a06 100644 --- a/sql-statements/sql-statement-show-privileges.md +++ b/sql-statements/sql-statement-show-privileges.md @@ -10,14 +10,18 @@ This statement shows a list of assignable privileges in TiDB. It is a static lis ## Synopsis -**ShowStmt:** - -![ShowStmt](/media/sqlgram/ShowStmt.png) +```ebnf+diagram +ShowPrivilegesStmt ::= + "SHOW" "PRIVILEGES" +``` ## Examples ```sql -mysql> show privileges; +SHOW PRIVILEGES; +``` + +``` +---------------------------------+---------------------------------------+-------------------------------------------------------+ | Privilege | Context | Comment | +---------------------------------+---------------------------------------+-------------------------------------------------------+ @@ -80,5 +84,18 @@ The `SHOW PRIVILEGES` statement in TiDB is fully compatible with MySQL. If you f ## See also + + * [SHOW GRANTS](/sql-statements/sql-statement-show-grants.md) +* [Privilege Management](/privilege-management.md) * [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) + + + + + +* [SHOW GRANTS](/sql-statements/sql-statement-show-grants.md) +* [Privilege Management](https://docs.pingcap.com/tidb/stable/privilege-management) +* [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) + + diff --git a/sql-statements/sql-statement-show-processlist.md b/sql-statements/sql-statement-show-processlist.md index 2614426960139..712e269852e50 100644 --- a/sql-statements/sql-statement-show-processlist.md +++ b/sql-statements/sql-statement-show-processlist.md @@ -10,13 +10,10 @@ This statement lists the current sessions connected to the same TiDB server. The ## Synopsis -**ShowProcesslistStmt:** - -![ShowProcesslistStmt](/media/sqlgram/ShowProcesslistStmt.png) - -**OptFull:** - -![OptFull](/media/sqlgram/OptFull.png) +```ebnf+diagram +ShowProcesslistStmt ::= + "SHOW" "FULL"? "PROCESSLIST" +``` ## Examples @@ -37,3 +34,4 @@ mysql> SHOW PROCESSLIST; ## See also * [KILL \[TIDB\]](/sql-statements/sql-statement-kill.md) +* [`INFORMATION_SCHEMA.PROCESSLIST`](/information-schema/information-schema-processlist.md) diff --git a/sql-statements/sql-statement-show-profiles.md b/sql-statements/sql-statement-show-profiles.md index 1179466688037..b6734b58f5583 100644 --- a/sql-statements/sql-statement-show-profiles.md +++ b/sql-statements/sql-statement-show-profiles.md @@ -10,14 +10,17 @@ The `SHOW PROFILES` statement currently only returns an empty result. ## Synopsis -**ShowStmt:** +```ebnf+diagram +ShowProfilesStmt ::= + "SHOW" "PROFILES" ShowLikeOrWhere? -![ShowStmt](/media/sqlgram/ShowStmt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples -{{< copyable "sql" >}} - ```sql SHOW PROFILES; ``` diff --git a/sql-statements/sql-statement-show-stats-buckets.md b/sql-statements/sql-statement-show-stats-buckets.md new file mode 100644 index 0000000000000..6d8540b2adb18 --- /dev/null +++ b/sql-statements/sql-statement-show-stats-buckets.md @@ -0,0 +1,64 @@ +--- +title: SHOW STATS_BUCKETS +summary: An overview of the usage of SHOW STATS_BUCKETS for TiDB database. +--- + +# SHOW STATS_BUCKETS + +The `SHOW STATS_BUCKETS` statement shows the bucket information in [statistics](/statistics.md). + +Currently, the `SHOW STATS_BUCKETS` statement returns the following columns: + +| Column name | Description | +| :-------- | :------------- | +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | +| `Is_index` | Whether it is an index column or not | +| `Bucket_id` | The ID of a bucket | +| `Count` | The number of all the values that falls on the bucket and the previous buckets | +| `Repeats` | The occurrence number of the maximum value | +| `Lower_bound` | The minimum value | +| `Upper_bound` | The maximum value | +| `Ndv` | The number of different values in the bucket. When `tidb_analyze_version` = `1`, `Ndv` is always `0`, which has no actual meaning. | + +## Synopsis + +```ebnf+diagram +ShowStatsBucketsStmt ::= + "SHOW" "STATS_BUCKETS" ShowLikeOrWhere? + +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` + +## Examples + +```sql +SHOW STATS_BUCKETS WHERE Table_name='t'; +``` + +``` ++---------+------------+----------------+-------------+----------+-----------+-------+---------+--------------------------+--------------------------+------+ +| Db_name | Table_name | Partition_name | Column_name | Is_index | Bucket_id | Count | Repeats | Lower_Bound | Upper_Bound | Ndv | ++---------+------------+----------------+-------------+----------+-----------+-------+---------+--------------------------+--------------------------+------+ +| test | t | | a | 0 | 0 | 1 | 1 | 2023-12-27 00:00:00 | 2023-12-27 00:00:00 | 0 | +| test | t | | a | 0 | 1 | 2 | 1 | 2023-12-28 00:00:00 | 2023-12-28 00:00:00 | 0 | +| test | t | | ia | 1 | 0 | 1 | 1 | (NULL, 2) | (NULL, 2) | 0 | +| test | t | | ia | 1 | 1 | 2 | 1 | (NULL, 4) | (NULL, 4) | 0 | +| test | t | | ia | 1 | 2 | 3 | 1 | (2023-12-27 00:00:00, 1) | (2023-12-27 00:00:00, 1) | 0 | +| test | t | | ia | 1 | 3 | 4 | 1 | (2023-12-28 00:00:00, 3) | (2023-12-28 00:00:00, 3) | 0 | ++---------+------------+----------------+-------------+----------+-----------+-------+---------+--------------------------+--------------------------+------+ +6 rows in set (0.00 sec) +``` + +## MySQL compatibility + +This statement is a TiDB extension to MySQL syntax. + +## See also + +* [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) +* [Introduction to Statistics](/statistics.md) \ No newline at end of file diff --git a/sql-statements/sql-statement-show-stats-healthy.md b/sql-statements/sql-statement-show-stats-healthy.md index 22e214cf61497..0339aac26fdd7 100644 --- a/sql-statements/sql-statement-show-stats-healthy.md +++ b/sql-statements/sql-statement-show-stats-healthy.md @@ -7,28 +7,32 @@ summary: An overview of the usage of SHOW STATS_HEALTHY for TiDB database. The `SHOW STATS_HEALTHY` statement shows an estimation of how accurate statistics are believed to be. Tables with a low percentage health may generate sub-optimal query execution plans. -The health of a table can be improved by running the `ANALYZE` table command. `ANALYZE` runs automatically when the health drops below the [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio) threshold. +The health of a table can be improved by running the [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) statement. `ANALYZE` runs automatically when the health drops below the [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio) threshold. -## Synopsis - -**ShowStmt** +Currently, the `SHOW STATS_HEALTHY` statement returns the following columns: -![ShowStmt](/media/sqlgram/ShowStmt.png) +| Column name | Description | +| -------- | ------------- | +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Healthy` | The healthy percentage between 0 and 100 | -**ShowTargetFiltertable** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) +## Synopsis -**ShowLikeOrWhereOpt** +```ebnf+diagram +ShowStatsHealthyStmt ::= + "SHOW" "STATS_HEALTHY" ShowLikeOrWhere? -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples Load example data and run `ANALYZE`: -{{< copyable "sql" >}} - ```sql CREATE TABLE t1 ( id INT NOT NULL PRIMARY KEY auto_increment, @@ -62,8 +66,6 @@ mysql> SHOW STATS_HEALTHY; Perform a bulk update deleting approximately 30% of the records. Check the health of the statistics: -{{< copyable "sql" >}} - ```sql DELETE FROM t1 WHERE id BETWEEN 101010 AND 201010; # delete about 30% of records SHOW STATS_HEALTHY; diff --git a/sql-statements/sql-statement-show-histograms.md b/sql-statements/sql-statement-show-stats-histograms.md similarity index 66% rename from sql-statements/sql-statement-show-histograms.md rename to sql-statements/sql-statement-show-stats-histograms.md index 333a791532c65..2214d15a2abb8 100644 --- a/sql-statements/sql-statement-show-histograms.md +++ b/sql-statements/sql-statement-show-stats-histograms.md @@ -1,33 +1,48 @@ --- title: SHOW STATS_HISTOGRAMS -summary: An overview of the usage of SHOW HISTOGRAMS for TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-histograms/'] +summary: An overview of the usage of SHOW STATS_HISTOGRAMS for TiDB database. +aliases: ['/docs/dev/sql-statements/sql-statement-show-histograms/','/tidb/dev/sql-statement-show-histograms'] --- # SHOW STATS_HISTOGRAMS -This statement shows the histogram information collected by the `ANALYZE` statement. +This statement shows the histogram information collected by the [`ANALYZE` statement](/sql-statements/sql-statement-analyze-table.md) as part of database [statistics](/statistics.md). + +Currently, the `SHOW STATS_HISTOGRAMS` statement returns the following columns: + +| Column name | Description | +| -------- | ------------- | +| Db_name | Database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Column_name` | The column name | +| `Is_index` | Whether it is an index column or not | +| `Update_time` | The update time | +| `Distinct_count` | The distinct count | +| `Null_count` | NULL count | +| `Avg_col_size` | The average col size | +| `Correlation` | Correlation | +| `Load_status` | Load status, such as `allEvicted` and `allLoaded` | +| `Total_mem_usage` | The total memory usage | +| `Hist_mem_usage` | The historical memory usage | +| `Topn_mem_usage` | The TopN memory usage | +| `Cms_mem_usage` | The CMS memory usage | ## Synopsis -**ShowStmt** +```ebnf+diagram +ShowStatsHistogramsStmt ::= + "SHOW" "STATS_HISTOGRAMS" ShowLikeOrWhere? -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFiltertable** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) - -**ShowLikeOrWhereOpt** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples -{{< copyable "sql" >}} - ```sql -show stats_histograms; +SHOW STATS_HISTOGRAMS; ``` ```sql @@ -41,10 +56,8 @@ show stats_histograms; 3 rows in set (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql -show stats_histograms where table_name = 't2'; +SHOW STATS_HISTOGRAMS WHERE table_name = 't2'; ``` ```sql diff --git a/sql-statements/sql-statement-show-stats-locked.md b/sql-statements/sql-statement-show-stats-locked.md index a3f9d66e41273..165578f5dfcd8 100644 --- a/sql-statements/sql-statement-show-stats-locked.md +++ b/sql-statements/sql-statement-show-stats-locked.md @@ -7,6 +7,15 @@ summary: An overview of the usage of SHOW STATS_LOCKED for the TiDB database. `SHOW STATS_LOCKED` shows the tables whose statistics are locked. +Currently, the `SHOW STATS_LOCKED` statement returns the following columns: + +| Column name | Description | +| -------- | ------------- | +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Status` | The statistics status, such as `locked` | + ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-show-stats-meta.md b/sql-statements/sql-statement-show-stats-meta.md index 20f1c25202878..b2c8f239d5db7 100644 --- a/sql-statements/sql-statement-show-stats-meta.md +++ b/sql-statements/sql-statement-show-stats-meta.md @@ -25,24 +25,19 @@ Currently, the `SHOW STATS_META` statement outputs 6 columns: ## Synopsis -**ShowStmt** +```ebnf+diagram +ShowStatsMetaStmt ::= + "SHOW" "STATS_META" ShowLikeOrWhere? -![ShowStmt](/media/sqlgram/ShowStmt.png) - -**ShowTargetFiltertable** - -![ShowTargetFilterable](/media/sqlgram/ShowTargetFilterable.png) - -**ShowLikeOrWhereOpt** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples -{{< copyable "sql" >}} - ```sql -show stats_meta; +SHOW STATS_META; ``` ```sql @@ -58,10 +53,8 @@ show stats_meta; 5 rows in set (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql -show stats_meta where table_name = 't2'; +SHOW STATS_META WHERE table_name = 't2'; ``` ```sql diff --git a/sql-statements/sql-statement-show-stats-topn.md b/sql-statements/sql-statement-show-stats-topn.md new file mode 100644 index 0000000000000..767c4de4daa08 --- /dev/null +++ b/sql-statements/sql-statement-show-stats-topn.md @@ -0,0 +1,60 @@ +--- +title: SHOW STATS_TOPN +summary: An overview of the usage of SHOW STATS_TOPN for TiDB database. +--- + +# SHOW STATS_TOPN + +The `SHOW STATS_TOPN` statement shows the Top-N information in [statistics](/statistics.md). + +Currently, the `SHOW STATS_TOPN` statement returns the following columns: + +| Column name | Description | +| ---- | ----| +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | +| `Is_index` | Whether it is an index column or not | +| `Value` | The value of this column | +| `Count` | How many times the value appears | + +## Synopsis + +```ebnf+diagram +ShowStatsTopnStmt ::= + "SHOW" "STATS_TOPN" ShowLikeOrWhere? + +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` + +## Example + +```sql +SHOW STATS_TOPN WHERE Table_name='t'; +``` + +``` ++---------+------------+----------------+-------------+----------+--------------------------+-------+ +| Db_name | Table_name | Partition_name | Column_name | Is_index | Value | Count | ++---------+------------+----------------+-------------+----------+--------------------------+-------+ +| test | t | | a | 0 | 2023-12-27 00:00:00 | 1 | +| test | t | | a | 0 | 2023-12-28 00:00:00 | 1 | +| test | t | | ia | 1 | (NULL, 2) | 1 | +| test | t | | ia | 1 | (NULL, 4) | 1 | +| test | t | | ia | 1 | (2023-12-27 00:00:00, 1) | 1 | +| test | t | | ia | 1 | (2023-12-28 00:00:00, 3) | 1 | ++---------+------------+----------------+-------------+----------+--------------------------+-------+ +6 rows in set (0.00 sec) +``` + +## MySQL compatibility + +This statement is a TiDB extension to MySQL syntax. + +## See also + +* [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) +* [Introduction to Statistics](/statistics.md) \ No newline at end of file diff --git a/sql-statements/sql-statement-show-table-next-rowid.md b/sql-statements/sql-statement-show-table-next-rowid.md index 98171492513f7..73ee64ae0fb49 100644 --- a/sql-statements/sql-statement-show-table-next-rowid.md +++ b/sql-statements/sql-statement-show-table-next-rowid.md @@ -8,34 +8,29 @@ aliases: ['/docs/dev/sql-statements/sql-statement-show-table-next-rowid/'] `SHOW TABLE NEXT_ROW_ID` is used to show the details of some special columns of a table, including: -* `AUTO_INCREMENT` column automatically created by TiDB, namely, `_tidb_rowid` column. +* [`AUTO_INCREMENT`](/auto-increment.md) column automatically created by TiDB, namely, `_tidb_rowid` column. * `AUTO_INCREMENT` column created by users. * [`AUTO_RANDOM`](/auto-random.md) column created by users. * [`SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) created by users. ## Synopsis -**ShowTableNextRowIDStmt:** - -![ShowTableNextRowIDStmt](/media/sqlgram/ShowTableNextRowIDStmt.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) +```ebnf+diagram +ShowTableNextRowIDStmt ::= + "SHOW" "TABLE" (SchemaName ".")? TableName "NEXT_ROW_ID" +``` ## Examples For newly created tables, `NEXT_GLOBAL_ROW_ID` is `1` because no Row ID is allocated. -{{< copyable "sql" >}} - ```sql -create table t(a int); +CREATE TABLE t(a int); Query OK, 0 rows affected (0.06 sec) ``` ```sql -show table t next_row_id; +SHOW TABLE t NEXT_ROW_ID; +---------+------------+-------------+--------------------+ | DB_NAME | TABLE_NAME | COLUMN_NAME | NEXT_GLOBAL_ROW_ID | +---------+------------+-------------+--------------------+ @@ -44,16 +39,16 @@ show table t next_row_id; 1 row in set (0.00 sec) ``` -Data have been written to the table. The TiDB server that inserts the data allocates and caches 30000 IDs at once. Thus, NEXT_GLOBAL_ROW_ID is 30001 now. +Data have been written to the table. The TiDB server that inserts the data allocates and caches 30000 IDs at once. Thus, NEXT_GLOBAL_ROW_ID is 30001 now. The number of IDs is controlled by [`AUTO_ID_CACHE`](/auto-increment.md#auto_id_cache). ```sql -insert into t values (), (), (); +INSERT INTO t VALUES (), (), (); Query OK, 3 rows affected (0.02 sec) Records: 3 Duplicates: 0 Warnings: 0 ``` ```sql -show table t next_row_id; +SHOW TABLE t NEXT_ROW_ID; +---------+------------+-------------+--------------------+ | DB_NAME | TABLE_NAME | COLUMN_NAME | NEXT_GLOBAL_ROW_ID | +---------+------------+-------------+--------------------+ diff --git a/sql-statements/sql-statement-show-table-regions.md b/sql-statements/sql-statement-show-table-regions.md index 7270900787702..b3ed1f3441750 100644 --- a/sql-statements/sql-statement-show-table-regions.md +++ b/sql-statements/sql-statement-show-table-regions.md @@ -21,25 +21,13 @@ SHOW TABLE [table_name] INDEX [index_name] REGIONS [WhereClauseOptional]; ## Synopsis -**ShowTableRegionStmt:** +```ebnf+diagram +ShowTableRegionStmt ::= + "SHOW" "TABLE" TableName PartitionNameList? ("INDEX" IndexName)? "REGIONS" ("WHERE" Expression)? -![ShowTableRegionStmt](/media/sqlgram/ShowTableRegionStmt.png) - -**TableName:** - -![TableName](/media/sqlgram/TableName.png) - -**PartitionNameListOpt:** - -![PartitionNameListOpt](/media/sqlgram/PartitionNameListOpt.png) - -**WhereClauseOptional:** - -![WhereClauseOptional](/media/sqlgram/WhereClauseOptional.png) - -**WhereClause:** - -![WhereClause](/media/sqlgram/WhereClause.png) +TableName ::= + (SchemaName ".")? Identifier +``` Executing `SHOW TABLE REGIONS` returns the following columns: @@ -151,7 +139,7 @@ The above output shows that Region 96 was split, with a new Region 98 being crea For a more detailed example: ```sql -mysql> show table t regions; +mysql> SHOW TABLE t REGIONS; +-----------+--------------+--------------+-----------+-----------------+---------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ | REGION_ID | START_KEY | END_KEY | LEADER_ID | LEADER_STORE_ID | PEERS | SCATTERING | WRITTEN_BYTES | READ_BYTES | APPROXIMATE_SIZE(MB) | APPROXIMATE_KEYS | SCHEDULING_CONSTRAINTS | SCHEDULING_STATE | +-----------+--------------+--------------+-----------+-----------------+---------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ @@ -175,7 +163,7 @@ In the above example: To check the Region that corresponds to table t in store 1, use the `WHERE` clause: ```sql -test> show table t regions where leader_store_id =1; +test> SHOW TABLE t REGIONS WHERE leader_store_id =1; +-----------+-----------+---------+-----------+-----------------+--------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ | REGION_ID | START_KEY | END_KEY | LEADER_ID | LEADER_STORE_ID | PEERS | SCATTERING | WRITTEN_BYTES | READ_BYTES | APPROXIMATE_SIZE(MB) | APPROXIMATE_KEYS | SCHEDULING_CONSTRAINTS | SCHEDULING_STATE | +-----------+-----------+---------+-----------+-----------------+--------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ @@ -186,7 +174,7 @@ test> show table t regions where leader_store_id =1; Use `SPLIT TABLE REGION` to split the index data into Regions. In the following example, the index data `name` of table t is split into two Regions in the range of `[a,z]`. ```sql -test> split table t index name between ("a") and ("z") regions 2; +test> SPLIT TABLE t INDEX name BETWEEN ("a") AND ("z") REGIONS 2; +--------------------+----------------------+ | TOTAL_SPLIT_REGION | SCATTER_FINISH_RATIO | +--------------------+----------------------+ @@ -198,7 +186,7 @@ test> split table t index name between ("a") and ("z") regions 2; Now table t corresponds to seven Regions. Five of them (`102`, `106`, `110`, `114`, `3`) store the record data of table t and another two (`135`, `98`) store the index data `name`. ```sql -test> show table t regions; +test> SHOW TABLE t REGIONS; +-----------+-----------------------------+-----------------------------+-----------+-----------------+---------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ | REGION_ID | START_KEY | END_KEY | LEADER_ID | LEADER_STORE_ID | PEERS | SCATTERING | WRITTEN_BYTES | READ_BYTES | APPROXIMATE_SIZE(MB) | APPROXIMATE_KEYS | SCHEDULING_CONSTRAINTS | SCHEDULING_STATE | +-----------+-----------------------------+-----------------------------+-----------+-----------------+---------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ diff --git a/sql-statements/sql-statement-show-table-status.md b/sql-statements/sql-statement-show-table-status.md index 8ed264627624b..967196674bb22 100644 --- a/sql-statements/sql-statement-show-table-status.md +++ b/sql-statements/sql-statement-show-table-status.md @@ -10,17 +10,14 @@ This statement shows various statistics about tables in TiDB. If the statistics ## Synopsis -**ShowTableStatusStmt:** +```ebnf+diagram +ShowTableStatusStmt ::= + "SHOW" "TABLE" "STATUS" ("FROM" Identifier | "IN" Identifier )? ShowLikeOrWhere? -![ShowTableStatusStmt](/media/sqlgram/ShowTableStatusStmt.png) - -**FromOrIn:** - -![FromOrIn](/media/sqlgram/FromOrIn.png) - -**StatusTableName:** - -![StatusTableName](/media/sqlgram/StatusTableName.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples @@ -54,7 +51,7 @@ Max_data_length: 0 Comment: 1 row in set (0.00 sec) -mysql> analyze table t1; +mysql> ANALYZE TABLE t1; Query OK, 0 rows affected (0.12 sec) mysql> SHOW TABLE STATUS LIKE 't1'\G diff --git a/sql-statements/sql-statement-show-tables.md b/sql-statements/sql-statement-show-tables.md index 0b01ca64d2f85..f41f6bc44d6ee 100644 --- a/sql-statements/sql-statement-show-tables.md +++ b/sql-statements/sql-statement-show-tables.md @@ -6,27 +6,20 @@ aliases: ['/docs/dev/sql-statements/sql-statement-show-tables/','/docs/dev/refer # SHOW [FULL] TABLES -This statement shows a list of tables and views in the currently selected database. The optional keyword `FULL` indicates if a table is of type `BASE TABLE` or `VIEW`. +This statement shows a list of tables and views in the currently selected database. The optional keyword `FULL` indicates if a table is of type `BASE TABLE`, `SEQUENCE`, or `VIEW`. To show tables in a different database, use `SHOW TABLES IN DatabaseName`. ## Synopsis -**ShowTablesStmt:** +```ebnf+diagram +ShowTableStmt ::= + "SHOW" "FULL"? "TABLES" ("FROM" Identifier | "IN" Identifier )? ShowLikeOrWhere? -![ShowTablesStmt](/media/sqlgram/ShowTablesStmt.png) - -**OptFull:** - -![OptFull](/media/sqlgram/OptFull.png) - -**ShowDatabaseNameOpt:** - -![ShowDatabaseNameOpt](/media/sqlgram/ShowDatabaseNameOpt.png) - -**ShowLikeOrWhereOpt:** - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` ## Examples @@ -92,3 +85,4 @@ The `SHOW [FULL] TABLES` statement in TiDB is fully compatible with MySQL. If yo * [CREATE TABLE](/sql-statements/sql-statement-create-table.md) * [DROP TABLE](/sql-statements/sql-statement-drop-table.md) * [SHOW CREATE TABLE](/sql-statements/sql-statement-show-create-table.md) +* [`INFORMATION_SCHEMA.TABLES`](/information-schema/information-schema-tables.md) diff --git a/sql-statements/sql-statement-show-warnings.md b/sql-statements/sql-statement-show-warnings.md index 187954a67b9fb..f9f14aedba8f4 100644 --- a/sql-statements/sql-statement-show-warnings.md +++ b/sql-statements/sql-statement-show-warnings.md @@ -10,7 +10,7 @@ This statement shows a list of warnings that occurred for previously executed st ## Synopsis -```ebnf+trace +```ebnf+diagram ShowWarningsStmt ::= "SHOW" "WARNINGS" ``` diff --git a/sql-statements/sql-statement-shutdown.md b/sql-statements/sql-statement-shutdown.md index 770aaa40da948..a0455f2245388 100644 --- a/sql-statements/sql-statement-shutdown.md +++ b/sql-statements/sql-statement-shutdown.md @@ -14,14 +14,13 @@ The `SHUTDOWN` statement is used to perform a shutdown operation in TiDB. Execut ## Synopsis -**Statement:** - -![Statement](/media/sqlgram/ShutdownStmt.png) +```ebnf+diagram +ShutdownStmt ::= + "SHUTDOWN" +``` ## Examples -{{< copyable "sql" >}} - ```sql SHUTDOWN; ``` diff --git a/sql-statements/sql-statement-split-region.md b/sql-statements/sql-statement-split-region.md index 5dda9c9cee9cc..c5a10cdacb017 100644 --- a/sql-statements/sql-statement-split-region.md +++ b/sql-statements/sql-statement-split-region.md @@ -18,33 +18,26 @@ To solve the hotspot problem in the above scenario, TiDB introduces the pre-spli ## Synopsis -**SplitRegionStmt:** +```ebnf+diagram +SplitRegionStmt ::= + "SPLIT" SplitSyntaxOption "TABLE" TableName PartitionNameList? ("INDEX" IndexName)? SplitOption -![SplitRegionStmt](/media/sqlgram/SplitRegionStmt.png) +SplitSyntaxOption ::= + ("REGION" "FOR")? "PARTITION"? -**SplitSyntaxOption:** +TableName ::= + (SchemaName ".")? Identifier -![SplitSyntaxOption](/media/sqlgram/SplitSyntaxOption.png) +PartitionNameList ::= + "PARTITION" "(" PartitionName ("," PartitionName)* ")" -**TableName:** +SplitOption ::= + ("BETWEEN" RowValue "AND" RowValue "REGIONS" NUM +| "BY" RowValue ("," RowValue)* ) -![TableName](/media/sqlgram/TableName.png) - -**PartitionNameListOpt:** - -![PartitionNameListOpt](/media/sqlgram/PartitionNameListOpt.png) - -**SplitOption:** - -![SplitOption](/media/sqlgram/SplitOption.png) - -**RowValue:** - -![RowValue](/media/sqlgram/RowValue.png) - -**Int64Num:** - -![Int64Num](/media/sqlgram/Int64Num.png) +RowValue ::= + "(" ValuesOpt ")" +``` ## Usage of Split Region @@ -52,8 +45,6 @@ There are two types of Split Region syntax: - The syntax of even split: - {{< copyable "sql" >}} - ```sql SPLIT TABLE table_name [INDEX index_name] BETWEEN (lower_value) AND (upper_value) REGIONS region_num ``` @@ -62,8 +53,6 @@ There are two types of Split Region syntax: - The syntax of uneven split: - {{< copyable "sql" >}} - ```sql SPLIT TABLE table_name [INDEX index_name] BY (value_list) [, (value_list)] ... ``` @@ -112,16 +101,12 @@ Because `row_id` is an integer, the value of the key to be split can be calculat For example, if you want 16 evenly split Regions split from key range`minInt64`~`maxInt64` for table t, you can use this statement: -{{< copyable "sql" >}} - ```sql SPLIT TABLE t BETWEEN (-9223372036854775808) AND (9223372036854775807) REGIONS 16; ``` This statement splits table t into 16 Regions between minInt64 and maxInt64. If the given primary key range is smaller than the specified one, for example, 0~1000000000, you can use 0 and 1000000000 take place of minInt64 and maxInt64 respectively to split Regions. -{{< copyable "sql" >}} - ```sql SPLIT TABLE t BETWEEN (0) AND (1000000000) REGIONS 16; ``` @@ -130,8 +115,6 @@ SPLIT TABLE t BETWEEN (0) AND (1000000000) REGIONS 16; If the known data is unevenly distributed, and you want a Region to be split respectively in key ranges -inf ~ 10000, 10000 ~ 90000, and 90000 ~ +inf, you can achieve this by setting fixed points, as shown below: -{{< copyable "sql" >}} - ```sql SPLIT TABLE t BY (10000), (90000); ``` @@ -160,8 +143,6 @@ The values of `upper` and `lower` are encoded into a byte array firstly. After r If the column of the `idx` index is of the integer type, you can use the following SQL statement to split index data: -{{< copyable "sql" >}} - ```sql SPLIT TABLE t INDEX idx BETWEEN (-9223372036854775808) AND (9223372036854775807) REGIONS 16; ``` @@ -170,8 +151,6 @@ This statement splits the Region of index idx in table t into 16 Regions from `m If the column of index idx1 is of varchar type, and you want to split index data by prefix letters. -{{< copyable "sql" >}} - ```sql SPLIT TABLE t INDEX idx1 BETWEEN ("a") AND ("z") REGIONS 25; ``` @@ -180,8 +159,6 @@ This statement splits index idx1 into 25 Regions from a~z. The range of Region 1 In the split method above, both data with the `y` and `z` prefixes are written into Region 25, because the upper bound is not `z`, but `{` (the character next to `z` in ASCII). Therefore, a more accurate split method is as follows: -{{< copyable "sql" >}} - ```sql SPLIT TABLE t INDEX idx1 BETWEEN ("a") AND ("{") REGIONS 26; ``` @@ -190,8 +167,6 @@ This statement splits index idx1 of the table `t` into 26 Regions from a~`{`. Th If the column of index `idx2` is of time type like timestamp/datetime, and you want to split index Region by year: -{{< copyable "sql" >}} - ```sql SPLIT TABLE t INDEX idx2 BETWEEN ("2010-01-01 00:00:00") AND ("2020-01-01 00:00:00") REGIONS 10; ``` @@ -200,8 +175,6 @@ This statement splits the Region of index `idx2` in table `t` into 10 Regions fr If you want to split the index Region by day, see the following example: -{{< copyable "sql" >}} - ```sql SPLIT TABLE t INDEX idx2 BETWEEN ("2020-06-01 00:00:00") AND ("2020-07-01 00:00:00") REGIONS 30; ``` @@ -214,16 +187,12 @@ For data Region split of joint indexes, the only difference is that you can spec For example, index `idx3 (a, b)` contains 2 columns, with column `a` of timestamp type and column `b` int. If you just want to do a time range split according to column `a`, you can use the SQL statement for splitting time index of a single column. In this case, do not specify the value of column `b` in `lower_value` and `upper_velue`. -{{< copyable "sql" >}} - ```sql SPLIT TABLE t INDEX idx3 BETWEEN ("2010-01-01 00:00:00") AND ("2020-01-01 00:00:00") REGIONS 10; ``` Within the same range of time, if you want to do one more split according to column b column. Just specify the value for column b when splitting. -{{< copyable "sql" >}} - ```sql SPLIT TABLE t INDEX idx3 BETWEEN ("2010-01-01 00:00:00", "a") AND ("2010-01-01 00:00:00", "z") REGIONS 10; ``` @@ -242,8 +211,6 @@ Index data can also be split by specified index values. For example, there is `idx4 (a,b)`, with column `a` of the varchar type and column `b` of the timestamp type. -{{< copyable "sql" >}} - ```sql SPLIT TABLE t1 INDEX idx4 BY ("a", "2000-01-01 00:00:01"), ("b", "2019-04-17 14:26:19"), ("c", ""); ``` @@ -263,16 +230,12 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or + The syntax of even split: - {{< copyable "sql" >}} - ```sql SPLIT [PARTITION] TABLE t [PARTITION] [(partition_name_list...)] [INDEX index_name] BETWEEN (lower_value) AND (upper_value) REGIONS region_num ``` + The syntax of uneven split: - {{< copyable "sql" >}} - ```sql SPLIT [PARTITION] TABLE table_name [PARTITION (partition_name_list...)] [INDEX index_name] BY (value_list) [, (value_list)] ... ``` @@ -281,18 +244,14 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or 1. Create a partitioned table `t`. Suppose that you want to create a Hash table divided into two partitions. The example statement is as follows: - {{< copyable "sql" >}} - ```sql - create table t (a int,b int,index idx(a)) partition by hash(a) partitions 2; + CREATE TABLE t (a INT, b INT, INDEX idx(a)) PARTITION BY HASH(a) PARTITIONS 2; ``` - After creating the table `t`, a Region is split for each partition. Use the `SHOW TABLE REGIONS` syntax to view the Regions of this table: - - {{< copyable "sql" >}} + After creating the table `t`, a Region is split for each partition. Use the [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) syntax to view the Regions of this table: ```sql - show table t regions; + SHOW TABLE t REGIONS; ``` ```sql @@ -306,8 +265,6 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or 2. Use the `SPLIT` syntax to split a Region for each partition. Suppose that you want to split the data in the `[0,10000]` range of each partition into four Regions. The example statement is as follows: - {{< copyable "sql" >}} - ```sql split partition table t between (0) and (10000) regions 4; ``` @@ -320,10 +277,8 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or 3. Use the `SHOW TABLE REGIONS` syntax to view the Regions of this table again. You can see that this table now has ten Regions, each partition with five Regions, four of which are the row data and one is the index data. - {{< copyable "sql" >}} - ```sql - show table t regions; + SHOW TABLE t REGIONS; ``` ```sql @@ -345,10 +300,8 @@ Splitting Regions for partitioned tables is the same as splitting Regions for or 4. You can also split Regions for the index of each partition. For example, you can split the `[1000,10000]` range of the `idx` index into two Regions. The example statement is as follows: - {{< copyable "sql" >}} - ```sql - split partition table t index idx between (1000) and (10000) regions 2; + SPLIT PARTITION TABLE t INDEX idx BETWEEN (1000) AND (10000) REGIONS 2; ``` #### Examples of Split Region for a single partition @@ -357,37 +310,29 @@ You can specify the partition to be split. 1. Create a partitioned table. Suppose that you want to create a Range partitioned table split into three partitions. The example statement is as follows: - {{< copyable "sql" >}} - ```sql - create table t ( a int, b int, index idx(b)) partition by range( a ) ( - partition p1 values less than (10000), - partition p2 values less than (20000), - partition p3 values less than (MAXVALUE) ); + CREATE TABLE t ( a INT, b INT, INDEX idx(b)) PARTITION BY RANGE( a ) ( + PARTITION p1 VALUES LESS THAN (10000), + PARTITION p2 VALUES LESS THAN (20000), + PARTITION p3 VALUES LESS THAN (MAXVALUE) ); ``` 2. Suppose that you want to split the data in the `[0,10000]` range of the `p1` partition into two Regions. The example statement is as follows: - {{< copyable "sql" >}} - ```sql - split partition table t partition (p1) between (0) and (10000) regions 2; + SPLIT PARTITION TABLE t PARTITION (p1) BETWEEN (0) AND (10000) REGIONS 2; ``` 3. Suppose that you want to split the data in the `[10000,20000]` range of the `p2` partition into two Regions. The example statement is as follows: - {{< copyable "sql" >}} - ```sql - split partition table t partition (p2) between (10000) and (20000) regions 2; + SPLIT PARTITION TABLE t PARTITION (p2) BETWEEN (10000) AND (20000) REGIONS 2; ``` 4. You can use the `SHOW TABLE REGIONS` syntax to view the Regions of this table: - {{< copyable "sql" >}} - ```sql - show table t regions; + SHOW TABLE t REGIONS; ``` ```sql @@ -404,10 +349,8 @@ You can specify the partition to be split. 5. Suppose that you want to split the `[0,20000]` range of the `idx` index of the `p1` and `p2` partitions into two Regions. The example statement is as follows: - {{< copyable "sql" >}} - ```sql - split partition table t partition (p1,p2) index idx between (0) and (20000) regions 2; + SPLIT PARTITION TABLE t PARTITION (p1,p2) INDEX idx BETWEEN (0) AND (20000) REGIONS 2; ``` ## pre_split_regions @@ -422,10 +365,8 @@ The `tidb_scatter_region` global variable affects the behavior of `PRE_SPLIT_REG ### Examples of pre_split_regions -{{< copyable "sql" >}} - ```sql -create table t (a int, b int,index idx1(a)) shard_row_id_bits = 4 pre_split_regions=2; +CREATE TABLE t (a INT, b INT, INDEX idx1(a)) SHARD_ROW_ID_BITS = 4 PRE_SPLIT_REGIONS=2; ``` After building the table, this statement splits `4 + 1` Regions for table t. `4 (2^2)` Regions are used to save table row data, and 1 Region is for saving the index data of `idx1`. diff --git a/sql-statements/sql-statement-unlock-stats.md b/sql-statements/sql-statement-unlock-stats.md index cd6446a8bafaf..937503347bee2 100644 --- a/sql-statements/sql-statement-unlock-stats.md +++ b/sql-statements/sql-statement-unlock-stats.md @@ -11,7 +11,7 @@ summary: An overview of the usage of UNLOCK STATS for the TiDB database. ```ebnf+diagram UnlockStatsStmt ::= - 'UNLOCK' 'STATS' (TableNameList) | (TableName 'PARTITION' PartitionNameList) + 'UNLOCK' 'STATS' (TableNameList | TableName 'PARTITION' PartitionNameList) TableNameList ::= TableName (',' TableName)* diff --git a/sql-statements/sql-statement-update.md b/sql-statements/sql-statement-update.md index 903de2e3230da..74e73671b84c3 100644 --- a/sql-statements/sql-statement-update.md +++ b/sql-statements/sql-statement-update.md @@ -10,29 +10,22 @@ The `UPDATE` statement is used to modify data in a specified table. ## Synopsis -**UpdateStmt:** +```ebnf+diagram +UpdateStmt ::= + "UPDATE" UpdateOption +( TableRef "SET" Assignment ("," Assignment)* WhereClause? OrderBy? Limit? +| TableRefs "SET" Assignment ("," Assignment)* WhereClause? +) -![UpdateStmt](/media/sqlgram/UpdateStmt.png) +UpdateOption ::= + OptimizerHints? ("LOW_PRIORITY" | "HIGH_PRIORITY" | "DELAYED")? "IGNORE"? -**PriorityOpt:** +TableRef ::= + ( TableFactor | JoinTable ) -![PriorityOpt](/media/sqlgram/PriorityOpt.png) - -**TableRef:** - -![TableRef](/media/sqlgram/TableRef.png) - -**TableRefs:** - -![TableRefs](/media/sqlgram/TableRefs.png) - -**AssignmentList:** - -![AssignmentList](/media/sqlgram/AssignmentList.png) - -**WhereClauseOptional:** - -![WhereClauseOptional](/media/sqlgram/WhereClauseOptional.png) +TableRefs ::= + EscapedTableRef ("," EscapedTableRef)* +``` ## Examples diff --git a/sql-statements/sql-statement-with.md b/sql-statements/sql-statement-with.md index 5501721dbd60d..2f0130337a95d 100644 --- a/sql-statements/sql-statement-with.md +++ b/sql-statements/sql-statement-with.md @@ -14,7 +14,7 @@ A Common Table Expression (CTE) is a temporary result set that can be referred m ```ebnf+diagram WithClause ::= "WITH" WithList -| "WITH" recursive WithList +| "WITH" "RECURSIVE" WithList ``` **WithList:** @@ -43,10 +43,8 @@ IdentListWithParenOpt ::= Non-recursive CTE: -{{< copyable "sql" >}} - ```sql -WITH CTE AS (SELECT 1, 2) SELECT * FROM cte t1, cte t2; +WITH cte AS (SELECT 1, 2) SELECT * FROM cte t1, cte t2; ``` ``` @@ -60,8 +58,6 @@ WITH CTE AS (SELECT 1, 2) SELECT * FROM cte t1, cte t2; Recursive CTE: -{{< copyable "sql" >}} - ```sql WITH RECURSIVE cte(a) AS (SELECT 1 UNION SELECT a+1 FROM cte WHERE a < 5) SELECT * FROM cte; ``` @@ -88,6 +84,7 @@ WITH RECURSIVE cte(a) AS (SELECT 1 UNION SELECT a+1 FROM cte WHERE a < 5) SELECT ## See also +* [Developer Guide: Common Table Expression](/develop/dev-guide-use-common-table-expression.md) * [SELECT](/sql-statements/sql-statement-select.md) * [INSERT](/sql-statements/sql-statement-insert.md) * [DELETE](/sql-statements/sql-statement-delete.md) diff --git a/statistics.md b/statistics.md index 7b6db2d637880..08540e8ab1b77 100644 --- a/statistics.md +++ b/statistics.md @@ -6,97 +6,53 @@ aliases: ['/docs/dev/statistics/','/docs/dev/reference/performance/statistics/'] # Introduction to Statistics -TiDB uses statistics to decide [which index to choose](/choose-index.md). +TiDB uses statistics as input to the optimizer to estimate the number of rows processed in each plan step for a SQL statement. The optimizer estimates the cost of each available plan choice, including [index accesses](/choose-index.md) and the sequence of table joins, and produces a cost for each available plan. The optimizer then picks the execution plan with the lowest overall cost. -## Versions of statistics - -The `tidb_analyze_version` variable controls the statistics collected by TiDB. Currently, two versions of statistics are supported: `tidb_analyze_version = 1` and `tidb_analyze_version = 2`. - -- For TiDB Self-Hosted, the default value of this variable changes from `1` to `2` starting from v5.3.0. -- For TiDB Cloud, the default value of this variable changes from `1` to `2` starting from v6.5.0. -- If your cluster is upgraded from an earlier version, the default value of `tidb_analyze_version` does not change after the upgrade. - -Compared to Version 1, Version 2 statistics avoids the potential inaccuracy caused by hash collision when the data volume is huge. It also maintains the estimate precision in most scenarios. - -These two versions include different information in TiDB: - -| Information | Version 1 | Version 2| -| --- | --- | ---| -| The total number of rows in the table | √ | √ | -| Column Count-Min Sketch | √ | × | -| Index Count-Min Sketch | √ | × | -| Column Top-N | √ | √ (Maintenance methods and precision are improved) | -| Index Top-N | √ (Insufficient maintenance precision might cause inaccuracy) | √ (Maintenance methods and precision are improved) | -| Column histogram | √ | √ (The histogram does not include Top-N values.) | -| Index histogram | √ | √ (The histogram buckets record the number of different values in each bucket, and the histogram does not include Top-N values.) | -| The number of `NULL`s in the column | √ | √ | -| The number of `NULL`s in the index | √ | √ | -| The average length of columns | √ | √ | -| The average length of indexes | √ | √ | - -When `tidb_analyze_version = 2`, if memory overflow occurs after `ANALYZE` is executed, you need to set `tidb_analyze_version = 1` to fall back to Version 1, and perform one of the following operations: - -- If the `ANALYZE` statement is executed manually, manually analyze every table to be analyzed. - - ```sql - SELECT DISTINCT(CONCAT('ANALYZE TABLE ', table_schema, '.', table_name, ';')) FROM information_schema.tables, mysql.stats_histograms WHERE stats_ver = 2 AND table_id = tidb_table_id; - ``` - -- If TiDB automatically executes the `ANALYZE` statement because the auto-analysis has been enabled, execute the following statement that generates the `DROP STATS` statement: - - ```sql - SELECT DISTINCT(CONCAT('DROP STATS ', table_schema, '.', table_name, ';')) FROM information_schema.tables, mysql.stats_histograms WHERE stats_ver = 2 AND table_id = tidb_table_id; - ``` - -- If the result of the preceding statement is too long to copy and paste, you can export the result to a temporary text file and then perform execution from the file like this: - - ```sql - SELECT DISTINCT ... INTO OUTFILE '/tmp/sql.txt'; - mysql -h ${TiDB_IP} -u user -P ${TIDB_PORT} ... < '/tmp/sql.txt' - ``` +## Collect statistics -This document briefly introduces the histogram, Count-Min Sketch, and Top-N, and details the collection and maintenance of statistics. +### Automatic update -## Histogram +For the [`INSERT`](/sql-statements/sql-statement-insert.md), [`DELETE`](/sql-statements/sql-statement-delete.md), or [`UPDATE`](/sql-statements/sql-statement-update.md) statements, TiDB automatically updates the number of rows and modified rows in statistics. -A histogram is an approximate representation of the distribution of data. It divides the entire range of values into a series of buckets, and uses simple data to describe each bucket, such as the number of values ​​falling in the bucket. In TiDB, an equal-depth histogram is created for the specific columns of each table. The equal-depth histogram can be used to estimate the interval query. + -Here "equal-depth" means that the number of values ​​falling into each bucket is as equal as possible. For example, for a given set {1.6, 1.9, 1.9, 2.0, 2.4, 2.6, 2.7, 2.7, 2.8, 2.9, 3.4, 3.5}, you want to generate 4 buckets. The equal-depth histogram is as follows. It contains four buckets [1.6, 1.9], [2.0, 2.6], [2.7, 2.8], [2.9, 3.5]. The bucket depth is 3. +TiDB persists the update information regularly and the update cycle is 20 * [`stats-lease`](/tidb-configuration-file.md#stats-lease). The default value of `stats-lease` is `3s`. If you specify the value as `0`, TiDB stops updating statistics automatically. -![Equal-depth Histogram Example](/media/statistics-1.png) + -For details about the parameter that determines the upper limit to the number of histogram buckets, refer to [Manual Collection](#manual-collection). When the number of buckets is larger, the accuracy of the histogram is higher; however, higher accuracy is at the cost of the usage of memory resources. You can adjust this number appropriately according to the actual scenario. + -## Count-Min Sketch +TiDB persists the update information every 60 seconds. -Count-Min Sketch is a hash structure. When an equivalence query contains `a = 1` or `IN` query (for example, `a in (1, 2, 3)`), TiDB uses this data structure for estimation. + -A hash collision might occur since Count-Min Sketch is a hash structure. In the `EXPLAIN` statement, if the estimate of the equivalent query deviates greatly from the actual value, it can be considered that a larger value and a smaller value have been hashed together. In this case, you can take one of the following ways to avoid the hash collision: +Based upon the number of changes to a table, TiDB will automatically schedule [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) to collect statistics on those tables. This is controlled by the [`tidb_enable_auto_anlyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610) system variable and the following `tidb_auto_analyze%` variables. -- Modify the `WITH NUM TOPN` parameter. TiDB stores the high-frequency (top x) data separately, with the other data stored in Count-Min Sketch. Therefore, to prevent a larger value and a smaller value from being hashed together, you can increase the value of `WITH NUM TOPN`. In TiDB, its default value is 20. The maximum value is 1024. For more information about this parameter, see [Manual collection](#manual-collection). -- Modify two parameters `WITH NUM CMSKETCH DEPTH` and `WITH NUM CMSKETCH WIDTH`. Both affect the number of hash buckets and the collision probability. You can increase the values of the two parameters appropriately according to the actual scenario to reduce the probability of hash collision, but at the cost of higher memory usage of statistics. In TiDB, the default value of `WITH NUM CMSKETCH DEPTH` is 5, and the default value of `WITH NUM CMSKETCH WIDTH` is 2048. For more information about the two parameters, see [Manual collection](#manual-collection). +| System Variable | Default Value | Description | +|---|---|---| +| [`tidb_enable_auto_anlyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610) | true | Controls whether TiDB automatically executes ANALYZE. | +| [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio) | 0.5 | The threshold value of automatic update | +| [`tidb_auto_analyze_start_time`](/system-variables.md#tidb_auto_analyze_start_time) | `00:00 +0000` | The start time in a day when TiDB can perform automatic update | +| [`tidb_auto_analyze_end_time`](/system-variables.md#tidb_auto_analyze_end_time) | `23:59 +0000` | The end time in a day when TiDB can perform automatic update | +| [`tidb_auto_analyze_partition_batch_size`](/system-variables.md#tidb_auto_analyze_partition_batch_size-new-in-v640) | `128` | The number of partitions that TiDB automatically analyzes when analyzing a partitioned table (that is, when automatically updating statistics on a partitioned table) | +| [`tidb_enable_auto_analyze_priority_queue`](/system-variables.md#tidb_enable_auto_analyze_priority_queue-new-in-v800) | `ON` | Controls whether to enable the priority queue to schedule the tasks of automatically collecting statistics. When this variable is enabled, TiDB prioritizes collecting statistics for tables that are more valuable to collect, such as newly created indexes and partitioned tables with partition changes. Additionally, TiDB prioritizes tables with lower health scores, placing them at the front of the queue. | -## Top-N values +When the ratio of the number of modified rows to the total number of rows of `tbl` in a table is greater than `tidb_auto_analyze_ratio`, and the current time is between `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time`, TiDB executes the `ANALYZE TABLE tbl` statement in the background to automatically update the statistics on this table. -Top-N values are values with the top N occurrences in a column or index. TiDB records the values and occurrences of Top-N values. +To avoid the situation that modifying data on a small table frequently triggers the automatic update, when a table has less than 1000 rows, modifications do not trigger the automatic update in TiDB. You can use the `SHOW STATS_META` statement to view the number of rows in a table. -## Collect statistics +> **Note:** +> +> Currently, the automatic update does not record the configuration items input at manual `ANALYZE`. Therefore, when you use the `WITH` syntax to control the collecting behavior of `ANALYZE`, you need to manually set scheduled tasks to collect statistics. ### Manual collection Currently, TiDB collects statistical information as a full collection. You can execute the `ANALYZE TABLE` statement to collect statistics. -> **Note:** -> -> - The execution time of `ANALYZE TABLE` in TiDB is longer than that in MySQL or InnoDB. In InnoDB, only a small number of pages are sampled, while in TiDB a comprehensive set of statistics is completely rebuilt. Scripts that were written for MySQL might mistakenly expect that `ANALYZE TABLE` will be a short-lived operation. -> - Starting from v7.5.0, the [Fast Analyze feature (`tidb_enable_fast_analyze`)](/system-variables.md#tidb_enable_fast_analyze) and the [incremental collection feature](https://docs.pingcap.com/tidb/v7.4/statistics#incremental-collection) for statistics are deprecated. - You can perform full collection using the following syntax. + To collect statistics of all the tables in `TableNameList`: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableNameList [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` @@ -110,54 +66,78 @@ You can perform full collection using the following syntax. `WITH NUM SAMPLES` and `WITH FLOAT_NUM SAMPLERATE` correspond to two different algorithms of collecting samples. -- `WITH NUM SAMPLES` specifies the size of the sampling set, which is implemented in the reservoir sampling method in TiDB. When a table is large, it is not recommended to use this method to collect statistics. Because the intermediate result set of the reservoir sampling contains redundant results, it causes additional pressure on resources such as memory. -- `WITH FLOAT_NUM SAMPLERATE` is a sampling method introduced in v5.3.0. With the value range `(0, 1]`, this parameter specifies the sampling rate. It is implemented in the way of Bernoulli sampling in TiDB, which is more suitable for sampling larger tables and performs better in collection efficiency and resource usage. +See [Histograms](#histogram), [Top-N](#top-n-values) and [CMSketch](#count-min-sketch) (Count-Min Sketch) for detailed explanations. For `SAMPLES`/`SAMPLERATE`, see [Improve collection performance](#improve-collection-performance). -Before v5.3.0, TiDB uses the reservoir sampling method to collect statistics. Since v5.3.0, the TiDB Version 2 statistics uses the Bernoulli sampling method to collect statistics by default. To re-use the reservoir sampling method, you can use the `WITH NUM SAMPLES` statement. +For information on persisting the options for easier reuse, see [Persist ANALYZE configurations](#persist-analyze-configurations). -The current sampling rate is calculated based on an adaptive algorithm. When you can observe the number of rows in a table using [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md), you can use this number of rows to calculate the sampling rate corresponding to 100,000 rows. If you cannot observe this number, you can use the `TABLE_KEYS` column in the [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) table as another reference to calculate the sampling rate. +## Types of statistics - +### Histogram + +Histogram statistics are used by the optimizer to estimate selectivity of an interval or range predicate, and might also be used to determine the number of distinct values within a column for estimation of equal/IN predicates in Version 2 of statistics (refer to [Versions of Statistics](#versions-of-statistics)). + +A histogram is an approximate representation of the distribution of data. It divides the entire range of values into a series of buckets, and uses simple data to describe each bucket, such as the number of values ​​falling in the bucket. In TiDB, an equal-depth histogram is created for the specific columns of each table. The equal-depth histogram can be used to estimate the interval query. + +Here "equal-depth" means that the number of values ​​falling into each bucket is as equal as possible. For example, for a given set {1.6, 1.9, 1.9, 2.0, 2.4, 2.6, 2.7, 2.7, 2.8, 2.9, 3.4, 3.5}, you want to generate 4 buckets. The equal-depth histogram is as follows. It contains four buckets [1.6, 1.9], [2.0, 2.6], [2.7, 2.8], [2.9, 3.5]. The bucket depth is 3. + +![Equal-depth Histogram Example](/media/statistics-1.png) + +For details about the parameter that determines the upper limit to the number of histogram buckets, refer to [Manual Collection](#manual-collection). When the number of buckets is larger, the accuracy of the histogram is higher; however, higher accuracy is at the cost of the usage of memory resources. You can adjust this number appropriately according to the actual scenario. + +### Count-Min Sketch > **Note:** > -> Normally, `STATS_META` is more credible than `TABLE_KEYS`. However, after importing data through the methods like [TiDB Lightning](https://docs.pingcap.com/tidb/stable/tidb-lightning-overview), the result of `STATS_META` is `0`. To handle this situation, you can use `TABLE_KEYS` to calculate the sampling rate when the result of `STATS_META` is much smaller than the result of `TABLE_KEYS`. +> Count-Min Sketch is used in statistics Version 1 only for equal/IN predicate selectivity estimation. In Version 2, other statistics are used due to challenges in managing Count-Min sketch to avoid collisions as discussed below. - +Count-Min Sketch is a hash structure. When an equivalence query contains `a = 1` or `IN` query (for example, `a IN (1, 2, 3)`), TiDB uses this data structure for estimation. - +A hash collision might occur since Count-Min Sketch is a hash structure. In the [`EXPLAIN`](/sql-statements/sql-statement-explain.md) statement, if the estimate of the equivalent query deviates greatly from the actual value, it can be considered that a larger value and a smaller value have been hashed together. In this case, you can take one of the following ways to avoid the hash collision: + +- Modify the `WITH NUM TOPN` parameter. TiDB stores the high-frequency (top x) data separately, with the other data stored in Count-Min Sketch. Therefore, to prevent a larger value and a smaller value from being hashed together, you can increase the value of `WITH NUM TOPN`. In TiDB, its default value is 20. The maximum value is 1024. For more information about this parameter, see [Manual collection](#manual-collection). +- Modify two parameters `WITH NUM CMSKETCH DEPTH` and `WITH NUM CMSKETCH WIDTH`. Both affect the number of hash buckets and the collision probability. You can increase the values of the two parameters appropriately according to the actual scenario to reduce the probability of hash collision, but at the cost of higher memory usage of statistics. In TiDB, the default value of `WITH NUM CMSKETCH DEPTH` is 5, and the default value of `WITH NUM CMSKETCH WIDTH` is 2048. For more information about the two parameters, see [Manual collection](#manual-collection). + +### Top-N values + +Top-N values are values with the top N occurrences in a column or index. Top-N statistics are often referred to as frequency statistics or data skew. + +TiDB records the values and occurrences of Top-N values. The default value is 20, meaning the top 20 most frequent values are collected. The maximum value is 1024. For details about the parameter that determines the number of values collected, see [Manual collection](#manual-collection). + +## Selective statistics collection + +### Collect statistics on indexes + +To collect statistics on all indexes in `IndexNameList` in `TableName`, use the following syntax: + +```sql +ANALYZE TABLE TableName INDEX [IndexNameList] [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; +``` + +When `IndexNameList` is empty, this syntax collects statistics on all indexes in `TableName`. > **Note:** > -> Normally, `STATS_META` is more credible than `TABLE_KEYS`. However, after importing data through TiDB Cloud console (see [Import Sample Data](/tidb-cloud/import-sample-data.md)), the result of `STATS_META` is `0`. To handle this situation, you can use `TABLE_KEYS` to calculate the sampling rate when the result of `STATS_META` is much smaller than the result of `TABLE_KEYS`. - - +> To ensure that the statistical information before and after the collection is consistent, when `tidb_analyze_version` is `2`, this syntax collects statistics on the entire table (including all columns and indexes), instead of only on indexes. -#### Collect statistics on some columns +### Collect statistics on some columns -In most cases, when executing SQL statements, the optimizer only uses statistics on some columns (such as columns in the `WHERE`, `JOIN`, `ORDER BY`, and `GROUP BY` statements). These columns are called `PREDICATE COLUMNS`. +In most cases, the optimizer only uses statistics on columns in the `WHERE`, `JOIN`, `ORDER BY`, and `GROUP BY` statements. These columns can be referred to as `PREDICATE COLUMNS`. -If a table has many columns, collecting statistics on all the columns can cause a large overhead. To reduce the overhead, you can collect statistics on only specific columns or `PREDICATE COLUMNS` to be used by the optimizer. +If a table has many columns, collecting statistics on all the columns can cause a large overhead. To reduce the overhead, you can collect statistics on only specific columns (that you choose) or `PREDICATE COLUMNS` to be used by the optimizer. To persist the column list of any subset of columns for reuse in future, see [Persist column configurations](#persist-column-configurations). > **Note:** > -> - Collecting statistics on some columns is only applicable for [`tidb_analyze_version = 2`](/system-variables.md#tidb_analyze_version-new-in-v510). -> - Starting from TiDB v7.2.0, TiDB introduces the [`tidb_analyze_skip_column_types`](/system-variables.md#tidb_analyze_skip_column_types-new-in-v720) system variable, indicating which types of columns are skipped for statistics collection when executing the `ANALYZE` command to collect statistics. The system variable is only applicable for `tidb_analyze_version = 2`. +> - Collecting statistics on predicate columns is only applicable for [`tidb_analyze_version = 2`](/system-variables.md#tidb_analyze_version-new-in-v510). +> - Starting from TiDB v7.2.0, TiDB also introduces the [`tidb_analyze_skip_column_types`](/system-variables.md#tidb_analyze_skip_column_types-new-in-v720) system variable, indicating which types of columns are skipped for statistics collection when executing the `ANALYZE` command to collect statistics. The system variable is only applicable for `tidb_analyze_version = 2`. - To collect statistics on specific columns, use the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName COLUMNS ColumnNameList [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` In the syntax, `ColumnNameList` specifies the name list of the target columns. If you need to specify more than one column, use comma `,` to separate the column names. For example, `ANALYZE table t columns a, b`. Besides collecting statistics on the specific columns in a specific table, this syntax collects statistics on the indexed columns and all indexes in that table at the same time. - > **Note:** - > - > The syntax above is a full collection. For example, after collecting statistics on columns `a` and `b` using this syntax, if you also want to collect statistics on column `c`, you need to specify all three columns using `ANALYZE table t columns a, b, c`, rather than only specifying the additional column `c` using `ANALYZE TABLE t COLUMNS c`. - - To collect statistics on `PREDICATE COLUMNS`, do the following: > **Warning:** @@ -180,8 +160,6 @@ If a table has many columns, collecting statistics on all the columns can cause 2. After the query pattern of your business is relatively stable, collect statistics on `PREDICATE COLUMNS` by using the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName PREDICATE COLUMNS [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` @@ -190,111 +168,25 @@ If a table has many columns, collecting statistics on all the columns can cause > **Note:** > - > - If the `mysql.column_stats_usage` system table does not contain any `PREDICATE COLUMNS` record for that table, the preceding syntax collects statistics on all columns and all indexes in that table. - > - After using this syntax to collect statistics, when executing a new type of SQL query, the optimizer might temporarily use the old or pseudo column statistics for this time, and TiDB will collect the statistics on the used columns from the next time. + > - If the [`mysql.column_stats_usage`](/mysql-schema.md) system table does not contain any `PREDICATE COLUMNS` recorded for that table, the preceding syntax collects statistics on all columns and all indexes in that table. + > - Any columns excluded from collection (either by manually listing columns or using `PREDICATE COLUMNS`) will not have their statistics overwritten. When executing a new type of SQL query, the optimizer will use the old statistics for such columns if it exists or pseudo column statistics if columns never had statistics collected. The next ANALYZE using `PREDICATE COLUMNS` will collect the statistics on those columns. - To collect statistics on all columns and indexes, use the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName ALL COLUMNS [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` -If you want to persist the column configuration in the `ANALYZE` statement (including `COLUMNS ColumnNameList`, `PREDICATE COLUMNS`, and `ALL COLUMNS`), set the value of the `tidb_persist_analyze_options` system variable to `ON` to enable the [ANALYZE configuration persistence](#persist-analyze-configurations) feature. After enabling the ANALYZE configuration persistence feature: - -- When TiDB collects statistics automatically or when you manually collect statistics by executing the `ANALYZE` statement without specifying the column configuration, TiDB continues using the previously persisted configuration for statistics collection. -- When you manually execute the `ANALYZE` statement multiple times with column configuration specified, TiDB overwrites the previously recorded persistent configuration using the new configuration specified by the latest `ANALYZE` statement. - -To locate `PREDICATE COLUMNS` and columns on which statistics have been collected, use the following syntax: - -{{< copyable "sql" >}} - -```sql -SHOW COLUMN_STATS_USAGE [ShowLikeOrWhere]; -``` - -The `SHOW COLUMN_STATS_USAGE` statement returns the following 6 columns: - -| Column name | Description | -| -------- | ------------- | -| `Db_name` | The database name | -| `Table_name` | The table name | -| `Partition_name` | The partition name | -| `Column_name` | The column name | -| `Last_used_at` | The last time when the column statistics were used in the query optimization | -| `Last_analyzed_at` | The last time when the column statistics were collected | - -In the following example, after executing `ANALYZE TABLE t PREDICATE COLUMNS;`, TiDB collects statistics on columns `b`, `c`, and `d`, where column `b` is a `PREDICATE COLUMN` and columns `c` and `d` are index columns. - -{{< copyable "sql" >}} - -```sql -SET GLOBAL tidb_enable_column_tracking = ON; -Query OK, 0 rows affected (0.00 sec) - -CREATE TABLE t (a INT, b INT, c INT, d INT, INDEX idx_c_d(c, d)); -Query OK, 0 rows affected (0.00 sec) - --- The optimizer uses the statistics on column b in this query. -SELECT * FROM t WHERE b > 1; -Empty set (0.00 sec) - --- After waiting for a period of time (100 * stats-lease), TiDB writes the collected `PREDICATE COLUMNS` to mysql.column_stats_usage. --- Specify `last_used_at IS NOT NULL` to show the `PREDICATE COLUMNS` collected by TiDB. -SHOW COLUMN_STATS_USAGE WHERE db_name = 'test' AND table_name = 't' AND last_used_at IS NOT NULL; -+---------+------------+----------------+-------------+---------------------+------------------+ -| Db_name | Table_name | Partition_name | Column_name | Last_used_at | Last_analyzed_at | -+---------+------------+----------------+-------------+---------------------+------------------+ -| test | t | | b | 2022-01-05 17:21:33 | NULL | -+---------+------------+----------------+-------------+---------------------+------------------+ -1 row in set (0.00 sec) - -ANALYZE TABLE t PREDICATE COLUMNS; -Query OK, 0 rows affected, 1 warning (0.03 sec) - --- Specify `last_analyzed_at IS NOT NULL` to show the columns for which statistics have been collected. -SHOW COLUMN_STATS_USAGE WHERE db_name = 'test' AND table_name = 't' AND last_analyzed_at IS NOT NULL; -+---------+------------+----------------+-------------+---------------------+---------------------+ -| Db_name | Table_name | Partition_name | Column_name | Last_used_at | Last_analyzed_at | -+---------+------------+----------------+-------------+---------------------+---------------------+ -| test | t | | b | 2022-01-05 17:21:33 | 2022-01-05 17:23:06 | -| test | t | | c | NULL | 2022-01-05 17:23:06 | -| test | t | | d | NULL | 2022-01-05 17:23:06 | -+---------+------------+----------------+-------------+---------------------+---------------------+ -3 rows in set (0.00 sec) -``` - -#### Collect statistics on indexes - -To collect statistics on all indexes in `IndexNameList` in `TableName`, use the following syntax: - -{{< copyable "sql" >}} - -```sql -ANALYZE TABLE TableName INDEX [IndexNameList] [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; -``` - -When `IndexNameList` is empty, this syntax collects statistics on all indexes in `TableName`. - -> **Note:** -> -> To ensure that the statistical information before and after the collection is consistent, when `tidb_analyze_version` is `2`, this syntax collects statistics on the entire table (including all columns and indexes), instead of only on indexes. - -#### Collect statistics on partitions +### Collect statistics on partitions - To collect statistics on all partitions in `PartitionNameList` in `TableName`, use the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName PARTITION PartitionNameList [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` - To collect index statistics on all partitions in `PartitionNameList` in `TableName`, use the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName PARTITION PartitionNameList INDEX [IndexNameList] [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` @@ -305,129 +197,72 @@ When `IndexNameList` is empty, this syntax collects statistics on all indexes in > > Currently, collecting statistics on `PREDICATE COLUMNS` is an experimental feature. It is not recommended that you use it in production environments. - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName PARTITION PartitionNameList [COLUMNS ColumnNameList|PREDICATE COLUMNS|ALL COLUMNS] [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` #### Collect statistics of partitioned tables in dynamic pruning mode -When accessing partitioned tables in [dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode), TiDB collects table-level statistics, which is called GlobalStats. Currently, GlobalStats is aggregated from statistics of all partitions. In dynamic pruning mode, a statistics update of any partitioned table can trigger the GlobalStats to be updated. - -> **Note:** -> -> - When GlobalStats update is triggered and [`tidb_skip_missing_partition_stats`](/system-variables.md#tidb_skip_missing_partition_stats-new-in-v730) is `OFF`: -> -> - If some partitions have no statistics (such as a new partition that has never been analyzed), GlobalStats generation is interrupted and a warning message is displayed saying that no statistics are available on partitions. -> - If statistics of some columns are absent in specific partitions (different columns are specified for analyzing in these partitions), GlobalStats generation is interrupted when statistics of these columns are aggregated, and a warning message is displayed saying that statistics of some columns are absent in specific partitions. -> -> - When GlobalStats update is triggered and [`tidb_skip_missing_partition_stats`](/system-variables.md#tidb_skip_missing_partition_stats-new-in-v730) is `ON`: -> -> If statistics of all or some columns are missing for some partitions, TiDB skips these missing partition statistics when generating GlobalStats so the generation of GlobalStats is not affected. -> -> - In dynamic pruning mode, the Analyze configurations of partitions and tables should be the same. Therefore, if you specify the `COLUMNS` configuration following the `ANALYZE TABLE TableName PARTITION PartitionNameList` statement or the `OPTIONS` configuration following `WITH`, TiDB will ignore them and return a warning. - -### Automatic update - - - -For the `INSERT`, `DELETE`, or `UPDATE` statements, TiDB automatically updates the number of rows and modified rows. TiDB persists this information regularly and the update cycle is 20 * [`stats-lease`](/tidb-configuration-file.md#stats-lease). The default value of `stats-lease` is `3s`. If you specify the value as `0`, TiDB stops updating statistics automatically. +When accessing partitioned tables in [dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode) (which is the default since v6.3.0), TiDB collects table-level statistics, which is called GlobalStats. Currently, GlobalStats is aggregated from statistics of all partitions. In dynamic pruning mode, a statistics update of any partitioned table can trigger the GlobalStats to be updated. - +If partitions are empty, or columns for some partitions are missing, then the collection behavior is controlled by the [`tidb_skip_missing_partition_stats`](/system-variables.md#tidb_skip_missing_partition_stats-new-in-v730) variable: - +- When GlobalStats update is triggered and [`tidb_skip_missing_partition_stats`](/system-variables.md#tidb_skip_missing_partition_stats-new-in-v730) is `OFF`: -For the `INSERT`, `DELETE`, or `UPDATE` statements, TiDB automatically updates the number of rows and modified rows. TiDB persists this information regularly and the update cycle is 20 * `stats-lease`. The default value of `stats-lease` is `3s`. + - If some partitions have no statistics (such as a new partition that has never been analyzed), GlobalStats generation is interrupted and a warning message is displayed saying that no statistics are available on partitions. - + - If statistics of some columns are absent in specific partitions (different columns are specified for analyzing in these partitions), GlobalStats generation is interrupted when statistics of these columns are aggregated, and a warning message is displayed saying that statistics of some columns are absent in specific partitions. -### Relevant system variables +- When GlobalStats update is triggered and [`tidb_skip_missing_partition_stats`](/system-variables.md#tidb_skip_missing_partition_stats-new-in-v730) is `ON`: -Three system variables related to automatic update of statistics are as follows: - -| System Variable | Default Value | Description | -|---|---|---| -| [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio) | 0.5 | The threshold value of automatic update | -| [`tidb_auto_analyze_start_time`](/system-variables.md#tidb_auto_analyze_start_time) | `00:00 +0000` | The start time in a day when TiDB can perform automatic update | -| [`tidb_auto_analyze_end_time`](/system-variables.md#tidb_auto_analyze_end_time) | `23:59 +0000` | The end time in a day when TiDB can perform automatic update | -| [`tidb_auto_analyze_partition_batch_size`](/system-variables.md#tidb_auto_analyze_partition_batch_size-new-in-v640) | `1` | The number of partitions that TiDB automatically analyzes when analyzing a partitioned table (that is, when automatically updating statistics on a partitioned table) | -| [`tidb_enable_auto_analyze_priority_queue`](/system-variables.md#tidb_enable_auto_analyze_priority_queue-new-in-v800) | `ON` | Controls whether to enable the priority queue to schedule the tasks of automatically collecting statistics. When this variable is enabled, TiDB prioritizes collecting statistics for tables that are more valuable to collect, such as newly created indexes and partitioned tables with partition changes. Additionally, TiDB prioritizes tables with lower health scores, placing them at the front of the queue. | + - If statistics of all or some columns are missing for some partitions, TiDB skips these missing partition statistics when generating GlobalStats so the generation of GlobalStats is not affected. -When the ratio of the number of modified rows to the total number of rows of `tbl` in a table is greater than `tidb_auto_analyze_ratio`, and the current time is between `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time`, TiDB executes the `ANALYZE TABLE tbl` statement in the background to automatically update the statistics on this table. +In dynamic pruning mode, the Analyze configurations of partitions and tables should be the same. Therefore, if you specify the `COLUMNS` configuration following the `ANALYZE TABLE TableName PARTITION PartitionNameList` statement or the `OPTIONS` configuration following `WITH`, TiDB will ignore them and return a warning. -To avoid the situation that modifying a small amount of data on a small table frequently triggers the automatic update, when a table has less than 1000 rows, such data modifying does not trigger the automatic update in TiDB. You can use the `SHOW STATS_META` statement to view the number of rows in a table. +## Improve collection performance > **Note:** > -> Currently, the automatic update does not record the configuration items input at manual `ANALYZE`. Therefore, when you use the `WITH` syntax to control the collecting behavior of `ANALYZE`, you need to manually set scheduled tasks to collect statistics. - -#### Disable automatic update - -If you find that automatic update of statistics consumes excessive resources and affects online application operations, you can disable it using the [`tidb_enable_auto_analyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610) system variable. - -#### Terminate background `ANALYZE` tasks - -Since TiDB v6.0, TiDB supports using the `KILL` statement to terminate an `ANALYZE` task running in the background. If you find that an `ANALYZE` task running in the background consumes a lot of resources and affects your application, you can terminate the `ANALYZE` task by taking the following steps: - -1. Execute the following SQL statement: - - {{< copyable "sql" >}} +> - The execution time of `ANALYZE TABLE` in TiDB might be longer than that in MySQL or InnoDB. In InnoDB, only a small number of pages are sampled, while by default in TiDB a comprehensive set of statistics are completely rebuilt. - ```sql - SHOW ANALYZE STATUS - ``` - - By checking the `instance` column and the `process_id` column in the result, you can get the TiDB instance address and the task `ID` of the background `ANALYZE` task. - -2. Terminate the `ANALYZE` task that is running in the background. - - - - - If [`enable-global-kill`](/tidb-configuration-file.md#enable-global-kill-new-in-v610) is `true` (`true` by default), you can execute the `KILL TIDB ${id};` statement directly, where `${id}` is the `ID` of the background `ANALYZE` task obtained from the previous step. - - If `enable-global-kill` is `false`, you need to use a client to connect to the TiDB instance that is executing the backend `ANALYZE` task, and then execute the `KILL TIDB ${id};` statement. If you use a client to connect to another TiDB instance, or if there is a proxy between the client and the TiDB cluster, the `KILL` statement cannot terminate the background `ANALYZE` task. - - - - - - To terminate the `ANALYZE` task, you can execute the `KILL TIDB ${id};` statement, where `${id}` is the `ID` of the background `ANALYZE` task obtained from the previous step. - - - -For more information on the `KILL` statement, see [`KILL`](/sql-statements/sql-statement-kill.md). - -### Control `ANALYZE` concurrency - -When you run the `ANALYZE` statement, you can adjust the concurrency using system variables, to control its effect on the system. - -The relationships of the relevant system variables are shown below: +TiDB provides two options to improve the performance of statistics collection: -![analyze_concurrency](/media/analyze_concurrency.png) +- Collecting statistics on a subset of the columns. See [Collecting statistics on some columns](#collect-statistics-on-some-columns). +- Sampling. -`tidb_build_stats_concurrency`, `tidb_build_sampling_stats_concurrency`, and `tidb_analyze_partition_concurrency` are in an upstream-downstream relationship, as shown in the preceding diagram. The actual total concurrency is: `tidb_build_stats_concurrency` * (`tidb_build_sampling_stats_concurrency` + `tidb_analyze_partition_concurrency`). When modifying these variables, you need to consider their respective values at the same time. It is recommended to adjust them one by one in the order of `tidb_analyze_partition_concurrency`, `tidb_build_sampling_stats_concurrency`, `tidb_build_stats_concurrency`, and observe the impact on the system. The larger the values of these three variables, the greater the resource overhead on the system. +### Statistics sampling -#### `tidb_build_stats_concurrency` +Sampling is available via two separate options of the `ANALYZE` statement - with each corresponding to a different collection algorithm: -When you run the `ANALYZE` statement, the task is divided into multiple small tasks. Each task only works on statistics of one column or index. You can use the [`tidb_build_stats_concurrency`](/system-variables.md#tidb_build_stats_concurrency) variable to control the number of simultaneous small tasks. The default value is `2`. The default value is `4` for v7.4.0 and earlier versions. +- `WITH NUM SAMPLES` specifies the size of the sampling set, which is implemented in the reservoir sampling method in TiDB. When a table is large, it is not recommended to use this method to collect statistics. Because the intermediate result set of the reservoir sampling contains redundant results, it causes additional pressure on resources such as memory. +- `WITH FLOAT_NUM SAMPLERATE` is a sampling method introduced in v5.3.0. With the value range `(0, 1]`, this parameter specifies the sampling rate. It is implemented in the way of Bernoulli sampling in TiDB, which is more suitable for sampling larger tables and performs better in collection efficiency and resource usage. -#### `tidb_build_sampling_stats_concurrency` +Before v5.3.0, TiDB uses the reservoir sampling method to collect statistics. Since v5.3.0, the TiDB Version 2 statistics uses the Bernoulli sampling method to collect statistics by default. To re-use the reservoir sampling method, you can use the `WITH NUM SAMPLES` statement. -When analyzing ordinary columns, you can use [`tidb_build_sampling_stats_concurrency`](/system-variables.md#tidb_build_sampling_stats_concurrency-new-in-v750) to control the concurrency of executing sampling tasks. The default value is `2`. +The current sampling rate is calculated based on an adaptive algorithm. When you can observe the number of rows in a table using [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md), you can use this number of rows to calculate the sampling rate corresponding to 100,000 rows. If you cannot observe this number, you can use the sum of all the values in the `APPROXIMATE_KEYS` column in the results of [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) of the table as another reference to calculate the sampling rate. -#### `tidb_analyze_partition_concurrency` +> **Note:** +> +> Normally, `STATS_META` is more credible than `APPROXIMATE_KEYS`. However, when the result of `STATS_META` is much smaller than the result of `APPROXIMATE_KEYS`, it is recommended that you use `APPROXIMATE_KEYS` to calculate the sampling rate. -When running the `ANALYZE` statement, you can use [`tidb_analyze_partition_concurrency`](/system-variables.md#tidb_analyze_partition_concurrency) to control the concurrency of reading and writing statistics for a partitioned table. The default value is `2`. The default value is `1` for v7.4.0 and earlier versions. +### The memory quota for collecting statistics -#### `tidb_distsql_scan_concurrency` +> **Warning:** +> +> Currently, the `ANALYZE` memory quota is an experimental feature, and the memory statistics might be inaccurate in production environments. -When you analyze regular columns, you can use the [`tidb_distsql_scan_concurrency`](/system-variables.md#tidb_distsql_scan_concurrency) variable to control the number of Regions to be read at one time. The default value is `15`. Note that changing the value will affect query performance. Adjust the value carefully. +Since TiDB v6.1.0, you can use the system variable [`tidb_mem_quota_analyze`](/system-variables.md#tidb_mem_quota_analyze-new-in-v610) to control the memory quota for collecting statistics in TiDB. -#### `tidb_index_serial_scan_concurrency` +To set a proper value of `tidb_mem_quota_analyze`, consider the data size of the cluster. When the default sampling rate is used, the main considerations are the number of columns, the size of column values, and the memory configuration of TiDB. Consider the following suggestions when you configure the maximum and minimum values: -When you analyze index columns, you can use the [`tidb_index_serial_scan_concurrency`](/system-variables.md#tidb_index_serial_scan_concurrency) variable to control the number of Regions to be read at one time. The default value is `1`. Note that changing the value will affect query performance. Adjust the value carefully. +> **Note:** +> +> The following suggestions are for reference only. You need to configure the values based on the real scenario. +> +> - Minimum value: should be greater than the maximum memory usage when TiDB collects statistics from the table with the most columns. An approximate reference: when TiDB collects statistics from a table with 20 columns using the default configuration, the maximum memory usage is about 800 MiB; when TiDB collects statistics from a table with 160 columns using the default configuration, the maximum memory usage is about 5 GiB. +> - Maximum value: should be less than the available memory when TiDB is not collecting statistics. -### Persist ANALYZE configurations +## Persist ANALYZE configurations Since v5.4.0, TiDB supports persisting some `ANALYZE` configurations. With this feature, the existing configurations can be easily reused for future statistics collection. @@ -442,7 +277,7 @@ The following are the `ANALYZE` configurations that support persistence: | The `ANALYZE` column type | AnalyzeColumnOption ::= ( 'ALL COLUMNS' \| 'PREDICATE COLUMNS' \| 'COLUMNS' ColumnNameList ) | | The `ANALYZE` column | ColumnNameList ::= Identifier ( ',' Identifier )* | -#### Enable ANALYZE configuration persistence +### Enable ANALYZE configuration persistence @@ -466,7 +301,7 @@ SELECT sample_num, sample_rate, buckets, topn, column_choice, column_ids FROM my TiDB will overwrite the previously recorded persistent configuration using the new configurations specified by the latest `ANALYZE` statement. For example, if you run `ANALYZE TABLE t WITH 200 TOPN;`, it will set the top 200 values in the `ANALYZE` statement. Subsequently, executing `ANALYZE TABLE t WITH 0.1 SAMPLERATE;` will set both the top 200 values and a sampling rate of 0.1 for auto `ANALYZE` statements, similar to `ANALYZE TABLE t WITH 200 TOPN, 0.1 SAMPLERATE;`. -#### Disable ANALYZE configuration persistence +### Disable ANALYZE configuration persistence To disable the `ANALYZE` configuration persistence feature, set the `tidb_persist_analyze_options` system variable to `OFF`. Because the `ANALYZE` configuration persistence feature is not applicable to `tidb_analyze_version = 1`, setting `tidb_analyze_version = 1` can also disable the feature. @@ -476,239 +311,157 @@ After disabling the `ANALYZE` configuration persistence feature, TiDB does not c > > When you enable the `ANALYZE` configuration persistence feature again, if the previously recorded persistence configurations are no longer applicable to the latest data, you need to execute the `ANALYZE` statement manually and specify the new persistence configurations. -### The memory quota for collecting statistics - -> **Warning:** -> -> Currently, the `ANALYZE` memory quota is an experimental feature, and the memory statistics might be inaccurate in production environments. - -Since TiDB v6.1.0, you can use the system variable [`tidb_mem_quota_analyze`](/system-variables.md#tidb_mem_quota_analyze-new-in-v610) to control the memory quota for collecting statistics in TiDB. - -To set a proper value of `tidb_mem_quota_analyze`, consider the data size of the cluster. When the default sampling rate is used, the main considerations are the number of columns, the size of column values, and the memory configuration of TiDB. Consider the following suggestions when you configure the maximum and minimum values: +### Persist column configurations -> **Note:** -> -> The following suggestions are for reference only. You need to configure the values based on the real scenario. - -- Minimum value: should be greater than the maximum memory usage when TiDB collects statistics from the table with the most columns. An approximate reference: when TiDB collects statistics from a table with 20 columns using the default configuration, the maximum memory usage is about 800 MiB; when TiDB collects statistics from a table with 160 columns using the default configuration, the maximum memory usage is about 5 GiB. -- Maximum value: should be less than the available memory when TiDB is not collecting statistics. +If you want to persist the column configuration in the `ANALYZE` statement (including `COLUMNS ColumnNameList`, `PREDICATE COLUMNS`, and `ALL COLUMNS`), set the value of the `tidb_persist_analyze_options` system variable to `ON` to enable the [ANALYZE configuration persistence](#persist-analyze-configurations) feature. After enabling the ANALYZE configuration persistence feature: -### View `ANALYZE` state +- When TiDB collects statistics automatically or when you manually collect statistics by executing the `ANALYZE` statement without specifying the column configuration, TiDB continues using the previously persisted configuration for statistics collection. +- When you manually execute the `ANALYZE` statement multiple times with column configuration specified, TiDB overwrites the previously recorded persistent configuration using the new configuration specified by the latest `ANALYZE` statement. -When executing the `ANALYZE` statement, you can view the current state of `ANALYZE` using the following SQL statement: +To locate `PREDICATE COLUMNS` and columns on which statistics have been collected, use the [`SHOW COLUMN_STATS_USAGE`](/sql-statements/sql-statement-show-column-stats-usage.md) statement. -{{< copyable "sql" >}} +In the following example, after executing `ANALYZE TABLE t PREDICATE COLUMNS;`, TiDB collects statistics on columns `b`, `c`, and `d`, where column `b` is a `PREDICATE COLUMN` and columns `c` and `d` are index columns. ```sql -SHOW ANALYZE STATUS [ShowLikeOrWhere] -``` - -This statement returns the state of `ANALYZE`. You can use `ShowLikeOrWhere` to filter the information you need. - -Currently, the `SHOW ANALYZE STATUS` statement returns the following 11 columns: - -| Column name | Description | -| :-------- | :------------- | -| table_schema | The database name | -| table_name | The table name | -| partition_name| The partition name | -| job_info | The task information. If an index is analyzed, this information will include the index name. When `tidb_analyze_version =2`, this information will include configuration items such as sample rate. | -| processed_rows | The number of rows that have been analyzed | -| start_time | The time at which the task starts | -| state | The state of a task, including `pending`, `running`, `finished`, and `failed` | -| fail_reason | The reason why the task fails. If the execution is successful, the value is `NULL`. | -| instance | The TiDB instance that executes the task | -| process_id | The process ID that executes the task | +SET GLOBAL tidb_enable_column_tracking = ON; +Query OK, 0 rows affected (0.00 sec) -Starting from TiDB v6.1.0, the `SHOW ANALYZE STATUS` statement supports showing cluster-level tasks. Even after a TiDB restart, you can still view task records before the restart using this statement. Before TiDB v6.1.0, the `SHOW ANALYZE STATUS` statement can only show instance-level tasks, and task records are cleared after a TiDB restart. +CREATE TABLE t (a INT, b INT, c INT, d INT, INDEX idx_c_d(c, d)); +Query OK, 0 rows affected (0.00 sec) -`SHOW ANALYZE STATUS` shows the most recent task records only. Starting from TiDB v6.1.0, you can view the history tasks within the last 7 days through the system table `mysql.analyze_jobs`. +-- The optimizer uses the statistics on column b in this query. +SELECT * FROM t WHERE b > 1; +Empty set (0.00 sec) -When [`tidb_mem_quota_analyze`](/system-variables.md#tidb_mem_quota_analyze-new-in-v610) is set and an automatic `ANALYZE` task running in the TiDB background uses more memory than this threshold, the task will be retried. You can see failed and retried tasks in the output of the `SHOW ANALYZE STATUS` statement. +-- After waiting for a period of time (100 * stats-lease), TiDB writes the collected `PREDICATE COLUMNS` to mysql.column_stats_usage. +-- Specify `last_used_at IS NOT NULL` to show the `PREDICATE COLUMNS` collected by TiDB. +SHOW COLUMN_STATS_USAGE +WHERE db_name = 'test' AND table_name = 't' AND last_used_at IS NOT NULL; ++---------+------------+----------------+-------------+---------------------+------------------+ +| Db_name | Table_name | Partition_name | Column_name | Last_used_at | Last_analyzed_at | ++---------+------------+----------------+-------------+---------------------+------------------+ +| test | t | | b | 2022-01-05 17:21:33 | NULL | ++---------+------------+----------------+-------------+---------------------+------------------+ +1 row in set (0.00 sec) -When [`tidb_max_auto_analyze_time`](/system-variables.md#tidb_max_auto_analyze_time-new-in-v610) is greater than 0 and an automatic `ANALYZE` task running in the TiDB background takes more time than this threshold, the task will be terminated. +ANALYZE TABLE t PREDICATE COLUMNS; +Query OK, 0 rows affected, 1 warning (0.03 sec) -```sql -mysql> SHOW ANALYZE STATUS [ShowLikeOrWhere]; -+--------------+------------+----------------+-------------------------------------------------------------------------------------------+----------------+---------------------+---------------------+----------+-------------------------------------------------------------------------------| -| Table_schema | Table_name | Partition_name | Job_info | Processed_rows | Start_time | End_time | State | Fail_reason | -+--------------+------------+----------------+-------------------------------------------------------------------------------------------+----------------+---------------------+---------------------+----------+-------------------------------------------------------------------------------| -| test | sbtest1 | | retry auto analyze table all columns with 100 topn, 0.055 samplerate | 2000000 | 2022-05-07 16:41:09 | 2022-05-07 16:41:20 | finished | NULL | -| test | sbtest1 | | auto analyze table all columns with 100 topn, 0.5 samplerate | 0 | 2022-05-07 16:40:50 | 2022-05-07 16:41:09 | failed | analyze panic due to memory quota exceeds, please try with smaller samplerate | +-- Specify `last_analyzed_at IS NOT NULL` to show the columns for which statistics have been collected. +SHOW COLUMN_STATS_USAGE +WHERE db_name = 'test' AND table_name = 't' AND last_analyzed_at IS NOT NULL; ++---------+------------+----------------+-------------+---------------------+---------------------+ +| Db_name | Table_name | Partition_name | Column_name | Last_used_at | Last_analyzed_at | ++---------+------------+----------------+-------------+---------------------+---------------------+ +| test | t | | b | 2022-01-05 17:21:33 | 2022-01-05 17:23:06 | +| test | t | | c | NULL | 2022-01-05 17:23:06 | +| test | t | | d | NULL | 2022-01-05 17:23:06 | ++---------+------------+----------------+-------------+---------------------+---------------------+ +3 rows in set (0.00 sec) ``` -## View statistics +## Versions of statistics -You can view the statistics status using the following statements. +The [`tidb_analyze_version`](/system-variables.md#tidb_analyze_version-new-in-v510) variable controls the statistics collected by TiDB. Currently, two versions of statistics are supported: `tidb_analyze_version = 1` and `tidb_analyze_version = 2`. -### Metadata of tables +- For TiDB Self-Hosted, the default value of this variable changes from `1` to `2` starting from v5.3.0. +- For TiDB Cloud, the default value of this variable changes from `1` to `2` starting from v6.5.0. +- If your cluster is upgraded from an earlier version, the default value of `tidb_analyze_version` does not change after the upgrade. -You can use the `SHOW STATS_META` statement to view the total number of rows and the number of updated rows. +Version 2 is preferred, and will continue to be enhanced to ultimately replace Version 1 completely. Compared to Version 1, Version 2 improves the accuracy of many of the statistics collected for larger data volumes. Version 2 also improves collection performance by removing the need to collect Count-Min sketch statistics for predicate selectivity estimation, and also supporting automated collection only on selected columns (see [Collecting statistics on some columns](#collect-statistics-on-some-columns)). -{{< copyable "sql" >}} +The following table lists the information collected by each version for usage in the optimizer estimates: -```sql -SHOW STATS_META [ShowLikeOrWhere]; -``` +| Information | Version 1 | Version 2| +| --- | --- | ---| +| The total number of rows in the table | √ | √ | +| Equal/IN predicate estimation | √ (Column/Index Top-N & Count-Min Sketch) | √ (Column/Index Top-N & Histogram) | +| Range predicate estimation | √ (Column/Index Top-N & Histogram) | √ (Column/Index Top-N & Histogram) | +| `NULL` predicate estimation | √ | √ | +| The average length of columns | √ | √ | +| The average length of indexes | √ | √ | -The syntax of `ShowLikeOrWhereOpt` is as follows: +### Switch between statistics versions -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) +It is recommended to ensure that all tables/indexes (and partitions) utilize statistics collection from the same version. Version 2 is recommended, however, it is not recommended to switch from one version to another without a justifiable reason such as an issue experienced with the version in use. A switch between versions might take a period of time when no statistics are available until all tables have been analyzed with the new version, which might negatively affect the optimizer plan choices if statistics are not available. -Currently, the `SHOW STATS_META` statement returns the following 6 columns: +Examples of justifications to switch might include - with Version 1, there could be inaccuracies in equal/IN predicate estimation due to hash collisions when collecting Count-Min sketch statistics. Solutions are listed in the [Count-Min Sketch](#count-min-sketch) section. Alternatively, setting `tidb_analyze_version = 2` and rerunning `ANALYZE` on all objects is also a solution. In the early release of Version 2, there was a risk of memory overflow after `ANALYZE`. This issue is resolved, but initially, one solution was to `set tidb_analyze_version = 1` and rerun `ANALYZE` on all objects. -| Column name | Description | -| :-------- | :------------- | -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name`| The partition name | -| `update_time` | The time of the update | -| `modify_count` | The number of modified rows | -| `row_count` | The total number of rows | +To prepare `ANALYZE` for switching between versions: -> **Note:** -> -> When TiDB automatically updates the total number of rows and the number of modified rows according to DML statements, `update_time` is also updated. Therefore, `update_time` does not necessarily indicate the last time when the `ANALYZE` statement is executed. +- If the `ANALYZE` statement is executed manually, manually analyze every table to be analyzed. -### Health state of tables + ```sql + SELECT DISTINCT(CONCAT('ANALYZE TABLE ', table_schema, '.', table_name, ';')) + FROM information_schema.tables JOIN mysql.stats_histograms + ON table_id = tidb_table_id + WHERE stats_ver = 2; + ``` -You can use the `SHOW STATS_HEALTHY` statement to check the health state of tables and roughly estimate the accuracy of the statistics. When `modify_count` >= `row_count`, the health state is 0; when `modify_count` < `row_count`, the health state is (1 - `modify_count`/`row_count`) * 100. +- If TiDB automatically executes the `ANALYZE` statement because the auto-analysis has been enabled, execute the following statement that generates the [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) statement: -The syntax is as follows: + ```sql + SELECT DISTINCT(CONCAT('DROP STATS ', table_schema, '.', table_name, ';')) + FROM information_schema.tables ON mysql.stats_histograms + ON table_id = tidb_table_id + WHERE stats_ver = 2; + ``` -{{< copyable "sql" >}} +- If the result of the preceding statement is too long to copy and paste, you can export the result to a temporary text file and then perform execution from the file like this: -```sql -SHOW STATS_HEALTHY [ShowLikeOrWhere]; -``` + ```sql + SELECT DISTINCT ... INTO OUTFILE '/tmp/sql.txt'; + mysql -h ${TiDB_IP} -u user -P ${TIDB_PORT} ... < '/tmp/sql.txt' + ``` -The synopsis of `SHOW STATS_HEALTHY` is: +## View statistics -![ShowStatsHealthy](/media/sqlgram/ShowStatsHealthy.png) +You can view the `ANALYZE` status and statistics information using the following statements. -Currently, the `SHOW STATS_HEALTHY` statement returns the following 4 columns: +### `ANALYZE` state -| Column name | Description | -| :-------- | :------------- | -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name` | The partition name | -| `healthy` | The health state of tables | +When executing the `ANALYZE` statement, you can view the current state of `ANALYZE` using [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md). -### Metadata of columns +Starting from TiDB v6.1.0, the `SHOW ANALYZE STATUS` statement supports showing cluster-level tasks. Even after a TiDB restart, you can still view task records before the restart using this statement. Before TiDB v6.1.0, the `SHOW ANALYZE STATUS` statement can only show instance-level tasks, and task records are cleared after a TiDB restart. -You can use the `SHOW STATS_HISTOGRAMS` statement to view the number of different values and the number of `NULL` in all the columns. +`SHOW ANALYZE STATUS` shows the most recent task records only. Starting from TiDB v6.1.0, you can view the history tasks within the last 7 days through the system table `mysql.analyze_jobs`. -Syntax as follows: +When [`tidb_mem_quota_analyze`](/system-variables.md#tidb_mem_quota_analyze-new-in-v610) is set and an automatic `ANALYZE` task running in the TiDB background uses more memory than this threshold, the task will be retried. You can see failed and retried tasks in the output of the `SHOW ANALYZE STATUS` statement. -{{< copyable "sql" >}} +When [`tidb_max_auto_analyze_time`](/system-variables.md#tidb_max_auto_analyze_time-new-in-v610) is greater than 0 and an automatic `ANALYZE` task running in the TiDB background takes more time than this threshold, the task will be terminated. ```sql -SHOW STATS_HISTOGRAMS [ShowLikeOrWhere] +mysql> SHOW ANALYZE STATUS [ShowLikeOrWhere]; ++--------------+------------+----------------+-------------------------------------------------------------------------------------------+----------------+---------------------+---------------------+----------+-------------------------------------------------------------------------------| +| Table_schema | Table_name | Partition_name | Job_info | Processed_rows | Start_time | End_time | State | Fail_reason | ++--------------+------------+----------------+-------------------------------------------------------------------------------------------+----------------+---------------------+---------------------+----------+-------------------------------------------------------------------------------| +| test | sbtest1 | | retry auto analyze table all columns with 100 topn, 0.055 samplerate | 2000000 | 2022-05-07 16:41:09 | 2022-05-07 16:41:20 | finished | NULL | +| test | sbtest1 | | auto analyze table all columns with 100 topn, 0.5 samplerate | 0 | 2022-05-07 16:40:50 | 2022-05-07 16:41:09 | failed | analyze panic due to memory quota exceeds, please try with smaller samplerate | ``` -This statement returns the number of different values and the number of `NULL` in all the columns. You can use `ShowLikeOrWhere` to filter the information you need. - -Currently, the `SHOW STATS_HISTOGRAMS` statement returns the following 10 columns: - -| Column name | Description | -| :-------- | :------------- | -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name` | The partition name | -| `column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | -| `is_index` | Whether it is an index column or not | -| `update_time` | The time of the update | -| `distinct_count` | The number of different values | -| `null_count` | The number of `NULL` | -| `avg_col_size` | The average length of columns | -| correlation | The Pearson correlation coefficient of the column and the integer primary key, which indicates the degree of association between the two columns| - -### Buckets of histogram - -You can use the `SHOW STATS_BUCKETS` statement to view each bucket of the histogram. - -The syntax is as follows: +### Metadata of tables -{{< copyable "sql" >}} +You can use the [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) statement to view the total number of rows and the number of updated rows. -```sql -SHOW STATS_BUCKETS [ShowLikeOrWhere] -``` +### Health state of tables -The diagram is as follows: +You can use the [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) statement to check the health state of tables and roughly estimate the accuracy of the statistics. When `modify_count` >= `row_count`, the health state is 0; when `modify_count` < `row_count`, the health state is (1 - `modify_count`/`row_count`) * 100. -![SHOW STATS_BUCKETS](/media/sqlgram/SHOW_STATS_BUCKETS.png) +### Metadata of columns -This statement returns information about all the buckets. You can use `ShowLikeOrWhere` to filter the information you need. +You can use the [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-stats-histograms.md) statement to view the number of different values and the number of `NULL` in all the columns. -Currently, the `SHOW STATS_BUCKETS` statement returns the following 11 columns: +### Buckets of histogram -| Column name | Description | -| :-------- | :------------- | -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name` | The partition name | -| `column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | -| `is_index` | Whether it is an index column or not | -| `bucket_id` | The ID of a bucket | -| `count` | The number of all the values that falls on the bucket and the previous buckets | -| `repeats` | The occurrence number of the maximum value | -| `lower_bound` | The minimum value | -| `upper_bound` | The maximum value | -| `ndv` | The number of different values in the bucket. When `tidb_analyze_version` = `1`, `ndv` is always `0`, which has no actual meaning. | +You can use the [`SHOW STATS_BUCKETS`](/sql-statements/sql-statement-show-stats-buckets.md statement to view each bucket of the histogram. ### Top-N information -You can use the `SHOW STATS_TOPN` statement to view the Top-N information currently collected by TiDB. - -The syntax is as follows: - -{{< copyable "sql" >}} - -```sql -SHOW STATS_TOPN [ShowLikeOrWhere]; -``` - -Currently, the `SHOW STATS_TOPN` statement returns the following 7 columns: - -| Column name | Description | -| ---- | ----| -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name` | The partition name | -| `column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | -| `is_index` | Whether it is an index column or not | -| `value` | The value of this column | -| `count` | How many times the value appears | +You can use the [`SHOW STATS_TOPN`](/sql-statements/sql-statement-show-stats-topn.md) statement to view the Top-N information currently collected by TiDB. ## Delete statistics -You can run the `DROP STATS` statement to delete statistics. - -{{< copyable "sql" >}} - -```sql -DROP STATS TableName -``` - -The preceding statement deletes all statistics of `TableName`. If a partitioned table is specified, this statement will delete statistics of all partitions in this table as well as GlobalStats generated in dynamic pruning mode. - -{{< copyable "sql" >}} - -```sql -DROP STATS TableName PARTITION PartitionNameList; -``` - -This preceding statement only deletes statistics of the specified partitions in `PartitionNameList`. - -{{< copyable "sql" >}} - -```sql -DROP STATS TableName GLOBAL; -``` - -The preceding statement only deletes GlobalStats generated in dynamic pruning mode of the specified table. +You can run the [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) statement to delete statistics. ## Load statistics @@ -746,7 +499,7 @@ The default value of `lite-init-stats` is `true`, which means to enable lightwei -After enabling the synchronously loading statistics feature, you can control how TiDB behaves when the waiting time of SQL optimization reaches the timeout by modifing the value of the [`tidb_stats_load_pseudo_timeout`](/system-variables.md#tidb_stats_load_pseudo_timeout-new-in-v540) system variable. The default value of this variable is `ON`, indicating that after the timeout, the SQL optimization process does not use any histogram, TopN, or CMSketch statistics on any columns. If this variable is set to `OFF`, after the timeout, SQL execution fails. +After enabling the synchronously loading statistics feature, you can control how TiDB behaves when the waiting time of SQL optimization reaches the timeout by modifying the value of the [`tidb_stats_load_pseudo_timeout`](/system-variables.md#tidb_stats_load_pseudo_timeout-new-in-v540) system variable. The default value of this variable is `ON`, indicating that after the timeout, the SQL optimization process does not use any histogram, TopN, or CMSketch statistics on any columns. If this variable is set to `OFF`, after the timeout, SQL execution fails. @@ -766,24 +519,18 @@ The interface to export statistics is as follows: + To obtain the JSON format statistics of the `${table_name}` table in the `${db_name}` database: - {{< copyable "" >}} - ``` http://${tidb-server-ip}:${tidb-server-status-port}/stats/dump/${db_name}/${table_name} ``` For example: - {{< copyable "" >}} - - ``` + ```shell curl -s http://127.0.0.1:10080/stats/dump/test/t1 -o /tmp/t1.json ``` + To obtain the JSON format statistics of the `${table_name}` table in the `${db_name}` database at specific time: - {{< copyable "" >}} - ``` http://${tidb-server-ip}:${tidb-server-status-port}/stats/dump/${db_name}/${table_name}/${yyyyMMddHHmmss} ``` @@ -796,11 +543,11 @@ The interface to export statistics is as follows: Generally, the imported statistics refer to the JSON file obtained using the export interface. -Syntax: +Loading statistics can be done with the [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) statement. -{{< copyable "sql" >}} +For example: -``` +```sql LOAD STATS 'file_name' ``` @@ -877,7 +624,7 @@ mysql> SHOW WARNINGS; 1 row in set (0.00 sec) ``` -In addition, you can also lock the statistics of a partition using `LOCK STATS`. For example: +In addition, you can also lock the statistics of a partition using [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md). For example: Create a partition table `t`, and insert data into it. When the statistics of partition `p1` are not locked, the `ANALYZE` statement can be successfully executed. @@ -964,6 +711,67 @@ The following table describes the behaviors of locking statistics: | A partitioned table and the whole table is locked | The lock is invalid | The lock is invalid because TiDB deletes the old table, so the lock information is also deleted | The old partition lock information is invalid, and the new partition is automatically locked | The new partition is automatically locked | The lock information of the deleted partition is cleared, and the lock of the whole table continues to take effect | The lock information of the deleted partition is cleared, and the new partition is automatically locked | The lock information is transferred to the exchanged table, and the new partition is automatically locked | | A partitioned table and only some partitions are locked | The lock is invalid | The lock is invalid because TiDB deletes the old table, so the lock information is also deleted | The lock is invalid because TiDB deletes the old table, so the lock information is also deleted | / | The deleted partition lock information is cleared | The deleted partition lock information is cleared | The lock information is transferred to the exchanged table | +## Manage `ANALYZE` tasks and concurrency + +### Terminate background `ANALYZE` tasks + +Since TiDB v6.0, TiDB supports using the `KILL` statement to terminate an `ANALYZE` task running in the background. If you find that an `ANALYZE` task running in the background consumes a lot of resources and affects your application, you can terminate the `ANALYZE` task by taking the following steps: + +1. Execute the following SQL statement: + + ```sql + SHOW ANALYZE STATUS + ``` + + By checking the `instance` column and the `process_id` column in the result, you can get the TiDB instance address and the task `ID` of the background `ANALYZE` task. + +2. Terminate the `ANALYZE` task that is running in the background. + + + + - If [`enable-global-kill`](/tidb-configuration-file.md#enable-global-kill-new-in-v610) is `true` (`true` by default), you can execute the `KILL TIDB ${id};` statement directly, where `${id}` is the `ID` of the background `ANALYZE` task obtained from the previous step. + - If `enable-global-kill` is `false`, you need to use a client to connect to the TiDB instance that is executing the backend `ANALYZE` task, and then execute the `KILL TIDB ${id};` statement. If you use a client to connect to another TiDB instance, or if there is a proxy between the client and the TiDB cluster, the `KILL` statement cannot terminate the background `ANALYZE` task. + + + + + + To terminate the `ANALYZE` task, you can execute the `KILL TIDB ${id};` statement, where `${id}` is the `ID` of the background `ANALYZE` task obtained from the previous step. + + + +For more information on the `KILL` statement, see [`KILL`](/sql-statements/sql-statement-kill.md). + +### Control `ANALYZE` concurrency + +When you run the `ANALYZE` statement, you can adjust the concurrency using system variables, to control its effect on the system. + +The relationships of the relevant system variables are shown below: + +![analyze_concurrency](/media/analyze_concurrency.png) + +`tidb_build_stats_concurrency`, `tidb_build_sampling_stats_concurrency`, and `tidb_analyze_partition_concurrency` are in an upstream-downstream relationship, as shown in the preceding diagram. The actual total concurrency is: `tidb_build_stats_concurrency` * (`tidb_build_sampling_stats_concurrency` + `tidb_analyze_partition_concurrency`). When modifying these variables, you need to consider their respective values at the same time. It is recommended to adjust them one by one in the order of `tidb_analyze_partition_concurrency`, `tidb_build_sampling_stats_concurrency`, `tidb_build_stats_concurrency`, and observe the impact on the system. The larger the values of these three variables, the greater the resource overhead on the system. + +#### `tidb_build_stats_concurrency` + +When you run the `ANALYZE` statement, the task is divided into multiple small tasks. Each task only works on statistics of one column or index. You can use the [`tidb_build_stats_concurrency`](/system-variables.md#tidb_build_stats_concurrency) variable to control the number of simultaneous small tasks. The default value is `2`. The default value is `4` for v7.4.0 and earlier versions. + +#### `tidb_build_sampling_stats_concurrency` + +When analyzing ordinary columns, you can use [`tidb_build_sampling_stats_concurrency`](/system-variables.md#tidb_build_sampling_stats_concurrency-new-in-v750) to control the concurrency of executing sampling tasks. The default value is `2`. + +#### `tidb_analyze_partition_concurrency` + +When running the `ANALYZE` statement, you can use [`tidb_analyze_partition_concurrency`](/system-variables.md#tidb_analyze_partition_concurrency) to control the concurrency of reading and writing statistics for a partitioned table. The default value is `2`. The default value is `1` for v7.4.0 and earlier versions. + +#### `tidb_distsql_scan_concurrency` + +When you analyze regular columns, you can use the [`tidb_distsql_scan_concurrency`](/system-variables.md#tidb_distsql_scan_concurrency) variable to control the number of Regions to be read at one time. The default value is `15`. Note that changing the value will affect query performance. Adjust the value carefully. + +#### `tidb_index_serial_scan_concurrency` + +When you analyze index columns, you can use the [`tidb_index_serial_scan_concurrency`](/system-variables.md#tidb_index_serial_scan_concurrency) variable to control the number of Regions to be read at one time. The default value is `1`. Note that changing the value will affect query performance. Adjust the value carefully. + ## See also diff --git a/sync-diff-inspector/sync-diff-inspector-overview.md b/sync-diff-inspector/sync-diff-inspector-overview.md index 56a2dd8937f37..22cf71b76a7c2 100644 --- a/sync-diff-inspector/sync-diff-inspector-overview.md +++ b/sync-diff-inspector/sync-diff-inspector-overview.md @@ -182,7 +182,7 @@ sync-diff-inspector sends progress information to `stdout` when running. Progres > > To ensure the display effect, keep the display window width above 80 characters. -```progress +``` A total of 2 tables need to be compared Comparing the table structure of ``sbtest`.`sbtest96`` ... equivalent @@ -193,7 +193,7 @@ _____________________________________________________________________________ Progress [==========================================================>--] 98% 193/200 ``` -```progress +``` A total of 2 tables need to be compared Comparing the table structure of ``sbtest`.`sbtest96`` ... equivalent @@ -245,7 +245,7 @@ The running sync-diff-inspector periodically (every 10 seconds) prints the progr After the check is finished, sync-diff-inspector outputs a report. It is located at `${output}/summary.txt`, and `${output}` is the value of `output-dir` in the `config.toml` file. -```summary +``` +---------------------+--------------------+----------------+---------+-----------+ | TABLE | STRUCTURE EQUALITY | DATA DIFF ROWS | UPCOUNT | DOWNCOUNT | +---------------------+--------------------+----------------+---------+-----------+ diff --git a/system-variables.md b/system-variables.md index eb1c7a924720d..07981bf18e5cc 100644 --- a/system-variables.md +++ b/system-variables.md @@ -1037,6 +1037,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; - Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: Integer - Default value: `4096` - Range: `[0, 9223372036854775807]` - Unit: Bytes @@ -1090,7 +1091,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; - Type: Boolean - Default value: `ON` - Controls whether to use the MPP mode of TiFlash to execute queries. The value options are as follows: - - `0` or `OFF`, which means that the MPP mode will not be used. + - `0` or `OFF`, which means that the MPP mode will not be used. For v7.3.0 or a later version, if you set the value of this variable to `0` or `OFF`, you also need to enable the [`tidb_allow_tiflash_cop`](/system-variables.md#tidb_allow_tiflash_cop-new-in-v730) variable. Otherwise, queries might return errors. - `1` or `ON`, which means that the optimizer determines whether to use the MPP mode based on the cost estimation (by default). MPP is a distributed computing framework provided by the TiFlash engine, which allows data exchange between nodes and provides high-performance, high-throughput SQL algorithms. For details about the selection of the MPP mode, refer to [Control whether to select the MPP mode](/tiflash/use-tiflash-mpp-mode.md#control-whether-to-select-the-mpp-mode). @@ -1123,10 +1124,6 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a ### tidb_analyze_partition_concurrency -> **Warning:** -> -> The feature controlled by this variable is not fully functional in the current TiDB version. Do not change the default value. - - Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -1229,7 +1226,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Float - Default value: `0.5` -- Range: `[0, 18446744073709551615]` +- Range: `(0, 1]`. The range for v8.0.0 and earlier versions is `[0, 18446744073709551615]`. - This variable is used to set the threshold when TiDB automatically executes [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) in a background thread to update table statistics. For example, a value of 0.5 means that auto-analyze is triggered when greater than 50% of the rows in a table have been modified. Auto-analyze can be restricted to only execute during certain hours of the day by specifying `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time`. > **Note:** @@ -1617,17 +1614,18 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Scope: GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No -- Default value: `OFF` +- Default value: `ON` - This variable is used to control whether to enable the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md). After the framework is enabled, the DXF tasks such as DDL and import will be distributedly executed and completed by multiple TiDB nodes in the cluster. - Starting from TiDB v7.1.0, the DXF supports distributedly executing the [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) statement for partitioned tables. - Starting from TiDB v7.2.0, the DXF supports distributedly executing the [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) statement for import jobs. +- Starting from TiDB v8.1.0, this variable is enabled by default. If you want to upgrade a cluster with the DXF enabled to v8.1.0 or later, disable the DXF (by setting `tidb_enable_dist_task` to `OFF`) before the upgrade, which avoids `ADD INDEX` operations during the upgrade causing data index inconsistency. After the upgrade, you can manually enable the DXF. - This variable is renamed from `tidb_ddl_distribute_reorg`. ### tidb_cloud_storage_uri New in v7.4.0 > **Note:** > -> Currently, the [Global Sort](/tidb-global-sort.md) process consumes a large amount of computing and memory resources of TiDB nodes. In scenarios such as adding indexes online while user business applications are running, it is recommended to add new TiDB nodes to the cluster and set the [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) variable of these nodes to `"background"`. In this way, the distributed framework schedules tasks to these nodes, isolating the workload from other TiDB nodes to reduce the impact of executing backend tasks such as `ADD INDEX` and `IMPORT INTO` on user business applications. +> Currently, the [Global Sort](/tidb-global-sort.md) process consumes a large amount of computing and memory resources of TiDB nodes. In scenarios such as adding indexes online while user business applications are running, it is recommended to add new TiDB nodes to the cluster, configure the [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) variable for these nodes, and connect to these nodes to create tasks. In this way, the distributed framework schedules tasks to these nodes, isolating the workload from other TiDB nodes to reduce the impact of executing backend tasks such as `ADD INDEX` and `IMPORT INTO` on user business applications. - Scope: GLOBAL - Persists to cluster: Yes @@ -1811,7 +1809,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Warning:** > -> The bulk DML execution mode (`tidb_dml_type = "bulk"`) is an experimental feature. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues). When TiDB performs large transactions using the bulk DML mode, it might affect the memory usage and execution efficiency of TiCDC, TiFlash, and the resolved-ts module of TiKV, and might cause OOM issues. Therefore, it is not recommended to use this mode when these components or features are enabled. +> The bulk DML execution mode (`tidb_dml_type = "bulk"`) is an experimental feature. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues). In the current version, when TiDB performs large transactions using the bulk DML mode, it might affect the memory usage and execution efficiency of TiCDC, TiFlash, and the resolved-ts module of TiKV, and might cause OOM issues. Additionally, BR might be blocked and fail to process when encountering locks. Therefore, it is not recommended to use this mode when these components or features are enabled. - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes @@ -1822,7 +1820,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - `"standard"` indicates the standard DML execution mode, where TiDB transactions are cached in memory before being committed. This mode is suitable for high-concurrency transaction scenarios with potential conflicts and is the default recommended execution mode. - `"bulk"` indicates the bulk DML execution mode, which is suitable for scenarios where a large amount of data is written, causing excessive memory usage in TiDB. - During the execution of TiDB transactions, the data is not fully cached in the TiDB memory, but is continuously written to TiKV to reduce memory usage and smooth the write pressure. - - Only `INSERT`, `UPDATE`, `REPLACE`, and `DELETE` statements are affected by the `"bulk"` mode. + - Only `INSERT`, `UPDATE`, `REPLACE`, and `DELETE` statements are affected by the `"bulk"` mode. Due to the pipelined execution in `"bulk"` mode, the usage of `INSERT IGNORE ... ON DUPLICATE UPDATE ...` might result in a `Duplicate entry` error when updates cause conflicts. In contrast, in `"standard"` mode, because the `IGNORE` keyword is set, this error would be ignored and not be returned to the user. - `"bulk"` mode is only suitable for scenarios where a large amount of **data is written without conflicts**. This mode is not efficient for handling write conflicts, as write-write conflicts might cause large transactions to fail and be rolled back. - `"bulk"` mode only takes effect on statements with auto-commit enabled, and requires the [`pessimistic-auto-commit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#pessimistic-auto-commit-new-in-v600) configuration item to be set to `false`. - When using the `"bulk"` mode to execute statements, ensure that the [metadata lock](/metadata-lock.md) remains enabled during the execution process. @@ -2774,6 +2772,8 @@ Query OK, 0 rows affected (0.09 sec) +- You can use the [`TIDB_ROW_CHECKSUM()`](/functions-and-operators/tidb-functions.md#tidb_row_checksum) function to get the checksum value of a row. + ### tidb_enforce_mpp New in v5.1 - Scope: SESSION @@ -4935,10 +4935,6 @@ SHOW WARNINGS; - When you set the variable to `ON`, the user information is hidden. For example, if the executed SQL statement is `INSERT INTO t VALUES (1,2)`, the statement is recorded as `INSERT INTO t VALUES (?,?)` in the log. - When you set the variable to `MARKER`, the user information is wrapped in `‹ ›`. For example, if the executed SQL statement is `INSERT INTO t VALUES (1,2)`, the statement is recorded as `INSERT INTO t VALUES (‹1›,‹2›)` in the log. If the input has `‹`, it is escaped as `‹‹`, and `›` is escaped as `››`. Based on the marked logs, you can decide whether to desensitize the marked information when the logs are displayed. -> **Warning:** -> -> The `MARKER` option is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. - ### tidb_regard_null_as_point New in v5.4.0 - Scope: SESSION | GLOBAL @@ -5073,7 +5069,7 @@ SHOW WARNINGS; - Scope: GLOBAL - Persists to cluster: Yes -- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `0` - Range: `[0, 9223372036854775807]` @@ -5145,14 +5141,8 @@ SHOW WARNINGS; - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: String - Default value: "" -- Optional Value: "", `background` -- This variable is an instance-level system variable. You can use it to control the service scope of TiDB nodes under the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md). When you set `tidb_service_scope` of a TiDB node to `background`, the DXF schedules that TiDB node to execute background tasks, such as [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). - -> **Note:** -> -> - If `tidb_service_scope` is not set for any TiDB node in a cluster, the DXF schedules all TiDB nodes to execute background tasks. If you are concerned about performance impact on current business, you can set `tidb_service_scope` to `background` for a few of the TiDB nodes. Only those nodes will execute background tasks. -> - In a cluster with several TiDB nodes, it is strongly recommended to set this system variable to `background` on two or more TiDB nodes. If `tidb_service_scope` is set on a single TiDB node only, when the node is restarted or fails, the task will be rescheduled to other TiDB nodes that lack the `background` setting, which will affect these TiDB nodes. -> - For newly scaled nodes, the DXF tasks are not executed by default to avoid consuming the resources of the scaled node. If you want this scaled node to execute background tasks, you need to manually set `tidb_service_scop` of this node to `background`. +- Optional value: a string with a length of up to 64 characters. Valid characters include digits `0-9`, letters `a-zA-Z`, underscores `_`, and hyphens `-`. +- This variable is an instance-level system variable. You can use it to control the service scope of each TiDB node under the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md). The DXF determines which TiDB nodes can be scheduled to execute distributed tasks based on the value of this variable. For specific rules, see [Task scheduling](/tidb-distributed-execution-framework.md#task-scheduling). ### tidb_session_alias New in v7.4.0 @@ -5232,10 +5222,10 @@ Query OK, 0 rows affected, 1 warning (0.00 sec) - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Boolean - Default value: `ON` -- When accesing a partitioned table in [dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode), TiDB aggregates the statistics of each partition to generate GlobalStats. This variable controls the generation of GlobalStats when partition statistics are missing. +- When accessing a partitioned table in [dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode), TiDB aggregates the statistics of each partition to generate GlobalStats. This variable controls the generation of GlobalStats when partition statistics are missing. - If this variable is `ON`, TiDB skips missing partition statistics when generating GlobalStats so the generation of GlobalStats is not affected. - - If this variable is `OFF`, TiDB stops generating GloablStats when it detects any missing partition statistics. + - If this variable is `OFF`, TiDB stops generating GlobalStats when it detects any missing partition statistics. ### tidb_skip_utf8_check @@ -5317,6 +5307,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer +- Unit: Byte - Default value: `0`, which means that the memory quota is automatically set to half of the total memory size of the TiDB instance. - Range: `[0, 1099511627776]` - This variable sets the memory quota for the TiDB statistics cache. @@ -6025,7 +6016,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). - When the window function is pushed down to TiFlash for execution, you can use this variable to control the concurrency level of the window function execution. The possible values are as follows: * -1: the Fine Grained Shuffle feature is disabled. The window function pushed down to TiFlash is executed in a single thread. - * 0: the Fine Grained Shuffle feature is enabled. If [`tidb_max_tiflash_threads`](/system-variables.md#tidb_max_tiflash_threads-new-in-v610) is set to a valid value (greater than 0), then `tiflash_fine_grained_shuffle_stream_count` is set to the value of [`tidb_max_tiflash_threads`](/system-variables.md#tidb_max_tiflash_threads-new-in-v610). Otherwise, it is set to 8. The actual concurrency level of the window function on TiFlash is: min(`tiflash_fine_grained_shuffle_stream_count`, the number of physical threads on TiFlash nodes). + * 0: the Fine Grained Shuffle feature is enabled. If [`tidb_max_tiflash_threads`](/system-variables.md#tidb_max_tiflash_threads-new-in-v610) is set to a valid value (greater than 0), then `tiflash_fine_grained_shuffle_stream_count` is set to the value of [`tidb_max_tiflash_threads`](/system-variables.md#tidb_max_tiflash_threads-new-in-v610). Otherwise, it is automatically estimated based on the CPU resources of the TiFlash compute node. The actual concurrency level of the window function on TiFlash is: min(`tiflash_fine_grained_shuffle_stream_count`, the number of physical threads on TiFlash nodes). * Integer greater than 0: the Fine Grained Shuffle feature is enabled. The window function pushed down to TiFlash is executed in multiple threads. The concurrency level is: min(`tiflash_fine_grained_shuffle_stream_count`, the number of physical threads on TiFlash nodes). - Theoretically, the performance of the window function increases linearly with this value. However, if the value exceeds the actual number of physical threads, it instead leads to performance degradation. @@ -6257,7 +6248,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Scope: NONE - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Default value: `8.0.11-TiDB-`(tidb version) -- This variable returns the MySQL version, followed by the TiDB version. For example '8.0.11-TiDB-v8.0.0'. +- This variable returns the MySQL version, followed by the TiDB version. For example '8.0.11-TiDB-v8.1.0'. ### version_comment diff --git a/table-filter.md b/table-filter.md index a6768fcd76a92..ec1a5cf97cab0 100644 --- a/table-filter.md +++ b/table-filter.md @@ -21,11 +21,11 @@ Table filters can be applied to the tools using multiple `-f` or `--filter` comm * [BR](/br/backup-and-restore-overview.md): ```shell - ./br backup full -f 'foo*.*' -f 'bar*.*' -s 'local:///tmp/backup' + tiup br backup full -f 'foo*.*' -f 'bar*.*' -s 'local:///tmp/backup' ``` ```shell - ./br restore full -f 'foo*.*' -f 'bar*.*' -s 'local:///tmp/backup' + tiup br restore full -f 'foo*.*' -f 'bar*.*' -s 'local:///tmp/backup' ``` @@ -33,7 +33,7 @@ Table filters can be applied to the tools using multiple `-f` or `--filter` comm * [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview): ```shell - ./dumpling -f 'foo*.*' -f 'bar*.*' -P 3306 -o /tmp/data/ + tiup dumpling -f 'foo*.*' -f 'bar*.*' -P 3306 -o /tmp/data/ ``` @@ -41,7 +41,7 @@ Table filters can be applied to the tools using multiple `-f` or `--filter` comm * [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md): ```shell - ./tidb-lightning -f 'foo*.*' -f 'bar*.*' -d /tmp/data/ --backend tidb + tiup tidb-lightning -f 'foo*.*' -f 'bar*.*' -d /tmp/data/ --backend tidb ``` @@ -51,7 +51,7 @@ Table filters can be applied to the tools using multiple `-f` or `--filter` comm * [TiDB Lightning](https://docs.pingcap.com/tidb/stable/tidb-lightning-overview): ```shell - ./tidb-lightning -f 'foo*.*' -f 'bar*.*' -d /tmp/data/ --backend tidb + tiup tidb-lightning -f 'foo*.*' -f 'bar*.*' -d /tmp/data/ --backend tidb ``` @@ -139,8 +139,8 @@ employees.* the following two invocations are equivalent: ```bash -./dumpling -f '@config/filter.txt' -./dumpling -f 'employees.*' -f '*.WorkOrder' +tiup dumpling -f '@config/filter.txt' +tiup dumpling -f 'employees.*' -f '*.WorkOrder' ``` A filter file cannot further import another file. @@ -235,10 +235,10 @@ To build a block list, an explicit `*.*` must be used as the first rule, otherwi ```bash # every table will be filtered out -./dumpling -f '!*.Password' +tiup dumpling -f '!*.Password' # only the "Password" table is filtered out, the rest are included. -./dumpling -f '*.*' -f '!*.Password' +tiup dumpling -f '*.*' -f '!*.Password' ``` In a filter list, if a table name matches multiple patterns, the last match decides the outcome. For instance: diff --git a/ticdc/deploy-ticdc.md b/ticdc/deploy-ticdc.md index 2fa74b732bb7f..98c6cea03a291 100644 --- a/ticdc/deploy-ticdc.md +++ b/ticdc/deploy-ticdc.md @@ -95,7 +95,7 @@ tiup cluster upgrade --transfer-timeout 600 > **Note:** > -> In the preceding command, you need to replace `` and `` with the actual cluster name and cluster version. For example, the version can be v8.0.0. +> In the preceding command, you need to replace `` and `` with the actual cluster name and cluster version. For example, the version can be v8.1.0. ### Upgrade cautions @@ -152,7 +152,7 @@ See [Enable TLS Between TiDB Components](/enable-tls-between-components.md). ## View TiCDC status using the command-line tool -Run the following command to view the TiCDC cluster status. Note that you need to replace `v` with the TiCDC cluster version, such as `v8.0.0`: +Run the following command to view the TiCDC cluster status. Note that you need to replace `v` with the TiCDC cluster version, such as `v8.1.0`: ```shell tiup cdc:v cli capture list --server=http://10.0.10.25:8300 diff --git a/ticdc/monitor-ticdc.md b/ticdc/monitor-ticdc.md index 66feb10fcd071..08f355e63f8bf 100644 --- a/ticdc/monitor-ticdc.md +++ b/ticdc/monitor-ticdc.md @@ -97,8 +97,8 @@ The description of each metric in the **Events** panel is as follows: - Entry sorter sort duration percentile: The time (P95, P99, and P999) spent by TiCDC sorting events within one second - Entry sorter merge duration: The histogram of the time spent by TiCDC nodes merging sorted events - Entry sorter merge duration percentile: The time (P95, P99, and P999) spent by TiCDC merging sorted events within one second -- Mounter unmarshal duration: The histogram of the time spent by TiCDC nodes unmarshaling events -- Mounter unmarshal duration percentile: The time (P95, P99, and P999) spent by TiCDC unmarshaling events within one second +- Mounter unmarshal duration: The histogram of the time spent by TiCDC nodes unmarshalling events +- Mounter unmarshal duration percentile: The time (P95, P99, and P999) spent by TiCDC unmarshalling events within one second - KV client dispatch events/s: The number of events that the KV client module dispatches among the TiCDC nodes - KV client batch resolved size: The batch size of resolved timestamp messages that TiKV sends to TiCDC diff --git a/ticdc/ticdc-alert-rules.md b/ticdc/ticdc-alert-rules.md index 10e9f0235271a..855819fcc8ae9 100644 --- a/ticdc/ticdc-alert-rules.md +++ b/ticdc/ticdc-alert-rules.md @@ -16,7 +16,7 @@ For critical alerts, you need to pay close attention to abnormal monitoring metr - Alert rule: - `(time() - ticdc_owner_checkpoint_ts / 1000) > 600` + `ticdc_owner_checkpoint_ts_lag > 600` - Description: @@ -30,7 +30,7 @@ For critical alerts, you need to pay close attention to abnormal monitoring metr - Alert rule: - `(time() - ticdc_owner_resolved_ts / 1000) > 300` + `ticdc_owner_resolved_ts_lag > 300` - Description: diff --git a/ticdc/ticdc-behavior-change.md b/ticdc/ticdc-behavior-change.md index 887ff14aa49f4..317b971071531 100644 --- a/ticdc/ticdc-behavior-change.md +++ b/ticdc/ticdc-behavior-change.md @@ -5,11 +5,11 @@ summary: Introduce the behavior changes of TiCDC changefeed, including the reaso # TiCDC Behavior Changes -## Split update events into delete and insert events +## Split `UPDATE` events into `DELETE` and `INSERT` events -### Transactions containing a single update change +### Transactions containing a single `UPDATE` change -Starting from v6.5.3, v7.1.1, and v7.2.0, when using a non-MySQL sink, for transactions that only contain a single update change, if the primary key or non-null unique index value is modified in the update event, TiCDC splits this event into delete and insert events. For more information, see GitHub issue [#9086](https://github.com/pingcap/tiflow/issues/9086). +Starting from v6.5.3, v7.1.1, and v7.2.0, when using a non-MySQL sink, for transactions that only contain a single update change, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. For more information, see GitHub issue [#9086](https://github.com/pingcap/tiflow/issues/9086). This change primarily addresses the following issues: @@ -24,16 +24,16 @@ INSERT INTO t VALUES (1, 1); UPDATE t SET a = 2 WHERE a = 1; ``` -In this example, the primary key `a` is updated from `1` to `2`. If the update event is not split: +In this example, the primary key `a` is updated from `1` to `2`. If the `UPDATE` event is not split: * When using the CSV and AVRO protocols, the consumer only obtains the new value `a = 2` and cannot obtain the old value `a = 1`. This might cause the downstream consumer to only insert the new value `2` without deleting the old value `1`. -* When using the index value dispatcher, the event for inserting `(1, 1)` might be sent to Partition 0, and the update event `(2, 1)` might be sent to Partition 1. If the consumption progress of Partition 1 is faster than that of Partition 0, an error might occur due to the absence of corresponding data in the downstream. Therefore, TiCDC splits the update event into delete and insert events. The event for deleting `(1, 1)` is sent to Partition 0, and the event for writing `(2, 1)` is sent to Partition 1, ensuring that the events are consumed successfully regardless of the progress of the consumer. +* When using the index value dispatcher, the event for inserting `(1, 1)` might be sent to Partition 0, and the `UPDATE` event `(2, 1)` might be sent to Partition 1. If the consumption progress of Partition 1 is faster than that of Partition 0, an error might occur due to the absence of corresponding data in the downstream. Therefore, TiCDC splits the `UPDATE` event into `DELETE` and `INSERT` events. The event for deleting `(1, 1)` is sent to Partition 0, and the event for writing `(2, 1)` is sent to Partition 1, ensuring that the events are consumed successfully regardless of the progress of the consumer. -### Transactions containing multiple update changes +### Transactions containing multiple `UPDATE` changes -Starting from v6.5.4, v7.1.2, and v7.4.0, for transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits the event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see GitHub issue [#9430](https://github.com/pingcap/tiflow/issues/9430). +Starting from v6.5.4, v7.1.2, and v7.4.0, for transactions containing multiple changes, if the primary key or non-null unique index value is modified in the `UPDATE` event, TiCDC splits the event into `DELETE` and `INSERT` events and ensures that all events follow the sequence of `DELETE` events preceding `INSERT` events. For more information, see GitHub issue [#9430](https://github.com/pingcap/tiflow/issues/9430). -This change primarily addresses the potential issue of primary key conflicts when using the MySQL sink to directly write these two events to the downstream, leading to changefeed errors. +This change primarily addresses the potential issue of primary key or unique key conflicts when using the MySQL sink to directly write these two events to the downstream, leading to changefeed errors. When using the Kafka sink or other sinks, you might encounter the same error if the consumer writes messages to a relational database or performs similar operation. Take the following SQL as an example: @@ -43,12 +43,75 @@ INSERT INTO t VALUES (1, 1); INSERT INTO t VALUES (2, 2); BEGIN; -UPDATE t SET a = 1 WHERE a = 3; -UPDATE t SET a = 2 WHERE a = 1; -UPDATE t SET a = 3 WHERE a = 2; +UPDATE t SET a = 3 WHERE a = 1; +UPDATE t SET a = 1 WHERE a = 2; +UPDATE t SET a = 2 WHERE a = 3; COMMIT; ``` -In this example, by executing three SQL statements to swap the primary keys of two rows, TiCDC only receives two update change events, that is, changing the primary key `a` from `1` to `2` and changing the primary key `a` from `2` to `1`. If the MySQL sink directly writes these two update events to the downstream, a primary key conflict might occur, leading to changefeed errors. +In this example, by executing three SQL statements to swap the primary keys of two rows, TiCDC only receives two update change events, that is, changing the primary key `a` from `1` to `2` and changing the primary key `a` from `2` to `1`. If the MySQL sink directly writes these two `UPDATE` events to the downstream, a primary key conflict might occur, leading to changefeed errors. Therefore, TiCDC splits these two events into four events, that is, deleting records `(1, 1)` and `(2, 2)` and writing records `(2, 1)` and `(1, 2)`. + +### MySQL sink + +Starting from v8.1.0, when using the MySQL sink, TiCDC fetches the current timestamp `thresholdTs` from PD at startup and decides whether to split `UPDATE` events based on the value of this timestamp: + +- For transactions containing one or multiple `UPDATE` changes, if the primary key or non-null unique index value is modified in an `UPDATE` event and the transaction `commitTS` is less than `thresholdTs`, TiCDC splits the `UPDATE` event into a `DELETE` event and an `INSERT` event before writing them to the Sorter module. +- For `UPDATE` events with the transaction `commitTS` greater than or equal to `thresholdTs`, TiCDC does not split them. For more information, see GitHub issue [#10918](https://github.com/pingcap/tiflow/issues/10918). + +This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. + +Take the following SQL statements as an example: + +```sql +CREATE TABLE t (a INT PRIMARY KEY, b INT); +INSERT INTO t VALUES (1, 1); +INSERT INTO t VALUES (2, 2); + +BEGIN; +UPDATE t SET a = 3 WHERE a = 2; +UPDATE t SET a = 2 WHERE a = 1; +COMMIT; +``` + +In this example, the two `UPDATE` statements within the transaction have a sequential dependency on execution. The primary key `a` is changed from `2` to `3`, and then the primary key `a` is changed from `1` to `2`. After this transaction is executed, the records in the upstream database are `(2, 1)` and `(3, 2)`. + +However, the order of `UPDATE` events received by TiCDC might differ from the actual execution order of the upstream transaction. For example: + +```sql +UPDATE t SET a = 2 WHERE a = 1; +UPDATE t SET a = 3 WHERE a = 2; +``` + +- Before this behavior change, TiCDC writes these `UPDATE` events to the Sorter module and then splits them into `DELETE` and `INSERT` events. After the split, the actual execution order of these events in the downstream is as follows: + + ```sql + BEGIN; + DELETE FROM t WHERE a = 1; + REPLACE INTO t VALUES (2, 1); + DELETE FROM t WHERE a = 2; + REPLACE INTO t VALUES (3, 2); + COMMIT; + ``` + + After the downstream executes the transaction, the records in the database are `(3, 2)`, which are different from the records in the upstream database (`(2, 1)` and `(3, 2)`), indicating a data inconsistency issue. + +- After this behavior change, if the transaction `commitTS` is less than the `thresholdTs` obtained by TiCDC at startup, TiCDC splits these `UPDATE` events into `DELETE` and `INSERT` events before writing them to the Sorter module. After the sorting by the Sorter module, the actual execution order of these events in the downstream is as follows: + + ```sql + BEGIN; + DELETE FROM t WHERE a = 1; + DELETE FROM t WHERE a = 2; + REPLACE INTO t VALUES (2, 1); + REPLACE INTO t VALUES (3, 2); + COMMIT; + ``` + + After the downstream executes the transaction, the records in the downstream database are the same as those in the upstream database, which are `(2, 1)` and `(3, 2)`, ensuring data consistency. + +As you can see from the preceding example, splitting the `UPDATE` event into `DELETE` and `INSERT` events before writing them to the Sorter module ensures that all `DELETE` events are executed before `INSERT` events after the split, thereby maintaining data consistency regardless of the order of `UPDATE` events received by TiCDC. + +> **Note:** +> +> After this behavior change, when using the MySQL sink, TiCDC does not split the `UPDATE` event in most cases. Consequently, there might be primary key or unique key conflicts during changefeed runtime, causing the changefeed to restart automatically. After the restart, TiCDC will split the conflicting `UPDATE` events into `DELETE` and `INSERT` events before writing them to the Sorter module. This ensures that all events within the same transaction are correctly ordered, with all `DELETE` events preceding `INSERT` events, thus correctly completing data replication. \ No newline at end of file diff --git a/ticdc/ticdc-changefeed-config.md b/ticdc/ticdc-changefeed-config.md index 3273c7e3875f9..c0002e7013ed2 100644 --- a/ticdc/ticdc-changefeed-config.md +++ b/ticdc/ticdc-changefeed-config.md @@ -16,7 +16,7 @@ cdc cli changefeed create --server=http://10.0.10.25:8300 --sink-uri="mysql://ro ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2024-03-28T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.0.0"} +Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2024-05-24T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.1.0"} ``` - `--changefeed-id`: The ID of the replication task. The format must match the `^[a-zA-Z0-9]+(\-[a-zA-Z0-9]+)*$` regular expression. If this ID is not specified, TiCDC automatically generates a UUID (the version 4 format) as the ID. @@ -68,6 +68,12 @@ case-sensitive = false # The default value is the same as the default SQL mode of TiDB. # sql-mode = "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" +# The duration for which the changefeed is allowed to automatically retry when internal errors or exceptions occur. The default value is 30 minutes. +# The changefeed enters the failed state if internal errors or exceptions occur in the changefeed and persist longer than the duration set by this parameter. +# When the changefeed is in the failed state, you need to restart the changefeed manually for recovery. +# The format of this parameter is "h m s", for example, "1h30m30s". +changefeed-error-stuck-duration = "30m" + [mounter] # The number of threads with which the mounter decodes KV data. The default value is 16. # worker-num = 16 @@ -198,6 +204,8 @@ enable-partition-separator = true # output-handle-key = false # Whether to output the value before the row data changes. The default value is false. # When it is enabled, the UPDATE event will output two rows of data: the first row is a DELETE event that outputs the data before the change; the second row is an INSERT event that outputs the changed data. +# When it is enabled (setting it to true), the "is-update" column will be added before the column with data changes. This added column is used to identify whether the data change of the current row comes from the UPDATE event or the original INSERT/DELETE event. +# If the data change of the current row comes from the UPDATE event, the value of the "is-update" column is true. Otherwise it is false. # output-old-value = false # Starting from v8.0.0, TiCDC supports the Simple message encoding protocol. The following are the configuration parameters for the Simple protocol. @@ -222,6 +230,14 @@ enable-partition-separator = true # The default value is "json". # encoding-format = "json" +[sink.open] +# Whether to output the value before the row data changes. The default value is true. When it is disabled, the UPDATE event does not output the "p" field. +# output-old-value = true + +[sink.debezium] +# Whether to output the value before the row data changes. The default value is true. When it is disabled, the UPDATE event does not output the "before" field. +# output-old-value = true + # Specifies the replication consistency configurations for a changefeed when using the redo log. For more information, see https://docs.pingcap.com/tidb/stable/ticdc-sink-to-mysql#eventually-consistent-replication-in-disaster-scenarios. # Note: The consistency-related configuration items only take effect when the downstream is a database and the redo log feature is enabled. [consistent] diff --git a/ticdc/ticdc-changefeed-overview.md b/ticdc/ticdc-changefeed-overview.md index 30517454d1a32..2aadb3368b913 100644 --- a/ticdc/ticdc-changefeed-overview.md +++ b/ticdc/ticdc-changefeed-overview.md @@ -17,7 +17,7 @@ The states in the preceding state transfer diagram are described as follows: - `Normal`: The replication task runs normally and the checkpoint-ts proceeds normally. A changefeed in this state blocks GC to advance. - `Stopped`: The replication task is stopped, because the user manually pauses the changefeed. The changefeed in this state blocks GC operations. -- `Warning`: The replication task returns an error. The replication cannot continue due to some recoverable errors. The changefeed in this state keeps trying to resume until the state transfers to `Normal`. The maximum retry time is 30 minutes. If it exceeds this time, the changefeed enters a failed state. The changefeed in this state blocks GC operations. +- `Warning`: The replication task returns an error. The replication cannot continue due to some recoverable errors. The changefeed in this state keeps trying to resume until the state transfers to `Normal`. The default retry time is 30 minutes (which can be adjusted by [`changefeed-error-stuck-duration`](/ticdc/ticdc-changefeed-config.md)). If it exceeds this time, the changefeed enters a failed state. The changefeed in this state blocks GC operations. - `Finished`: The replication task is finished and has reached the preset `TargetTs`. The changefeed in this state does not block GC operations. - `Failed`: The replication task fails. The changefeed in this state does not keep trying to resume. To give you enough time to handle the failure, the changefeed in this state blocks GC operations. The duration of the blockage is specified by the `gc-ttl` parameter, with a default value of 24 hours. If the underlying issue is resolved within this duration, you can manually resume the changefeed. Otherwise, if the changefeed remains in this state beyond the `gc-ttl` duration, the replication task cannot resume and cannot be recovered. @@ -44,4 +44,4 @@ You can manage a TiCDC cluster and its replication tasks using the command-line You can also use the HTTP interface (the TiCDC OpenAPI feature) to manage a TiCDC cluster and its replication tasks. For details, see [TiCDC OpenAPI](/ticdc/ticdc-open-api.md). -If your TiCDC is deployed using TiUP, you can start `cdc cli` by running the `tiup cdc:v cli` command. Replace `v` with the TiCDC cluster version, such as `v8.0.0`. You can also run `cdc cli` directly. +If your TiCDC is deployed using TiUP, you can start `cdc cli` by running the `tiup cdc:v cli` command. Replace `v` with the TiCDC cluster version, such as `v8.1.0`. You can also run `cdc cli` directly. diff --git a/ticdc/ticdc-client-authentication.md b/ticdc/ticdc-client-authentication.md new file mode 100644 index 0000000000000..c3e41fb740e3a --- /dev/null +++ b/ticdc/ticdc-client-authentication.md @@ -0,0 +1,122 @@ +--- +title: TiCDC Client Authentication +summary: Introduce how to perform TiCDC client authentication using the command-line tool or OpenAPI. +--- + +# TiCDC Client Authentication + +Starting from v8.1.0, TiCDC supports client authentication using Mutual Transport Layer Security (mTLS) or TiDB username and password. + +- mTLS authentication provides security control at the transport layer, enabling TiCDC to verify the client identity. +- TiDB username and password authentication provides security control at the application layer, ensuring that only authorized users can log in through the TiCDC node. + +These two authentication methods can be used either independently or in combination to meet different scenarios and security requirements. + +> **Note:** +> +> To ensure the security of network access, it is strongly recommended to use TiCDC client authentication only when [TLS is enabled](/enable-tls-between-clients-and-servers.md). If TLS is not enabled, the username and password are transmitted as plaintext over the network, which can lead to serious credential leaks. + +## Use mTLS for client authentication + +1. In the TiCDC server, configure the `security.mtls` parameter as `true` to enable mTLS authentication: + + ```toml + [security] + # This parameter controls whether to enable the TLS client authentication. The default value is false. + mtls = true + ``` + +2. Configure the client certificate. + + +
+ + When using the [TiCDC command-line tool](/ticdc/ticdc-manage-changefeed.md), you can specify the client certificate using the following methods. TiCDC will attempt to read the client certificate in the following order: + + 1. Specify the certificate and private key using the command-line parameters `--cert` and `--key`. If the server uses a self-signed certificate, you also need to specify the trusted CA certificate using the `--ca` parameter. + + ```bash + cdc cli changefeed list --cert client.crt --key client.key --ca ca.crt + ``` + + 2. Specify the paths to the certificate, private key, and CA certificate using the environment variables `TICDC_CERT_PATH`, `TICDC_KEY_PATH`, and `TICDC_CA_PATH`. + + ```bash + export TICDC_CERT_PATH=client.crt + export TICDC_KEY_PATH=client.key + export TICDC_CA_PATH=ca.crt + ``` + + 3. Specify the certificate using the shared credential file `~/.ticdc/credentials`. You can modify the configuration using the `cdc cli configure-credentials` command. + +
+ +
+ + When using [TiCDC OpenAPI](/ticdc/ticdc-open-api-v2.md), you can specify the client certificate and private key using `--cert` and `--key`. If the server uses a self-signed certificate, you also need to specify the trusted CA certificate using the `--cacert` parameter. For example: + + ```bash + curl -X GET http://127.0.0.1:8300/api/v2/status --cert client.crt --key client.key --cacert ca.crt + ``` + +
+ + +## Use TiDB username and password for client authentication + +1. [Create a user](/sql-statements/sql-statement-create-user.md) in TiDB and grant the user permission to log in from the TiCDC node. + + ```sql + CREATE USER 'test'@'ticdc_ip_address' IDENTIFIED BY 'password'; + ``` + +2. In the TiCDC server, configure `security.client-user-required` and `security.client-allowed-user` to enable username and password authentication: + + ```toml + [security] + # This parameter controls whether to use username and password for client authentication. The default value is false. + client-user-required = true + # This parameter lists the usernames that are allowed for client authentication. Authentication requests with usernames not in this list will be rejected. The default value is null. + client-allowed-user = ["test"] + ``` + +3. Specify the username and password of the user created in step 1. + + +
+ + When using the [TiCDC command-line tool](/ticdc/ticdc-manage-changefeed.md), you can specify the username and password using the following methods. TiCDC will attempt to read the client certificate in the following order: + + 1. Specify the username and password using the command-line parameters `--user` and `--password`: + + ```bash + cdc cli changefeed list --user test --password password + ``` + + 2. Specify the username using the command-line parameter `--user`. Then, enter the password in the terminal: + + ```bash + cdc cli changefeed list --user test + ``` + + 3. Specify the username and password using the environment variables `TICDC_USER` and `TICDC_PASSWORD`: + + ```bash + export TICDC_USER=test + export TICDC_PASSWORD=password + ``` + + 4. Specify the username and password using the shared credential file `~/.ticdc/credentials`. You can modify the configuration using the `cdc cli configure-credentials` command. + +
+ +
+ + When using [TiCDC OpenAPI](/ticdc/ticdc-open-api-v2.md), you can specify the username and password using `--user :`. For example: + + ```bash + curl -X GET http://127.0.0.1:8300/api/v2/status --user test:password + ``` + +
+ diff --git a/ticdc/ticdc-csv.md b/ticdc/ticdc-csv.md index a55c78b89e693..12278eec59ca0 100644 --- a/ticdc/ticdc-csv.md +++ b/ticdc/ticdc-csv.md @@ -27,6 +27,7 @@ delimiter = ',' # Before v7.6.0, you can only set the delimiter to a single char quote = '"' null = '\N' include-commit-ts = true +output-old-value = false ``` ## Transactional constraints @@ -47,7 +48,8 @@ In the CSV file, each column is defined as follows: - Column 2: Table name. - Column 3: Schema name. - Column 4: The `commit-ts` of the source transaction. This column is optional. -- Column 5 to the last column: One or more columns that represent data to be changed. +- Column 5: The `is-update` column only exists when the value of `output-old-value` is true, which is used to identify whether the row data change comes from the UPDATE event (the value of the column is true) or the INSERT/DELETE event (the value is false). +- Column 6 to the last column: One or more columns with data changes. Assume that table `hr.employee` is defined as follows: @@ -61,7 +63,7 @@ CREATE TABLE `employee` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; ``` -The DML events of this table are stored in the CSV format as follows: +When `include-commit-ts = true` and `output-old-value = false`, the DML events of this table are stored in the CSV format as follows: ```shell "I","employee","hr",433305438660591626,101,"Smith","Bob","2014-06-04","New York" @@ -71,6 +73,18 @@ The DML events of this table are stored in the CSV format as follows: "U","employee","hr",433305438660591630,102,"Alex","Alice","2018-06-15","Beijing" ``` +When `include-commit-ts = true` and `output-old-value = true`, the DML events of this table are stored in the CSV format as follows: + +``` +"I","employee","hr",433305438660591626,false,101,"Smith","Bob","2014-06-04","New York" +"D","employee","hr",433305438660591627,true,101,"Smith","Bob","2015-10-08","Shanghai" +"I","employee","hr",433305438660591627,true,101,"Smith","Bob","2015-10-08","Los Angeles" +"D","employee","hr",433305438660591629,false,101,"Smith","Bob","2017-03-13","Dallas" +"I","employee","hr",433305438660591630,false,102,"Alex","Alice","2017-03-14","Shanghai" +"D","employee","hr",433305438660591630,true,102,"Alex","Alice","2017-03-14","Beijing" +"I","employee","hr",433305438660591630,true,102,"Alex","Alice","2018-06-15","Beijing" +``` + ## Data type mapping | MySQL type | CSV type | Example | Description | diff --git a/ticdc/ticdc-ddl.md b/ticdc/ticdc-ddl.md index 1d05faf4d2c58..ef396d4f55692 100644 --- a/ticdc/ticdc-ddl.md +++ b/ticdc/ticdc-ddl.md @@ -102,25 +102,6 @@ TiCDC processes this type of DDL as follows: | `RENAME TABLE test.t1 TO ignore.t1, test.t2 TO test.t22;` | Report an error | The new database name `ignore` does not match the filter rule. | | `RENAME TABLE test.t1 TO test.t4, test.t3 TO test.t1, test.t4 TO test.t3;` | Report an error | The `RENAME TABLE` DDL swaps the names of `test.t1` and `test.t3` in one DDL statement, which TiCDC cannot handle correctly. In this case, refer to the error message for handling. | -### SQL mode +### DDL statement considerations -By default, TiCDC uses the default SQL mode of TiDB to parse DDL statements. If your upstream TiDB cluster uses a non-default SQL mode, you must specify the SQL mode in the TiCDC configuration file. Otherwise, TiCDC might fail to parse DDL statements correctly. For more information about TiDB SQL mode, see [SQL Mode](/sql-mode.md). - -For example, if the upstream TiDB cluster uses the `ANSI_QUOTES` mode, you must specify the SQL mode in the changefeed configuration file as follows: - -```toml -# In the value, "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" is the default SQL mode of TiDB. -# "ANSI_QUOTES" is the SQL mode added to your upstream TiDB cluster. - -sql-mode = "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,ANSI_QUOTES" -``` - -If the SQL mode is not configured, TiCDC might fail to parse some DDL statements correctly. For example: - -```sql -CREATE TABLE "t1" ("a" int PRIMARY KEY); -``` - -Because in the default SQL mode of TiDB, double quotation marks are treated as strings rather than identifiers, TiCDC fails to parse the DDL statement correctly. - -Therefore, when creating a replication task, it is recommended that you specify the SQL mode used by the upstream TiDB cluster in the configuration file. +When executing cross-database DDL statements (such as `CREATE TABLE db1.t1 LIKE t2`) in the upstream, it is recommended that you explicitly specify all relevant database names in DDL statements (such as `CREATE TABLE db1.t1 LIKE db2.t2`). Otherwise, cross-database DDL statements might not be executed correctly in the downstream due to the lack of database name information. diff --git a/ticdc/ticdc-faq.md b/ticdc/ticdc-faq.md index 058dd245998ad..474ca3616cb7e 100644 --- a/ticdc/ticdc-faq.md +++ b/ticdc/ticdc-faq.md @@ -136,7 +136,7 @@ For more information, refer to [TiCDC changefeed configurations](/ticdc/ticdc-ch ## When TiCDC replicates data to Kafka, can I control the maximum size of a single message in TiDB? -When `protocol` is set to `avro` or `canal-json`, messages are sent per row change. A single Kafka message contains only one row change and is generally no larger than Kafka's limit. Therefore, there is no need to limit the size of a single message. If the size of a single Kafka message does exceed Kakfa's limit, refer to [Why does the latency from TiCDC to Kafka become higher and higher?](/ticdc/ticdc-faq.md#why-does-the-latency-from-ticdc-to-kafka-become-higher-and-higher). +When `protocol` is set to `avro` or `canal-json`, messages are sent per row change. A single Kafka message contains only one row change and is generally no larger than Kafka's limit. Therefore, there is no need to limit the size of a single message. If the size of a single Kafka message does exceed Kafka's limit, refer to [Why does the latency from TiCDC to Kafka become higher and higher?](/ticdc/ticdc-faq.md#why-does-the-latency-from-ticdc-to-kafka-become-higher-and-higher). When `protocol` is set to `open-protocol`, messages are sent in batches. Therefore, one Kafka message might be excessively large. To avoid this situation, you can configure the `max-message-bytes` parameter to control the maximum size of data sent to the Kafka broker each time (optional, `10MB` by default). You can also configure the `max-batch-size` parameter (optional, `16` by default) to specify the maximum number of change records in each Kafka message. diff --git a/ticdc/ticdc-manage-changefeed.md b/ticdc/ticdc-manage-changefeed.md index 7c4695d4ed9fc..47de10c21e33c 100644 --- a/ticdc/ticdc-manage-changefeed.md +++ b/ticdc/ticdc-manage-changefeed.md @@ -19,7 +19,7 @@ cdc cli changefeed create --server=http://10.0.10.25:8300 --sink-uri="mysql://ro ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2024-03-28T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.0.0"} +Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2024-05-24T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.1.0"} ``` ## Query the replication task list @@ -136,7 +136,7 @@ cdc cli changefeed query --server=http://10.0.10.25:8300 --changefeed-id=simple- } ``` -In the preceding command and result: +In the preceding command and result: - `info` is the replication configuration of the queried changefeed. - `status` is the replication state of the queried changefeed. @@ -144,7 +144,7 @@ In the preceding command and result: - `checkpoint-ts`: The largest transaction `TS` in the current `changefeed`. Note that this `TS` has been successfully written to the downstream. - `admin-job-type`: The status of a changefeed: - `0`: The state is normal. - - `1`: The task is paused. When the task is paused, all replicated `processor`s exit. The configuration and the replication status of the task are retained, so you can resume the task from `checkpiont-ts`. + - `1`: The task is paused. When the task is paused, all replicated `processor`s exit. The configuration and the replication status of the task are retained, so you can resume the task from `checkpoint-ts`. - `2`: The task is resumed. The replication task resumes from `checkpoint-ts`. - `3`: The task is removed. When the task is removed, all replicated `processor`s are ended, and the configuration information of the replication task is cleared up. Only the replication status is retained for later queries. - `task-status` indicates the state of each replication sub-task in the queried changefeed. diff --git a/ticdc/ticdc-open-api-v2.md b/ticdc/ticdc-open-api-v2.md index dd8d281b6ee54..b47e09e65c4f1 100644 --- a/ticdc/ticdc-open-api-v2.md +++ b/ticdc/ticdc-open-api-v2.md @@ -92,7 +92,7 @@ curl -X GET http://127.0.0.1:8300/api/v2/status ```json { - "version": "v8.0.0", + "version": "v8.1.0", "git_hash": "10413bded1bdb2850aa6d7b94eb375102e9c44dc", "id": "d2912e63-3349-447c-90ba-72a4e04b5e9e", "pid": 1447, @@ -316,6 +316,8 @@ The `sink` parameters are described as follows: | `transaction_atomicity` | `STRING` type. The atomicity level of the transaction. (Optional) | | `only_output_updated_columns` | `BOOLEAN` type. For MQ sinks using the `canal-json` or `open-protocol` protocol, you can specify whether only output the modified columns. The default value is `false`. (Optional) | | `cloud_storage_config` | The storage sink configuration. (Optional) | +| `open` | The Open Protocol configuration. (Optional) | +| `debezium` | The Debezium Protocol configuration. (Optional) | `sink.column_selectors` is an array. The parameters are described as follows: @@ -349,7 +351,7 @@ The `sink.csv` parameters are described as follows: | `partition` | `STRING` type. The target partition for dispatching events. | | `topic` | `STRING` type. The target topic for dispatching events. | -`sink.cloud_storage_config` parameters are described as follows: +`sink.cloud_storage_config` parameters are described as follows: | Parameter name | Description | |:-----------------|:---------------------------------------| @@ -360,6 +362,18 @@ The `sink.csv` parameters are described as follows: | `file_cleanup_cron_spec` | `STRING` type. The running cycle of the scheduled cleanup task, compatible with the crontab configuration, with a format of ` `. | | `flush_concurrency` | `INT` type. The concurrency for uploading a single file. | +`sink.open` parameters are described as follows: + +| Parameter name | Description | +|:-------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `output_old_value` | `BOOLEAN` type. It controls whether to output the value before the row data changes. The default value is `true`. When it is disabled, the UPDATE event does not output the "p" field. | + +`sink.debezium` parameters are described as follows: + +| Parameter name | Description | +|:-------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `output_old_value` | `BOOLEAN` type. It controls whether to output the value before the row data changes. The default value is true. When it is disabled, the UPDATE event does not output the "before" field. | + ### Example The following request creates a replication task with an ID of `test5` and `sink_uri` of `blackhome://`. diff --git a/ticdc/ticdc-open-api.md b/ticdc/ticdc-open-api.md index 331528f7d31e9..789baa924a791 100644 --- a/ticdc/ticdc-open-api.md +++ b/ticdc/ticdc-open-api.md @@ -150,7 +150,7 @@ The configuration parameters of sink are as follows: {"matcher":["test1.*", "test2.*"], "dispatcher":"ts"}, {"matcher":["test3.*", "test4.*"], "dispatcher":"index-value"} ], - "protocal":"canal-json" + "protocol":"canal-json" } ``` diff --git a/ticdc/ticdc-overview.md b/ticdc/ticdc-overview.md index c9ad339ac7795..5960e4c8190e3 100644 --- a/ticdc/ticdc-overview.md +++ b/ticdc/ticdc-overview.md @@ -87,6 +87,61 @@ As shown in the architecture diagram, TiCDC supports replicating data to TiDB, M - To ensure eventual consistency when using TiCDC for disaster recovery, you need to configure [redo log](/ticdc/ticdc-sink-to-mysql.md#eventually-consistent-replication-in-disaster-scenarios) and ensure that the storage system where the redo log is written can be read normally when a disaster occurs in the upstream. +## Implementation of processing data changes + +This section mainly describes how TiCDC processes data changes generated by upstream DML operations. + +For data changes generated by upstream DDL operations, TiCDC obtains the complete DDL SQL statement, converts it into the corresponding format based on the sink type of the downstream, and sends it to the downstream. This section does not elaborate on this. + +> **Note:** +> +> The logic of how TiCDC processes data changes might be adjusted in subsequent versions. + +MySQL binlog directly records all DML SQL statements executed in the upstream. Unlike MySQL, TiCDC listens to the real-time information of each Region Raft Log in the upstream TiKV, and generates data change information based on the difference between the data before and after each transaction, which corresponds to multiple SQL statements. TiCDC only guarantees that the output change events are equivalent to the changes in the upstream TiDB, and does not guarantee that it can accurately restore the SQL statements that caused the data changes in the upstream TiDB. + +Data change information includes the data change types and the data values before and after the change. The difference between the data before and after the transaction can result in three types of events: + +1. The `DELETE` event: corresponds to a `DELETE` type data change message, which contains the data before the change. + +2. The `INSERT` event: corresponds to a `PUT` type data change message, which contains the data after the change. + +3. The `UPDATE` event: corresponds to a `PUT` type data change message, which contains the data both before and after the change. + +Based on the data change information, TiCDC generates data in the appropriate formats for different downstream types, and sends the data to the downstream. For example, it generates data in formats such as Canal-JSON and Avro, and writes the data to Kafka, or converts the data back into SQL statements and sends them to the downstream MySQL or TiDB. + +Currently, when TiCDC adapts data change information for the corresponding protocol, for specific `UPDATE` events, it might split them into one `DELETE` event and one `INSERT` event. For more information, see [Split update events into delete and insert events](/ticdc/ticdc-behavior-change.md#split-update-events-into-delete-and-insert-events). + +When the downstream is MySQL or TiDB, TiCDC cannot guarantee that the SQL statements written to the downstream are exactly the same as the SQL statements executed in the upstream. This is because TiCDC does not directly obtain the original DML statements executed in the upstream, but generates SQL statements based on the data change information. However, TiCDC ensures the consistency of the final results. + +For example, the following SQL statement is executed in the upstream: + +```sql +Create Table t1 (A int Primary Key, B int); + +BEGIN; +Insert Into t1 values(1,2); +Insert Into t1 values(2,2); +Insert Into t1 values(3,3); +Commit; + +Update t1 set b = 4 where b = 2; +``` + +TiCDC generates the following two SQL statements based on the data change information, and writes them to the downstream: + +```sql +INSERT INTO `test.t1` (`A`,`B`) VALUES (1,2),(2,2),(3,3); +UPDATE `test`.`t1` +SET `A` = CASE + WHEN `A` = 1 THEN 1 + WHEN `A` = 2 THEN 2 +END, `B` = CASE + WHEN `A` = 1 THEN 4 + WHEN `A` = 2 THEN 4 +END +WHERE `A` = 1 OR `A` = 2; +``` + ## Unsupported scenarios Currently, the following scenarios are not supported: diff --git a/ticdc/ticdc-server-config.md b/ticdc/ticdc-server-config.md index 0a0aadbb971b9..23b7506014af6 100644 --- a/ticdc/ticdc-server-config.md +++ b/ticdc/ticdc-server-config.md @@ -47,6 +47,12 @@ gc-tuner-memory-threshold = 0 ca-path = "" cert-path = "" key-path = "" + # This parameter controls whether to enable the TLS client authentication. The default value is false. + mtls = false + # This parameter controls whether to use username and password for client authentication. The default value is false. + client-user-required = false + # This parameter lists the usernames that are allowed for client authentication. Authentication requests with usernames not in this list will be rejected. The default value is null. + client-allowed-user = ["username_1", "username_2"] # The session duration between TiCDC and etcd services, measured in seconds. This parameter is optional and its default value is 10. capture-session-ttl = 10 # 10s diff --git a/ticdc/ticdc-sink-to-cloud-storage.md b/ticdc/ticdc-sink-to-cloud-storage.md index 15ca22c11cf9a..7a2f68496e9f9 100644 --- a/ticdc/ticdc-sink-to-cloud-storage.md +++ b/ticdc/ticdc-sink-to-cloud-storage.md @@ -24,7 +24,7 @@ cdc cli changefeed create \ The output is as follows: ```shell -Info: {"upstream_id":7171388873935111376,"namespace":"default","id":"simple-replication-task","sink_uri":"s3://logbucket/storage_test?protocol=canal-json","create_time":"2024-03-28T18:52:05.566016967+08:00","start_ts":437706850431664129,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.0.0"} +Info: {"upstream_id":7171388873935111376,"namespace":"default","id":"simple-replication-task","sink_uri":"s3://logbucket/storage_test?protocol=canal-json","create_time":"2024-05-24T18:52:05.566016967+08:00","start_ts":437706850431664129,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.1.0"} ``` - `--server`: The address of any TiCDC server in the TiCDC cluster. @@ -59,24 +59,83 @@ For `[query_parameters]` in the URI, the following parameters can be configured: ### Configure sink URI for external storage +When storing data in a cloud storage system, you need to set different authentication parameters depending on the cloud service provider. This section describes the authentication methods when using Amazon S3, Google Cloud Storage (GCS), and Azure Blob Storage, and how to configure accounts to access the corresponding storage services. + + +
+ The following is an example configuration for Amazon S3: ```shell --sink-uri="s3://bucket/prefix?protocol=canal-json" ``` +Before replicating data, you need to set appropriate access permissions for the directory in Amazon S3: + +- Minimum permissions required by TiCDC: `s3:ListBucket`, `s3:PutObject`, and `s3:GetObject`. +- If the changefeed configuration item `sink.cloud-storage-config.flush-concurrency` is greater than 1, which means parallel uploading of single files is enabled, you need to additionally add permissions related to [ListParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html): + - `s3:AbortMultipartUpload` + - `s3:ListMultipartUploadParts` + - `s3:ListBucketMultipartUploads` + +If you have not created a replication data storage directory, refer to [Create a bucket](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-bucket.html) to create an S3 bucket in the specified region. If necessary, you can also create a folder in the bucket by referring to [Organize objects in the Amazon S3 console by using folders](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-folder.html). + +You can configure an account to access Amazon S3 in the following ways: + +- Method 1: Specify the access key + + If you specify an access key and a secret access key, authentication is performed according to them. In addition to specifying the key in the URI, the following methods are supported: + + - TiCDC reads the `$AWS_ACCESS_KEY_ID` and `$AWS_SECRET_ACCESS_KEY` environment variables. + - TiCDC reads the `$AWS_ACCESS_KEY` and `$AWS_SECRET_KEY` environment variables. + - TiCDC reads the shared credentials file in the path specified by the `$AWS_SHARED_CREDENTIALS_FILE` environment variable. + - TiCDC reads the shared credentials file in the `~/.aws/credentials` path. + +- Method 2: Access based on an IAM role + + Associate an [IAM role with configured permissions to access Amazon S3](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html) to the EC2 instance running the TiCDC server. After successful setup, TiCDC can directly access the corresponding directories in Amazon S3 without additional settings. + +
+
+ The following is an example configuration for GCS: ```shell --sink-uri="gcs://bucket/prefix?protocol=canal-json" ``` +You can configure the account used to access GCS by specifying an access key. Authentication is performed according to the specified `credentials-file`. In addition to specifying the key in the URI, the following methods are supported: + +- TiCDC reads the file in the path specified by the `$GOOGLE_APPLICATION_CREDENTIALS` environment variable. +- TiCDC reads the file `~/.config/gcloud/application_default_credentials.json`. +- TiCDC obtains credentials from the metadata server when the cluster is running in GCE or GAE. + +
+
+ The following is an example configuration for Azure Blob Storage: ```shell --sink-uri="azure://bucket/prefix?protocol=canal-json" ``` +You can configure an account to access Azure Blob Storage in the following ways: + +- Method 1: Specify a shared access signature + + If you configure `account-name` and `sas-token` in the URI, the storage account name and shared access signature token specified by this parameter are used. Because the shared access signature token has the `&` character, you need to encode it as `%26` before adding it to the URI. You can also directly encode the entire `sas-token` using percent-encoding. + +- Method 2: Specify the access key + + If you configure `account-name` and `account-key` in the URI, the storage account name and key specified by this parameter are used. In addition to specifying the key file in the URI, TiCDC can also read the key from the environment variable `$AZURE_STORAGE_KEY`. + +- Method 3: Use Azure AD to restore the backup + + Configure the environment variables `$AZURE_CLIENT_ID`, `$AZURE_TENANT_ID`, and `$AZURE_CLIENT_SECRET`. + +
+ + > **Tip:** > > For more information about the URI parameters of Amazon S3, GCS, and Azure Blob Storage in TiCDC, see [URI Formats of External Storage Services](/external-storage-uri.md). diff --git a/ticdc/ticdc-sink-to-kafka.md b/ticdc/ticdc-sink-to-kafka.md index 3fc45116afdd0..b329d11c9c8ea 100644 --- a/ticdc/ticdc-sink-to-kafka.md +++ b/ticdc/ticdc-sink-to-kafka.md @@ -36,9 +36,17 @@ Info: {"sink-uri":"kafka://127.0.0.1:9092/topic-name?protocol=canal-json&kafka-v Sink URI is used to specify the connection information of the TiCDC target system. The format is as follows: ```shell -[scheme]://[userinfo@][host]:[port][/path]?[query_parameters] +[scheme]://[host]:[port][/path]?[query_parameters] ``` +> **Tip:** +> +> If there are multiple hosts or ports for the downstream Kafka, you can configure multiple `[host]:[port]` in the sink URI. For example: +> +> ```shell +> [scheme]://[host]:[port],[host]:[port],[host]:[port][/path]?[query_parameters] +> ``` + Sample configuration: ```shell diff --git a/ticdc/ticdc-sink-to-pulsar.md b/ticdc/ticdc-sink-to-pulsar.md index e8d62d9484c42..a3568beb57731 100644 --- a/ticdc/ticdc-sink-to-pulsar.md +++ b/ticdc/ticdc-sink-to-pulsar.md @@ -23,7 +23,7 @@ cdc cli changefeed create \ Create changefeed successfully! ID: simple-replication-task -Info: {"upstream_id":7277814241002263370,"namespace":"default","id":"simple-replication-task","sink_uri":"pulsar://127.0.0.1:6650/consumer-test?protocol=canal-json","create_time":"2024-03-28T14:42:32.000904+08:00","start_ts":444203257406423044,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["pulsar_test.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["pulsar_test.*"],"partition":"","topic":"test_{schema}_{table}"}],"encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":30,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"use_file_backend":false},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"}},"state":"normal","creator_version":"v8.0.0","resolved_ts":444203257406423044,"checkpoint_ts":444203257406423044,"checkpoint_time":"2024-01-25 14:42:31.410"} +Info: {"upstream_id":7277814241002263370,"namespace":"default","id":"simple-replication-task","sink_uri":"pulsar://127.0.0.1:6650/consumer-test?protocol=canal-json","create_time":"2024-05-24T14:42:32.000904+08:00","start_ts":444203257406423044,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["pulsar_test.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["pulsar_test.*"],"partition":"","topic":"test_{schema}_{table}"}],"encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":30,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"use_file_backend":false},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"}},"state":"normal","creator_version":"v8.1.0","resolved_ts":444203257406423044,"checkpoint_ts":444203257406423044,"checkpoint_time":"2024-05-24 14:42:31.410"} ``` The meaning of each parameter is as follows: diff --git a/tidb-binlog/get-started-with-tidb-binlog.md b/tidb-binlog/get-started-with-tidb-binlog.md index b560d15684a65..b686f48aada4c 100644 --- a/tidb-binlog/get-started-with-tidb-binlog.md +++ b/tidb-binlog/get-started-with-tidb-binlog.md @@ -43,7 +43,7 @@ sudo yum install -y mariadb-server ``` ```bash -curl -L https://download.pingcap.org/tidb-community-server-v8.0.0-linux-amd64.tar.gz | tar xzf - +curl -L https://download.pingcap.org/tidb-community-server-v8.1.0-linux-amd64.tar.gz | tar xzf - cd tidb-latest-linux-amd64 ``` diff --git a/tidb-cloud/data-service-oas-with-nextjs.md b/tidb-cloud/data-service-oas-with-nextjs.md index d1d3b4b3e20b3..d2a9c5a741a3c 100644 --- a/tidb-cloud/data-service-oas-with-nextjs.md +++ b/tidb-cloud/data-service-oas-with-nextjs.md @@ -182,7 +182,7 @@ You can use the generated client code to develop your Next.js application. > **Note:** > - > If the linked clusters of your Data App are hosted in different regions, you wil see multiple items in the `servers` section of the downloaded OpenAPI Specification file. In this case, you also need to configure the endpoint path in the `config` object as follows: + > If the linked clusters of your Data App are hosted in different regions, you will see multiple items in the `servers` section of the downloaded OpenAPI Specification file. In this case, you also need to configure the endpoint path in the `config` object as follows: > > ```js > const config = new Configuration({ diff --git a/tidb-cloud/dev-guide-bi-looker-studio.md b/tidb-cloud/dev-guide-bi-looker-studio.md new file mode 100644 index 0000000000000..29a446a68f752 --- /dev/null +++ b/tidb-cloud/dev-guide-bi-looker-studio.md @@ -0,0 +1,137 @@ +--- +title: Connect to TiDB Serverless with Looker Studio +summary: Learn how to connect to TiDB Serverless using Looker Studio. +--- + +# Connect to TiDB Serverless with Looker Studio + +TiDB is a MySQL-compatible database, TiDB Serverless is a fully managed TiDB offering, and [Looker Studio](https://lookerstudio.google.com/) is a free web-based BI tool that can visualize data from various sources. + +In this tutorial, you can learn how to connect to your TiDB Serverless cluster with Looker Studio. + +> **Note:** +> +> Most steps in this tutorial work with TiDB Dedicated as well. However, for TiDB Dedicated, you need to note the following: +> +> - Import your dataset following [Import data from files to TiDB Cloud](/tidb-cloud/tidb-cloud-migration-overview.md#import-data-from-files-to-tidb-cloud). +> - Get the connection information for your cluster following [Connect to TiDB Dedicated](/tidb-cloud/connect-via-standard-connection.md). When connecting to TiDB Dedicated, you need to allow access from `142.251.74.0/23`. For more information about connections from Looker Studio, see [Looker Studio documentation](https://support.google.com/looker-studio/answer/7088031#zippy=%2Cin-this-article). + +## Prerequisites + +To complete this tutorial, you need: + +- A Google account +- A TiDB Serverless cluster + +**If you don't have a TiDB Serverless cluster, you can create one as follows:** + +- [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster) + +## Step 1. Import a dataset + +You can import the S&P 500 dataset provided in the interactive tutorial of TiDB Serverless. + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and click **?** in the lower-right corner. A **Help** dialog is displayed. + +2. In the dialog, click **Interactive Tutorials**, and then click **S&P 500 Analysis**. + +3. Select your TiDB Serverless cluster, and then click **Import Dataset** to import the S&P 500 dataset to your cluster. + +4. After the import status changes to **IMPORTED**, click **Exit Tutorial** to close this dialog. + +If you encounter any issues during import, you can cancel this import task as follows: + +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your TiDB Serverless cluster to go to its overview page. +2. In the left navigation pane, click **Import**. +3. Find the import task named **sp500-insight**, click **...** in the **Action** column, and then click **Cancel**. + +## Step 2. Get the connection information for your cluster + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. In the connection dialog, set **Connect With** to `General`, and then click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, use the original password or click **Reset Password** to generate a new one. + +4. Download the [CA cert](https://letsencrypt.org/certs/isrgrootx1.pem). + + > **Tip:** + > + > TiDB Serverless requires a secure TLS connection between the client and the cluster, so you need this CA cert for connection settings in Looker Studio. + +## Step 3. Connect to your TiDB cluster with Looker Studio + +1. Log into [Looker Studio](https://lookerstudio.google.com/), and then click **Create** > **Report** in the left navigation pane. + +2. On the displayed page, search and select the **MySQL** connector, and then click **AUTHORIZE**. + +3. In the **BASIC** setting pane, configure the connection parameters. + + - **Host Name or IP**: enter the `HOST` parameter from the TiDB Serverless connection dialog. + - **Port(Optional)**: enter the `PORT` parameter from the TiDB Serverless connection dialog. + - **Database**: enter the database you want to connect to. For this tutorial, enter `sp500insight`. + - **Username**: enter the `USERNAME` parameter from the TiDB Serverless connection dialog. + - **Password**: enter the `PASSWORD` parameter from the TiDB Serverless connection dialog. + - **Enable SSL**: select this option, and then click the upload icon to the right of **MySQL SSL Client Configuration Files** to upload the CA file downloaded from [Step 2](#step-2-get-the-connection-information-for-your-cluster). + + ![Looker Studio: configure connection settings for TiDB Serverless](/media/tidb-cloud/looker-studio-configure-connection.png) + +4. Click **AUTHENTICATE**. + +If the authentication succeeds, you can see tables in the database. + +## Step 4. Create a simple chart + +Now, you can use the TiDB cluster as a data source and create a simple chart with data. + +1. In the right pane, click **CUSTOM QUERY**. + + ![Looker Studio: custom query](/media/tidb-cloud/looker-studio-custom-query.png) + +2. Copy the following code to the **Enter Custom Query** area, and then click **Add** in the lower-right corner. + + ```sql + SELECT sector, + COUNT(*) AS companies, + ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC ) AS companies_ranking, + SUM(market_cap) AS total_market_cap, + ROW_NUMBER() OVER (ORDER BY SUM(market_cap) DESC ) AS total_market_cap_ranking, + SUM(revenue_growth * weight) / SUM(weight) AS avg_revenue_growth, + ROW_NUMBER() OVER (ORDER BY SUM(revenue_growth * weight) / SUM(weight) DESC ) AS avg_revenue_growth_ranking + FROM companies + LEFT JOIN index_compositions ic ON companies.stock_symbol = ic.stock_symbol + GROUP BY sector + ORDER BY 5 ASC; + ``` + + If you see the **You are about to add data to this report** dialog, click **ADD TO REPORT**. Then, a table is displayed in the report. + +3. In the toolbar of the report, click **Add a chart**, and then select `Combo chart` in the `Line` category. + +4. In the **Chart** settings pane on the right, configure the following parameters: + + - In the **SETUP** Tab: + - **Dimension**: `sector`. + - **Metric**: `companies` and `total_market_cap`. + - In the **STYLE** Tab: + - Series #1: select the `Line` option and the `Right` axis. + - Series #2: select the `Bars` option and the `Left` axis. + - Leave other fields as defaults. + +Then, you can see a combo chart similar as follows: + +![Looker Studio: A simple Combo chart](/media/tidb-cloud/looker-studio-simple-chart.png) + +## Next steps + +- Learn more usage of Looker Studio from [Looker Studio Help](https://support.google.com/looker-studio). +- Learn the best practices for TiDB application development with the chapters in the [Developer guide](/develop/dev-guide-overview.md), such as [Insert data](/develop/dev-guide-insert-data.md), [Update data](/develop/dev-guide-update-data.md), [Delete data](/develop/dev-guide-delete-data.md), [Single table reading](/develop/dev-guide-get-data-from-single-table.md), [Transactions](/develop/dev-guide-transaction-overview.md), and [SQL performance optimization](/develop/dev-guide-optimize-sql-overview.md). +- Learn through the professional [TiDB developer courses](https://www.pingcap.com/education/) and earn [TiDB certifications](https://www.pingcap.com/education/certification/) after passing the exam. + +## Need help? + +Ask questions on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [create a support ticket](https://support.pingcap.com/). diff --git a/tidb-cloud/import-snapshot-files.md b/tidb-cloud/import-snapshot-files.md index 9d0e01d63d7d8..e31daac06fc85 100644 --- a/tidb-cloud/import-snapshot-files.md +++ b/tidb-cloud/import-snapshot-files.md @@ -7,4 +7,4 @@ summary: Learn how to import Amazon Aurora or RDS for MySQL snapshot files into You can import Amazon Aurora or RDS for MySQL snapshot files into TiDB Cloud. Note that all source data files with the `.parquet` suffix in the `{db_name}.{table_name}/` folder must conform to the [naming convention](/tidb-cloud/naming-conventions-for-data-import.md). -The process of importing snapshot files is similiar to that of importing Parquet files. For more information, see [Import Apache Parquet Files from Amazon S3 or GCS into TiDB Cloud](/tidb-cloud/import-parquet-files.md). +The process of importing snapshot files is similar to that of importing Parquet files. For more information, see [Import Apache Parquet Files from Amazon S3 or GCS into TiDB Cloud](/tidb-cloud/import-parquet-files.md). diff --git a/tidb-cloud/manage-user-access.md b/tidb-cloud/manage-user-access.md index d3f87b9a27728..47b651404122d 100644 --- a/tidb-cloud/manage-user-access.md +++ b/tidb-cloud/manage-user-access.md @@ -104,6 +104,7 @@ At the project level, TiDB Cloud defines three roles, in which `Project Owner` c | Manage [spending limit](/tidb-cloud/manage-serverless-spend-limit.md) for all TiDB Serverless clusters in the project. | ✅ | ❌ | ❌ | | Manage cluster operations in the project, such as cluster creation, modification, and deletion. | ✅ | ❌ | ❌ | | Manage branches for TiDB Serverless clusters in the project, such as branch creation, connection, and deletion. | ✅ | ❌ | ❌ | +| Manage [recovery groups](/tidb-cloud/recovery-group-overview.md) for TiDB Dedicated clusters in the project, such as recovery group creation and deletion. | ✅ | ❌ | ❌ | | Manage cluster data such as data import, data backup and restore, and data migration. | ✅ | ✅ | ❌ | | Manage [Data Service](/tidb-cloud/data-service-overview.md) for data read-only operations such as using or creating endpoints to read data. | ✅ | ✅ | ✅ | | Manage [Data Service](/tidb-cloud/data-service-overview.md) for data read and write operations. | ✅ | ✅ | ❌ | diff --git a/tidb-cloud/migrate-from-op-tidb.md b/tidb-cloud/migrate-from-op-tidb.md index a3bc2a058203d..f96f59668f203 100644 --- a/tidb-cloud/migrate-from-op-tidb.md +++ b/tidb-cloud/migrate-from-op-tidb.md @@ -172,7 +172,7 @@ Do the following to export data from the upstream TiDB cluster to Amazon S3 usin 3. Run Dumpling to export data to the Amazon S3 bucket. - ```ymal + ```shell dumpling \ -u root \ -P 4000 \ @@ -187,7 +187,7 @@ Do the following to export data from the upstream TiDB cluster to Amazon S3 usin The `-t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, do not set a too large number for this parameter. - For mor information, see [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview#export-to-sql-files). + For more information, see [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview#export-to-sql-files). 4. Check the export data. Usually the exported data includes the following: diff --git a/tidb-cloud/notification-2023-09-26-console-maintenance.md b/tidb-cloud/notification-2023-09-26-console-maintenance.md index e85b29135c439..5e00966b1da98 100644 --- a/tidb-cloud/notification-2023-09-26-console-maintenance.md +++ b/tidb-cloud/notification-2023-09-26-console-maintenance.md @@ -20,7 +20,7 @@ This notification describes the details that you need to know about the [TiDB Cl ## Reason for maintenance -We're upgrading the management infrastucture of the TiDB Cloud Serverless to enhance performance and efficiency, delivering a better experience for all users. This is part of our ongoing commitment to providing high-quality services. +We're upgrading the management infrastructure of the TiDB Cloud Serverless to enhance performance and efficiency, delivering a better experience for all users. This is part of our ongoing commitment to providing high-quality services. ## Impact diff --git a/tidb-cloud/recovery-group-delete.md b/tidb-cloud/recovery-group-delete.md new file mode 100644 index 0000000000000..45cb9ff0162aa --- /dev/null +++ b/tidb-cloud/recovery-group-delete.md @@ -0,0 +1,27 @@ +--- +title: Delete a Recovery Group +summary: Learn how to delete a Recovery Group when it is no longer needed. +--- + +# Delete a Recovery Group + +When a recovery group is no longer needed, you can delete it. + +## Delete a recovery group + +When a recovery group is no longer needed to manage the replication of a set of databases, you can delete it from the system. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. + +2. In the **Project Settings** navigation pane, click **Recovery Group**. + +3. On the **Recovery Group** page, locate the name of the recovery group that you wish to delete. + +4. Click the **Action** menu for the recovery group, and then click **Delete**. The deletion dialog is displayed. + + > **Warning** + > + > - Deleting a recovery group also removes all associated replication relationships associated with that recovery group. + > - The databases associated with the recovery group are no longer protected against disasters. + +5. Confirm that you understand the impact of the deletion by typing the name of the recovery group and clicking **I understand, delete it**. diff --git a/tidb-cloud/recovery-group-failover.md b/tidb-cloud/recovery-group-failover.md new file mode 100644 index 0000000000000..38d84ed7246e6 --- /dev/null +++ b/tidb-cloud/recovery-group-failover.md @@ -0,0 +1,69 @@ +--- +title: Failover and Reprotect Databases +summary: Learn how to use a Recovery Group to failover and reprotect databases between TiDB Cloud clusters. +--- + +# Failover and Reprotect Databases + +Databases in a recovery group are replicated from one cluster to another, typically in a different region of the cloud service provider. + +The **Failover** action promotes the replicated databases in the secondary region to be the new primary copy, ensuring ongoing availability during a regional outage. + +When the regional outage is resolved, the ability to reverse the replication from the recovery region back to the original region is done using the **Reprotect** action. This ensures that the databases are protected against future disasters impacting their new region, and prepares them for migration back to the original region if desired. + +## Prerequisites + +Before performing a failover, a recovery group should have been created and be successfully replicating to the secondary cluster. For more information, see [Get Started with Recovery Groups](/tidb-cloud/recovery-group-get-started.md). + +![Protected Recovery Group](/media/tidb-cloud/recovery-group/recovery-group-protected.png) + +## Failover databases using a recovery group + +In the event of a disaster, you can use the recovery group to failover databases to the secondary cluster. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. + +2. In the **Project Settings** navigation pane, click **Recovery Group**. + +3. On the **Recovery Group** page, locate the name of the recovery group that you wish to failover. + +4. Click the **Action** menu for the recovery group, and then click **Failover**. The failover dialog is displayed. + + > **Warning** + > + > Performing a failover will sever the existing replication relationship. + +5. Select the secondary TiDB Cloud cluster to be promoted to the primary copy. Ensure that the selected cluster is in a healthy state. + +6. Confirm that you understand the potentially disruptive nature of a failover by typing **Failover** into the confirmation entry and clicking **I understand, failover group** to begin the failover. + + ![Fail Over Recovery Group](/media/tidb-cloud/recovery-group/recovery-group-failover.png) + +## Reprotect databases using a recovery group + +After a failover completes, the replica databases on the secondary cluster are now the primary copy. However, these databases are unprotected against future disasters as the replication relationship is stopped by the failover process. + +If the original primary cluster that was affected by the disaster can be brought online again, you can re-establish replication from the recovery region back to the original region using the **Reprotect** action. + +![Unprotected Recovery Group](/media/tidb-cloud/recovery-group/recovery-group-unprotected.png) + +1. In the [TiDB Cloud console](https://tidbcloud.com/), click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. + +2. In the **Project Settings** navigation pane, click **Recovery Group**. + +3. On the **Recovery Group** page, locate the name of the recovery group that you wish to reprotect. + + > **Note** + > + > The **Recovery Group Detail** page provides information about the recovery group, including current status and replication topology. + > During the reprotect synchronization, due to the volume of data transferred, the online query performance at the primary or secondary clusters might be affected. It is recommended that you schedule the reprotection of databases for a less busy period. + + > **Warning** + > + > As part of the data replication necessary to perform the reprotect operation, the content of the selected databases will be replaced at the target cluster by the content of the databases from the (new) primary cluster. If you wish to preserve the unique content on the target cluster, complete a backup before performing the Reprotect operation. + +4. Click the **Action** menu for the recovery group, and then click **Reprotect**. The reprotect dialog is displayed. + +5. Confirm the reprotect operation by clicking **Reprotect** to begin the reprotect operation. + + ![Reprotect Recovery Group](/media/tidb-cloud/recovery-group/recovery-group-reprotected.png) diff --git a/tidb-cloud/recovery-group-get-started.md b/tidb-cloud/recovery-group-get-started.md new file mode 100644 index 0000000000000..c221169d9c78f --- /dev/null +++ b/tidb-cloud/recovery-group-get-started.md @@ -0,0 +1,89 @@ +--- +title: Get Started with Recovery Groups +summary: Learn how to create a recovery group in TiDB Cloud and view its details. +--- + +# Get Started with Recovery Groups + +This document describes how to create a recovery group to protect your databases running on TiDB Cloud Dedicated clusters using the [TiDB Cloud console](https://tidbcloud.com/). It also shows how to view details of a recovery group. + +## Prerequisites + +- A recovery group replicates your databases to another cluster to protect your databases from regional disasters. Before creating a recovery group, you need to have two TiDB Cloud Dedicated clusters. One cluster hosts the primary databases, and a second cluster hosts the replicas of the primary databases. If you have not done so already, follow the steps in [Create a TiDB Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md) to create the necessary clusters. +- To create a recovery group, you must be in the `Organization Owner` role of your organization or the `Project Owner` role of the target project. + +> **Note** +> +> Currently, only TiDB Dedicated clusters hosted on AWS support recovery groups. + +## Create a new recovery group + +To create a recovery group, perform the following steps: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. + +2. In the **Project Settings** navigation pane, click **Recovery Group**. + +3. On the **Recovery Group** page, click **Create Recovery Group**. + +4. On the **Create Recovery Group** page, enter a name for the recovery group. + + > **Note** + > + > Currently only one resiliency level is supported. For more information, see [About resiliency levels](#about-resiliency-levels). + +5. Select the TiDB Dedicated cluster that will be the primary cluster for this group. + +6. Select the TiDB Dedicated cluster that will be the secondary cluster where databases will be replicated for this group. + +7. Select which databases you wish to replicate as part of this recovery group. + + > **Note** + > + > When assigning databases to the group, you can select specific databases, or select all (non-system) databases on the primary cluster (current and future). + > + > - If you **Assign all databases (current and future)**, any future databases added to the cluster will be automatically included in this recovery group and replicated to the secondary cluster. + > - If you **Assign specific databases**, select the specific databases on the primary cluster that you want to replicate to the secondary cluster. If any databases are added to the primary cluster in the future, these new databases will not be automatically replicated as part of this recovery group. + > + > During the initial replication, due to the volume of data transferred, the online query performance at the primary or secondary clusters might be affected. Schedule the initial protection of databases for a less busy period. + + > **Warning** + > + > During the initial replication, the content of the selected databases at the primary cluster will replace the content of the databases at the secondary cluster. If you wish to preserve the unique content at the secondary cluster, complete a backup before setting up the recovery group. + +8. Review the summary information, and then click **Create** to begin protecting the databases as part of the new recovery group. + +## View recovery group details + +After creating a recovery group, you can view its status information on the **Recovery Group Detail** page: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. + +2. In the **Project Settings** navigation pane, click **Recovery Group**. + +3. On the **Recovery Group** page, click the name of the recovery group that you wish to view. + + The **Recovery Group Detail** page provides information about a recovery group, including its configuration details, status, and metrics on the replication throughput and latency. + +4. When a replication relationship is fully established and functioning, the status is displayed as **Available**. + + > **Warning** + > + > During the setup of a recovery group, an account named following the pattern `cloud-rg-*` will be created on the secondary cluster for the replication process. Deleting or modifying this account will interrupt the replication. + +## About resiliency levels + +A resiliency level defines the consistency characteristics of data reading in different scenarios of a recovery group. Currently, TiDB Cloud only provides the following resiliency level: + +- No consistency guaranteed. During the replication of a recovery group, the downstream cluster does not guarantee transaction consistency read. When the upstream cluster becomes unavailable, you can not restore the data in the downstream cluster to a transaction consistency state. + +TiDB Cloud will provide two additional resiliency levels in the near future: + +- Eventual consistency. During the replication of a recovery group, the downstream cluster does not guarantee transaction consistency read. However, when the upstream cluster becomes unavailable, you can restore the data in the downstream cluster to a transaction consistency state. +- Near real-time consistency. During the replication of a recovery group, the downstream cluster provides approximately real-time transaction consistency read. When the upstream cluster becomes unavailable, you can restore the data in the downstream cluster to a transaction consistency state. + +## What's next + +After creating the recovery group, you might want to familiarize yourself with the failover and reprotect operations. These operations are used to **Failover** the primary cluster for the replicated databases from one cluster to the other, and then to later re-establish replication in the opposite direction to **Reprotect** the failed over databases. + +- [Failover and Reprotect Databases](/tidb-cloud/recovery-group-failover.md) diff --git a/tidb-cloud/recovery-group-overview.md b/tidb-cloud/recovery-group-overview.md new file mode 100644 index 0000000000000..0ef67556fa849 --- /dev/null +++ b/tidb-cloud/recovery-group-overview.md @@ -0,0 +1,37 @@ +--- +title: Recovery Group Overview (Beta) +summary: Learn how to protect your databases against disasters by using TiDB Cloud recovery groups. +--- + +# Recovery Group Overview (Beta) + +A TiDB Cloud recovery group allows you to replicate your databases between TiDB Cloud Dedicated clusters for protection against regional disasters. You can orchestrate the failover of databases from one cluster to another. After a failover to the secondary cluster, if the original primary cluster becomes available again, you can re-establish replication in the reverse direction to reprotect your databases. + +## Architecture + +A recovery group consists of a set of replicated databases that can be failed over together between two TiDB Dedicated clusters. Each recovery group is assigned a primary cluster, and databases on this primary cluster are associated with the group and are then replicated to the secondary cluster. + +![Recovery Group](/media/tidb-cloud/recovery-group/recovery-group-overview.png) + +- Recovery Group: a group of databases that are replicated between two clusters +- Primary Cluster: the cluster where the database is actively written by the application +- Secondary Cluster: the cluster where replicas of the database are located + +> **Note** +> +> Client connections to the replica databases are not explicitly forced to be read-only by the Recovery Group feature. Ensuring that the application connecting to the replica databases only performs read-only queries is the responsibility of the application. + +## Key features and limitations + +- Currently, only TiDB Dedicated clusters hosted on AWS support recovery groups. +- Recovery groups are established between two clusters. +- Bi-directional replication of a database is not supported with recovery groups. + +> **Warning** +> +> This feature is in beta and not recommended for production environments. + +## What's next + +- To get started with recovery groups, see [Create Database Recovery Group](/tidb-cloud/recovery-group-get-started.md). +- To learn how to use a recovery group, see [Failover and Reprotect Databases](/tidb-cloud/recovery-group-failover.md). diff --git a/tidb-cloud/serverless-driver-node-example.md b/tidb-cloud/serverless-driver-node-example.md index 926bfdd562564..b6a9fb7509012 100644 --- a/tidb-cloud/serverless-driver-node-example.md +++ b/tidb-cloud/serverless-driver-node-example.md @@ -75,7 +75,7 @@ The serverless driver supports both CommonJS and ES modules. The following steps node index.js ``` -## Compatability with earlier versions of Node.js +## Compatibility with earlier versions of Node.js If you are using Node.js earlier than 18.0.0, which does not have a global `fetch` function, you can take the following steps to get `fetch`: diff --git a/tidb-cloud/terraform-use-cluster-resource.md b/tidb-cloud/terraform-use-cluster-resource.md index 48104f6e4addc..1f223d17ce471 100644 --- a/tidb-cloud/terraform-use-cluster-resource.md +++ b/tidb-cloud/terraform-use-cluster-resource.md @@ -604,7 +604,7 @@ You can scale a TiDB cluster when its status is `AVAILABLE`. 1. In the `cluster.tf` file that is used when you [create the cluster](#create-a-cluster-using-the-cluster-resource), edit the `components` configurations. - For example, to add one more node for TiDB, 3 more nodes for TiKV (The number of TiKV nodes needs to be a multiple of 3 for its step is 3. You can [get this information from the cluster specifcation](#get-cluster-specification-information-using-the-tidbcloud_cluster_specs-data-source)), and one more node for TiFlash, you can edit the configurations as follows: + For example, to add one more node for TiDB, 3 more nodes for TiKV (The number of TiKV nodes needs to be a multiple of 3 for its step is 3. You can [get this information from the cluster specification](#get-cluster-specification-information-using-the-tidbcloud_cluster_specs-data-source)), and one more node for TiFlash, you can edit the configurations as follows: ``` components = { diff --git a/tidb-cloud/tidb-cloud-billing-recovery-group.md b/tidb-cloud/tidb-cloud-billing-recovery-group.md new file mode 100644 index 0000000000000..4197b257b90c9 --- /dev/null +++ b/tidb-cloud/tidb-cloud-billing-recovery-group.md @@ -0,0 +1,14 @@ +--- +title: Recovery Group Billing +summary: Learn about billing for recovery groups in TiDB Cloud. +--- + +# Recovery Group Billing + +TiDB Cloud bills for recovery groups based on the deployed size of your TiKV nodes in the primary cluster of the recovery group. When you [create a recovery group](/tidb-cloud/recovery-group-get-started.md) for a cluster, you can select the primary cluster for the recovery group. The larger the TiKV configuration, the higher the cost for recovery group protection. + +TiDB Cloud also bills for data processing per GiB basis. The data processing price varies depending on whether the data is replicated to a secondary cluster in another region, or within the same region. + +## Pricing + +To learn about the supported regions and the pricing for TiDB Cloud recovery groups, see [Recovery Group Cost](https://www.pingcap.com/tidb-cloud-pricing-details/#recovery-group-cost). diff --git a/tidb-cloud/tidb-cloud-console-auditing.md b/tidb-cloud/tidb-cloud-console-auditing.md index 4e48ab2c2c0e8..7c73d1836aa7c 100644 --- a/tidb-cloud/tidb-cloud-console-auditing.md +++ b/tidb-cloud/tidb-cloud-console-auditing.md @@ -121,7 +121,7 @@ The console audit logs record various user activities on the TiDB Cloud console | PauseCluster | Pause a cluster | | ResumeCluster | Resume a cluster | | ScaleCluster | Scale a cluster | -| DownloadTiDBClusterCA | Download TiDB cluster CA certificate | +| DownloadTiDBClusterCA | Download CA certificate | | OpenWebSQLConsole | Connect to a TiDB cluster through Web SQL | | SetRootPassword | Set the root password of a TiDB cluster | | UpdateIPAccessList | Update the IP access list of a TiDB cluster | diff --git a/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md b/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md index 06d0e87b2f670..7f419a96114d7 100644 --- a/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md +++ b/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md @@ -16,33 +16,33 @@ The solutions vary depending on your upstream database. ### Error message: Check whether mysql server_id has been greater than 0 - Amazon Aurora MySQL or Amazon RDS: `server_id` is configured by default. You do not need to configure it. Make sure you are using Amazon Aurora MySQL writer instances to support both full and incremental data migration. -- MySQL: to configure `server_id` for MySQL, see [Setting the Replication Source Configuration](https://dev.mysql.com/doc/refman/5.7/en/replication-howto-masterbaseconfig.html). +- MySQL: to configure `server_id` for MySQL, see [Setting the Replication Source Configuration](https://dev.mysql.com/doc/refman/8.0/en/replication-howto-masterbaseconfig.html). ### Error message: Check whether mysql binlog is enabled - Amazon Aurora MySQL: see [How do I turn on binary logging for my Amazon Aurora MySQL-Compatible cluster](https://aws.amazon.com/premiumsupport/knowledge-center/enable-binary-logging-aurora/?nc1=h_ls). Make sure you are using Amazon Aurora MySQL writer instances to support both full and incremental data migration. - Amazon RDS: see [Configuring MySQL binary logging](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.MySQL.BinaryFormat.html). - Google Cloud SQL for MySQL: Google enables binary logging through point-in-time recovery for MySQL master databases. See [Enable point-in-time recovery](https://cloud.google.com/sql/docs/mysql/backup-recovery/pitr#enablingpitr). -- MySQL: see [Setting the Replication Source Configuration](https://dev.mysql.com/doc/refman/5.7/en/replication-howto-masterbaseconfig.html). +- MySQL: see [Setting the Replication Source Configuration](https://dev.mysql.com/doc/refman/8.0/en/replication-howto-masterbaseconfig.html). ### Error message: Check whether mysql binlog_format is ROW - Amazon Aurora MySQL: see [How do I turn on binary logging for my Amazon Aurora MySQL-Compatible cluster](https://aws.amazon.com/premiumsupport/knowledge-center/enable-binary-logging-aurora/?nc1=h_ls). Make sure you are using Amazon Aurora MySQL writer instances to support both full and incremental data migration. - Amazon RDS: see [Configuring MySQL binary logging](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.MySQL.BinaryFormat.html). -- MySQL: execute `set global binlog_format=ROW;`. See [Setting The Binary Log Format](https://dev.mysql.com/doc/refman/5.7/en/binary-log-setting.html). +- MySQL: execute `set global binlog_format=ROW;`. See [Setting The Binary Log Format](https://dev.mysql.com/doc/refman/8.0/en/binary-log-setting.html). ### Error message: Check whether mysql binlog_row_image is FULL - Amazon Aurora MySQL: `binlog_row_image` is not configurable. This precheck item does not fail for it. Make sure you are using Amazon Aurora MySQL writer instances to support both full and incremental data migration. - Amazon RDS: the process is similar to setting the `binlog_format` parameter. The only difference is that the parameter you need to change is `binlog_row_image` instead of `binlog_format`. See [Configuring MySQL binary logging](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.MySQL.BinaryFormat.html). -- MySQL: `set global binlog_row_image = FULL;`. See [Binary Logging Options and Variables](https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html#sysvar_binlog_row_image). +- MySQL: `set global binlog_row_image = FULL;`. See [Binary Logging Options and Variables](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#sysvar_binlog_row_image). ### Error message: Check whether migrated dbs are in binlog_do_db/binlog_ignore_db Make sure that binlog has been enabled in the upstream database. See [Check whether mysql binlog is enabled](#error-message-check-whether-mysql-binlog-is-enabled). After that, resolve the issue according to the message you get: -- If the message is similar to `These dbs xxx are not in binlog_do_db xxx`, make sure all the databases that you want to migrate are in the list. See [--binlog-do-db=db_name](https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html#option_mysqld_binlog-do-db). -- If the message is similar to `These dbs xxx are in binlog_ignore_db xxx`, make sure all the databases that you want to migrate are not in the ignore list. See [--binlog-ignore-db=db_name](https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html#option_mysqld_binlog-ignore-db). +- If the message is similar to `These dbs xxx are not in binlog_do_db xxx`, make sure all the databases that you want to migrate are in the list. See [--binlog-do-db=db_name](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#option_mysqld_binlog-do-db). +- If the message is similar to `These dbs xxx are in binlog_ignore_db xxx`, make sure all the databases that you want to migrate are not in the ignore list. See [--binlog-ignore-db=db_name](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#option_mysqld_binlog-ignore-db). For Amazon Aurora MySQL, this precheck item does not fail for it. Make sure you are using Amazon Aurora MySQL writer instances to support both full and incremental data migration. @@ -54,7 +54,7 @@ If the error occurs in the upstream database, set `max_connections` as follows: - Amazon Aurora MySQL: the process is similar to setting the `binlog_format`. The only difference is that the parameter you change is `max_connections` instead of `binlog_format`. See [How do I turn on binary logging for my Amazon Aurora MySQL-Compatible cluster](https://aws.amazon.com/premiumsupport/knowledge-center/enable-binary-logging-aurora/?nc1=h_ls). - Amazon RDS: the process is similar to setting the `binlog_format`. The only difference is that the parameter you change is `max_connections` instead of `binlog_format`. See [Configuring MySQL binary logging](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_LogAccess.MySQL.BinaryFormat.html). -- MySQL: configure `max_connections` following the document [max_connections](https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_connections). +- MySQL: configure `max_connections` following the document [max_connections](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_connections). If the error occurs in the TiDB Cloud cluster, configure `max_connections` following the document [max_connections](https://docs.pingcap.com/tidb/stable/system-variables#max_connections). diff --git a/tidb-cloud/tidb-cloud-import-local-files.md b/tidb-cloud/tidb-cloud-import-local-files.md index ed20166acf53f..89c0e514a1a7c 100644 --- a/tidb-cloud/tidb-cloud-import-local-files.md +++ b/tidb-cloud/tidb-cloud-import-local-files.md @@ -18,7 +18,7 @@ Currently, this method supports importing one CSV file for one task into either - If the extra columns are not the primary keys or the unique keys, no error will be reported. Instead, these extra columns will be populated with their [default values](/data-type-default-values.md). - If the extra columns are the primary keys or the unique keys and do not have the `auto_increment` or `auto_random` attribute, an error will be reported. In that case, it is recommended that you choose one of the following strategies: - Provide a source file that includes these the primary keys or the unique keys columns. - - Set the attributes of the the primary key or the unique key columns to `auto_increment` or `auto_random`. + - Set the attributes of the primary key or the unique key columns to `auto_increment` or `auto_random`. - If a column name is a reserved [keyword](/keywords.md) in TiDB, TiDB Cloud automatically adds backticks `` ` `` to enclose the column name. For example, if the column name is `order`, TiDB Cloud automatically adds backticks `` ` `` to change it to `` `order` `` and imports the data into the target table. ## Import local files diff --git a/tidb-cloud/tidb-cloud-release-notes.md b/tidb-cloud/tidb-cloud-release-notes.md index 2737ac68c1c9f..1ccd2955548e3 100644 --- a/tidb-cloud/tidb-cloud-release-notes.md +++ b/tidb-cloud/tidb-cloud-release-notes.md @@ -56,9 +56,9 @@ This page lists the release notes of [TiDB Cloud](https://www.pingcap.com/tidb-c **General changes** -- When you restore data from TiDB Dedicated clusters, the default behavior is now modified from restoring without user accounts to restoring with all user accounts, including the `cloud_admin@'%'` account. +- When you restore data from TiDB Dedicated clusters, the default behavior is now modified from restoring without user accounts to restoring with all user accounts. - For more information, see [BackUp and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md). + For more information, see [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md). - Introduce event filters for changefeeds. diff --git a/tidb-configuration-file.md b/tidb-configuration-file.md index 96b79531c62ac..f824ef26d7cae 100644 --- a/tidb-configuration-file.md +++ b/tidb-configuration-file.md @@ -122,7 +122,7 @@ The TiDB configuration file supports more options than command-line parameters. - When the built-in `VERSION()` function is used. - When TiDB establishes the initial connection to the client and returns the initial handshake packet with version string of the server. For details, see [MySQL Initial Handshake Packet](https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase.html#sect_protocol_connection_phase_initial_handshake). + Default value: "" -+ By default, the format of the TiDB version string is `5.7.${mysql_latest_minor_version}-TiDB-${tidb_version}`. ++ By default, the format of the TiDB version string is `8.0.11-TiDB-${tidb_version}`. > **Note:** > @@ -202,6 +202,16 @@ The TiDB configuration file supports more options than command-line parameters. - Default value: `0` - When TiDB is waiting for shutdown (in the grace period), the HTTP status will indicate a failure, which allows the load balancers to reroute traffic. +> **Note:** +> +> The duration that TiDB waits before shutting down the server is also affected by the following parameters: +> +> - When you use a platform that employs SystemD, the default stop timeout is 90 seconds. If you need a longer timeout, you can set [`TimeoutStopSec=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#TimeoutStopSec=). +> +> - When you use the TiUP Cluster component, the default [`--wait-timeout`](/tiup/tiup-component-cluster.md#--wait-timeout) is 120 seconds. +> +> - When you use Kubernetes, the default [`terminationGracePeriodSeconds`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#lifecycle) is 30 seconds. + ### `enable-global-kill` New in v6.1.0 + Controls whether to enable the Global Kill (terminating queries or connections across instances) feature. @@ -513,7 +523,7 @@ Configuration items related to performance. - In a single transaction, the total size of key-value records cannot exceed this value. The maximum value of this parameter is `1099511627776` (1 TB). Note that if you have used the binlog to serve the downstream consumer Kafka (such as the `arbiter` cluster), the value of this parameter must be no more than `1073741824` (1 GB). This is because 1 GB is the upper limit of a single message size that Kafka can process. Otherwise, an error is returned if this limit is exceeded. - In TiDB v6.5.0 and later versions, this configuration is no longer recommended. The memory size of a transaction will be accumulated into the memory usage of the session, and the [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) variable will take effect when the session memory threshold is exceeded. To be compatible with previous versions, this configuration works as follows when you upgrade from an earlier version to TiDB v6.5.0 or later: - If this configuration is not set or is set to the default value (`104857600`), after an upgrade, the memory size of a transaction will be accumulated into the memory usage of the session, and the `tidb_mem_quota_query` variable will take effect. - - If this configuration is not defaulted (`104857600`), it still takes effect and its behavior on controling the size of a single transaction remains unchanged before and after the upgrade. This means that the memory size of the transaction is not controlled by the `tidb_mem_quota_query` variable. + - If this configuration is not defaulted (`104857600`), it still takes effect and its behavior on controlling the size of a single transaction remains unchanged before and after the upgrade. This means that the memory size of the transaction is not controlled by the `tidb_mem_quota_query` variable. ### `tcp-keep-alive` @@ -591,6 +601,11 @@ Configuration items related to performance. + Default value: `1000` + Currently, the valid value range is `[1, 100000]`. +### `concurrently-init-stats` New in v8.1.0 and v7.5.2 + ++ Controls whether to initialize statistics concurrently during TiDB startup. ++ Default value: `false` + ### `lite-init-stats` New in v7.1.0 + Controls whether to use lightweight statistics initialization during TiDB startup. @@ -762,7 +777,7 @@ This section introduces configuration items related to the Coprocessor Cache fea ## txn-local-latches -Configuration related to the transaction latch. It is recommended to enable it when many local transaction conflicts occur. +Configuration items related to the transaction latch. These configuration items might be deprecated in the future. It is not recommended to use them. ### `enabled` diff --git a/tidb-distributed-execution-framework.md b/tidb-distributed-execution-framework.md index 70b68989e923a..a25317f755863 100644 --- a/tidb-distributed-execution-framework.md +++ b/tidb-distributed-execution-framework.md @@ -9,13 +9,13 @@ summary: Learn the use cases, limitations, usage, and implementation principles > > This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. -TiDB adopts a computing-storage separation architecture with excellent scalability and elasticity. Starting from v7.1.0, TiDB introduces a Distributed eXecution Framework (DXF) to further leverage the resource advantages of the distributed architecture. The goal of the DXF is to implement unified scheduling and distributed execution of tasks, and to provide unified resource management capabilities for both overall and individual tasks, which better meets users' expectations for resource usage. +TiDB adopts a computing-storage separation architecture with excellent scalability and elasticity. Starting from v7.1.0, TiDB introduces a **Distributed eXecution Framework (DXF)** to further leverage the resource advantages of the distributed architecture. The goal of the DXF is to implement unified scheduling and distributed execution of tasks, and to provide unified resource management capabilities for both overall and individual tasks, which better meets users' expectations for resource usage. This document describes the use cases, limitations, usage, and implementation principles of the DXF. ## Use cases -In a database management system, in addition to the core transactional processing (TP) and analytical processing (AP) workloads, there are other important tasks, such as DDL operations, IMPORT INTO, TTL, Analyze, and Backup/Restore. These tasks need to process a large amount of data in database objects (tables), so they typically have the following characteristics: +In a database management system, in addition to the core transactional processing (TP) and analytical processing (AP) workloads, there are other important tasks, such as DDL operations, [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md), [TTL](/time-to-live.md), [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md), and Backup/Restore. These tasks need to process a large amount of data in database objects (tables), so they typically have the following characteristics: - Need to process all data in a schema or a database object (table). - Might need to be executed periodically, but at a low frequency. @@ -27,29 +27,28 @@ Enabling the DXF can solve the above problems and has the following three advant - The DXF supports distributed execution of tasks, which can flexibly schedule the available computing resources of the entire TiDB cluster, thereby better utilizing the computing resources in a TiDB cluster. - The DXF provides unified resource usage and management capabilities for both overall and individual tasks. -Currently, the DXF supports the distributed execution of the `ADD INDEX` and `IMPORT INTO` statements. +Currently, the DXF supports the distributed execution of the [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) statements. -- `ADD INDEX` is a DDL statement used to create indexes. For example: +- [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) is a DDL statement used to create indexes. For example: ```sql ALTER TABLE t1 ADD INDEX idx1(c1); CREATE INDEX idx1 ON table t1(c1); ``` -- `IMPORT INTO` is used to import data in formats such as `CSV`, `SQL`, and `PARQUET` into an empty table. For more information, see [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). +- [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) is used to import data in formats such as CSV, SQL, and Parquet into an empty table. ## Limitation -The DXF can only schedule up to 16 tasks (including `ADD INDEX` tasks and `IMPORT INTO` tasks) simultaneously. +The DXF can only schedule up to 16 tasks (including [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) tasks and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) tasks) simultaneously. ## `ADD INDEX` limitation -- For each cluster, only one `ADD INDEX` task is allowed for distributed execution at a time. If a new `ADD INDEX` task is submitted before the current `ADD INDEX` distributed task has finished, the new `ADD INDEX` task is executed through a transaction instead of being scheduled by DXF. - Adding indexes on columns with the `TIMESTAMP` data type through the DXF is not supported, because it might lead to inconsistency between the index and the data. ## Prerequisites -Before using the DXF to execute `ADD INDEX` tasks, you need to enable the [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) mode. +Before using the DXF to execute [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) tasks, you need to enable the [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) mode. @@ -79,7 +78,7 @@ Adjust the following system variables related to Fast Online DDL: ## Usage -1. To enable the DXF, set the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: +1. To enable the DXF, set the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`. Starting from v8.1.0, this variable is enabled by default. For newly created clusters of v8.1.0 or later versions, you can skip this step. ```sql SET GLOBAL tidb_enable_dist_task = ON; @@ -94,12 +93,20 @@ Adjust the following system variables related to Fast Online DDL: * [`tidb_ddl_error_count_limit`](/system-variables.md#tidb_ddl_error_count_limit) * [`tidb_ddl_reorg_batch_size`](/system-variables.md#tidb_ddl_reorg_batch_size): use the default value. The recommended maximum value is `1024`. -3. Starting from v7.4.0, for TiDB Self-Hosted, you can adjust the number of TiDB nodes that perform the DXF tasks according to actual needs. After deploying a TiDB cluster, you can set the instance-level system variable [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) for each TiDB node in the cluster. When `tidb_service_scope` of a TiDB node is set to `background`, the TiDB node can execute the DXF tasks. When `tidb_service_scope` of a TiDB node is set to the default value "", the TiDB node cannot execute the DXF tasks. If `tidb_service_scope` is not set for any TiDB node in a cluster, the DXF schedules all TiDB nodes to execute tasks by default. +## Task scheduling - > **Note:** - > - > - In a cluster with several TiDB nodes, it is strongly recommended to set `tidb_service_scope` to `background` on two or more TiDB nodes. If `tidb_service_scope` is set on a single TiDB node only, when the node is restarted or fails, the task will be rescheduled to other TiDB nodes that lack the `background` setting, which will affect these TiDB nodes. - > - During the execution of a distributed task, changes to the `tidb_service_scope` configuration will not take effect for the current task, but will take effect from the next task. +By default, the DXF schedules all TiDB nodes to execute distributed tasks. Starting from v7.4.0, for TiDB Self-Hosted clusters, you can control which TiDB nodes can be scheduled by the DXF to execute distributed tasks by configuring [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740). + +- For versions from v7.4.0 to v8.0.0, the optional values of [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) are `''` or `background`. If the current cluster has TiDB nodes with `tidb_service_scope = 'background'`, the DXF schedules tasks to these nodes for execution. If the current cluster does not have TiDB nodes with `tidb_service_scope = 'background'`, whether due to faults or normal scaling in, the DXF schedules tasks to nodes with `tidb_service_scope = ''` for execution. + +- Starting from v8.1.0, you can set [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) to any valid value. When a distributed task is submitted, the task binds to the [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) value of the currently connected TiDB node, and the DXF only schedules the task to the TiDB nodes with the same [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) value for execution. However, for configuration compatibility with earlier versions, if a distributed task is submitted on a node with `tidb_service_scope = ''` and the current cluster has TiDB nodes with `tidb_service_scope = 'background'`, the DXF schedules the task to TiDB nodes with `tidb_service_scope = 'background'` for execution. + +Starting from v8.1.0, if new nodes are added during task execution, the DXF determines whether to schedule tasks to the new nodes for execution based on the preceding rules. If you do not want newly added nodes to execute tasks, it is recommended to set a different [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) for those newly added nodes in advance. + +> **Note:** +> +> - For versions from v7.4.0 to v8.0.0, in clusters with multiple TiDB nodes, it is strongly recommended to set [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) to `background` on two or more TiDB nodes. If this variable is set only on a single TiDB node, when that node restarts or fails, tasks will be rescheduled to TiDB nodes with `tidb_service_scope = ''`, which affects applications running on these TiDB nodes. +> - During the execution of a distributed task, changes to the [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) configuration do not take effect for the current task, but take effect from the next task. ## Implementation principles @@ -125,4 +132,4 @@ As shown in the preceding diagram, the execution of tasks in the DXF is mainly h * [Execution Principles and Best Practices of DDL Statements](https://docs.pingcap.com/tidb/stable/ddl-introduction) - + \ No newline at end of file diff --git a/tidb-global-sort.md b/tidb-global-sort.md index 3094178c46877..c80f57af664c5 100644 --- a/tidb-global-sort.md +++ b/tidb-global-sort.md @@ -10,7 +10,7 @@ summary: Learn the use cases, limitations, usage, and implementation principles > **Note:** > -> - Currently, the Global Sort process consumes a large amount of computing and memory resources of TiDB nodes. In scenarios such as adding indexes online while user business applications are running, it is recommended to add new TiDB nodes to the cluster and set the [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) variable of these nodes to `"background"`. In this way, the distributed framework schedules tasks to these nodes, isolating the workload from other TiDB nodes to reduce the impact of executing backend tasks such as `ADD INDEX` and `IMPORT INTO` on user business applications. +> - Currently, the Global Sort process consumes a large amount of computing and memory resources of TiDB nodes. In scenarios such as adding indexes online while user business applications are running, it is recommended to add new TiDB nodes to the cluster, configure the [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) variable for these nodes, and connect to these nodes to create tasks. In this way, the distributed framework schedules tasks to these nodes, isolating the workload from other TiDB nodes to reduce the impact of executing backend tasks such as `ADD INDEX` and `IMPORT INTO` on user business applications. > - When the Global Sort feature is used, it is recommended to use TiDB nodes with at least 16 cores of CPU and 32 GiB of memory to avoid OOM. > **Note:** @@ -37,7 +37,7 @@ Currently, the Global Sort feature is not used as a component of the query execu To enable Global Sort, follow these steps: -1. Enable the DXF by setting the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`: +1. Enable the DXF by setting the value of [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `ON`. Starting from v8.1.0, this variable is enabled by default. For newly created clusters of v8.1.0 or later versions, you can skip this step. ```sql SET GLOBAL tidb_enable_dist_task = ON; diff --git a/tidb-lightning/import-into-vs-tidb-lightning.md b/tidb-lightning/import-into-vs-tidb-lightning.md new file mode 100644 index 0000000000000..69e9cadd52927 --- /dev/null +++ b/tidb-lightning/import-into-vs-tidb-lightning.md @@ -0,0 +1,133 @@ +--- +title: IMPORT INTO vs. TiDB Lightning +summary: Learn about the differences between `IMPORT INTO` and TiDB Lightning. +--- + +# IMPORT INTO vs. TiDB Lightning + +Many users have provided feedback that the deployment, configuration, and maintenance of [TiDB Lightning](/tidb-lightning/tidb-lightning-configuration.md) are complex, especially in scenarios involving [parallel importing](/tidb-lightning/tidb-lightning-distributed-import.md) of large datasets. + +Based on the feedback, TiDB has gradually integrated some functionalities of TiDB Lightning into the [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) SQL statement. You can directly import data by executing `IMPORT INTO`, thereby improving the efficiency of data import. In addition, `IMPORT INTO` supports some functionalities that TiDB Lightning does not, such as automatic distributed task scheduling and [TiDB Global Sort](/tidb-global-sort.md). + +`IMPORT INTO` is introduced in v7.2.0 and becomes generally available (GA) in v7.5.0. It will continue to be improved and optimized in future versions. Once the `IMPORT INTO` capability can fully replace TiDB Lightning, TiDB Lightning will be deprecated. At that time, relevant notification will be provided in advance in TiDB Release Notes and documentation. + +## Comparison between `IMPORT INTO` and TiDB Lightning + +The following sections describe the differences between `IMPORT INTO` and TiDB Lightning in multiple dimensions. + +### Deployment cost + +#### `IMPORT INTO` + +`IMPORT INTO` does not require separate deployment. You can run it directly on TiDB nodes, which eliminates additional deployment work. + +#### TiDB Lightning + +By contrast, TiDB Lightning requires [separate server deployment](/tidb-lightning/deploy-tidb-lightning.md). + +### Resource utilization + +#### `IMPORT INTO` + +The `IMPORT INTO` task and other business workloads can share TiDB resources or utilize them at different times to fully leverage the TiDB resources. To ensure stable operation of business workloads while maintaining the performance and stability of the `IMPORT INTO` task, you can specify [specific TiDB nodes](/system-variables.md#tidb_service_scope-new-in-v740) dedicated to `IMPORT INTO` for data import. + +When you use [TiDB Global Sort](/tidb-global-sort.md), there is no need to mount large local disks. TiDB Global Sort can use Amazon S3 as the storage. Once the import task is completed, the temporary data stored on Amazon S3 for global sorting will be automatically deleted to save storage costs. + +#### TiDB Lightning + +You need separate servers to deploy and run TiDB Lightning. When no import tasks are executed, these resources remain idle. The total idle time is even longer in scenarios where import tasks are executed periodically, resulting in a waste of resources. + +If the dataset to be imported is large, you also need to prepare large local disks to sort the data to be imported. + +### Task configuration and integration + +#### `IMPORT INTO` + +You can directly write SQL statements to submit import tasks, which are easy to call and integrate. + +#### TiDB Lightning + +By contrast, TiDB Lightning requires you to write [task configuration files](/tidb-lightning/tidb-lightning-configuration.md). These configuration files are complex and not easily called by third parties. + +### Task scheduling + +#### `IMPORT INTO` + +`IMPORT INTO` supports distributed execution. For example, when you import 40 TiB of source data files into one target table, after submitting the SQL statement, TiDB will automatically split the import task into multiple sub-tasks and schedule different TiDB nodes to execute these sub-tasks. + +#### TiDB Lightning + +By contrast, the configuration for TiDB Lightning is complex, inefficient, and prone to errors. + +Assume that you start 10 TiDB Lightning instances to import data in parallel, then you need to create 10 TiDB Lightning configuration files. In each file, you need to configure the range of source files to be read by the corresponding TiDB Lightning instance. For example, TiDB Lightning instance 1 reads the first 100 files, instance 2 reads the next 100 files, and so on. + +In addition, you need to configure the shared metadata table and other configuration information for these 10 TiDB Lightning instances, which is complex. + +### Global Sort vs. local sort + +#### `IMPORT INTO` + +With TiDB Global Sort, `IMPORT INTO` can transmit tens of TiB of source data to multiple TiDB nodes, encode the data KV pairs and index KV pairs, and then transfer these pairs to Amazon S3 for global sorting before writing them into TiKV. + +Because these KV pairs are globally sorted, data imported from various TiDB nodes into TiKV does not overlap, allowing it to be written directly into the RocksDB. This eliminates the need for TiKV to perform compaction operations, resulting in significant improvement in both write performance and stability of TiKV. + +After the import is completed, the data used for global sorting on Amazon S3 will be automatically deleted, saving storage costs. + +#### TiDB Lightning + +TiDB Lightning only supports local sort. For example, for tens of TiB of source data, if TiDB Lightning does not have large local disks configured, or if multiple TiDB Lightning instances are used for parallel import, each TiDB Lightning instance can only use local disks to sort the data to be imported. Due to the inability to perform global sort, there will be an overlap between the data imported into TiKV by multiple TiDB Lightning instances, especially in scenarios where index data is more prevalent, triggering TiKV to perform compaction operations. Compaction operations are resource-intensive, which will lead to a decrease in TiKV's write performance and stability. + +If you want to continue importing data later, you will need to keep the TiDB Lightning server and the disks on the server for the next import. The cost of using preallocated disks is relatively high, compared with `IMPORT INTO` using Amazon S3 on a pay-as-you-go basis. + +### Performance + +Currently, there are no performance test comparison results under equivalent test environments between `IMPORT INTO` and TiDB Lightning. + +When Amazon S3 is used as the storage for global sorting, the performance test results for `IMPORT INTO` are as follows: + +| Source dataset | Node configuration | Average import speed per TiDB node | +|------------------------------------|--------------------------------------------------------------|------------------------------------| +| 40 TiB data (2.26 billion rows, 19 KiB per row) | 10 TiDB (16C32G) nodes and 20 TiKV (16C27G) nodes | 222 GiB/h | +| 10 TiB data (565 million rows, 19 KiB per row) | 5 TiDB (16C32G) nodes and 10 TiKV (16C27G) nodes | 307 GiB/h | + +### High availability + +#### `IMPORT INTO` + +After a TiDB node fails, tasks on that node are automatically transferred to the remaining TiDB nodes to continue running. + +#### TiDB Lightning + +After a TiDB Lightning instance node fails, you need to perform manual recovery of tasks on a new node based on previously recorded checkpoints. + +### Scalability + +#### `IMPORT INTO` + +Due to the use of Global Sort, data imported into TiKV does not overlap, resulting in better scalability compared with TiDB Lightning. + +#### TiDB Lightning + +Due to only supporting local sort, data imported into TiKV might overlap when new TiDB Lightning instances are added, resulting in more compaction operations for TiKV and limiting scalability relative to `IMPORT INTO`. + +## Functionalities not supported by `IMPORT INTO` + +Currently, `IMPORT INTO` still lacks some functionalities and cannot fully replace TiDB Lightning in some scenarios, such as: + +- Logical import + + Before importing data with `IMPORT INTO`, the target table must be empty. If you need to import data into a table that already contains data, it is recommended to use methods such as [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) or direct insertion. Starting from v8.0, TiDB supports [bulk DML](/system-variables.md#tidb_dml_type-new-in-v800) for executing large transactions. + +- Conflict data handling + + `IMPORT INTO` currently does not support conflict data handling. Before the data import, you need to define the table schema properly to ensure that the data to be imported does not conflict with primary keys (PK) or unique keys (UK). Otherwise, it might cause task failures. + +- Importing data into multiple target tables + + Currently, only one target table is allowed for one `IMPORT INTO` SQL statement. If you want to import data into multiple target tables, you need to submit multiple `IMPORT INTO` SQL statements. + +In future versions, these functionalities will be supported by `IMPORT INTO`, along with additional enhancements to its capabilities, such as allowing modification of concurrency during task execution and adjusting throughput for writing to TiKV. This will make it more convenient for you to manage tasks. + +## Summary + +Compared with TiDB Lightning, `IMPORT INTO`can be directly executed on TiDB nodes, supports automated distributed task scheduling and [TiDB Global Sort](/tidb-global-sort.md), and offers significant improvements in deployment, resource utilization, task configuration convenience, ease of invocation and integration, high availability, and scalability. It is recommended that you consider using `IMPORT INTO` instead of TiDB Lightning in appropriate scenarios. diff --git a/tidb-lightning/tidb-lightning-command-line-full.md b/tidb-lightning/tidb-lightning-command-line-full.md index 4f4b3778a4998..6af6b5016fc13 100644 --- a/tidb-lightning/tidb-lightning-command-line-full.md +++ b/tidb-lightning/tidb-lightning-command-line-full.md @@ -38,7 +38,7 @@ You can configure the following parameters using `tidb-lightning`: | `--key ` | Private key path for TLS connection | `security.key-path` | | `--server-mode` | Start TiDB Lightning in server mode | `lightning.server-mode` | -If you specify both a command line parameter and the corresponding setting in the configuration file, the command line parameter takes precedence. For example, running `./tidb-lightning -L debug --config cfg.toml` would always set the log level to "debug" regardless of the content of `cfg.toml`. +If you specify both a command line parameter and the corresponding setting in the configuration file, the command line parameter takes precedence. For example, running `tiup tidb-lightning -L debug --config cfg.toml` would always set the log level to "debug" regardless of the content of `cfg.toml`. ## `tidb-lightning-ctl` diff --git a/tidb-lightning/tidb-lightning-compatibility-and-scenarios.md b/tidb-lightning/tidb-lightning-compatibility-and-scenarios.md new file mode 100644 index 0000000000000..b01c240e29e24 --- /dev/null +++ b/tidb-lightning/tidb-lightning-compatibility-and-scenarios.md @@ -0,0 +1,100 @@ +--- +title: Compatibility of TiDB Lightning and IMPORT INTO with TiCDC and Log Backup +summary: Learn about compatibility of IMPORT INTO and TiDB Lightning with log backup and TiCDC. +--- + +# Compatibility of TiDB Lightning and IMPORT INTO with TiCDC and Log Backup + +This document describes TiDB Lightning and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) compatibility with [log backup](/br/br-pitr-guide.md) and [TiCDC](/ticdc/ticdc-overview.md), as well as some special usage scenarios. + +## `IMPORT INTO` vs. TiDB Lightning + +[`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) integrates with the physical import mode of TiDB Lightning, but there are some differences. See [`IMPORT INTO` vs. TiDB Lightning](/tidb-lightning/import-into-vs-tidb-lightning.md) for details. + +## Compatibility with log backup and TiCDC + +- TiDB Lightning [logical import mode](/tidb-lightning/tidb-lightning-logical-import-mode.md) is compatible with log backup and TiCDC. + +- TiDB Lightning [physical import mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) is not compatible with log backup and TiCDC. The reason is that physical import mode directly ingests encoded KV pairs of the source data to TiKV, causing TiKV not to generate corresponding change logs during this process. Without such change logs, the relevant data cannot be backed up via log backup and cannot be replicated by TiCDC. + +- `IMPORT INTO` is not compatible with log backup and TiCDC. The reason is that `IMPORT INTO` also ingests encoded KV pairs of the source data directly to TiKV. + +## Scenarios for TiDB Lightning logical import mode + +If TiDB Lightning logical import mode can meet your application's performance requirements and your application requires imported tables to be backed up or replicated downstream using TiCDC, it is recommended to use TiDB Lightning logical import mode. + +## Scenarios for TiDB Lightning physical import mode + +This section describes how to use TiDB Lightning together with [log backup](/br/br-pitr-guide.md) and [TiCDC](/ticdc/ticdc-overview.md). + +If TiDB Lightning logical import mode does not meet your application's performance requirements, you have to use TiDB Lightning physical import mode, and imported tables need to be backed up or replicated downstream using TiCDC, then the following scenarios are recommended. + +### Used with log backup + +You can perform in different scenarios as follows: + +- Scenario 1: tables in physical import mode do not need to be backed up + + In this scenario, if [PITR](/br/br-log-architecture.md#process-of-pitr) is enabled, the compatibility check will report an error after TiDB Lightning starts. If you are sure that these tables do not need backup or [log backup](/br/br-pitr-guide.md), you can change the `Lightning.check-requirements` parameter in the [TiDB Lightning configuration file](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) to `false` and restart the import task. + +- Scenario 2: after the physical import is finished, there will be no new DML operations on the table + + This scenario does not involve incremental data writes, so it is sufficient to perform a table-level snapshot backup of the table after completing the data import in TiDB Lightning physical import mode, as described in [Back up a table](/br/br-snapshot-manual.md#back-up-a-table). + + During data recovery, the snapshot data of the table is restored. See [Restore a table](/br/br-snapshot-manual.md#restore-a-table) for the procedure. + +- Scenario 3: after the physical import is finished, there will be new DML operations on the table (not supported) + + In this scenario, you can only choose either [full snapshot backup](/br/br-snapshot-guide.md) or [log backup](/br/br-pitr-guide.md) for this table. You cannot back up and restore both the full snapshot data and log backup data of this table. + +### Used with TiCDC + +Using TiCDC with physical import mode is not compatible in the short term, because TiCDC cannot keep up with the write speed of TiDB Lightning physical import mode, which might result in increasing cluster replication latency. + +You can perform in different scenarios as follows: + +- Scenario 1: the table does not need to be replicated downstream by TiCDC. + + In this scenario, if TiCDC changefeed is enabled, the compatibility check will report an error after TiDB Lightning starts. If you are sure that these tables do not need backup or [log backup](/br/br-pitr-guide.md), you can change the `Lightning.check-requirements` parameter in the [TiDB Lightning configuration file](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) to `false` and restart the import task. + +- Scenario 2: the table needs to be replicated downstream by TiCDC. + + In this scenario, if TiCDC changefeed is enabled, the compatibility check will report an error after TiDB Lightning starts. You need to change the `Lightning.check-requirements` parameter in the [TiDB Lightning configuration file](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) in the upstream TiDB cluster to `false` and restart the import task. + + After the import task for the upstream TiDB cluster is finished, use TiDB Lightning to import the same data in the downstream TiDB cluster. If you have databases such as Redshift and Snowflake in the downstream, you can configure them to read CSV, SQL, or Parquet files from a cloud storage service and then write the data to the database. + +## Scenarios for `IMPORT INTO` + +This section describes how to use `IMPORT INTO` together with [log backup](/br/br-pitr-guide.md) and [TiCDC](/ticdc/ticdc-overview.md). + +### Used with log backup + +You can perform in different scenarios as follows: + +- Scenario 1: tables do not need to be backed up + + In this scenario, if [PITR](/br/br-log-architecture.md#process-of-pitr) is enabled, the compatibility check will report an error after you submit the `IMPORT INTO` statement. If you are sure that these tables do not need backup or [log backup](/br/br-pitr-guide.md), you can include `DISABLE_PRECHECK` (introduced in v8.0.0) in [`WithOptions`](/sql-statements/sql-statement-import-into.md#withoptions) of that statement, and then resubmit it. In this way, the data import task ignores the compatibility check and imports the data directly. + +- Scenario 2: after the import is finished, there will be no new DML operations on the table + + This scenario does not involve incremental data writes, so it is sufficient to perform a table-level snapshot backup of the table after completing the data import, as described in [Back up a table](/br/br-snapshot-manual.md#back-up-a-table). + + During data recovery, the snapshot data of the table is restored. See [Restore a table](/br/br-snapshot-manual.md#restore-a-table) for the procedure. + +- Scenario 3: after the import is finished, there will be new DML operations on the table (not supported) + + In this scenario, you can only choose either [full snapshot backup](/br/br-snapshot-guide.md) or [log backup](/br/br-pitr-guide.md) for this table. You cannot back up and restore both the full snapshot data and log backup data of this table. + +### Used with TiCDC + +You can perform in different scenarios as follows: + +- Scenario 1: the table does not need to be replicated downstream by TiCDC. + + In this scenario, if TiCDC changefeed is enabled, the compatibility check will report an error after you submit the `IMPORT INTO` statement. If you are sure that these tables do not need to be replicated by TiCDC, you can include `DISABLE_PRECHECK` (introduced in v8.0.0) in [`WithOptions`](/sql-statements/sql-statement-import-into.md#withoptions) of that statement, and then resubmit it. In this way, the data import task ignores the compatibility check and imports the data directly. + +- Scenario 2: the table needs to be replicated downstream by TiCDC. + + In this scenario, if TiCDC changefeed is enabled, the compatibility check will report an error after you submit the `IMPORT INTO` statement. You can include `DISABLE_PRECHECK` (introduced in v8.0.0) in [`WithOptions`](/sql-statements/sql-statement-import-into.md#withoptions) of that statement, and then resubmit it. In this way, the data import task ignores the compatibility check and imports the data directly. + + After the import task for the upstream TiDB cluster is finished, use `IMPORT INTO` to import the same data in the downstream TiDB cluster. If you have databases such as Redshift and Snowflake in the downstream, you can configure them to read CSV, SQL, or Parquet files from a cloud storage service and then write the data to the database. diff --git a/tidb-lightning/tidb-lightning-configuration.md b/tidb-lightning/tidb-lightning-configuration.md index 59ec10b04bb95..2d14f5cc4ce5c 100644 --- a/tidb-lightning/tidb-lightning-configuration.md +++ b/tidb-lightning/tidb-lightning-configuration.md @@ -125,20 +125,24 @@ driver = "file" # keep-after-success = false [conflict] -# Starting from v7.3.0, a new version of strategy is introduced to handle conflicting data. The default value is "". Starting from v8.0.0, TiDB Lightning optimizes the conflict strategy for both physical and logical import modes (experimental). +# Starting from v7.3.0, a new version of strategy is introduced to handle conflicting data. The default value is "". Starting from v8.0.0, TiDB Lightning optimizes the conflict strategy for both physical and logical import modes. # - "": in the physical import mode, TiDB Lightning does not detect or handle conflicting data. If the source file contains conflicting primary or unique key records, the subsequent step reports an error. In the logical import mode, TiDB Lightning converts the "" strategy to the "error" strategy for processing. # - "error": when detecting conflicting primary or unique key records in the imported data, TiDB Lightning terminates the import and reports an error. # - "replace": when encountering conflicting primary or unique key records, TiDB Lightning retains the latest data and overwrites the old data. # The conflicting data are recorded in the `lightning_task_info.conflict_error_v2` table (recording conflicting data detected by post-import conflict detection in the physical import mode) and the `conflict_records` table (recording conflicting data detected by preprocess conflict detection in both logical and physical import modes) of the target TiDB cluster. +# If you set `conflict.strategy = "replace"` in physical import mode, the conflicting data can be checked in the `lightning_task_info.conflict_view` view. # You can manually insert the correct records into the target table based on your application requirements. Note that the target TiKV must be v5.2.0 or later versions. # - "ignore": when encountering conflicting primary or unique key records, TiDB Lightning retains the old data and ignores the new data. This option can only be used in the logical import mode. strategy = "" -# Controls whether to enable preprocess conflict detection, which checks conflicts in data before importing it to TiDB. In scenarios where the ratio of conflict records is greater than or equal to 1%, it is recommended to enable preprocess conflict detection for better performance in conflict detection. In other scenarios, it is recommended to disable it. The default value is false, indicating that TiDB Lightning only checks conflicts after the import. If you set it to true, TiDB Lightning checks conflicts both before and after the import. This parameter is experimental, and it can be used only in the physical import mode. +# Controls whether to enable preprocess conflict detection, which checks conflicts in data before importing it to TiDB. The default value is false, indicating that TiDB Lightning only checks conflicts after the import. If you set it to true, TiDB Lightning checks conflicts both before and after the import. This parameter can be used only in the physical import mode. In scenarios where the number of conflict records is greater than 1,000,000, it is recommended to set `precheck-conflict-before-import = true` for better performance in conflict detection. In other scenarios, it is recommended to disable it. # precheck-conflict-before-import = false -# Controls the maximum number of conflict errors that can be handled when the strategy is "replace" or "ignore". You can set it only when the strategy is "replace" or "ignore". The default value is 9223372036854775807, which means that almost all errors are tolerant. -# threshold = 9223372036854775807 -# Controls the maximum number of records in the `conflict_records` table. The default value is 100. In the physical import mode, if the strategy is "replace", the conflict records that are overwritten are recorded. In the logical import mode, if the strategy is "ignore", the conflict records that are ignored are recorded; if the strategy is "replace", the conflict records are not recorded. -# max-record-rows = 100 +# Controls the maximum number of conflict errors that can be handled when strategy is "replace" or "ignore". You can set it only when the strategy is "replace" or "ignore". The default value is 10000. If you set a value larger than 10000, the import process might experience performance degradation. +# threshold = 10000 +# Controls the maximum number of records in the `conflict_records` table. The default value is 10000. +# Starting from v8.1.0, there is no need to configure `max-record-rows` manually, because TiDB Lightning automatically assigns the value of `max-record-rows` with the value of `threshold`, regardless of the user input. `max-record-rows` will be deprecated in a future release. +# In the physical import mode, if the strategy is "replace", the conflict records that are overwritten are recorded. +# In the logical import mode, if the strategy is "ignore", the conflict records that are ignored are recorded; if the strategy is "replace", the conflict records are not recorded. +# max-record-rows = 10000 [tikv-importer] # "local": Physical import mode, used by default. It applies to large dataset import, @@ -459,54 +463,3 @@ log-progress = "5m" # The default value is 60 seconds. # check-disk-quota = "60s" ``` - -## Command line parameters - -### Usage of `tidb-lightning` - -| Parameter | Explanation | Corresponding setting | -|:----|:----|:----| -| --config *file* | Reads global configuration from *file*. If not specified, the default configuration would be used. | | -| -V | Prints program version | | -| -d *directory* | Directory or [external storage URI](/external-storage-uri.md) of the data dump to read from | `mydumper.data-source-dir` | -| -L *level* | Log level: debug, info, warn, error, fatal (default = info) | `lightning.log-level` | -| -f *rule* | [Table filter rules](/table-filter.md) (can be specified multiple times) | `mydumper.filter` | -| --backend *[backend](/tidb-lightning/tidb-lightning-overview.md)* | Select an import mode. `local` refers to the physical import mode; `tidb` refers to the logical import mode. | `local` | -| --log-file *file* | Log file path. By default, it is `/tmp/lightning.log.{timestamp}`. If set to '-', it means that the log files will be output to stdout. | `lightning.log-file` | -| --status-addr *ip:port* | Listening address of the TiDB Lightning server | `lightning.status-port` | -| --pd-urls *host:port* | PD endpoint address | `tidb.pd-addr` | -| --tidb-host *host* | TiDB server host | `tidb.host` | -| --tidb-port *port* | TiDB server port (default = 4000) | `tidb.port` | -| --tidb-status *port* | TiDB status port (default = 10080) | `tidb.status-port` | -| --tidb-user *user* | User name to connect to TiDB | `tidb.user` | -| --tidb-password *password* | Password to connect to TiDB. The password can either be plaintext or Base64 encoded. | `tidb.password` | -| --enable-checkpoint *bool* | Whether to enable checkpoints (default = true) | `checkpoint.enable` | -| --analyze *level* | Analyze tables after importing. Available values are "required", "optional" (default value), and "off" | `post-restore.analyze` | -| --checksum *level* | Compare checksum after importing. Available values are "required" (default value), "optional", and "off" | `post-restore.checksum` | -| --check-requirements *bool* | Check cluster version compatibility before starting the task, and check whether TiKV has more than 10% free space left during running time. (default = true) | `lightning.check-requirements` | -| --ca *file* | CA certificate path for TLS connection | `security.ca-path` | -| --cert *file* | Certificate path for TLS connection | `security.cert-path` | -| --key *file* | Private key path for TLS connection | `security.key-path` | -| --server-mode | Start TiDB Lightning in server mode | `lightning.server-mode` | - -If a command line parameter and the corresponding setting in the configuration file are both provided, the command line parameter will be used. For example, running `./tidb-lightning -L debug --config cfg.toml` would always set the log level to "debug" regardless of the content of `cfg.toml`. - -## Usage of `tidb-lightning-ctl` - -This tool can execute various actions given one of the following parameters: - -| Parameter | Explanation | -|:----|:----| -| --compact | Performs a full compaction | -| --switch-mode *mode* | Switches every TiKV store to the given mode: normal, import | -| --fetch-mode | Prints the current mode of every TiKV store | -| --import-engine *uuid* | Imports the closed engine file from TiKV Importer into the TiKV cluster | -| --cleanup-engine *uuid* | Deletes the engine file from TiKV Importer | -| --checkpoint-dump *folder* | Dumps current checkpoint as CSVs into the folder | -| --checkpoint-error-destroy *tablename* | Removes the checkpoint and drops the table if it caused error | -| --checkpoint-error-ignore *tablename* | Ignores any error recorded in the checkpoint involving the given table | -| --checkpoint-remove *tablename* | Unconditionally removes the checkpoint of the table | - -The *tablename* must either be a qualified table name in the form `` `db`.`tbl` `` (including the backquotes), or the keyword "all". - -Additionally, all parameters of `tidb-lightning` described in the section above are valid in `tidb-lightning-ctl`. diff --git a/tidb-lightning/tidb-lightning-data-source.md b/tidb-lightning/tidb-lightning-data-source.md index 1bede6ce99bc4..ff7722a070816 100644 --- a/tidb-lightning/tidb-lightning-data-source.md +++ b/tidb-lightning/tidb-lightning-data-source.md @@ -410,35 +410,35 @@ The following examples show how to import data from Amazon S3 using TiDB Lightni + Use the locally configured permissions to access S3 data: ```bash - ./tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ + tiup tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ -d 's3://my-bucket/sql-backup' ``` + Use the path-style request to access S3 data: ```bash - ./tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ + tiup tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ -d 's3://my-bucket/sql-backup?force-path-style=true&endpoint=http://10.154.10.132:8088' ``` + Use a specific AWS IAM role ARN to access S3 data: ```bash - ./tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ + tiup tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ -d 's3://my-bucket/test-data?role-arn=arn:aws:iam::888888888888:role/my-role' ``` * Use access keys of an AWS IAM user to access S3 data: ```bash - ./tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ + tiup tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ -d 's3://my-bucket/test-data?access_key={my_access_key}&secret_access_key={my_secret_access_key}' ``` * Use the combination of AWS IAM role access keys and session tokens to access S3 data: ```bash - ./tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ + tiup tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ -d 's3://my-bucket/test-data?access_key={my_access_key}&secret_access_key={my_secret_access_key}&session-token={my_session_token}' ``` diff --git a/tidb-lightning/tidb-lightning-distributed-import.md b/tidb-lightning/tidb-lightning-distributed-import.md index 20d6219d0ea09..b80dbc82f3ccf 100644 --- a/tidb-lightning/tidb-lightning-distributed-import.md +++ b/tidb-lightning/tidb-lightning-distributed-import.md @@ -107,7 +107,7 @@ sorted-kv-dir = "/path/to/sorted-dir" If the data source is stored in external storage such as Amazon S3 or GCS, you need to configure additional parameters for connection. You can specify parameters for such configuration. For example, the following example assumes that data is stored in Amazon S3: ``` -./tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ +tiup tidb-lightning --tidb-port=4000 --pd-urls=127.0.0.1:2379 --backend=local --sorted-kv-dir=/tmp/sorted-kvs \ -d 's3://my-bucket/sql-backup' ``` diff --git a/tidb-lightning/tidb-lightning-error-resolution.md b/tidb-lightning/tidb-lightning-error-resolution.md index 21fb3b56640f3..683bb05392f51 100644 --- a/tidb-lightning/tidb-lightning-error-resolution.md +++ b/tidb-lightning/tidb-lightning-error-resolution.md @@ -47,7 +47,7 @@ The following errors are always fatal, and cannot be skipped by changing `lightn ## Conflict errors -You can use the [`conflict.threshold`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) configuration item to increase the tolerance of errors related to data conflict. If this configuration item is set to *N*, TiDB Lightning allows and skips up to *N* conflict errors from the data source before it exits. The default value is `9223372036854775807`, which means that almost all errors are tolerant. +You can use the [`conflict.threshold`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) configuration item to increase the tolerance of errors related to data conflict. If this configuration item is set to *N*, TiDB Lightning allows and skips up to *N* conflict errors from the data source before it exits. The default value is `10000`, which means that 10000 errors are tolerant. These errors are recorded in a table. After the import is completed, you can view the errors in the database and process them manually. For more information, see [Error Report](#error-report) diff --git a/tidb-lightning/tidb-lightning-logical-import-mode-usage.md b/tidb-lightning/tidb-lightning-logical-import-mode-usage.md index d7f53d5abb6d2..419cc529c832f 100644 --- a/tidb-lightning/tidb-lightning-logical-import-mode-usage.md +++ b/tidb-lightning/tidb-lightning-logical-import-mode-usage.md @@ -56,9 +56,9 @@ Conflicting data refers to two or more records with the same data in the PK or U | `"error"` | Terminating the import when conflicting data is detected. | `INSERT INTO ...` | | `""` | Converted to `"error"`, which means terminating the import when conflicting data is detected. | None | -When the strategy is `"error"`, errors caused by conflicting data directly terminates the import task. When the strategy is `"replace"` or `"ignore"`, you can control the maximum tolerant conflicts by configuring [`conflict.threshold`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task). The default value is `9223372036854775807`, which means that almost all errors are tolerant. +When the strategy is `"error"`, errors caused by conflicting data directly terminate the import task. When the strategy is `"replace"` or `"ignore"`, you can control the maximum tolerant conflicts by configuring [`conflict.threshold`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task). The default value is `10000`, which means that 10000 errors are tolerant. -When the strategy is `"ignore"`, conflicting data is recorded in the downstream `conflict_records` table. For further details, see [Error report](/tidb-lightning/tidb-lightning-error-resolution.md#error-report). In this case, you can limit the records by configuring [`conflict.max-record-rows`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task), and conflicting data that exceeds the limit is skipped and not recorded. The default value is `100`. +When the strategy is `"ignore"`, conflicting data is recorded in the downstream `conflict_records` table. For further details, see [Error report](/tidb-lightning/tidb-lightning-error-resolution.md#error-report). Before v8.1.0, you can limit the records by configuring [`conflict.max-record-rows`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task), and conflicting data that exceeds the limit is skipped and not recorded. Starting from v8.1.0, you need to configure [`conflict.threshold`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) instead, because TiDB Lightning automatically assigns the value of `max-record-rows` with the value of `threshold`, regardless of the user input. ## Performance tuning diff --git a/tidb-lightning/tidb-lightning-overview.md b/tidb-lightning/tidb-lightning-overview.md index 9e1b8eed49ed9..f8e9d63791547 100644 --- a/tidb-lightning/tidb-lightning-overview.md +++ b/tidb-lightning/tidb-lightning-overview.md @@ -20,6 +20,10 @@ TiDB Lightning can read data from the following sources: - [Amazon S3](/external-storage-uri.md#amazon-s3-uri-format) - [Google Cloud Storage](/external-storage-uri.md#gcs-uri-format) +> **Note:** +> +> Compared with TiDB Lightning, the [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) statement can be directly executed on TiDB nodes, supports automated distributed task scheduling and [TiDB Global Sort](/tidb-global-sort.md), and offers significant improvements in deployment, resource utilization, task configuration convenience, ease of invocation and integration, high availability, and scalability. It is recommended that you consider using `IMPORT INTO` instead of TiDB Lightning in appropriate scenarios. + ## TiDB Lightning architecture ![Architecture of TiDB Lightning tool set](/media/tidb-lightning-architecture.png) diff --git a/tidb-lightning/tidb-lightning-physical-import-mode-usage.md b/tidb-lightning/tidb-lightning-physical-import-mode-usage.md index a9682462f46c4..0a6f22cde8b28 100644 --- a/tidb-lightning/tidb-lightning-physical-import-mode-usage.md +++ b/tidb-lightning/tidb-lightning-physical-import-mode-usage.md @@ -30,17 +30,19 @@ check-requirements = true data-source-dir = "/data/my_database" [conflict] -# Starting from v7.3.0, a new version of strategy is introduced to handle conflicting data. The default value is "". Starting from v8.0.0, TiDB Lightning optimizes the conflict strategy for both physical and logical import modes (experimental). +# Starting from v7.3.0, a new version of strategy is introduced to handle conflicting data. The default value is "". Starting from v8.0.0, TiDB Lightning optimizes the conflict strategy for both physical and logical import modes. # - "": TiDB Lightning does not detect or handle conflicting data. If the source file contains conflicting primary or unique key records, the subsequent step reports an error. # - "error": when detecting conflicting primary or unique key records in the imported data, TiDB Lightning terminates the import and reports an error. # - "replace": when encountering conflicting primary or unique key records, TiDB Lightning retains the latest data and overwrites the old data. # The conflicting data are recorded in the `lightning_task_info.conflict_error_v2` table (recording conflicting data detected by post-import conflict detection) and the `conflict_records` table (recording conflicting data detected by preprocess conflict detection) of the target TiDB cluster. +# If you set `conflict.strategy = "replace"` in physical import mode, the conflicting data can be checked in the `lightning_task_info.conflict_view` view. # You can manually insert the correct records into the target table based on your application requirements. Note that the target TiKV must be v5.2.0 or later versions. strategy = "" -# Controls whether to enable preprocess conflict detection, which checks conflicts in data before importing it to TiDB. In scenarios where the ratio of conflict records is greater than or equal to 1%, it is recommended to enable preprocess conflict detection for better performance in conflict detection. In other scenarios, it is recommended to disable it. The default value is false, indicating that TiDB Lightning only checks conflicts after the import. If you set it to true, TiDB Lightning checks conflicts both before and after the import. This parameter is experimental. +# Controls whether to enable preprocess conflict detection, which checks conflicts in data before importing it to TiDB. The default value is false, indicating that TiDB Lightning only checks conflicts after the import. If you set it to true, TiDB Lightning checks conflicts both before and after the import. This parameter can be used only in the physical import mode. In scenarios where the number of conflict records is greater than 1,000,000, it is recommended to set `precheck-conflict-before-import = true` for better performance in conflict detection. In other scenarios, it is recommended to disable it. # precheck-conflict-before-import = false -# threshold = 9223372036854775807 -# max-record-rows = 100 +# threshold = 10000 +# Starting from v8.1.0, there is no need to configure `max-record-rows` manually, because TiDB Lightning automatically assigns the value of `max-record-rows` with the value of `threshold`, regardless of the user input. `max-record-rows` will be deprecated in a future release. +# max-record-rows = 10000 [tikv-importer] # Import mode. "local" means using the physical import mode. @@ -210,6 +212,25 @@ store-write-bwlimit = "128MiB" distsql-scan-concurrency = 3 ``` +You can measure the impact of data import on TPCC results by simulating the online application using TPCC and importing data into a TiDB cluster using TiDB Lightning. The test result is as follows: + +| Concurrency | TPM | P99 | P90 | AVG | +| ----- | --- | --- | --- | --- | +| 1 | 20%~30% | 60%~80% | 30%~50% | 30%~40% | +| 8 | 15%~25% | 70%~80% | 35%~45% | 20%~35% | +| 16 | 20%~25% | 55%~85% | 35%~40% | 20%~30% | +| 64 | No significant impact | +| 256 | No significant impact | + +The percentage in the preceding table indicates the impact of data import on TPCC results. + +* For the TPM column, the number indicates the percentage of TPM decrease. +* For the P99, P90, and AVG columns, the number indicates the percentage of latency increase. + +The test results show that the smaller the concurrency, the larger the impact of data import on TPCC results. When the concurrency is 64 or more, the impact of data import on TPCC results is negligible. + +Therefore, if your TiDB cluster has a latency-sensitive application and a low concurrency, it is strongly recommended **not** to use TiDB Lightning to import data into the cluster. This will cause a significant impact on the online application. + ## Performance tuning **The most direct and effective ways to improve import performance of the physical import mode are as follows:** diff --git a/tidb-lightning/tidb-lightning-physical-import-mode.md b/tidb-lightning/tidb-lightning-physical-import-mode.md index 6e017e0ba944b..032b9fecdfa3b 100644 --- a/tidb-lightning/tidb-lightning-physical-import-mode.md +++ b/tidb-lightning/tidb-lightning-physical-import-mode.md @@ -21,7 +21,7 @@ The backend for the physical import mode is `local`. You can modify it in `tidb- 1. Before importing data, TiDB Lightning automatically switches the TiKV nodes to "import mode", which improves write performance and stops auto-compaction. TiDB Lightning determines whether to pause global scheduling according to the TiDB Lightning version. - - Starting from v7.1.0, you can you can control the scope of pausing scheduling by using the TiDB Lightning parameter [`pause-pd-scheduler-scope`](/tidb-lightning/tidb-lightning-configuration.md). + - Starting from v7.1.0, you can control the scope of pausing scheduling by using the TiDB Lightning parameter [`pause-pd-scheduler-scope`](/tidb-lightning/tidb-lightning-configuration.md). - For TiDB Lightning versions between v6.2.0 and v7.0.0, the behavior of pausing global scheduling depends on the TiDB cluster version. When the TiDB cluster >= v6.1.0, TiDB Lightning pauses scheduling for the Region that stores the target table data. After the import is completed, TiDB Lightning recovers scheduling. For other versions, TiDB Lightning pauses global scheduling. - When TiDB Lightning < v6.2.0, TiDB Lightning pauses global scheduling. diff --git a/tidb-lightning/tidb-lightning-prechecks.md b/tidb-lightning/tidb-lightning-prechecks.md index c5ae306246c71..c55455afc9bce 100644 --- a/tidb-lightning/tidb-lightning-prechecks.md +++ b/tidb-lightning/tidb-lightning-prechecks.md @@ -14,7 +14,7 @@ The following table describes each check item and detailed explanation. | Cluster version and status| >= 5.3.0 | Check whether the cluster can be connected in the configuration, and whether the TiKV/PD/TiFlash version supports the physical import mode. | | Permissions | >= 5.3.0 | When the data source is cloud storage (Amazon S3), check whether TiDB Lightning has the necessary permissions and make sure that the import will not fail due to lack of permissions. | | Disk space | >= 5.3.0 | Check whether there is enough space on the local disk and on the TiKV cluster for importing data. TiDB Lightning samples the data sources and estimates the percentage of the index size from the sample result. Because indexes are included in the estimation, there might be cases where the size of the source data is less than the available space on the local disk, but still, the check fails. In the physical import mode, TiDB Lightning also checks whether the local storage is sufficient because external sorting needs to be done locally. For more details about the TiKV cluster space and local storage space (controlled by `sort-kv-dir`), see [Downstream storage space requirements](/tidb-lightning/tidb-lightning-requirements.md#storage-space-of-the-target-database) and [Resource requirements](/tidb-lightning/tidb-lightning-physical-import-mode.md#environment-requirements). | -| Region distribution status | >= 5.3.0 | Check whether the Regions in the TiKV cluster are distributed evenly and whether there are too many empty Regions. If the number of empty Regions exceeds max(1000, number of tables * 3), i.e. greater than the bigger one of "1000" or "3 times the number of tables ", then the import cannot be executed. | +| Region distribution status | >= 5.3.0 | Check whether the Regions in the TiKV cluster are distributed evenly and whether there are too many empty Regions. If the number of empty Regions exceeds max(1000, number of tables * 3), i.e. greater than the bigger one of "1000" or "3 times the number of tables", then the import cannot be executed. | | Exceedingly Large CSV files in the data file | >= 5.3.0 | When there are CSV files larger than 10 GiB in the backup file and auto-slicing is not enabled (StrictFormat=false), it will impact the import performance. The purpose of this check is to remind you to ensure the data is in the right format and to enable auto-slicing. | | Recovery from breakpoints | >= 5.3.0 | This check ensures that no changes are made to the source file or schema in the database during the breakpoint recovery process that would result in importing the wrong data. | | Import into an existing table | >= 5.3.0 | When importing into an already created table, it checks, as much as possible, whether the source file matches the existing table. Check if the number of columns matches. If the source file has column names, check if the column names match. When there are default columns in the source file, it checks if the default columns have Default Value, and if they have, the check passes. | diff --git a/tidb-lightning/troubleshoot-tidb-lightning.md b/tidb-lightning/troubleshoot-tidb-lightning.md index 0748a5de36c54..57aa0deeb9a88 100644 --- a/tidb-lightning/troubleshoot-tidb-lightning.md +++ b/tidb-lightning/troubleshoot-tidb-lightning.md @@ -47,7 +47,7 @@ It is potentially caused by starting `tidb-lightning` incorrectly, which causes [2018/08/10 07:29:08.310 +08:00] [INFO] [main.go:41] ["got signal to exit"] [signal=hangup] ``` -It is not recommended to directly use `nohup` in the command line to start `tidb-lightning`. You can [start `tidb-lightning`](/tidb-lightning/deploy-tidb-lightning.md) by executing a script. +It is not recommended to directly use `nohup` in the command line to start `tidb-lightning`. You can [start `tidb-lightning`](/get-started-with-tidb-lightning.md#step-4-start-tidb-lightning) by executing a script. In addition, if the last log of TiDB Lightning shows that the error is "Context canceled", you need to search for the first "ERROR" level log. This "ERROR" level log is usually followed by "got signal to exit", which indicates that TiDB Lightning received an interrupt signal and then exited. diff --git a/tidb-monitoring-api.md b/tidb-monitoring-api.md index 0179b1d4de566..c79c5f7e7356d 100644 --- a/tidb-monitoring-api.md +++ b/tidb-monitoring-api.md @@ -28,7 +28,7 @@ The following example uses `http://${host}:${port}/status` to get the current st curl http://127.0.0.1:10080/status { connections: 0, # The current number of clients connected to the TiDB server. - version: "8.0.11-TiDB-v8.0.0", # The TiDB version number. + version: "8.0.11-TiDB-v8.1.0", # The TiDB version number. git_hash: "778c3f4a5a716880bcd1d71b257c8165685f0d70" # The Git Hash of the current TiDB code. } ``` diff --git a/tidb-resource-control.md b/tidb-resource-control.md index 63c8dc9cae67d..966d0881b126b 100644 --- a/tidb-resource-control.md +++ b/tidb-resource-control.md @@ -258,10 +258,6 @@ SELECT /*+ RESOURCE_GROUP(rg1) */ * FROM t limit 10; ### Manage queries that consume more resources than expected (Runaway Queries) -> **Warning:** -> -> This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. - A runaway query is a query (`SELECT` statement only) that consumes more time or resources than expected. The term **runaway queries** is used in the following to describe the feature of managing the runaway query. - Starting from v7.2.0, the resource control feature introduces the management of runaway queries. You can set criteria for a resource group to identify runaway queries and automatically take actions to prevent them from exhausting resources and affecting other queries. You can manage runaway queries for a resource group by including the `QUERY_LIMIT` field in [`CREATE RESOURCE GROUP`](/sql-statements/sql-statement-create-resource-group.md) or [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md). @@ -380,7 +376,7 @@ You can get more information about runaway queries from the following system tab + The `mysql.tidb_runaway_queries` table contains the history records of all runaway queries identified in the past 7 days. Take one of the rows as an example: ```sql - MySQL [(none)]> SELECT * FROM mysql.tidb_runaway_queries LIMIT 1\G; + MySQL [(none)]> SELECT * FROM mysql.tidb_runaway_queries LIMIT 1\G *************************** 1. row *************************** resource_group_name: rg1 time: 2023-06-16 17:40:22 diff --git a/tidb-roadmap.md b/tidb-roadmap.md index 8e2c5d6fa5c72..41333efb2b866 100644 --- a/tidb-roadmap.md +++ b/tidb-roadmap.md @@ -7,7 +7,7 @@ summary: Learn about what's coming in the future for TiDB. This roadmap provides a look into the proposed future. This will be continually updated as we release long-term stable (LTS) versions. The purpose is to provide visibility into what is coming, so that you can more closely follow the progress, learn about the key milestones on the way, and give feedback as the development work goes on. -In the course of development, this roadmap is subject to change based on user needs and feedback. As expected, as the columns move right, the items under them are less committed. If you have a feature request or want to prioritize a feature, please file an issue on [GitHub](https://github.com/pingcap/tidb/issues). +In the course of development, this roadmap is subject to change based on user needs and feedback. **DO NOT** schedule your release plans according to the content of the roadmap. If you have a feature request or want to prioritize a feature, please file an issue on [GitHub](https://github.com/pingcap/tidb/issues). ## Rolling roadmap highlights @@ -15,8 +15,8 @@ In the course of development, this roadmap is subject to change based on user ne Category - End of CY23 LTS release - Mid of CY24 LTS release + End of CY24 LTS release + Mid of CY25 LTS release Future releases @@ -27,39 +27,77 @@ In the course of development, this roadmap is subject to change based on user ne
    +
  • + TiKV in-memory data cache
    + TiKV maintains recent versions of data in memory to reduce redundant MVCC scans, thus improving performance. +
    • +
  • - Distributed eXecution Framework (DXF)
    - In v7.1.0, TiDB introduced the DXF for tasks (such as DDL and analyze). This is the foundation for parallelizing these tasks across compute nodes. v7.4.0 introduces global sorting in distributed re-organization tasks (such as DDL and import), which greatly mitigates extra resource consumption in storage. Optionally, external shared storage can be leveraged for simplicity and cost savings. + Global indexing for partitioned tables

    • +
  • + Adaptive concurrency for statistics collection
    + TiDB automatically adjusts the parallelism and scan concurrency of statistics collection tasks based on the number of deployed nodes and hardware specifications, improving the speed of statistics collection. +

    • +
  • + Rapid database recovery
    + Reduce the time required for full database recovery and point-in-time recovery (PITR). +
    • +
    +
  • + Unlimited-size transactions
    + The volume of data processed by uncommitted transactions is no longer limited by the available memory of TiDB nodes, thus improving the success rate of transactions and batch tasks. +
    • +
    +
  • + Load-based traffic routing by TiProxy
    + TiProxy forwards traffic based on the workload of the target TiDB node, maximizing hardware resource utilization. +
  • - Enhancements to performance and generalizability of plan cache
    + Microservice for PD heartbeat
    + Heartbeat services in PD can be independently deployed and scaled, preventing PD from becoming a bottleneck for the cluster's resources. +
    • +
    +
  • + Less I/O consumption for statistics collection
    + Users can choose to scan only a portion of the data samples on TiKV during statistics collection, reducing time and resource consumption. +
    • +
    +
  • + Remove the limitation for pushing down Limit operator to TiKV

  • - Dynamic node scaling via the DXF
    - Automatically adjust node allocation to meet resource costs of background tasks, while maintaining stability and performance expectations + Cascades optimizer framework
    + Introduce a more mature and powerful optimizer framework to expand the capabilities of the current optimizer.

    • +
  • + Single DM task reaches 150 MiB/s during full data migration
    +

    • +
  • + Enhanced DDL execution framework
    + Provide a scalable parallel DDL execution framework to improve the performance and stability of DDL operations. +
  • - Unlimited transaction size + Table-level load balancing
    + PD determines data scheduling strategies based on the workload of each Region on every table.
    • -
  • - Federated query -
    TiDB query planner support for multiple storage engines in HTAP use cases. + Improve performance of handling system tables with large data volumes
    + Enhance query performance and reduce query overhead for system tables with large data volumes.
    • -
@@ -71,35 +109,73 @@ In the course of development, this roadmap is subject to change based on user ne
  • - Resource control for background tasks
    - - Control over how background tasks, such as imports, DDL, TTL, auto-analyze, and compactions, can affect foreground traffic - + Limit memory consumption for backup tasks
    +
    • +
    +
  • + Limit memory consumption for statistics collection
    +
    • +
    +
  • + Manage massive SQL bindings
    + Improve the user experience of SQL binding, making it convenient for users to create and manage a large number of execution plans to stabilize database performance. +
    • +
    +
  • + Enhance resource group control over complex SQL
    + Regularly assess the Request Unit (RU) consumption of complex SQL before completion of SQL execution to prevent excessively large impacts on the entire system during execution.

  • - Runaway query control
    - An operator-controlled way to greatly enhance performance stability for workloads with unexpectedly expensive queries - + Automatically switch resource groups for runaway queries
    + When a query is identified as a runaway query, users can choose to switch it to a specific resource group and set an upper limit on resource consumption.
  • - Disaggregation of Placement Driver (PD) -
    Enhance cluster scalability and resilience + Limit memory consumption of schema metadata
    + Enhance the stability of large-scale clusters. +
    • +
    +
  • + Distributed statistics collection
    + Statistics collection supports parallel execution across multiple TiDB nodes to improve collection efficiency.

    • +
  • + Multi-version statistics
    + After statistics are updated, users can view the historical versions and choose to restore them to an earlier version. +
    • +
    +
  • + Reliable data backup
    + Reduce potential issues like insufficient memory during data backup and ensure the availability of backup data. +
    • +
    +
  • + Common operators support spilling to disk
    + Common operators such as HashAgg, Sort, TopN, HashJoin, WindowFunction, IndexJoin, and IndexHashJoin support spilling to disk, reducing the risk of out-of-memory (OOM). +
  • - Multi-tenancy -
    Resource isolation on top of resource control + Adaptive resource group
    + Resource groups automatically adjust their Request Unit (RU) settings based on past execution patterns. +
    • +
    +
  • + Enhanced memory protection
    + TiDB actively monitors the memory usage of all components and prevents memory operations that might impact system stability.

    • +
  • + Instance-level execution plan cache
    + All sessions within the same TiDB instance can share the execution plan cache, improving memory utilization. +
@@ -110,15 +186,22 @@ In the course of development, this roadmap is subject to change based on user ne
    +
  • + Reliable query termination
    + Running SQL statements can be immediately terminated, and the corresponding resources are released from TiDB and TiKV. +
    • +
  • - TiCDC integrations with data warehouse or data lake systems -
    + Permission control for switching resource groups +
    Only users with specific permissions can switch their resource groups, thus preventing resource abuse.

  • - TiDB node labels -
    Assign existing or newly added TiDB nodes for DDL operations to isolate DDL tasks from the compute resources used by online traffic - + Mapping tables or SQL with hot Regions +
    • +
    +
  • + Logical data import mode with IMPORT INTO

@@ -126,25 +209,49 @@ In the course of development, this roadmap is subject to change based on user ne
  • - SQL plan management -
    Mechanism for controlling SQL plan regression + Fine-grained customization of statistics collection +
    Users can modify the statistics collection strategy for specific tables, such as healthiness and parallelism. +
    • +
    +
  • + Workload Repository +
    TiDB persists workload information in memory to permanent volume, including cumulative and real-time statistic data, which aids in troubleshooting and analysis. +
    • +
    +
  • + Automatic index advisor +
    TiDB automatically analyzes SQL statements that can be optimized and recommends creating or dropping indexes. +
    • +
    +
  • + Support modifying column types for partitioned tables +
    Users can modify the data type of columns in partitioned tables, regardless of whether a column is a partitioning key. +
    • +
    +
  • + Conflict strategy for IMPORT INTO +
    Users can set the conflict resolution strategy when importing data, such as exiting with an error, ignoring, or replacing in case of conflicts.

  • - Index Advisor -
    Offer index recommendations to users based on workload, statistics, and execution plans + End-to-End monitoring +
    Track the time consumption of individual SQL statements throughout their entire lifecycle, including consumption on TiDB, TiKV, PD, and TiFlash components.
  • - Materialized views -
    Store pre-computed results as a persistent data view to boost query performance + Workload analysis +
    Analyze historical workload data from the Workload Repository and provide optimization recommendations, such as SQL tuning and statistics collection.

  • - Heterogeneous database migration support + Revisable primary key +
    • +
    +
  • + Export data as SQL statements

@@ -158,32 +265,40 @@ In the course of development, this roadmap is subject to change based on user ne
  • - Key management via Azure Key Vault -
    Static encryption managed by Azure Key Vault + Google Cloud KMS +
    Enhance the key management mechanism for static encryption based on Google Cloud KMS, making it generally available (GA). +
    • +
    +
  • + Improved dynamic privilege +
    Improve the dynamic privilege design and limit the implementation of Super privilege.

  • - Column-level access control -
    Grant and restrict access to specific columns + Marker-based log desensitization +
    Support marking sensitive information in the cluster log. Then, you can determine whether to desensitize it according to the usage scenario.

    • +
  • + FIPS +
    Encryption scenarios comply with FIPS. +
    -
  • +
  • IAM authentication for AWS -
    TiDB as AWS third-party ARN for AWS IAM access +
    TiDB as AWS third-party ARN for AWS IAM access.

  • - Unified TLS CA/Key rotation policy -
    Unified certificate management mechanism for all TiDB components + Kerberos +
    Support Kerberos-based authentication.
    • -

  • - AWS FIPS support -
    Enable FedRAMP compliance + MFA +
    Support the multi-factor authentication mechanism.
@@ -191,49 +306,24 @@ In the course of development, this roadmap is subject to change based on user ne
  • Label-based access control -
    Access permissions granted by configured labels +
    Support data access control by configuring labels.

  • Enhanced client-side encryption +
    Support client-side encryption of key fields to enhance data security.

  • - Enhanced data masking + Dynamic desensitization of business data +
    Support desensitizing data based on different data application scenarios to ensure data security in important fields.
    • -
-These are non-exhaustive plans and are subject to change. Features might differ per service subscriptions. - -## Previously delivered roadmap items - -You might have been waiting on some items from the last version. The following lists some previously delivered features. For more details, refer to the [v7.1.0 release notes](/releases/release-7.1.0.md). - -- Foundation of multi-tenancy framework: resource control quotas and scheduling for resource groups -- TiCDC supports object storage sink, including Amazon S3 and Azure Blob Storage (GA) -- Fastest online `ADD INDEX` (GA) -- TiFlash late materialization (GA) -- TiFlash supports spill to disk (GA) -- LDAP authentication -- SQL audit log enhancement (Enterprise-only) -- Partitioned Raft KV storage engine (experimental) -- General session-level plan cache (experimental) -- TiCDC distributed per table with Kafka downstream (experimental) - -## Recently shipped - -- [TiDB 8.0.0 Release Notes](https://docs.pingcap.com/tidb/v8.0/release-8.0.0) -- [TiDB 7.6.0 Release Notes](https://docs.pingcap.com/tidb/v7.6/release-7.6.0) -- [TiDB 7.5.0 Release Notes](https://docs.pingcap.com/tidb/v7.5/release-7.5.0) -- [TiDB 7.4.0 Release Notes](https://docs.pingcap.com/tidb/v7.4/release-7.4.0) -- [TiDB 7.3.0 Release Notes](https://docs.pingcap.com/tidb/v7.3/release-7.3.0) -- [TiDB 7.2.0 Release Notes](https://docs.pingcap.com/tidb/v7.2/release-7.2.0) -- [TiDB 7.1.0 Release Notes](https://docs.pingcap.com/tidb/v7.1/release-7.1.0) -- [TiDB 7.0.0 Release Notes](https://docs.pingcap.com/tidb/v7.0/release-7.0.0) -- [TiDB 6.6.0 Release Notes](https://docs.pingcap.com/tidb/v6.6/release-6.6.0) -- [TiDB 6.5.0 Release Notes](https://docs.pingcap.com/tidb/v6.5/release-6.5.0) +> **Note:** +> +> These are non-exhaustive plans and are subject to change. Features might differ per service subscriptions. \ No newline at end of file diff --git a/tidb-troubleshooting-map.md b/tidb-troubleshooting-map.md index 8b88e375925ed..8d73f3151c494 100644 --- a/tidb-troubleshooting-map.md +++ b/tidb-troubleshooting-map.md @@ -243,15 +243,13 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - 4.3.1 TiKV RocksDB encounters `write stall`. - A TiKV instance has two RocksDB instances, one in `data/raft` to save the Raft log, another in `data/db` to save the real data. You can check the specific cause for stall by running `grep "Stalling" RocksDB` in the log. The RocksDB log is a file starting with `LOG`, and `LOG` is the current log. + A TiKV instance has two RocksDB instances, one in `data/raft` to store the Raft log, and the other in `data/db` to store the real data. You can check the specific cause for the stall by running `grep "Stalling" RocksDB` in the log. The RocksDB log is a file starting with `LOG`, and `LOG` is the current log. `write stall` is a performance degradation mechanism natively built in RocksDB. When `write stall` occurs in RocksDB, the system performance drops significantly. In versions before v5.2.0, TiDB tries to block all write requests by returning a `ServerIsBusy` error directly to the client when encountering `write stall`, but this could lead to a sharp decrease in QPS performance. Starting from v5.2.0, TiKV introduces a new flow control mechanism that suppresses writes by delaying write requests dynamically in the scheduling layer, which replaces the previous mechanism of returning `server is busy` to clients when `write stall` occurs. The new flow control mechanism is enabled by default, and TiKV automatically disables the `write stall` mechanism for `KvDB` and `RaftDB` (except for memtables). However, when the number of pending requests exceeds a certain threshold, the flow control mechanism still takes effect, beginning to reject some or all write requests and returning a `server is busy` error to clients. For detailed explanations and thresholds, see [Flow control configuration](/tikv-configuration-file.md#storageflow-control). - - Too many `level0 sst` causes stall. You can add the `[rocksdb] max-sub-compactions = 2` (or 3) parameter to speed up `level0 sst` compaction. The compaction task from level0 to level1 is divided into several subtasks (the max number of subtasks is the value of `max-sub-compactions`) to be executed concurrently. See [case-815](https://github.com/pingcap/tidb-map/blob/master/maps/diagnose-case-study/case815.md) in Chinese. + - If the `server is busy` error is triggered by too many pending compaction bytes, you can alleviate this issue by increasing the values of the [`soft-pending-compaction-bytes-limit`](/tikv-configuration-file.md#soft-pending-compaction-bytes-limit) and [`hard-pending-compaction-bytes-limit`](/tikv-configuration-file.md#hard-pending-compaction-bytes-limit) parameters. - - Too many `pending compaction bytes` causes stall. The disk I/O fails to keep up with the write operations in business peaks. You can mitigate this problem by increasing the `soft-pending-compaction-bytes-limit` and `hard-pending-compaction-bytes-limit` of the corresponding CF. + - When pending compaction bytes reach the value of the `soft-pending-compaction-bytes-limit` parameter (`192GiB` by default), the flow control mechanism begins to reject some write requests (by returning `ServerIsBusy` to clients). In this case, you can increase the value of this parameter, for example, `[storage.flow-control] soft-pending-compaction-bytes-limit = "384GiB"`. - - The default value of `[rocksdb.defaultcf] soft-pending-compaction-bytes-limit` is `64GB`. If the pending compaction bytes reaches the threshold, RocksDB slows down the write speed. You can set `[rocksdb.defaultcf] soft-pending-compaction-bytes-limit` to `128GB`. - - - The default value of `hard-pending-compaction-bytes-limit` is `256GB`. If the pending compaction bytes reaches the threshold (this is not likely to happen, because RocksDB slows down the write after the pending compaction bytes reaches `soft-pending-compaction-bytes-limit`), RocksDB stops the write operation. You can set `hard-pending-compaction-bytes-limit` to `512GB`. + - When pending compaction bytes reach the value of the `hard-pending-compaction-bytes-limit` parameter (`1024GiB` by default), the flow control mechanism begins to reject all write requests (by returning `ServerIsBusy` to clients). This scenario is less likely to occur because the flow control mechanism intervenes to slow down write speed after reaching the threshold of `soft-pending-compaction-bytes-limit`. If it occurs, you can increase the value of this parameter, for example, `[storage.flow-control] hard-pending-compaction-bytes-limit = "2048GiB"`. - If the disk I/O capacity fails to keep up with the write for a long time, it is recommended to scale up your disk. If the disk throughput reaches the upper limit and causes write stall (for example, the SATA SSD is much lower than NVME SSD), while the CPU resources is sufficient, you may apply a compression algorithm of higher compression ratio. This way, the CPU resources is traded for disk resources, and the pressure on the disk is eased. @@ -416,7 +414,7 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - The binlog data is too large, so the single message written to Kafka is too large. You need to modify the following configuration of Kafka: - ```conf + ```properties message.max.bytes=1073741824 replica.fetch.max.bytes=1073741824 fetch.message.max.bytes=1073741824 diff --git a/tiflash-upgrade-guide.md b/tiflash-upgrade-guide.md index cd8ff9b62afc8..0a4fcbb78c7e0 100644 --- a/tiflash-upgrade-guide.md +++ b/tiflash-upgrade-guide.md @@ -23,11 +23,39 @@ To learn the standard upgrade process, see the following documents: > > - PingCAP does not provide bug fixes for non-LTS versions, such as v6.0. It is recommended that you upgrade to v6.1 and later LTS versions whenever possible. > -> - To upgrade TiFlash from versions earlier than v5.3.0 to v5.3.0 or later, you should stop TiFlash and then upgrade it. The following steps help you upgrade TiFlash without interrupting other components: -> -> - Stop the TiFlash instance: `tiup cluster stop -R tiflash` -> - Upgrade the TiDB cluster without restarting it (only updating the files): `tiup cluster upgrade --offline`, such as `tiup cluster upgrade v5.3.0 --offline` -> - Reload the TiDB cluster: `tiup cluster reload `. After the reload, the TiFlash instance is started and you do not need to manually start it. + +## Upgrade TiFlash using TiUP + +To upgrade TiFlash from versions earlier than v5.3.0 to v5.3.0 or later, you must stop TiFlash and then upgrade it. When you upgrade TiFlash using TiUP, note the following: + +- If the TiUP cluster version is v1.12.0 or later, you cannot stop TiFlash and then upgrade it. If the target version requires a TiUP cluster version of v1.12.0 or later, it is recommended that you first use `tiup cluster:v1.11.3 ` to upgrade TiFlash to an intermediate version, perform an online upgrade of the TiDB cluster, upgrade the TiUP version, and then upgrade the TiDB cluster to the target version directly without stopping it. +- If the TiUP cluster version is earlier than v1.12.0, perform the following steps to upgrade TiFlash. + +The following steps help you use TiUP to upgrade TiFlash without interrupting other components: + +1. Stop the TiFlash instance: + + ```shell + tiup cluster stop -R tiflash + ``` + +2. Upgrade the TiDB cluster without restarting it (only updating the files): + + ```shell + tiup cluster upgrade --offline + ``` + + For example: + + ```shell + tiup cluster upgrade v5.3.0 --offline + ``` + +3. Reload the TiDB cluster. After the reload, the TiFlash instance is started and you do not need to manually start it. + + ```shell + tiup cluster reload + ``` ## From 5.x or v6.0 to v6.1 diff --git a/tiflash/create-tiflash-replicas.md b/tiflash/create-tiflash-replicas.md index aaff056997f88..be4cb974f45dc 100644 --- a/tiflash/create-tiflash-replicas.md +++ b/tiflash/create-tiflash-replicas.md @@ -143,10 +143,10 @@ Before TiFlash replicas are added, each TiKV instance performs a full table scan tiup ctl:v pd -u http://:2379 store limit all engine tiflash 60 add-peer ``` - > In the preceding command, you need to replace `v` with the actual cluster version, such as `v8.0.0` and `:2379` with the address of any PD node. For example: + > In the preceding command, you need to replace `v` with the actual cluster version, such as `v8.1.0` and `:2379` with the address of any PD node. For example: > > ```shell - > tiup ctl:v8.0.0 pd -u http://192.168.1.4:2379 store limit all engine tiflash 60 add-peer + > tiup ctl:v8.1.0 pd -u http://192.168.1.4:2379 store limit all engine tiflash 60 add-peer > ``` Within a few minutes, you will observe a significant increase in CPU and disk IO resource usage of the TiFlash nodes, and TiFlash should create replicas faster. At the same time, the TiKV nodes' CPU and disk IO resource usage increases as well. diff --git a/tiflash/tiflash-spill-disk.md b/tiflash/tiflash-spill-disk.md index d1809951aac59..00d22eaf80006 100644 --- a/tiflash/tiflash-spill-disk.md +++ b/tiflash/tiflash-spill-disk.md @@ -17,8 +17,8 @@ Starting from v7.0.0, TiFlash supports spilling intermediate data to disk to rel TiFlash provides two triggering mechanisms for spilling data to disk. -* Operator-level spilling: by specifing the data spilling threshold for each operator, you can control when TiFlash spills data of that operator to disk. -* Query-level spilling: by specifing the maximum memory usage of a query on a TiFlash node and the memory ratio for spilling, you can control when TiFlash spills data of supported operators in a query to disk as needed. +* Operator-level spilling: by specifying the data spilling threshold for each operator, you can control when TiFlash spills data of that operator to disk. +* Query-level spilling: by specifying the maximum memory usage of a query on a TiFlash node and the memory ratio for spilling, you can control when TiFlash spills data of supported operators in a query to disk as needed. ### Operator-level spilling diff --git a/tiflash/tune-tiflash-performance.md b/tiflash/tune-tiflash-performance.md index 8c19bf7d96d07..c2ef222a5327f 100644 --- a/tiflash/tune-tiflash-performance.md +++ b/tiflash/tune-tiflash-performance.md @@ -230,7 +230,7 @@ ALTER TABLE employees COMPACT PARTITION pNorth, pEast TIFLASH REPLICA; ### Replace Shuffled Hash Join with Broadcast Hash Join -For `Join` operations with small tables, the Broadcast Hash Join algorithm can avoid transfering large tables, thereby improving the computing performance. +For `Join` operations with small tables, the Broadcast Hash Join algorithm can avoid transferring large tables, thereby improving the computing performance. - The [`tidb_broadcast_join_threshold_size`](/system-variables.md#tidb_broadcast_join_threshold_size-new-in-v50) variable controls whether to use the Broadcast Hash Join algorithm. If the table size (unit: byte) is smaller than the value of this variable, the Broadcast Hash Join algorithm is used. Otherwise, the Shuffled Hash Join algorithm is used. @@ -244,7 +244,7 @@ For `Join` operations with small tables, the Broadcast Hash Join algorithm can a set @@tidb_broadcast_join_threshold_count = 100000; ``` -The following example shows the query result before and after `tidb_broadcast_join_threshold_size` is re-configured. Before the re-configuration, the `ExchangeType` of `ExchangeSender_29` is `HashPartition`. After the value of this variable chages to `10000000`, the `ExchangeType` of `ExchangeSender_29` changes to `Broadcast`. +The following example shows the query result before and after `tidb_broadcast_join_threshold_size` is re-configured. Before the re-configuration, the `ExchangeType` of `ExchangeSender_29` is `HashPartition`. After the value of this variable changes to `10000000`, the `ExchangeType` of `ExchangeSender_29` changes to `Broadcast`. Before `tidb_broadcast_join_threshold_size` is re-configured: diff --git a/tikv-configuration-file.md b/tikv-configuration-file.md index 4ddca277f7023..18b194d0bf992 100644 --- a/tikv-configuration-file.md +++ b/tikv-configuration-file.md @@ -37,7 +37,7 @@ This document only describes the parameters that are not included in command-lin ### `slow-log-threshold` -+ The threshold for outputing slow logs. If the processing time is longer than this threshold, slow logs are output. ++ The threshold for outputting slow logs. If the processing time is longer than this threshold, slow logs are output. + Default value: `"1s"` ### `memory-usage-limit` @@ -1008,7 +1008,7 @@ Configuration items related to Raftstore. + At a certain interval, TiKV inspects the latency of the Raftstore component. This parameter specifies the interval of the inspection. If the latency exceeds this value, this inspection is marked as timeout. + Judges whether the TiKV node is slow based on the ratio of timeout inspection. -+ Default value: `"500ms"` ++ Default value: `"100ms"` + Minimum value: `"1ms"` ### `raft-write-size-limit` New in v5.3.0 @@ -1039,7 +1039,8 @@ Configuration items related to Raftstore. + Set the specific times that TiKV initiates periodic full compaction. You can specify multiple time schedules in an array. For example: + `periodic-full-compact-start-times = ["03:00", "23:00"]` indicates that TiKV performs full compaction daily at 03:00 AM and 11:00 PM, based on the local time zone of the TiKV node. - + `periodic-full-compact-start-times = ["03:00 +0000", "23:00 +0000"]` indicates that TiKV performs full compaction daily at 03:00 AM and 11:00 PM in UTC time. + + `periodic-full-compact-start-times = ["03:00 +0000", "23:00 +0000"]` indicates that TiKV performs full compaction daily at 03:00 AM and 11:00 PM in UTC timezone. + + `periodic-full-compact-start-times = ["03:00 +0800", "23:00 +0800"]` indicates that TiKV performs full compaction daily at 03:00 AM and 11:00 PM in UTC+08:00 timezone. + Default value: `[]`, which means periodic full compaction is disabled by default. ### `periodic-full-compact-start-max-cpu` New in v7.6.0 @@ -1272,6 +1273,10 @@ Configuration items related to RocksDB ### `info-log-max-size` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-size`](#max-size-new-in-v540). + + The maximum size of Info log + Default value: `"1GiB"` + Minimum value: `0` @@ -1279,11 +1284,19 @@ Configuration items related to RocksDB ### `info-log-roll-time` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated. TiKV no longer supports automatic log splitting based on time. Instead, you can use the configuration item [`log.file.max-size`](#max-size-new-in-v540) to set the threshold for automatic log splitting based on file size. + + The time interval at which Info logs are truncated. If the value is `0s`, logs are not truncated. + Default value: `"0s"` ### `info-log-keep-log-file-num` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-backups`](#max-backups-new-in-v540). + + The maximum number of kept log files + Default value: `10` + Minimum value: `0` @@ -1295,6 +1308,10 @@ Configuration items related to RocksDB ### `info-log-level` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.level`](#level-new-in-v540). + + Log levels of RocksDB + Default value: `"info"` @@ -1325,6 +1342,14 @@ Configuration items related to RocksDB + Unit: KiB|MiB|GiB +### `track-and-verify-wals-in-manifest` New in v6.5.9, v7.1.5, v7.5.2, and v8.0.0 + ++ Controls whether to record information about Write Ahead Log (WAL) files in the RocksDB MANIFEST file and whether to verify the integrity of WAL files during startup. For more information, see RocksDB [Track WAL in MANIFEST](https://github.com/facebook/rocksdb/wiki/Track-WAL-in-MANIFEST). ++ Default value: `true` ++ Value options: + + `true`: records information about WAL files in the MANIFEST file and verifies the integrity of WAL files during startup. + + `false`: does not record information about WAL files in the MANIFEST file and does not verify the integrity of WAL files during startup. + ## rocksdb.titan Configuration items related to Titan. @@ -1835,6 +1860,10 @@ Configuration items related to `raftdb` ### `info-log-max-size` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-size`](#max-size-new-in-v540). + + The maximum size of Info logs + Default value: `"1GiB"` + Minimum value: `0` @@ -1842,11 +1871,19 @@ Configuration items related to `raftdb` ### `info-log-roll-time` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated. TiKV no longer supports automatic log splitting based on time. Instead, you can use the configuration item [`log.file.max-size`](#max-size-new-in-v540) to set the threshold for automatic log splitting based on file size. + + The interval at which Info logs are truncated. If the value is `0s`, logs are not truncated. + Default value: `"0s"` (which means logs are not truncated) ### `info-log-keep-log-file-num` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-backups`](#max-backups-new-in-v540). + + The maximum number of Info log files kept in RaftDB + Default value: `10` + Minimum value: `0` @@ -1858,6 +1895,10 @@ Configuration items related to `raftdb` ### `info-log-level` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.level`](#level-new-in-v540). + + Log levels of RaftDB + Default value: `"info"` diff --git a/time-to-live.md b/time-to-live.md index 6b3e74b200c98..d0ab5106631fe 100644 --- a/time-to-live.md +++ b/time-to-live.md @@ -176,29 +176,32 @@ In addition, TiDB provides three tables to obtain more information about TTL job + The `mysql.tidb_ttl_table_status` table contains information about the previously executed TTL job and ongoing TTL job for all TTL tables ```sql - MySQL [(none)]> SELECT * FROM mysql.tidb_ttl_table_status LIMIT 1\G; + TABLE mysql.tidb_ttl_table_status LIMIT 1\G + ``` + + ``` *************************** 1. row *************************** table_id: 85 - parent_table_id: 85 + parent_table_id: 85 table_statistics: NULL - last_job_id: 0b4a6d50-3041-4664-9516-5525ee6d9f90 - last_job_start_time: 2023-02-15 20:43:46 + last_job_id: 0b4a6d50-3041-4664-9516-5525ee6d9f90 + last_job_start_time: 2023-02-15 20:43:46 last_job_finish_time: 2023-02-15 20:44:46 - last_job_ttl_expire: 2023-02-15 19:43:46 + last_job_ttl_expire: 2023-02-15 19:43:46 last_job_summary: {"total_rows":4369519,"success_rows":4369519,"error_rows":0,"total_scan_task":64,"scheduled_scan_task":64,"finished_scan_task":64} current_job_id: NULL current_job_owner_id: NULL current_job_owner_addr: NULL - current_job_owner_hb_time: NULL + current_job_owner_hb_time: NULL current_job_start_time: NULL current_job_ttl_expire: NULL - current_job_state: NULL + current_job_state: NULL current_job_status: NULL current_job_status_update_time: NULL 1 row in set (0.040 sec) ``` - The column `table_id` is the ID of the partitioned table, and the `parent_table_id` is the ID of the table, corresponding with the ID in `infomation_schema.tables`. If the table is not a partitioned table, the two IDs are the same. + The column `table_id` is the ID of the partitioned table, and the `parent_table_id` is the ID of the table, corresponding with the ID in [`information_schema.tables`](/information-schema/information-schema-tables.md). If the table is not a partitioned table, the two IDs are the same. The columns `{last, current}_job_{start_time, finish_time, ttl_expire}` describe respectively the start time, finish time, and expiration time used by the TTL job of the last or current execution. The `last_job_summary` column describes the execution status of the last TTL task, including the total number of rows, the number of successful rows, and the number of failed rows. @@ -206,25 +209,28 @@ In addition, TiDB provides three tables to obtain more information about TTL job + The `mysql.tidb_ttl_job_history` table contains information about the TTL jobs that have been executed. The record of TTL job history is kept for 90 days. ```sql - MySQL [(none)]> SELECT * FROM mysql.tidb_ttl_job_history LIMIT 1\G; + TABLE mysql.tidb_ttl_job_history LIMIT 1\G + ``` + + ``` *************************** 1. row *************************** - job_id: f221620c-ab84-4a28-9d24-b47ca2b5a301 - table_id: 85 + job_id: f221620c-ab84-4a28-9d24-b47ca2b5a301 + table_id: 85 parent_table_id: 85 - table_schema: test_schema - table_name: TestTable - partition_name: NULL + table_schema: test_schema + table_name: TestTable + partition_name: NULL create_time: 2023-02-15 17:43:46 finish_time: 2023-02-15 17:45:46 - ttl_expire: 2023-02-15 16:43:46 - summary_text: {"total_rows":9588419,"success_rows":9588419,"error_rows":0,"total_scan_task":63,"scheduled_scan_task":63,"finished_scan_task":63} - expired_rows: 9588419 - deleted_rows: 9588419 + ttl_expire: 2023-02-15 16:43:46 + summary_text: {"total_rows":9588419,"success_rows":9588419,"error_rows":0,"total_scan_task":63,"scheduled_scan_task":63,"finished_scan_task":63} + expired_rows: 9588419 + deleted_rows: 9588419 error_delete_rows: 0 - status: finished + status: finished ``` - The column `table_id` is the ID of the partitioned table, and the `parent_table_id` is the ID of the table, corresponding with the ID in `infomation_schema.tables`. `table_schema`, `table_name`, and `partition_name` correspond to the database, table name, and partition name. `create_time`, `finish_time`, and `ttl_expire` indicate the creation time, end time, and expiration time of the TTL task. `expired_rows` and `deleted_rows` indicate the number of expired rows and the number of rows deleted successfully. + The column `table_id` is the ID of the partitioned table, and the `parent_table_id` is the ID of the table, corresponding with the ID in `information_schema.tables`. `table_schema`, `table_name`, and `partition_name` correspond to the database, table name, and partition name. `create_time`, `finish_time`, and `ttl_expire` indicate the creation time, end time, and expiration time of the TTL task. `expired_rows` and `deleted_rows` indicate the number of expired rows and the number of rows deleted successfully. ## Compatibility with TiDB tools @@ -252,7 +258,6 @@ Currently, the TTL feature has the following limitations: * A table with the TTL attribute does not support being referenced by other tables as the primary table in a foreign key constraint. * It is not guaranteed that all expired data is deleted immediately. The time when expired data is deleted depends on the scheduling interval and scheduling window of the background cleanup job. * For tables that use [clustered indexes](/clustered-indexes.md), if the primary key is neither an integer nor a binary string type, the TTL job cannot be split into multiple tasks. This will cause the TTL job to be executed sequentially on a single TiDB node. If the table contains a large amount of data, the execution of the TTL job might become slow. -* TTL is not available for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). ## FAQs diff --git a/tiproxy/tiproxy-api.md b/tiproxy/tiproxy-api.md new file mode 100644 index 0000000000000..ee05aea62ab25 --- /dev/null +++ b/tiproxy/tiproxy-api.md @@ -0,0 +1,128 @@ +--- +title: TiProxy API +summary: Learn how to use the TiProxy API to access its configuration, health status, and monitoring data. +--- + +# TiProxy API + +[TiProxy](/tiproxy/tiproxy-overview.md) provides API endpoints for accessing its configuration, health status, and monitoring data. + +> **Note:** +> +> TiProxy API is specifically designed for debugging purposes and might not be fully compatible with future capabilities introduced in TiProxy. It's not recommended to include this tool in application or utility development to get information. + +The address for accessing the TiProxy API is `http://${host}:${port}${path}`, where `${host}:${port}` is specified by the TiProxy configuration item [`api.addr`](/tiproxy/tiproxy-configuration.md#addr-1), and `${path}` is the specific API endpoint you want to access. For example: + +```bash +curl http://127.0.0.1:3080/api/admin/config/ +``` + +## Get TiProxy configuration + +### Request URI + +`GET /api/admin/config/` + +### Parameter descriptions + +The query parameter is as follows: + +- `format`: (optional) specifies the format of the returned configuration. Value options are `json` and `toml`. The default value is `toml`. + +### Example + +The following example gets the TiProxy configuration in JSON format: + +```bash +curl "http://127.0.0.1:3080/api/admin/config/?format=json" +``` + +## Set TiProxy configuration + +Currently, you can only use the TOML format to modify TiProxy configuration. Unspecified configuration items will remain unchanged, so you only need to specify the items that you want to modify. + +### Request URI + +`PUT /api/admin/config/` + +### Request body + +You need to provide the TiProxy configuration in TOML format. For example: + +```toml +[log] +level='warning' +``` + +### Example + +The following example sets `log.level` as `'warning'`, while leaving other configuration items unchanged. + +1. Get the current TiProxy configuration: + + ```bash + curl http://127.0.0.1:3080/api/admin/config/ + ``` + + The output is as follows: + + ```toml + [log] + encoder = 'tidb' + level = 'info' + ``` + +2. Specify the value of `log.level` in the `test.toml` file, and then send a `PUT /api/admin/config/` request to update the value of `log.level`: + + ```shell + $ cat test.toml + [log] + level='warning' + $ curl -X PUT --data-binary @test.toml http://127.0.0.1:3080/api/admin/config/ + ``` + +3. Get the modified TiProxy configuration: + + ```bash + curl http://127.0.0.1:3080/api/admin/config/ + ``` + + The output is as follows: + + ```toml + [log] + encoder = 'tidb' + level = 'warning' + ``` + +## Get TiProxy health status + +This endpoint is used to get the health status of TiProxy and the checksum of the configuration. When TiProxy is running normally, this endpoint returns the checksum of the configuration. When TiProxy is shutting down or offline, it returns an error. + +### Request URI + +`GET /api/debug/health` + +### Example + +```bash +curl http://127.0.0.1:3080/api/debug/health +``` + +The output is as follows: + +```bash +{"config_checksum":3006078629} +``` + +## Get TiProxy monitoring data + +### Request URI + +`GET /metrics/` + +### Example + +```bash +curl http://127.0.0.1:3080/metrics/ +``` diff --git a/tiproxy/tiproxy-command-line-flags.md b/tiproxy/tiproxy-command-line-flags.md index 76099961b68a7..5468a5c30e17b 100644 --- a/tiproxy/tiproxy-command-line-flags.md +++ b/tiproxy/tiproxy-command-line-flags.md @@ -18,11 +18,29 @@ This section lists the flags of the server program `tiproxy`. + Default: `""` + You must specify the configuration file. For detailed configuration items, refer to [Configure TiProxy](/tiproxy/tiproxy-configuration.md). Note that TiProxy automatically reloads the configuration when the configuration file is modified. Therefore, do not directly modify the configuration file. It is recommended to modify the configuration by executing [`tiup cluster edit-config`](/tiup/tiup-component-cluster-edit-config.md) or [`kubectl edit tc`](https://docs.pingcap.com/tidb-in-kubernetes/stable/modify-tidb-configuration). -## TiProxy control +## TiProxy Control -This section lists the flags of the client program `tiproxyctl`. +This section introduces the syntax, options, and commands of the client program `tiproxyctl`. -### `--log_encoder` +> **Note:** +> +> TiProxy Control is specifically designed for debugging purposes and might not be fully compatible with future capabilities introduced in TiProxy. It's not recommended to include this tool in application or utility development to get information. + +### Syntax + +``` +tiproxyctl [flags] [command] +``` + +For example: + +``` +tiproxyctl --curls 127.0.0.1:3080 config get +``` + +### Options + +#### `--log_encoder` + Specifies the log format of `tiproxyctl`. + Type: `string` @@ -32,35 +50,67 @@ This section lists the flags of the client program `tiproxyctl`. - `console`: a more human-readable format - `json`: a structured log format -### `--log_level` +#### `--log_level` + Specifies the log level of tiproxyctl. + Type: `string` + Default: `"warn"` + You can specify `debug`, `info`, `warn`, `error`, `panic`. -### `--curls` +#### `--curls` + Specifies the server addresses. You can add multiple listening addresses. -+ Type: `comma seperated lists of ip:port` ++ Type: `comma separated lists of ip:port` + Default: `localhost:3080` + Server API gateway addresses. -### `-k`, `--insecure` +#### `-k, --insecure` + Specifies whether to skip TLS CA verification when dialing to the server. + Type: `boolean` + Default: `false` + Used for testing. -### `--ca` +#### `--ca` + Specifies the CA when dialing to the server. + Type: `string` + Default: `""` -### `--cert` +#### `--cert` + Specifies the certificate when dialing to the server. + Type: `string` + Default: `""` + +### Commands + +#### `config set` + +The `tiproxyctl config set` command reads a TOML-formatted configuration file from standard input and sets these configuration items to TiProxy. Unspecified configuration items will remain unchanged, so you only need to specify the items that you want to modify. + +The following example sets `log.level` as `'warning'`, while leaving other configuration items unchanged. + +```bash +$ cat test.toml +[log] +level='warning' +$ cat test.toml | tiproxyctl config set +"" +$ tiproxyctl config get | grep level +level = 'warning' +``` + +#### `config get` + +The `tiproxyctl config get` command is used to get the current TiProxy configuration in TOML format. + +#### `health` + +The `tiproxyctl health` command is used to get the health status of TiProxy and the checksum of the configuration. When TiProxy is running normally, it returns the checksum of the configuration. When TiProxy is shutting down or offline, it returns an error. + +Example output: + +```json +{"config_checksum":3006078629} +``` diff --git a/tiproxy/tiproxy-configuration.md b/tiproxy/tiproxy-configuration.md index 7fd480cef1bfb..e436315fbac58 100644 --- a/tiproxy/tiproxy-configuration.md +++ b/tiproxy/tiproxy-configuration.md @@ -84,19 +84,13 @@ Configuration for SQL port. + Possible values: ``, `v2` + Enable the [PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) on the port. By enabling the PROXY protocol, TiProxy can pass the real client IP address to TiDB. `v2` indicates using the PROXY protocol version 2, and `` indicates disabling the PROXY protocol. If the PROXY protocol is enabled on TiProxy, you need to also enable the [PROXY protocol](/tidb-configuration-file.md#proxy-protocol) on the TiDB server. -#### `require-backend-tls` - -+ Default value: `true` -+ Support hot-reload: yes, but only for new connections -+ Require TLS between TiProxy and TiDB servers. If the TiDB server doesn't support TLS, clients will report an error when connecting to TiProxy. - ### api Configurations for HTTP gateway. #### `addr` -+ Default value: `0.0.0.0:3090` ++ Default value: `0.0.0.0:3080` + Support hot-reload: no + API gateway address. You can specify `ip:port`. @@ -194,6 +188,12 @@ For server TLS object: A client TLS object. It is used to access TiDB or PD. +#### `require-backend-tls` + ++ Default value: `false` ++ Support hot-reload: yes, but only for new connections ++ Require TLS between TiProxy and TiDB servers. If the TiDB server does not support TLS, clients will report an error when connecting to TiProxy. + #### `sql-tls` A client TLS object. It is used to access TiDB SQL port (4000). diff --git a/tiproxy/tiproxy-overview.md b/tiproxy/tiproxy-overview.md index 32eda9fb4040f..0438e32b3c8a5 100644 --- a/tiproxy/tiproxy-overview.md +++ b/tiproxy/tiproxy-overview.md @@ -11,7 +11,7 @@ TiProxy is an optional component. You can also use a third-party proxy component The following figure shows the architecture of TiProxy: -TiProxy architecture +TiProxy architecture ## Main features @@ -23,7 +23,7 @@ TiProxy can migrate connections from one TiDB server to another without breaking As shown in the following figure, the client originally connects to TiDB 1 through TiProxy. After the connection migration, the client actually connects to TiDB 2. When TiDB 1 is about to be offline or the ratio of connections on TiDB 1 to connections on TiDB 2 exceeds the set threshold, the connection migration is triggered. The client is unaware of the connection migration. -TiProxy connection migration +TiProxy connection migration Connection migration usually occurs in the following scenarios: @@ -55,7 +55,7 @@ It is recommended that you use TiProxy for the scenarios that TiProxy is suitabl ## Installation and usage -This section describes how to deploy and change TiProxy using TiUP. For how to deploy TiProxy using TiDB Operator in Kubernetes, see [TiDB Operator documentation](https://docs.pingcap.com/tidb-in-kubernetes/stable). +This section describes how to deploy and change TiProxy using TiUP. For how to deploy TiProxy using TiDB Operator in Kubernetes, see [TiDB Operator documentation](https://docs.pingcap.com/tidb-in-kubernetes/stable/deploy-tiproxy). ### Deploy TiProxy @@ -126,7 +126,7 @@ When using TiUP to change the TiProxy configuration, if the configuration item t When you deploy TiProxy, it is recommended to specify the version of TiProxy so that TiProxy will not be upgraded when you upgrade the TiDB cluster. -If you need to upgrade TiProxy, add [`--tiproxy-version`](/tiup/tiup-component-cluster-upgrade.md) in the upgrade command to specify the version of TiProxy: +If you need to upgrade TiProxy, add [`--tiproxy-version`](/tiup/tiup-component-cluster-upgrade.md#--tiproxy-version) in the upgrade command to specify the version of TiProxy: ```shell tiup cluster upgrade --tiproxy-version diff --git a/tispark-overview.md b/tispark-overview.md index 8323afd2c3740..747c124539bc2 100644 --- a/tispark-overview.md +++ b/tispark-overview.md @@ -148,9 +148,9 @@ Add the following configuration in `spark-defaults.conf`: ``` spark.sql.extensions org.apache.spark.sql.TiExtensions -spark.tispark.pd.addresses ${your_pd_adress} +spark.tispark.pd.addresses ${your_pd_address} spark.sql.catalog.tidb_catalog org.apache.spark.sql.catalyst.catalog.TiCatalog -spark.sql.catalog.tidb_catalog.pd.addresses ${your_pd_adress} +spark.sql.catalog.tidb_catalog.pd.addresses ${your_pd_address} ``` Start spark-shell with the `--jars` option. diff --git a/tiup/tiup-cluster.md b/tiup/tiup-cluster.md index 6e862a92dd132..d81a58f5cf800 100644 --- a/tiup/tiup-cluster.md +++ b/tiup/tiup-cluster.md @@ -62,7 +62,7 @@ To deploy the cluster, run the `tiup cluster deploy` command. The usage of the c tiup cluster deploy [flags] ``` -This command requires you to provide the cluster name, the TiDB cluster version (such as `v8.0.0`), and a topology file of the cluster. +This command requires you to provide the cluster name, the TiDB cluster version (such as `v8.1.0`), and a topology file of the cluster. To write a topology file, refer to [the example](https://github.com/pingcap/tiup/blob/master/embed/examples/cluster/topology.example.yaml). The following file is an example of the simplest topology: @@ -122,12 +122,12 @@ tidb_servers: ... ``` -Save the file as `/tmp/topology.yaml`. If you want to use TiDB v8.0.0 and your cluster name is `prod-cluster`, run the following command: +Save the file as `/tmp/topology.yaml`. If you want to use TiDB v8.1.0 and your cluster name is `prod-cluster`, run the following command: {{< copyable "shell-regular" >}} ```shell -tiup cluster deploy -p prod-cluster v8.0.0 /tmp/topology.yaml +tiup cluster deploy -p prod-cluster v8.1.0 /tmp/topology.yaml ``` During the execution, TiUP asks you to confirm your topology again and requires the root password of the target machine (the `-p` flag means inputting password): @@ -135,7 +135,7 @@ During the execution, TiUP asks you to confirm your topology again and requires ```bash Please confirm your topology: TiDB Cluster: prod-cluster -TiDB Version: v8.0.0 +TiDB Version: v8.1.0 Type Host Ports OS/Arch Directories ---- ---- ----- ------- ----------- pd 172.16.5.134 2379/2380 linux/x86_64 deploy/pd-2379,data/pd-2379 @@ -179,7 +179,7 @@ tiup cluster list Starting /root/.tiup/components/cluster/v1.12.3/cluster list Name User Version Path PrivateKey ---- ---- ------- ---- ---------- -prod-cluster tidb v8.0.0 /root/.tiup/storage/cluster/clusters/prod-cluster /root/.tiup/storage/cluster/clusters/prod-cluster/ssh/id_rsa +prod-cluster tidb v8.1.0 /root/.tiup/storage/cluster/clusters/prod-cluster /root/.tiup/storage/cluster/clusters/prod-cluster/ssh/id_rsa ``` ## Start the cluster @@ -209,7 +209,7 @@ tiup cluster display prod-cluster ``` Starting /root/.tiup/components/cluster/v1.12.3/cluster display prod-cluster TiDB Cluster: prod-cluster -TiDB Version: v8.0.0 +TiDB Version: v8.1.0 ID Role Host Ports OS/Arch Status Data Dir Deploy Dir -- ---- ---- ----- ------- ------ -------- ---------- 172.16.5.134:3000 grafana 172.16.5.134 3000 linux/x86_64 Up - deploy/grafana-3000 @@ -284,7 +284,7 @@ tiup cluster display prod-cluster ``` Starting /root/.tiup/components/cluster/v1.12.3/cluster display prod-cluster TiDB Cluster: prod-cluster -TiDB Version: v8.0.0 +TiDB Version: v8.1.0 ID Role Host Ports OS/Arch Status Data Dir Deploy Dir -- ---- ---- ----- ------- ------ -------- ---------- 172.16.5.134:3000 grafana 172.16.5.134 3000 linux/x86_64 Up - deploy/grafana-3000 @@ -396,12 +396,12 @@ Global Flags: -y, --yes Skip all confirmations and assumes 'yes' ``` -For example, the following command upgrades the cluster to v8.0.0: +For example, the following command upgrades the cluster to v8.1.0: {{< copyable "shell-regular" >}} ```bash -tiup cluster upgrade tidb-test v8.0.0 +tiup cluster upgrade tidb-test v8.1.0 ``` ## Update configuration @@ -586,11 +586,11 @@ tiup cluster audit Starting component `cluster`: /home/tidb/.tiup/components/cluster/v1.12.3/cluster audit ID Time Command -- ---- ------- -4BLhr0 2024-03-28T23:55:09+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.0.0 /tmp/topology.yaml -4BKWjF 2024-03-28T23:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.0.0 /tmp/topology.yaml -4BKVwH 2024-03-28T23:02:08+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.0.0 /tmp/topology.yaml -4BKKH1 2024-03-28T16:39:04+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster destroy test -4BKKDx 2024-03-28T16:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.0.0 /tmp/topology.yaml +4BLhr0 2024-05-24T23:55:09+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.1.0 /tmp/topology.yaml +4BKWjF 2024-05-24T23:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.1.0 /tmp/topology.yaml +4BKVwH 2024-05-24T23:02:08+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.1.0 /tmp/topology.yaml +4BKKH1 2024-05-24T16:39:04+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster destroy test +4BKKDx 2024-05-24T16:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.1.0 /tmp/topology.yaml ``` The first column is `audit-id`. To view the execution log of a certain command, pass the `audit-id` of a command as the flag as follows: @@ -706,7 +706,7 @@ All operations above performed on the cluster machine use the SSH client embedde Then you can use the `--ssh=system` command-line flag to enable the system-native command-line tool: -- Deploy a cluster: `tiup cluster deploy --ssh=system`. Fill in the name of your cluster for ``, the TiDB version to be deployed (such as `v8.0.0`) for ``, and the topology file for ``. +- Deploy a cluster: `tiup cluster deploy --ssh=system`. Fill in the name of your cluster for ``, the TiDB version to be deployed (such as `v8.1.0`) for ``, and the topology file for ``. - Start a cluster: `tiup cluster start --ssh=system` - Upgrade a cluster: `tiup cluster upgrade ... --ssh=system` diff --git a/tiup/tiup-component-cluster-deploy.md b/tiup/tiup-component-cluster-deploy.md index 2c44333b7cdb0..01b1656a03415 100644 --- a/tiup/tiup-component-cluster-deploy.md +++ b/tiup/tiup-component-cluster-deploy.md @@ -14,7 +14,7 @@ tiup cluster deploy [flags] ``` - ``: the name of the new cluster, which cannot be the same as the existing cluster names. -- ``: the version number of the TiDB cluster to deploy, such as `v8.0.0`. +- ``: the version number of the TiDB cluster to deploy, such as `v8.1.0`. - ``: the prepared [topology file](/tiup/tiup-cluster-topology-reference.md). ## Options diff --git a/tiup/tiup-component-cluster-patch.md b/tiup/tiup-component-cluster-patch.md index 69ac1d04602b6..aff4fb3076618 100644 --- a/tiup/tiup-component-cluster-patch.md +++ b/tiup/tiup-component-cluster-patch.md @@ -29,7 +29,7 @@ Before running the `tiup cluster patch` command, you need to pack the binary pac 1. Determine the following variables: - `${component}`: the name of the component to be replaced (such as `tidb`, `tikv`, or `pd`). - - `${version}`: the version of the component (such as `v8.0.0` or `v7.5.1`). + - `${version}`: the version of the component (such as `v8.1.0` or `v7.5.1`). - `${os}`: the operating system (`linux`). - `${arch}`: the platform on which the component runs (`amd64`, `arm64`). diff --git a/tiup/tiup-component-cluster-upgrade.md b/tiup/tiup-component-cluster-upgrade.md index 90699ec1607dc..0942579584444 100644 --- a/tiup/tiup-component-cluster-upgrade.md +++ b/tiup/tiup-component-cluster-upgrade.md @@ -14,7 +14,7 @@ tiup cluster upgrade [flags] ``` - ``: the cluster name to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-cluster-list.md) command. -- ``: the target version to upgrade to, such as `v8.0.0`. Currently, it is only allowed to upgrade to a version higher than the current cluster, that is, no downgrade is allowed. It is also not allowed to upgrade to the nightly version. +- ``: the target version to upgrade to, such as `v8.1.0`. Currently, it is only allowed to upgrade to a version higher than the current cluster, that is, no downgrade is allowed. It is also not allowed to upgrade to the nightly version. ## Options @@ -44,17 +44,83 @@ tiup cluster upgrade [flags] - Data type: `BOOLEAN` - Default: false +### --ignore-version-check + +- Before upgrading, TiUP checks whether the target version is greater than or equal to the current version. To skip this check, you can use the `--ignore-version-check` option. +- Data type: `BOOLEAN` +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. + ### --offline - Declares that the current cluster is not running. When this option is specified, TiUP does not evict the service leader to another node or restart the service, but only replaces the binary files of the cluster components. - Data type: `BOOLEAN` - This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. +### --pd-version + +- Specifies the version of PD. If this option is set, the version of PD will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of PD remains consistent with the cluster version. + +### --tikv-version + +- Specifies the version of TiKV. If this option is set, the version of TiKV will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiKV remains consistent with the cluster version. + +### --tikv-cdc-version + +- Specifies the version of TiKV CDC. If this option is set, the version of TiKV CDC will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiKV CDC remains consistent with the cluster version. + +### --tiflash-version + +- Specifies the version of TiFlash. If this option is set, the version of TiFlash will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiFlash remains consistent with the cluster version. + +### --cdc-version + +- Specifies the version of TiCDC. If this option is set, the version of TiCDC will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiCDC remains consistent with the cluster version. + +### --tiproxy-version + +- Specifies the version of TiProxy. If this option is set, the version of TiProxy will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiProxy remains consistent with the cluster version. + +### --tidb-dashboard-version + +- Specifies the version of TiDB Dashboard. If this option is set, the version of TiDB Dashboard will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of TiDB Dashboard remains consistent with the cluster version. + +### --alertmanager-version + +- Specifies the version of alert manager. If this option is set, the version of alert manager will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of alert manager remains consistent with the cluster version. + +### --blackbox-exporter-version + +- Specifies the version of Blackbox Exporter. If this option is set, the version of Blackbox Exporter will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of Blackbox Exporter remains consistent with the cluster version. + +### --node-exporter-version + +- Specifies the version of Node Exporter. If this option is set, the version of Node Exporter will no longer be consistent with the cluster version. +- Data type: `STRINGS` +- If this option is not set, the version of Node Exporter remains consistent with the cluster version. + ### -h, --help - Prints the help information. - Data type: `BOOLEAN` -- Default: false +- This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. ## Output diff --git a/tiup/tiup-component-dm-upgrade.md b/tiup/tiup-component-dm-upgrade.md index 4e578aae8f37e..b0eda892c7f46 100644 --- a/tiup/tiup-component-dm-upgrade.md +++ b/tiup/tiup-component-dm-upgrade.md @@ -14,7 +14,7 @@ tiup dm upgrade [flags] ``` - `` is the name of the cluster to be operated on. If you forget the cluster name, you can check it using the [`tiup dm list`](/tiup/tiup-component-dm-list.md) command. -- `` is the target version to be upgraded to, such as `v8.0.0`. Currently, only upgrading to a later version is allowed, and upgrading to an earlier version is not allowed, which means the downgrade is not allowed. Upgrading to a nightly version is not allowed either. +- `` is the target version to be upgraded to, such as `v8.1.0`. Currently, only upgrading to a later version is allowed, and upgrading to an earlier version is not allowed, which means the downgrade is not allowed. Upgrading to a nightly version is not allowed either. ## Options diff --git a/tiup/tiup-component-management.md b/tiup/tiup-component-management.md index 84a0324bc0a0b..91d433d96934b 100644 --- a/tiup/tiup-component-management.md +++ b/tiup/tiup-component-management.md @@ -72,12 +72,12 @@ Example 2: Use TiUP to install the nightly version of TiDB. tiup install tidb:nightly ``` -Example 3: Use TiUP to install TiKV v8.0.0. +Example 3: Use TiUP to install TiKV v8.1.0. {{< copyable "shell-regular" >}} ```shell -tiup install tikv:v8.0.0 +tiup install tikv:v8.1.0 ``` ## Upgrade components @@ -130,12 +130,12 @@ Before the component is started, TiUP creates a directory for it, and then puts If you want to start the same component multiple times and reuse the previous working directory, you can use `--tag` to specify the same name when the component is started. After the tag is specified, the working directory will *not be automatically deleted* when the instance is terminated, which makes it convenient to reuse the working directory. -Example 1: Operate TiDB v8.0.0. +Example 1: Operate TiDB v8.1.0. {{< copyable "shell-regular" >}} ```shell -tiup tidb:v8.0.0 +tiup tidb:v8.1.0 ``` Example 2: Specify the tag with which TiKV operates. @@ -221,12 +221,12 @@ The following flags are supported in this command: - If the version is ignored, adding `--all` means to uninstall all versions of this component. - If the version and the component are both ignored, adding `--all` means to uninstall all components of all versions. -Example 1: Uninstall TiDB v8.0.0. +Example 1: Uninstall TiDB v8.1.0. {{< copyable "shell-regular" >}} ```shell -tiup uninstall tidb:v8.0.0 +tiup uninstall tidb:v8.1.0 ``` Example 2: Uninstall TiKV of all versions. diff --git a/tiup/tiup-mirror-reference.md b/tiup/tiup-mirror-reference.md index 4eb30cd2fcb14..520c6d50232d3 100644 --- a/tiup/tiup-mirror-reference.md +++ b/tiup/tiup-mirror-reference.md @@ -154,7 +154,7 @@ The index file's format is as follows: } }, "name": "{owner-name}", # The name of the owner. - "threshod": {N} # Indicates that the components owned by the owner must have at least N valid signatures. + "threshold": {N} # Indicates that the components owned by the owner must have at least N valid signatures. }, ... "{ownerN}": { # The ID of the Nth owner. diff --git a/tiup/tiup-mirror.md b/tiup/tiup-mirror.md index 714fe1f30a9fd..5b0b7896e6759 100644 --- a/tiup/tiup-mirror.md +++ b/tiup/tiup-mirror.md @@ -87,9 +87,9 @@ The `tiup mirror clone` command provides many optional flags (might provide more If you want to clone only one version (not all versions) of a component, use `--=` to specify this version. For example: - - Execute the `tiup mirror clone --tidb v8.0.0` command to clone the v8.0.0 version of the TiDB component. - - Run the `tiup mirror clone --tidb v8.0.0 --tikv all` command to clone the v8.0.0 version of the TiDB component and all versions of the TiKV component. - - Run the `tiup mirror clone v8.0.0` command to clone the v8.0.0 version of all components in a cluster. + - Execute the `tiup mirror clone --tidb v8.1.0` command to clone the v8.1.0 version of the TiDB component. + - Run the `tiup mirror clone --tidb v8.1.0 --tikv all` command to clone the v8.1.0 version of the TiDB component and all versions of the TiKV component. + - Run the `tiup mirror clone v8.1.0` command to clone the v8.1.0 version of all components in a cluster. After cloning, signing keys are set up automatically. diff --git a/tiup/tiup-playground.md b/tiup/tiup-playground.md index bf3e0b325eb43..202985e9b7bf1 100644 --- a/tiup/tiup-playground.md +++ b/tiup/tiup-playground.md @@ -22,7 +22,7 @@ This command actually performs the following operations: - Because this command does not specify the version of the playground component, TiUP first checks the latest version of the installed playground component. Assume that the latest version is v1.12.3, then this command works the same as `tiup playground:v1.12.3`. - If you have not used TiUP playground to install the TiDB, TiKV, and PD components, the playground component installs the latest stable version of these components, and then start these instances. -- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v8.0.0, then this command works the same as `tiup playground:v1.12.3 v8.0.0`. +- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v8.1.0, then this command works the same as `tiup playground:v1.12.3 v8.1.0`. - Because this command does not specify the number of each component, TiUP playground, by default, starts a smallest cluster that consists of one TiDB instance, one TiKV instance, one PD instance, and one TiFlash instance. - After starting each TiDB component, TiUP playground reminds you that the cluster is successfully started and provides you some useful information, such as how to connect to the TiDB cluster through the MySQL client and how to access the [TiDB Dashboard](/dashboard/dashboard-intro.md). diff --git a/troubleshoot-high-disk-io.md b/troubleshoot-high-disk-io.md index 997750736894a..1ef2c2f1d6b84 100644 --- a/troubleshoot-high-disk-io.md +++ b/troubleshoot-high-disk-io.md @@ -52,7 +52,7 @@ In addition, some other panel metrics might help you determine whether the bottl - Whether the value of `store-pool-size` of `[raftstore]` is too small. It is recommended to set this value between `[1,5]` and not too large. - Whether the CPU resource of the machine is insufficient. -- `append log` is slow. TiKV Grafana's `Raft I/O` and `append log duration` metrics are relatively high, which might usually occur along with relatively high `Raft Propose`/`apply wait duration`. The possible causes are as follows: +- `apply log` is slow. TiKV Grafana's `Raft I/O` and `apply log duration` metrics are relatively high, which might usually occur along with relatively high `Raft Propose`/`apply wait duration`. The possible causes are as follows: - The value of `apply-pool-size` of `[raftstore]` is too small. It is recommended to set this value between `[1, 5]` and not too large. The value of `Thread CPU`/`apply cpu` is also relatively high. - Insufficient CPU resources on the machine. @@ -74,7 +74,7 @@ In addition, some other panel metrics might help you determine whether the bottl If the disk's I/O capability fails to keep up with the write, it is recommended to scale up the disk. If the throughput of the disk reaches the upper limit (for example, the throughput of SATA SSD is much lower than that of NVMe SSD), which results in write stall, but the CPU resource is relatively sufficient, you can try to use a compression algorithm of higher compression ratio to relieve the pressure on the disk, that is, use CPU resources to make up for disk resources. - For example, when the pressure of `default cf compaction` is relatively high, you can change the parameter`[rocksdb.defaultcf] compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd" , "zstd"]` to `compression-per-level = ["no", "no", "zstd", "zstd", "zstd", "zstd", "zstd"]`. + For example, when the pressure of `default cf compaction` is relatively high, you can change the parameter`[rocksdb.defaultcf] compression-per-level = ["no", "no", "lz4", "lz4", "lz4", "zstd", "zstd"]` to `compression-per-level = ["no", "no", "zstd", "zstd", "zstd", "zstd", "zstd"]`. ### I/O issues found in alerts diff --git a/tune-tikv-memory-performance.md b/tune-tikv-memory-performance.md index 4bd4e9d1ef00e..7c1c45fa60c81 100644 --- a/tune-tikv-memory-performance.md +++ b/tune-tikv-memory-performance.md @@ -109,7 +109,7 @@ job = "tikv" region-split-check-diff = "32MB" [coprocessor] -## If the size of a Region with the range of [a,e) is larger than the value of `region_max_size`, TiKV trys to split the Region to several Regions, for example, the Regions with the ranges of [a,b), [b,c), [c,d), and [d,e). +## If the size of a Region with the range of [a,e) is larger than the value of `region_max_size`, TiKV tries to split the Region to several Regions, for example, the Regions with the ranges of [a,b), [b,c), [c,d), and [d,e). ## After the Region split, the size of the split Regions is equal to the value of `region_split_size` (or slightly larger than the value of `region_split_size`). # region-max-size = "144MB" # region-split-size = "96MB" diff --git a/two-data-centers-in-one-city-deployment.md b/two-data-centers-in-one-city-deployment.md index 1c6d71fc5df6e..6f8268b6e2beb 100644 --- a/two-data-centers-in-one-city-deployment.md +++ b/two-data-centers-in-one-city-deployment.md @@ -221,6 +221,7 @@ The replication mode is controlled by PD. You can configure the replication mode primary-replicas = 3 dr-replicas = 2 wait-store-timeout = "1m" + wait-recover-timeout = "0s" pause-region-split = false ``` @@ -244,6 +245,7 @@ Descriptions of configuration items: + `primary-replicas` is the number of Voter replicas in the primary AZ. + `dr-replicas` is the number of Voter replicas in the disaster recovery (DR) AZ. + `wait-store-timeout` is the waiting time for switching to asynchronous replication mode when network isolation or failure occurs. If the time of network failure exceeds the waiting time, asynchronous replication mode is enabled. The default waiting time is 60 seconds. ++ `wait-recover-timeout` is the waiting time for switching back to the `sync-recover` status after the network recovers. The default value is 0 seconds. + `pause-region-split` controls whether to pause Region split operations in the `async_wait` and `async` statuses. Pausing Region split can prevent temporary partial data loss in the DR AZ when synchronizing data in the `sync-recover` status. The default value is `false`. To check the current replication status of the cluster, use the following API: diff --git a/upgrade-monitoring-services.md b/upgrade-monitoring-services.md index 98e3938648a8e..8b8cb02ffd3cb 100644 --- a/upgrade-monitoring-services.md +++ b/upgrade-monitoring-services.md @@ -28,7 +28,16 @@ Download a new installation package from the [Prometheus download page](https:// ### Step 2. Download the Prometheus installation package provided by TiDB -1. Download the TiDB **Server Package** from the [TiDB download page](https://www.pingcap.com/download/) and extract it. +1. Download the TiDB server package and extract it. Note that your downloading means you agree to the [Privacy Policy](https://www.pingcap.com/privacy-policy/). + + ``` + https://download.pingcap.org/tidb-community-server-{version}-linux-{arch}.tar.gz + ``` + + > **Tip:** + > + > `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.1.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.1.0-linux-amd64.tar.gz`. + 2. In the extracted files, locate `prometheus-v{version}-linux-amd64.tar.gz` and extract it. ```bash @@ -50,7 +59,7 @@ Download a new installation package from the [Prometheus download page](https:// Execute the following command to upgrade Prometheus: ```bash -tiup cluster patch prometheus-v{new-version}.tar.gz -R prometheus +tiup cluster patch prometheus-v{new-version}.tar.gz -R prometheus --overwrite ``` After the upgrade, you can go to the home page of the Prometheus server (usually at `http://:9090`), click **Status** in the top navigation menu, and then open the **Runtime & Build Information** page to check the Prometheus version and confirm whether the upgrade is successful. @@ -68,7 +77,16 @@ In the following upgrade steps, you need to download the Grafana installation pa ### Step 2. Download the Grafana installation package provided by TiDB -1. Download the TiDB **Server Package** package from the [TiDB download page](https://www.pingcap.com/download) and extract it. +1. Download the TiDB server package and extract it. Note that your downloading means you agree to the [Privacy Policy](https://www.pingcap.com/privacy-policy/). + + ``` + https://download.pingcap.org/tidb-community-server-{version}-linux-{arch}.tar.gz + ``` + + > **Tip:** + > + > `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.1.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.1.0-linux-amd64.tar.gz`. + 2. In the extracted files, locate `grafana-v{version}-linux-amd64.tar.gz` and extract it. ```bash @@ -90,7 +108,7 @@ In the following upgrade steps, you need to download the Grafana installation pa Execute the following command to upgrade Grafana: ```bash -tiup cluster patch grafana-v{new-version}.tar.gz -R grafana +tiup cluster patch grafana-v{new-version}.tar.gz -R grafana --overwrite ``` @@ -109,7 +127,7 @@ Download the `alertmanager` installation package from the [Prometheus download p Execute the following command to upgrade Alertmanager: ```bash -tiup cluster patch alertmanager-v{new-version}-linux-amd64.tar.gz -R alertmanager +tiup cluster patch alertmanager-v{new-version}-linux-amd64.tar.gz -R alertmanager --overwrite ``` After the upgrade, you can go to the home page of the Alertmanager server (usually at `http://:9093`), click **Status** in the top navigation menu, and then check the Alertmanager version to confirm whether the upgrade is successful. \ No newline at end of file diff --git a/upgrade-tidb-using-tiup.md b/upgrade-tidb-using-tiup.md index 79a3efa7834d5..2d6e107db1c92 100644 --- a/upgrade-tidb-using-tiup.md +++ b/upgrade-tidb-using-tiup.md @@ -8,10 +8,11 @@ aliases: ['/docs/dev/upgrade-tidb-using-tiup/','/docs/dev/how-to/upgrade/using-t This document is targeted for the following upgrade paths: -- Upgrade from TiDB 4.0 versions to TiDB 8.0. -- Upgrade from TiDB 5.0-5.4 versions to TiDB 8.0. -- Upgrade from TiDB 6.0-6.6 to TiDB 8.0. -- Upgrade from TiDB 7.0-7.6 to TiDB 8.0. +- Upgrade from TiDB 4.0 versions to TiDB 8.1. +- Upgrade from TiDB 5.0-5.4 versions to TiDB 8.1. +- Upgrade from TiDB 6.0-6.6 to TiDB 8.1. +- Upgrade from TiDB 7.0-7.6 to TiDB 8.1. +- Upgrade from TiDB 8.0 to TiDB 8.1. > **Warning:** > @@ -23,7 +24,7 @@ This document is targeted for the following upgrade paths: > **Note:** > -> - If your cluster to be upgraded is v3.1 or an earlier version (v3.0 or v2.1), the direct upgrade to v8.0.0 is not supported. You need to upgrade your cluster first to v4.0 and then to v8.0.0. +> - If your cluster to be upgraded is v3.1 or an earlier version (v3.0 or v2.1), the direct upgrade to v8.1.0 is not supported. You need to upgrade your cluster first to v4.0 and then to v8.1.0. > - If your cluster to be upgraded is earlier than v6.2, the upgrade might get stuck when you upgrade the cluster to v6.2 or later versions in some scenarios. You can refer to [How to fix the issue](#how-to-fix-the-issue-that-the-upgrade-gets-stuck-when-upgrading-to-v620-or-later-versions). > - TiDB nodes use the value of the [`server-version`](/tidb-configuration-file.md#server-version) configuration item to verify the current TiDB version. Therefore, to avoid unexpected behaviors, before upgrading the TiDB cluster, you need to set the value of `server-version` to empty or the real version of the current TiDB cluster. > - Setting the [`performance.force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v657-and-v710) configuration item to `ON` prolongs the TiDB startup time, which might cause startup timeouts and upgrade failures. To avoid this issue, it is recommended to set a longer waiting timeout for TiUP. @@ -55,12 +56,12 @@ This document is targeted for the following upgrade paths: ## Upgrade caveat - TiDB currently does not support version downgrade or rolling back to an earlier version after the upgrade. -- For the v4.0 cluster managed using TiDB Ansible, you need to import the cluster to TiUP (`tiup cluster`) for new management according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). Then you can upgrade the cluster to v8.0.0 according to this document. -- To update versions earlier than v3.0 to v8.0.0: +- For the v4.0 cluster managed using TiDB Ansible, you need to import the cluster to TiUP (`tiup cluster`) for new management according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). Then you can upgrade the cluster to v8.1.0 according to this document. +- To update versions earlier than v3.0 to v8.1.0: 1. Update this version to 3.0 using [TiDB Ansible](https://docs.pingcap.com/tidb/v3.0/upgrade-tidb-using-ansible). 2. Use TiUP (`tiup cluster`) to import the TiDB Ansible configuration. 3. Update the 3.0 version to 4.0 according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). - 4. Upgrade the cluster to v8.0.0 according to this document. + 4. Upgrade the cluster to v8.1.0 according to this document. - Support upgrading the versions of TiDB Binlog, TiCDC, TiFlash, and other components. - When upgrading TiFlash from versions earlier than v6.3.0 to v6.3.0 and later versions, note that the CPU must support the AVX2 instruction set under the Linux AMD64 architecture and the ARMv8 instruction set architecture under the Linux ARM64 architecture. For details, see the description in [v6.3.0 Release Notes](/releases/release-6.3.0.md#others). - For detailed compatibility changes of different versions, see the [Release Notes](/releases/release-notes.md) of each version. Modify your cluster configuration according to the "Compatibility Changes" section of the corresponding release notes. @@ -72,7 +73,7 @@ This section introduces the preparation works needed before upgrading your TiDB ### Step 1: Review compatibility changes -Review [the compatibility changes](/releases/release-8.0.0.md#compatibility-changes) in TiDB v8.0.0 release notes. If any changes affect your upgrade, take actions accordingly. +Review [the compatibility changes](/releases/release-8.1.0.md#compatibility-changes) in TiDB v8.1.0 release notes. If any changes affect your upgrade, take actions accordingly. ### Step 2: Upgrade TiUP or TiUP offline mirror @@ -147,7 +148,7 @@ Now, the offline mirror has been upgraded successfully. If an error occurs durin > Skip this step if one of the following situations applies: > > + You have not modified the configuration parameters of the original cluster. Or you have modified the configuration parameters using `tiup cluster` but no more modification is needed. -> + After the upgrade, you want to use v8.0.0's default parameter values for the unmodified configuration items. +> + After the upgrade, you want to use v8.1.0's default parameter values for the unmodified configuration items. 1. Enter the `vi` editing mode to edit the topology file: @@ -163,7 +164,7 @@ Now, the offline mirror has been upgraded successfully. If an error occurs durin > **Note:** > -> Before you upgrade the cluster to v6.6.0, make sure that the parameters you have modified in v4.0 are compatible in v8.0.0. For details, see [TiKV Configuration File](/tikv-configuration-file.md). +> Before you upgrade the cluster to v6.6.0, make sure that the parameters you have modified in v4.0 are compatible in v8.1.0. For details, see [TiKV Configuration File](/tikv-configuration-file.md). ### Step 4: Check the DDL and backup status of the cluster @@ -207,12 +208,12 @@ If your application has a maintenance window for the database to be stopped for tiup cluster upgrade ``` -For example, if you want to upgrade the cluster to v8.0.0: +For example, if you want to upgrade the cluster to v8.1.0: {{< copyable "shell-regular" >}} ```shell -tiup cluster upgrade v8.0.0 +tiup cluster upgrade v8.1.0 ``` > **Note:** @@ -223,10 +224,7 @@ tiup cluster upgrade v8.0.0 > > + To keep a stable performance, make sure that all leaders in a TiKV instance are evicted before stopping the instance. You can set `--transfer-timeout` to a larger value, for example, `--transfer-timeout 3600` (unit: second). > -> + To upgrade TiFlash from versions earlier than 5.3 to 5.3 or later, you should stop TiFlash and then upgrade it. The following steps help you upgrade TiFlash without interrupting other components: -> 1. Stop the TiFlash instance: `tiup cluster stop -R tiflash` -> 2. Upgrade the TiDB cluster without restarting it (only updating the files): `tiup cluster upgrade --offline`, such as `tiup cluster upgrade v6.3.0 --offline` -> 3. Reload the TiDB cluster: `tiup cluster reload `. After the reload, the TiFlash instance is started and you do not need to manually start it. +> + To upgrade TiFlash from versions earlier than v5.3.0 to v5.3.0 or later, you must stop TiFlash and then upgrade it, and the TiUP version must be earlier than v1.12.0. For more information, see [Upgrade TiFlash using TiUP](/tiflash-upgrade-guide.md#upgrade-tiflash-using-tiup). > > + Try to avoid creating a new clustered index table when you apply rolling updates to the clusters using TiDB Binlog. @@ -263,7 +261,7 @@ tiup cluster upgrade -h | grep "version" tiup cluster stop ``` -2. Use the `upgrade` command with the `--offline` option to perform the offline upgrade. Fill in the name of your cluster for `` and the version to upgrade to for ``, such as `v8.0.0`. +2. Use the `upgrade` command with the `--offline` option to perform the offline upgrade. Fill in the name of your cluster for `` and the version to upgrade to for ``, such as `v8.1.0`. {{< copyable "shell-regular" >}} @@ -292,7 +290,7 @@ tiup cluster display ``` Cluster type: tidb Cluster name: -Cluster version: v8.0.0 +Cluster version: v8.1.0 ``` ## FAQ @@ -347,7 +345,7 @@ Starting from v6.2.0, TiDB enables the [concurrent DDL framework](/ddl-introduct ### The evict leader has waited too long during the upgrade. How to skip this step for a quick upgrade? -You can specify `--force`. Then the processes of transferring PD leader and evicting TiKV leader are skipped during the upgrade. The cluster is directly restarted to update the version, which has a great impact on the cluster that runs online. In the following command, `` is the version to upgrade to, such as `v8.0.0`. +You can specify `--force`. Then the processes of transferring PD leader and evicting TiKV leader are skipped during the upgrade. The cluster is directly restarted to update the version, which has a great impact on the cluster that runs online. In the following command, `` is the version to upgrade to, such as `v8.1.0`. {{< copyable "shell-regular" >}} @@ -362,5 +360,5 @@ You can upgrade the tool version by using TiUP to install the `ctl` component of {{< copyable "shell-regular" >}} ```shell -tiup install ctl:v8.0.0 +tiup install ctl:v8.1.0 ``` diff --git a/user-account-management.md b/user-account-management.md index c53352651b347..feffdfc245d26 100644 --- a/user-account-management.md +++ b/user-account-management.md @@ -159,7 +159,7 @@ TiDB creates the `'root'@'%'` default account during the database initialization ## Set account resource limits -Currently, TiDB does not support setting account resource limits. +TiDB can limit the resources consumed by users using resource groups. For more information, see [Use resource control to achieve resource isolation](/tidb-resource-control.md). ## Assign account passwords diff --git a/user-defined-variables.md b/user-defined-variables.md index 5a18349c13dd7..3f2ebe74c8ea8 100644 --- a/user-defined-variables.md +++ b/user-defined-variables.md @@ -18,48 +18,36 @@ The user-defined variables are session-specific, which means a user variable def ## Set the user-defined variables -You can use the `SET` statement to set a user-defined variable, and the syntax is `SET @var_name = expr [, @var_name = expr] ...;`. For example: - -{{< copyable "sql" >}} +You can use the [`SET` statement](/sql-statements/sql-statement-set-variable.md) to set a user-defined variable, and the syntax is `SET @var_name = expr [, @var_name = expr] ...;`. For example: ```sql SET @favorite_db = 'TiDB'; ``` -{{< copyable "sql" >}} - ```sql SET @a = 'a', @b = 'b', @c = 'c'; ``` For the assignment operator, you can also use `:=`. For example: -{{< copyable "sql" >}} - ```sql SET @favorite_db := 'TiDB'; ``` The content to the right of the assignment operator can be any valid expression. For example: -{{< copyable "sql" >}} - ```sql SET @c = @a + @b; ``` -{{< copyable "sql" >}} - ```sql -set @c = b'1000001' + b'1000001'; +SET @c = b'1000001' + b'1000001'; ``` ## Read the user-defined variables To read a user-defined variable, you can use the `SELECT` statement to query: -{{< copyable "sql" >}} - ```sql SELECT @a1, @a2, @a3 ``` @@ -90,16 +78,12 @@ Before the variable `@a4` is modified or the connection is closed, its value is If a hexadecimal literal or binary literal is used when setting the user-defined variable, TiDB will treat it as a binary string. If you want to set it to a number, you can manually add the `CAST` conversion, or use the numeric operator in the expression: -{{< copyable "sql" >}} - ```sql SET @v1 = b'1000001'; SET @v2 = b'1000001'+0; SET @v3 = CAST(b'1000001' AS UNSIGNED); ``` -{{< copyable "sql" >}} - ```sql SELECT @v1, @v2, @v3; ``` @@ -114,8 +98,6 @@ SELECT @v1, @v2, @v3; If you refer to a user-defined variable that has not been initialized, it has a value of NULL and a type of string. -{{< copyable "sql" >}} - ```sql SELECT @not_exist; ``` @@ -130,8 +112,6 @@ SELECT @not_exist; In addition to using the `SELECT` statement to read the user-defined variables, another common usage is the `PREPARE` statement. For example: -{{< copyable "sql" >}} - ```sql SET @s = 'SELECT SQRT(POW(?,2) + POW(?,2)) AS hypotenuse'; PREPARE stmt FROM @s; @@ -150,8 +130,6 @@ EXECUTE stmt USING @a, @b; The contents of the user-defined variables are not recognized as identifiers in the SQL statements. For example: -{{< copyable "sql" >}} - ```sql SELECT * from t; ``` @@ -164,8 +142,6 @@ SELECT * from t; +---+ ``` -{{< copyable "sql" >}} - ```sql SET @col = "`a`"; SELECT @col FROM t;