diff --git a/go/mysql/collations/charset/filename.go b/go/mysql/collations/charset/filename.go new file mode 100644 index 00000000000..50395645f98 --- /dev/null +++ b/go/mysql/collations/charset/filename.go @@ -0,0 +1,375 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package charset + +import ( + "strings" + "unicode" +) + +/* +*s++ = MY_FILENAME_ESCAPE; +if ((wc >= 0x00C0 && wc <= 0x05FF && (code = uni_0C00_05FF[wc - 0x00C0])) || + (wc >= 0x1E00 && wc <= 0x1FFF && (code = uni_1E00_1FFF[wc - 0x1E00])) || + (wc >= 0x2160 && wc <= 0x217F && (code = uni_2160_217F[wc - 0x2160])) || + (wc >= 0x24B0 && wc <= 0x24EF && (code = uni_24B0_24EF[wc - 0x24B0])) || + (wc >= 0xFF20 && wc <= 0xFF5F && (code = uni_FF20_FF5F[wc - 0xFF20]))) { + *s++ = (code / 80) + 0x30; + *s++ = (code % 80) + 0x30; + return 3; +} +*/ + +// TablenameToFilename is a rewrite of MySQL's `tablename_to_filename` utility function. +// InnoDB table names are in the form of schema_name/table_name, except each of the tokens are encoded using this function. +// For simple characters there is no change, but special characters get encoded. Thus, the table `tbl-fts` in `test` db +// will be encoded as `test/tbl@002dfts` +// +// Original encoding function: +// +// https://github.com/mysql/mysql-server/blob/89e1c722476deebc3ddc8675e779869f6da654c0/strings/ctype-utf8.cc#L6961-L6984 +// +// static int my_wc_mb_filename(const CHARSET_INFO *cs [[maybe_unused]], +// my_wc_t wc, uchar *s, uchar *e) { +// int code; +// char hex[] = "0123456789abcdef"; +// +// if (s >= e) return MY_CS_TOOSMALL; +// +// if (wc < 128 && filename_safe_char[wc]) { +// *s = (uchar)wc; +// return 1; +// } +// +// if (s + 3 > e) return MY_CS_TOOSMALL3; +// +// *s++ = MY_FILENAME_ESCAPE; +// if ((wc >= 0x00C0 && wc <= 0x05FF && (code = uni_0C00_05FF[wc - 0x00C0])) || +// (wc >= 0x1E00 && wc <= 0x1FFF && (code = uni_1E00_1FFF[wc - 0x1E00])) || +// (wc >= 0x2160 && wc <= 0x217F && (code = uni_2160_217F[wc - 0x2160])) || +// (wc >= 0x24B0 && wc <= 0x24EF && (code = uni_24B0_24EF[wc - 0x24B0])) || +// (wc >= 0xFF20 && wc <= 0xFF5F && (code = uni_FF20_FF5F[wc - 0xFF20]))) { +// *s++ = (code / 80) + 0x30; +// *s++ = (code % 80) + 0x30; +// return 3; +// } +// +// /* Non letter */ +// if (s + 5 > e) return MY_CS_TOOSMALL5; +// +// *s++ = hex[(wc >> 12) & 15]; +// *s++ = hex[(wc >> 8) & 15]; +// *s++ = hex[(wc >> 4) & 15]; +// *s++ = hex[(wc)&15]; +// return 5; +// } +// +// See also MySQL docs: https://dev.mysql.com/doc/refman/8.0/en/identifier-mapping.html +func TablenameToFilename(name string) string { + var b strings.Builder + for _, wc := range name { + if wc < 128 && filename_safe_char[wc] == 1 { + b.WriteRune(wc) + continue + } + + b.WriteRune('@') + + var code uint16 + switch { + case wc >= 0x00C0 && wc <= 0x05FF: + code = uni_0C00_05FF[wc-0x00C0] + case wc >= 0x1E00 && wc <= 0x1FFF: + code = uni_1E00_1FFF[wc-0x1E00] + case wc >= 0x2160 && wc <= 0x217F: + code = uni_2160_217F[wc-0x2160] + case wc >= 0x24B0 && wc <= 0x24EF: + code = uni_24B0_24EF[wc-0x24B0] + case wc >= 0xFF20 && wc <= 0xFF5F: + code = uni_FF20_FF5F[wc-0xFF20] + } + if code != 0 { + // One of specifically addressed character sets + b.WriteRune(unicode.ToLower(rune(code/80) + 0x30)) + b.WriteRune(unicode.ToLower(rune(code%80) + 0x30)) + continue + } + // Other characters + b.WriteByte(hexSequence[(wc>>12)&15]) + b.WriteByte(hexSequence[(wc>>8)&15]) + b.WriteByte(hexSequence[(wc>>4)&15]) + b.WriteByte(hexSequence[(wc)&15]) + } + return b.String() +} + +var ( + hexSequence = "0123456789abcdef" + + filename_safe_char = []byte{ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* ................ */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* ................ */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* !"#$%&'()*+,-./ */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0123456789:;<=>? */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* @ABCDEFGHIJKLMNO */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* PQRSTUVWXYZ[\]^_ */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* `abcdefghijklmno */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* pqrstuvwxyz{|}~. */ + } + + /* 00C0-05FF */ + uni_0C00_05FF = []uint16{ + 0x0017, 0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, + 0x0029, 0x002A, 0x0067, 0x0068, 0x0069, 0x0000, 0x006B, 0x006C, 0x006D, + 0x006E, 0x006F, 0x0070, 0x0071, 0x008A, 0x0037, 0x0038, 0x0039, 0x003A, + 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, 0x0048, 0x0049, 0x004A, 0x0087, 0x0088, + 0x0089, 0x0000, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, 0x0090, 0x0091, + 0x0092, 0x0073, 0x0093, 0x0074, 0x0094, 0x0075, 0x0095, 0x0076, 0x0096, + 0x0077, 0x0097, 0x0078, 0x0098, 0x0079, 0x0099, 0x007A, 0x009A, 0x00B7, + 0x00D7, 0x00B8, 0x00D8, 0x00B9, 0x00D9, 0x00BA, 0x00DA, 0x00BB, 0x00DB, + 0x00BC, 0x00DC, 0x00BD, 0x00DD, 0x00BE, 0x00DE, 0x00BF, 0x00DF, 0x00C0, + 0x00E0, 0x00C1, 0x00E1, 0x00C2, 0x00E2, 0x00C3, 0x00E3, 0x00C4, 0x00E4, + 0x00C5, 0x00E5, 0x00C6, 0x00E6, 0x0000, 0x00E7, 0x00C8, 0x00E8, 0x00C9, + 0x00E9, 0x00CA, 0x00EA, 0x0127, 0x0108, 0x0128, 0x0109, 0x0129, 0x010A, + 0x012A, 0x010B, 0x012B, 0x010C, 0x012C, 0x010D, 0x012D, 0x010E, 0x012E, + 0x010F, 0x012F, 0x0130, 0x0111, 0x0131, 0x0112, 0x0132, 0x0113, 0x0133, + 0x0114, 0x0134, 0x0115, 0x0135, 0x0116, 0x0136, 0x0117, 0x0137, 0x0118, + 0x0138, 0x0119, 0x0139, 0x011A, 0x013A, 0x0157, 0x0177, 0x0158, 0x0178, + 0x0159, 0x0179, 0x015A, 0x017A, 0x015B, 0x017B, 0x015C, 0x017C, 0x015D, + 0x017D, 0x015E, 0x017E, 0x015F, 0x017F, 0x0160, 0x0180, 0x0161, 0x0181, + 0x0162, 0x0182, 0x0163, 0x0183, 0x0072, 0x0164, 0x0184, 0x0165, 0x0185, + 0x0166, 0x0186, 0x0187, 0x1161, 0x0A86, 0x07B1, 0x11B1, 0x0801, 0x1201, + 0x0AD6, 0x0851, 0x1251, 0x0B76, 0x0BC6, 0x08A1, 0x12A1, 0x12F1, 0x0D52, + 0x0C66, 0x0D06, 0x0941, 0x1341, 0x0857, 0x0947, 0x1391, 0x0B27, 0x0AD7, + 0x09E1, 0x13E1, 0x1431, 0x1481, 0x0D07, 0x07B8, 0x14D1, 0x08A8, 0x0B21, + 0x1521, 0x0B71, 0x1571, 0x0BC1, 0x15C1, 0x0C18, 0x0C11, 0x1611, 0x0D08, + 0x1661, 0x16B1, 0x0D01, 0x1701, 0x0859, 0x0D51, 0x1751, 0x08F9, 0x0949, + 0x0762, 0x1162, 0x07B2, 0x11B2, 0x0B79, 0x0802, 0x1202, 0x1252, 0x12A2, + 0x0992, 0x1392, 0x1342, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x09E2, + 0x0000, 0x13E2, 0x0A32, 0x0000, 0x1432, 0x0A82, 0x0000, 0x1482, 0x0AD2, + 0x14D2, 0x0B22, 0x1522, 0x0B72, 0x1572, 0x0BC2, 0x15C2, 0x0C12, 0x1612, + 0x0C62, 0x1662, 0x0CB2, 0x16B2, 0x0D02, 0x1702, 0x1752, 0x0763, 0x1163, + 0x07B3, 0x11B3, 0x0803, 0x1203, 0x0853, 0x1253, 0x08A3, 0x12A3, 0x08F3, + 0x12F3, 0x0943, 0x1343, 0x0993, 0x1393, 0x09E3, 0x13E3, 0x1433, 0x0A83, + 0x0000, 0x1483, 0x0AD3, 0x14D3, 0x0991, 0x0000, 0x0B23, 0x1523, 0x0B73, + 0x1573, 0x0BC3, 0x15C3, 0x0C13, 0x1613, 0x0C63, 0x1663, 0x0CB3, 0x16B3, + 0x0D03, 0x1703, 0x0D53, 0x1753, 0x0764, 0x1164, 0x07B4, 0x11B4, 0x0804, + 0x1204, 0x0854, 0x1254, 0x08A4, 0x12A4, 0x08F4, 0x12F4, 0x0944, 0x1344, + 0x0994, 0x1394, 0x09E4, 0x13E4, 0x0A34, 0x1434, 0x0A84, 0x1484, 0x0AD4, + 0x14D4, 0x0AD1, 0x1524, 0x0B74, 0x1574, 0x0BC4, 0x15C4, 0x0C14, 0x1614, + 0x0C64, 0x1664, 0x0CB4, 0x16B4, 0x0D04, 0x1704, 0x0D54, 0x1754, 0x0765, + 0x1165, 0x07B5, 0x11B5, 0x1205, 0x1255, 0x12A5, 0x12F5, 0x1345, 0x1395, + 0x09E5, 0x0A35, 0x1435, 0x0A31, 0x0A85, 0x14D5, 0x1525, 0x0C19, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x1396, 0x13E6, 0x1436, 0x1486, 0x14D6, + 0x1526, 0x1576, 0x15C6, 0x1616, 0x1666, 0x16B6, 0x1706, 0x1756, 0x1167, + 0x11B7, 0x1207, 0x1257, 0x12A7, 0x12F7, 0x1347, 0x1397, 0x13E7, 0x1437, + 0x1487, 0x14D7, 0x1527, 0x1577, 0x15C7, 0x1617, 0x1667, 0x16B7, 0x1707, + 0x1757, 0x1168, 0x11B8, 0x1208, 0x1258, 0x12A8, 0x12F8, 0x1348, 0x1398, + 0x13E8, 0x1438, 0x1488, 0x14D8, 0x1528, 0x1578, 0x15C8, 0x1618, 0x1668, + 0x16B8, 0x1708, 0x1758, 0x1169, 0x11B9, 0x1209, 0x1259, 0x12A9, 0x12F9, + 0x1349, 0x1399, 0x13E9, 0x1439, 0x1489, 0x14D9, 0x1529, 0x1579, 0x15C9, + 0x1619, 0x1669, 0x16B9, 0x1709, 0x1759, 0x116A, 0x11BA, 0x120A, 0x125A, + 0x12AA, 0x12FA, 0x134A, 0x139A, 0x13EA, 0x143A, 0x148A, 0x14DA, 0x152A, + 0x157A, 0x15CA, 0x161A, 0x166A, 0x16BA, 0x170A, 0x175A, 0x116B, 0x11BB, + 0x120B, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x01F7, + 0x0000, 0x01F8, 0x01F9, 0x01FA, 0x0000, 0x0253, 0x0000, 0x0254, 0x0255, + 0x01D9, 0x01FC, 0x0257, 0x01FE, 0x01FF, 0x0200, 0x0201, 0x0202, 0x0258, + 0x0204, 0x02A7, 0x0206, 0x0207, 0x0208, 0x0209, 0x020A, 0x0299, 0x0248, + 0x0000, 0x02A9, 0x024B, 0x024C, 0x0298, 0x024E, 0x024F, 0x0250, 0x0251, + 0x0252, 0x0217, 0x0218, 0x0219, 0x021A, 0x021B, 0x021C, 0x021D, 0x021E, + 0x021F, 0x0220, 0x0221, 0x0222, 0x0223, 0x0224, 0x0225, 0x0226, 0x0227, + 0x0228, 0x0229, 0x022A, 0x0267, 0x0268, 0x0269, 0x026A, 0x026B, 0x026C, + 0x026D, 0x026E, 0x026F, 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, + 0x0000, 0x0277, 0x0278, 0x0259, 0x025A, 0x0297, 0x02B8, 0x02B9, 0x02BA, + 0x0000, 0x02BB, 0x029C, 0x02BC, 0x029D, 0x02BD, 0x029E, 0x02BE, 0x029F, + 0x02BF, 0x02A0, 0x02C0, 0x02A1, 0x02C1, 0x02A2, 0x02C2, 0x02A3, 0x02C3, + 0x02A4, 0x02C4, 0x02A5, 0x02C5, 0x02A6, 0x02C6, 0x02C7, 0x02C8, 0x02C9, + 0x02CA, 0x0000, 0x0307, 0x0308, 0x0000, 0x0309, 0x0000, 0x0000, 0x030A, + 0x030B, 0x02EC, 0x02ED, 0x02EE, 0x0AF1, 0x0B41, 0x0B91, 0x0BE1, 0x0C31, + 0x0C81, 0x0CD1, 0x0D21, 0x0732, 0x0782, 0x07D2, 0x0822, 0x0872, 0x08C2, + 0x0912, 0x0962, 0x0730, 0x0780, 0x07D0, 0x0820, 0x0870, 0x08C0, 0x0910, + 0x0960, 0x09B0, 0x0A00, 0x0A50, 0x0AA0, 0x0AF0, 0x0B40, 0x0B90, 0x0BE0, + 0x0C30, 0x0C80, 0x0CD0, 0x0D20, 0x0731, 0x0781, 0x07D1, 0x0821, 0x0871, + 0x08C1, 0x0911, 0x0961, 0x09B1, 0x0A01, 0x0A51, 0x0AA1, 0x1130, 0x1180, + 0x11D0, 0x1220, 0x1270, 0x12C0, 0x1310, 0x1360, 0x13B0, 0x1400, 0x1450, + 0x14A0, 0x14F0, 0x1540, 0x1590, 0x15E0, 0x1630, 0x1680, 0x16D0, 0x1720, + 0x1131, 0x1181, 0x11D1, 0x1221, 0x1271, 0x12C1, 0x1311, 0x1361, 0x13B1, + 0x1401, 0x1451, 0x14A1, 0x14F1, 0x1541, 0x1591, 0x15E1, 0x1631, 0x1681, + 0x16D1, 0x1721, 0x1132, 0x1182, 0x11D2, 0x1222, 0x1272, 0x12C2, 0x1312, + 0x1362, 0x09B2, 0x13B2, 0x0A02, 0x1402, 0x0A52, 0x1452, 0x0AA2, 0x14A2, + 0x0AF2, 0x14F2, 0x0B42, 0x1542, 0x0B92, 0x1592, 0x0BE2, 0x15E2, 0x0C32, + 0x1632, 0x0C82, 0x1682, 0x0CD2, 0x16D2, 0x0D22, 0x1722, 0x0733, 0x1133, + 0x0783, 0x1183, 0x07D3, 0x11D3, 0x0823, 0x1223, 0x0873, 0x1273, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0913, 0x1313, + 0x0963, 0x1363, 0x09B3, 0x13B3, 0x0A03, 0x1403, 0x0A53, 0x1453, 0x0AA3, + 0x14A3, 0x0AF3, 0x14F3, 0x0B43, 0x1543, 0x0B93, 0x1593, 0x0BE3, 0x15E3, + 0x0C33, 0x1633, 0x0C83, 0x1683, 0x0CD3, 0x16D3, 0x0D23, 0x1723, 0x0734, + 0x1134, 0x0784, 0x1184, 0x07D4, 0x11D4, 0x0824, 0x1224, 0x0874, 0x1274, + 0x08C4, 0x12C4, 0x0914, 0x1314, 0x0964, 0x1364, 0x09B4, 0x13B4, 0x0A04, + 0x1404, 0x0A54, 0x1454, 0x0AA4, 0x14A4, 0x0AF4, 0x14F4, 0x0B44, 0x0B94, + 0x1594, 0x0BE4, 0x15E4, 0x0C34, 0x1634, 0x0C84, 0x1684, 0x0CD4, 0x16D4, + 0x0D24, 0x1724, 0x0735, 0x1135, 0x0000, 0x07D5, 0x11D5, 0x0825, 0x1225, + 0x0875, 0x1275, 0x08C5, 0x12C5, 0x0915, 0x1315, 0x0965, 0x1365, 0x09B5, + 0x13B5, 0x0A05, 0x1405, 0x0A55, 0x1455, 0x0AA5, 0x14A5, 0x0AF5, 0x14F5, + 0x0B45, 0x1545, 0x0B95, 0x1595, 0x0BE5, 0x15E5, 0x0C35, 0x1635, 0x0C85, + 0x1685, 0x0CD5, 0x16D5, 0x0D25, 0x1725, 0x0736, 0x1136, 0x0786, 0x1186, + 0x07D6, 0x11D6, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0A06, + 0x1406, 0x0A56, 0x1456, 0x0AA6, 0x14A6, 0x0AF6, 0x14F6, 0x0B46, 0x1546, + 0x0B96, 0x1596, 0x0BE6, 0x15E6, 0x0C36, 0x1636, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0787, 0x07D7, 0x0827, 0x0877, 0x08C7, 0x0917, + 0x0967, 0x09B7, 0x0A07, 0x0A57, 0x0AA7, 0x0AF7, 0x0B47, 0x0B97, 0x0BE7, + 0x0C37, 0x0C87, 0x0CD7, 0x0D27, 0x0738, 0x0788, 0x07D8, 0x0828, 0x0878, + 0x08C8, 0x0918, 0x0968, 0x09B8, 0x0A08, 0x0A58, 0x0AA8, 0x0AF8, 0x0B48, + 0x0B98, 0x0BE8, 0x0C38, 0x0C88, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x1187, 0x11D7, 0x1227, + 0x1277, 0x12C7, 0x1317, 0x1367, 0x13B7, 0x1407, 0x1457, 0x14A7, 0x14F7, + 0x1547, 0x1597, 0x15E7, 0x1637, 0x1687, 0x16D7, 0x1727, 0x1138, 0x1188, + 0x11D8, 0x1228, 0x1278, 0x12C8, 0x1318, 0x1368, 0x13B8, 0x1408, 0x1458, + 0x14A8, 0x14F8, 0x1548, 0x1598, 0x15E8, 0x1638, 0x1688, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000} + + /* 1E00-1FFF */ + uni_1E00_1FFF = []uint16{ + 0x076C, 0x116C, 0x07BC, 0x11BC, 0x080C, 0x120C, 0x085C, 0x125C, 0x08AC, + 0x12AC, 0x08FC, 0x12FC, 0x094C, 0x134C, 0x099C, 0x139C, 0x09EC, 0x13EC, + 0x0A3C, 0x143C, 0x0A8C, 0x148C, 0x0ADC, 0x14DC, 0x0B2C, 0x152C, 0x0B7C, + 0x157C, 0x0BCC, 0x15CC, 0x0C1C, 0x161C, 0x0C6C, 0x166C, 0x0CBC, 0x16BC, + 0x0D0C, 0x170C, 0x0D5C, 0x175C, 0x076D, 0x116D, 0x07BD, 0x11BD, 0x080D, + 0x120D, 0x085D, 0x125D, 0x08AD, 0x12AD, 0x08FD, 0x12FD, 0x094D, 0x134D, + 0x099D, 0x139D, 0x09ED, 0x13ED, 0x0A3D, 0x143D, 0x0A8D, 0x148D, 0x0ADD, + 0x14DD, 0x0B2D, 0x152D, 0x0B7D, 0x157D, 0x0BCD, 0x15CD, 0x0C1D, 0x161D, + 0x0C6D, 0x166D, 0x0CBD, 0x16BD, 0x0D0D, 0x170D, 0x0D5D, 0x175D, 0x076E, + 0x116E, 0x07BE, 0x11BE, 0x080E, 0x120E, 0x085E, 0x125E, 0x08AE, 0x12AE, + 0x08FE, 0x12FE, 0x094E, 0x134E, 0x099E, 0x139E, 0x0770, 0x13EE, 0x0A3E, + 0x143E, 0x0A8E, 0x148E, 0x0ADE, 0x14DE, 0x0B2E, 0x152E, 0x0B7E, 0x157E, + 0x0BCE, 0x15CE, 0x0C1E, 0x161E, 0x0C6E, 0x166E, 0x0CBE, 0x16BE, 0x0D0E, + 0x170E, 0x0D5E, 0x175E, 0x076F, 0x116F, 0x07BF, 0x11BF, 0x080F, 0x120F, + 0x085F, 0x125F, 0x08AF, 0x12AF, 0x08FF, 0x12FF, 0x094F, 0x134F, 0x099F, + 0x139F, 0x09EF, 0x13EF, 0x0A3F, 0x143F, 0x0A8F, 0x148F, 0x0ADF, 0x14DF, + 0x0B2F, 0x152F, 0x0B7F, 0x157F, 0x0BCF, 0x15CF, 0x161F, 0x166F, 0x16BF, + 0x170F, 0x175F, 0x1170, 0x0000, 0x0000, 0x0000, 0x0000, 0x0900, 0x1300, + 0x0950, 0x1350, 0x09A0, 0x13A0, 0x09F0, 0x13F0, 0x0A40, 0x1440, 0x0A90, + 0x1490, 0x0AE0, 0x14E0, 0x0B30, 0x1530, 0x0B80, 0x1580, 0x0BD0, 0x15D0, + 0x0C20, 0x1620, 0x0C70, 0x1670, 0x0CC0, 0x16C0, 0x0D10, 0x1710, 0x0D60, + 0x1760, 0x0771, 0x1171, 0x07C1, 0x11C1, 0x0811, 0x1211, 0x0861, 0x1261, + 0x08B1, 0x12B1, 0x0901, 0x1301, 0x0951, 0x1351, 0x09A1, 0x13A1, 0x09F1, + 0x13F1, 0x0A41, 0x1441, 0x0A91, 0x1491, 0x0AE1, 0x14E1, 0x0B31, 0x1531, + 0x0B81, 0x1581, 0x0BD1, 0x15D1, 0x0C21, 0x1621, 0x0C71, 0x1671, 0x0CC1, + 0x16C1, 0x0D11, 0x1711, 0x0D61, 0x1761, 0x0772, 0x1172, 0x07C2, 0x11C2, + 0x0812, 0x1212, 0x0862, 0x1262, 0x08B2, 0x12B2, 0x0902, 0x1302, 0x0952, + 0x1352, 0x09A2, 0x13A2, 0x09F2, 0x13F2, 0x0A42, 0x1442, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x1173, 0x11C3, 0x1213, 0x1263, 0x12B3, + 0x1303, 0x1353, 0x13A3, 0x0773, 0x07C3, 0x0813, 0x0863, 0x08B3, 0x0903, + 0x0953, 0x09A3, 0x13F3, 0x1443, 0x1493, 0x14E3, 0x1533, 0x1583, 0x0000, + 0x0000, 0x09F3, 0x0A43, 0x0A93, 0x0AE3, 0x0B33, 0x0B83, 0x0000, 0x0000, + 0x1713, 0x1763, 0x1174, 0x11C4, 0x1214, 0x1264, 0x12B4, 0x1304, 0x0D13, + 0x0D63, 0x0774, 0x07C4, 0x0814, 0x0864, 0x08B4, 0x0904, 0x1354, 0x13A4, + 0x13F4, 0x1444, 0x1494, 0x14E4, 0x1534, 0x1584, 0x0954, 0x09A4, 0x09F4, + 0x0A44, 0x0A94, 0x0AE4, 0x0B34, 0x0B84, 0x15D4, 0x1624, 0x1674, 0x16C4, + 0x1714, 0x1764, 0x0000, 0x0000, 0x0BD4, 0x0C24, 0x0C74, 0x0CC4, 0x0D14, + 0x0D64, 0x0000, 0x0000, 0x12B5, 0x1305, 0x1355, 0x13A5, 0x13F5, 0x1445, + 0x1495, 0x14E5, 0x0000, 0x0905, 0x0000, 0x09A5, 0x0000, 0x0A45, 0x0000, + 0x0AE5, 0x1675, 0x16C5, 0x1715, 0x1765, 0x1176, 0x11C6, 0x1216, 0x1266, + 0x0C75, 0x0CC5, 0x0D15, 0x0D65, 0x0776, 0x07C6, 0x0816, 0x0866, 0x12B6, + 0x1306, 0x1356, 0x13A6, 0x13F6, 0x1446, 0x1496, 0x14E6, 0x1536, 0x1586, + 0x15D6, 0x1626, 0x1676, 0x16C6, 0x0000, 0x0000, 0x1177, 0x11C7, 0x1217, + 0x1267, 0x12B7, 0x1307, 0x1357, 0x13A7, 0x0777, 0x07C7, 0x0817, 0x0867, + 0x08B7, 0x0907, 0x0957, 0x09A7, 0x13F7, 0x1447, 0x1497, 0x14E7, 0x1537, + 0x1587, 0x15D7, 0x1627, 0x09F7, 0x0A47, 0x0A97, 0x0AE7, 0x0B37, 0x0B87, + 0x0BD7, 0x0C27, 0x1677, 0x16C7, 0x1717, 0x1767, 0x1178, 0x11C8, 0x1218, + 0x1268, 0x0C77, 0x0CC7, 0x0D17, 0x0D67, 0x0778, 0x07C8, 0x0818, 0x0868, + 0x12B8, 0x1308, 0x1358, 0x13A8, 0x13F8, 0x0000, 0x1498, 0x14E8, 0x08B8, + 0x0908, 0x08B6, 0x0906, 0x09A8, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1538, 0x1588, 0x15D8, 0x0000, 0x1678, 0x16C8, 0x0956, 0x09A6, 0x09F6, + 0x0A46, 0x0B88, 0x0000, 0x0000, 0x0000, 0x1718, 0x1768, 0x1179, 0x11C9, + 0x0000, 0x0000, 0x12B9, 0x1309, 0x0D18, 0x0D68, 0x0A96, 0x0AE6, 0x0000, + 0x0000, 0x0000, 0x0000, 0x13A9, 0x13F9, 0x1449, 0x1499, 0x14E9, 0x1539, + 0x1589, 0x15D9, 0x09A9, 0x09F9, 0x0BD6, 0x0C26, 0x0B39, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x16C9, 0x1719, 0x0000, 0x0000, 0x11CA, 0x121A, + 0x0B36, 0x0B86, 0x0C76, 0x0CC6, 0x0D19, 0x0000, 0x0000, 0x0000} + + /* 2160-217F */ + uni_2160_217F = []uint16{ + 0x0739, 0x0789, 0x07D9, 0x0829, 0x0879, 0x08C9, 0x0919, 0x0969, + 0x09B9, 0x0A09, 0x0A59, 0x0AA9, 0x0AF9, 0x0B49, 0x0B99, 0x0BE9, + 0x1139, 0x1189, 0x11D9, 0x1229, 0x1279, 0x12C9, 0x1319, 0x1369, + 0x13B9, 0x1409, 0x1459, 0x14A9, 0x14F9, 0x1549, 0x1599, 0x15E9} + + /* 24B0-24EF */ + uni_24B0_24EF = []uint16{ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0511, 0x0512, + 0x0513, 0x0514, 0x0515, 0x0516, 0x0517, 0x0518, 0x0519, 0x051A, + 0x051B, 0x051C, 0x051D, 0x051E, 0x051F, 0x0520, 0x0521, 0x0522, + 0x0523, 0x0524, 0x0525, 0x0526, 0x0527, 0x0528, 0x0529, 0x052A, + 0x0531, 0x0532, 0x0533, 0x0534, 0x0535, 0x0536, 0x0537, 0x0538, + 0x0539, 0x053A, 0x053B, 0x053C, 0x053D, 0x053E, 0x053F, 0x0540, + 0x0541, 0x0542, 0x0543, 0x0544, 0x0545, 0x0546, 0x0547, 0x0548, + 0x0549, 0x054A, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000} + + /* FF20-FF5F */ + uni_FF20_FF5F = []uint16{ + 0x0000, 0x0560, 0x05B0, 0x0600, 0x0650, 0x06A0, 0x06F0, 0x0740, + 0x0790, 0x07E0, 0x0830, 0x0880, 0x08D0, 0x0920, 0x0970, 0x09C0, + 0x0A10, 0x0A60, 0x0AB0, 0x0B00, 0x0B50, 0x0BA0, 0x0BF0, 0x0C40, + 0x0C90, 0x0CE0, 0x0D30, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0F60, 0x0FB0, 0x1000, 0x1050, 0x10A0, 0x10F0, 0x1140, + 0x1190, 0x11E0, 0x1230, 0x1280, 0x12D0, 0x1320, 0x1370, 0x13C0, + 0x1410, 0x1460, 0x14B0, 0x1500, 0x1550, 0x15A0, 0x15F0, 0x1640, + 0x1690, 0x16E0, 0x1730, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000} +) diff --git a/go/mysql/collations/charset/filename_test.go b/go/mysql/collations/charset/filename_test.go new file mode 100644 index 00000000000..a6f52cbf530 --- /dev/null +++ b/go/mysql/collations/charset/filename_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package charset + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTablenameToFilename(t *testing.T) { + testCases := []struct { + tablename string + filename string + }{ + { + tablename: "my_table123", + filename: "my_table123", + }, + { + tablename: "my-table", + filename: "my@002dtable", + }, + { + tablename: "my$table", + filename: "my@0024table", + }, + { + tablename: "myát", + filename: "my@0ht", + }, + { + tablename: "myÃt", + filename: "my@0jt", + }, + { + tablename: "myאt", + filename: "my@05d0t", + }, + } + + for _, tc := range testCases { + t.Run(tc.tablename, func(t *testing.T) { + filename := TablenameToFilename(tc.tablename) + assert.Equal(t, tc.filename, filename, "original bytes: %x", []byte(tc.tablename)) + }) + } +} diff --git a/go/mysql/config/config.go b/go/mysql/config/config.go index cc08107f0a3..6070d0d6248 100644 --- a/go/mysql/config/config.go +++ b/go/mysql/config/config.go @@ -2,3 +2,4 @@ package config const DefaultSQLMode = "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION" const DefaultMySQLVersion = "8.0.30" +const LegacyMySQLVersion = "5.7.31" diff --git a/go/mysql/fakesqldb/server.go b/go/mysql/fakesqldb/server.go index cd1080c862c..cc53ee19008 100644 --- a/go/mysql/fakesqldb/server.go +++ b/go/mysql/fakesqldb/server.go @@ -166,6 +166,11 @@ type ExpectedExecuteFetch struct { // New creates a server, and starts listening. func New(t testing.TB) *DB { + return NewWithEnv(t, vtenv.NewTestEnv()) +} + +// NewWithEnv creates a server, and starts listening. +func NewWithEnv(t testing.TB, env *vtenv.Environment) *DB { // Pick a path for our socket. socketDir, err := os.MkdirTemp("", "fakesqldb") if err != nil { @@ -185,7 +190,7 @@ func New(t testing.TB) *DB { queryPatternUserCallback: make(map[*regexp.Regexp]func(string)), patternData: make(map[string]exprResult), lastErrorMu: sync.Mutex{}, - env: vtenv.NewTestEnv(), + env: env, } db.Handler = db diff --git a/go/mysql/flavor.go b/go/mysql/flavor.go index a0f9e8cc4b1..4baccc78c28 100644 --- a/go/mysql/flavor.go +++ b/go/mysql/flavor.go @@ -151,6 +151,7 @@ type flavor interface { baseShowTables() string baseShowTablesWithSizes() string + baseShowInnodbTableSizes() string supportsCapability(capability capabilities.FlavorCapability) (bool, error) } @@ -454,6 +455,11 @@ func (c *Conn) BaseShowTablesWithSizes() string { return c.flavor.baseShowTablesWithSizes() } +// BaseShowInnodbTableSizes returns a query that shows innodb-internal FULLTEXT index tables and their sizes +func (c *Conn) BaseShowInnodbTableSizes() string { + return c.flavor.baseShowInnodbTableSizes() +} + // SupportsCapability checks if the database server supports the given capability func (c *Conn) SupportsCapability(capability capabilities.FlavorCapability) (bool, error) { return c.flavor.supportsCapability(capability) diff --git a/go/mysql/flavor_filepos.go b/go/mysql/flavor_filepos.go index 01d748c66f3..03d745468be 100644 --- a/go/mysql/flavor_filepos.go +++ b/go/mysql/flavor_filepos.go @@ -366,6 +366,10 @@ func (*filePosFlavor) baseShowTablesWithSizes() string { return TablesWithSize56 } +func (filePosFlavor) baseShowInnodbTableSizes() string { + return "" +} + // supportsCapability is part of the Flavor interface. func (f *filePosFlavor) supportsCapability(capability capabilities.FlavorCapability) (bool, error) { switch capability { diff --git a/go/mysql/flavor_mariadb_binlog_playback.go b/go/mysql/flavor_mariadb_binlog_playback.go index b7fa18e434f..55655e01498 100644 --- a/go/mysql/flavor_mariadb_binlog_playback.go +++ b/go/mysql/flavor_mariadb_binlog_playback.go @@ -22,6 +22,10 @@ func (mariadbFlavor) baseShowTables() string { return mysqlFlavor{}.baseShowTables() } +func (mariadbFlavor) baseShowInnodbTableSizes() string { + return "" +} + // baseShowTablesWithSizes is part of the Flavor interface. func (mariadbFlavor101) baseShowTablesWithSizes() string { return TablesWithSize56 diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index b840c3cca36..b109e5ca385 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -412,48 +412,67 @@ const BaseShowTables = `SELECT t.table_name, t.table_schema = database() ` -// TablesWithSize80 is a query to select table along with size for mysql 8.0 -// Note the following: -// - `TABLES`.`TABLE_NAME` has `utf8mb4_0900_ai_ci` collation. `INNODB_TABLESPACES`.`NAME` has `utf8mb3_general_ci`. -// We normalize the collation to get better query performance (we force the casting at the time of our choosing) -// - InnoDB has different table names than MySQL does, in particular for partitioned tables. As far as InnoDB -// is concerned, each partition is its own table. -// - We use a `UNION ALL` approach to handle two distinct scenarios: tables that are partitioned and those that are not. -// Since we `LEFT JOIN` from `TABLES` to `INNODB_TABLESPACES`, we know we already do full table scan on `TABLES`. We therefore -// don't mind spending some extra computation time (as in `CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8mb3_general_ci`) -// to make things easier for the JOIN. -// - We utilize `INFORMATION_SCHEMA`.`TABLES`.`CREATE_OPTIONS` column to tell if the table is partitioned or not. The column -// may be `NULL` or may have multiple attributes, one of which is "partitioned", which we are looking for. -// - In a partitioned table, InnoDB will return multiple rows for the same table name, one for each partition, which we successively SUM. -// We also `SUM` the sizes in the non-partitioned case. This is not because we need to, but because it makes the query -// symmetric and less prone to future edit errors. -const TablesWithSize80 = `SELECT t.table_name, - t.table_type, - UNIX_TIMESTAMP(t.create_time), - t.table_comment, - SUM(i.file_size), - SUM(i.allocated_size) - FROM information_schema.tables t - LEFT JOIN (SELECT name, file_size, allocated_size FROM information_schema.innodb_tablespaces WHERE name LIKE CONCAT(database(), '/%')) i - ON i.name = CONCAT(t.table_schema, '/', t.table_name) COLLATE utf8mb3_general_ci - WHERE - t.table_schema = database() AND IFNULL(t.create_options, '') NOT LIKE '%partitioned%' - GROUP BY - t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment -UNION ALL -SELECT t.table_name, - t.table_type, - UNIX_TIMESTAMP(t.create_time), - t.table_comment, - SUM(i.file_size), - SUM(i.allocated_size) - FROM information_schema.tables t - LEFT JOIN (SELECT name, file_size, allocated_size FROM information_schema.innodb_tablespaces WHERE name LIKE CONCAT(database(), '/%')) i - ON i.name LIKE (CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8mb3_general_ci) - WHERE - t.table_schema = database() AND t.create_options LIKE '%partitioned%' - GROUP BY - t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment +// InnoDBTableSizes: a query to return file/allocated sizes for InnoDB tables. +// File sizes and allocated sizes are found in information_schema.innodb_tablespaces +// Table names in information_schema.innodb_tablespaces match those in information_schema.tables, even for table names +// with special characters. This, a innodb_tablespaces.name could be `my-db/my-table`. +// These tablespaces will have one entry for every InnoDB table, hidden or internal. This means: +// - One entry for every partition in a partitioned table. +// - Several entries for any FULLTEXT index (FULLTEXT indexes are not BTREEs and are implemented using multiple hidden tables) +// So a single table wih a FULLTEXT index will have one entry for the "normal" table, plus multiple more entries for +// every FTS index hidden tables. +// Thankfully FULLTEXT does not work with Partitioning so this does not explode too much. +// Next thing is that FULLTEXT hidden table names do not resemble the original table name, and could look like: +// `a-b/fts_000000000000075e_00000000000005f9_index_2`. +// To unlock the identify of this table we turn to information_schema.innodb_tables. These table similarly has one entry for +// every InnoDB table, normal or hidden. It also has a `TABLE_ID` value. Given some table with FULLTEXT keys, its TABLE_ID +// is encoded in the names of the hidden tables in information_schema.innodb_tablespaces: `000000000000075e` in the +// example above. +// +// The query below is a two part: +// 1. Finding the "normal" tables only, those that the user created. We note their file size and allocated size. +// 2. Finding the hidden tables only, those that implement FTS keys. We aggregate their file size and allocated size grouping +// by the original table name with which they're associated. +// +// A table that has a FULLTEXT index will have two entries in the result set: +// - one for the "normal" table size (actual rows, texts, etc.) +// - and one for the aggregated hidden table size +// The code that reads the results of this query will need to add the two. +// Similarly, the code will need to know how to aggregate the sizes of partitioned tables, which could appear as: +// - `mydb/tbl_part#p#p0` +// - `mydb/tbl_part#p#p1` +// - `mydb/tbl_part#p#p2` +// - `mydb/tbl_part#p#p3` +// +// Lastly, we note that table name in information_schema.innodb_tables are encoded. A table that shows as +// `my-db/my-table` in information_schema.innodb_tablespaces will show as `my@002ddb/my@002dtable` in information_schema.innodb_tables. +// So this query returns InnoDB-encoded table names. The golang code reading those will have to decode the names. +const InnoDBTableSizes = ` + SELECT + it.name, + its.file_size as normal_tables_sum_file_size, + its.allocated_size as normal_tables_sum_allocated_size + FROM + information_schema.innodb_tables it + JOIN information_schema.innodb_tablespaces its + ON (its.space = it.space) + WHERE + its.name LIKE CONCAT(database(), '/%') + AND its.name NOT LIKE CONCAT(database(), '/fts_%') + UNION ALL + SELECT + it.name, + SUM(its.file_size) as hidden_tables_sum_file_size, + SUM(its.allocated_size) as hidden_tables_sum_allocated_size + FROM + information_schema.innodb_tables it + JOIN information_schema.innodb_tablespaces its + ON ( + its.name LIKE CONCAT(database(), '/fts_', CONVERT(LPAD(HEX(table_id), 16, '0') USING utf8mb3) COLLATE utf8mb3_general_ci, '_%') + ) + WHERE + its.name LIKE CONCAT(database(), '/fts_%') + GROUP BY it.name ` // baseShowTablesWithSizes is part of the Flavor interface. @@ -461,6 +480,11 @@ func (mysqlFlavor57) baseShowTablesWithSizes() string { return TablesWithSize57 } +// baseShowInnodbTableSizes is part of the Flavor interface. +func (mysqlFlavor57) baseShowInnodbTableSizes() string { + return "" +} + // supportsCapability is part of the Flavor interface. func (f mysqlFlavor) supportsCapability(capability capabilities.FlavorCapability) (bool, error) { return capabilities.MySQLVersionHasCapability(f.serverVersion, capability) @@ -468,7 +492,12 @@ func (f mysqlFlavor) supportsCapability(capability capabilities.FlavorCapability // baseShowTablesWithSizes is part of the Flavor interface. func (mysqlFlavor) baseShowTablesWithSizes() string { - return TablesWithSize80 + return "" // Won't be used, as InnoDBTableSizes is defined, and schema.Engine will use that, instead. +} + +// baseShowInnodbTableSizes is part of the Flavor interface. +func (mysqlFlavor) baseShowInnodbTableSizes() string { + return InnoDBTableSizes } func (mysqlFlavor) setReplicationSourceCommand(params *ConnParams, host string, port int32, heartbeatInterval float64, connectRetry int) string { diff --git a/go/mysql/flavor_mysql_legacy.go b/go/mysql/flavor_mysql_legacy.go index a5639cc944e..2263a9bff11 100644 --- a/go/mysql/flavor_mysql_legacy.go +++ b/go/mysql/flavor_mysql_legacy.go @@ -72,12 +72,13 @@ GROUP BY table_name, // We join with a subquery that materializes the data from `information_schema.innodb_sys_tablespaces` // early for performance reasons. This effectively causes only a single read of `information_schema.innodb_sys_tablespaces` // per query. +// Note that 5.7 has NULL for a VIEW's create_time, so we use IFNULL to make it 1 (non NULL and non zero). const TablesWithSize57 = `SELECT t.table_name, t.table_type, - UNIX_TIMESTAMP(t.create_time), + IFNULL(UNIX_TIMESTAMP(t.create_time), 1), t.table_comment, - IFNULL(SUM(i.file_size), SUM(t.data_length + t.index_length)), - IFNULL(SUM(i.allocated_size), SUM(t.data_length + t.index_length)) + IFNULL(SUM(i.file_size), SUM(t.data_length + t.index_length)) AS file_size, + IFNULL(SUM(i.allocated_size), SUM(t.data_length + t.index_length)) AS allocated_size FROM information_schema.tables t LEFT OUTER JOIN ( SELECT space, file_size, allocated_size, name diff --git a/go/mysql/flavor_mysqlgr.go b/go/mysql/flavor_mysqlgr.go index 98516e9cc9f..9cb9f887e39 100644 --- a/go/mysql/flavor_mysqlgr.go +++ b/go/mysql/flavor_mysqlgr.go @@ -254,7 +254,12 @@ func (mysqlGRFlavor) baseShowTables() string { } func (mysqlGRFlavor) baseShowTablesWithSizes() string { - return TablesWithSize80 + return "" // Won't be used, as InnoDBTableSizes is defined, and schema.Engine will use that, instead. +} + +// baseShowInnodbTableSizes is part of the Flavor interface. +func (mysqlGRFlavor) baseShowInnodbTableSizes() string { + return InnoDBTableSizes } // supportsCapability is part of the Flavor interface. diff --git a/go/mysql/schema.go b/go/mysql/schema.go index 03d558d2637..6863e1fb988 100644 --- a/go/mysql/schema.go +++ b/go/mysql/schema.go @@ -93,6 +93,29 @@ var BaseShowTablesWithSizesFields = append(BaseShowTablesFields, &querypb.Field{ Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), }) +var BaseInnoDBTableSizesFields = []*querypb.Field{{ + Name: "it.name", + Type: querypb.Type_VARCHAR, + Table: "tables", + OrgTable: "TABLES", + Database: "information_schema", + OrgName: "TABLE_NAME", + ColumnLength: 192, + Charset: uint32(collations.SystemCollation.Collation), + Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), +}, { + Name: "i.file_size", + Type: querypb.Type_INT64, + ColumnLength: 11, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), +}, { + Name: "i.allocated_size", + Type: querypb.Type_INT64, + ColumnLength: 11, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), +}} // BaseShowTablesRow returns the fields from a BaseShowTables or // BaseShowTablesForTable command. @@ -116,6 +139,14 @@ func BaseShowTablesWithSizesRow(tableName string, isView bool, comment string) [ ) } +func BaseInnoDBTableSizesRow(dbName string, tableName string) []sqltypes.Value { + return []sqltypes.Value{ + sqltypes.MakeTrusted(sqltypes.VarChar, []byte(dbName+"/"+tableName)), + sqltypes.MakeTrusted(sqltypes.Int64, []byte("100")), // file_size + sqltypes.MakeTrusted(sqltypes.Int64, []byte("150")), // allocated_size + } +} + // ShowPrimaryFields contains the fields for a BaseShowPrimary. var ShowPrimaryFields = []*querypb.Field{{ Name: "table_name", diff --git a/go/vt/vtenv/vtenv.go b/go/vt/vtenv/vtenv.go index 1371affff52..6218c96c715 100644 --- a/go/vt/vtenv/vtenv.go +++ b/go/vt/vtenv/vtenv.go @@ -67,6 +67,12 @@ func NewTestEnv() *Environment { } } +func NewLegacyTestEnv() *Environment { + env := NewTestEnv() + env.mysqlVersion = config.LegacyMySQLVersion + return env +} + func (e *Environment) CollationEnv() *collations.Environment { return e.collationEnv } diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index 644135f1e3b..38a3ca7bbb3 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -430,6 +430,7 @@ func newTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options, collatio showTableRows := make([][]sqltypes.Value, 0, len(ddls)) showTableWithSizesRows := make([][]sqltypes.Value, 0, len(ddls)) + innodbTableSizesRows := make([][]sqltypes.Value, 0, len(ddls)) for _, ddl := range ddls { table := ddl.GetTable().Name.String() @@ -455,9 +456,9 @@ func newTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options, collatio Fields: mysql.BaseShowTablesWithSizesFields, Rows: showTableWithSizesRows, }) - tEnv.addResult(mysql.TablesWithSize80, &sqltypes.Result{ - Fields: mysql.BaseShowTablesWithSizesFields, - Rows: showTableWithSizesRows, + tEnv.addResult(mysql.InnoDBTableSizes, &sqltypes.Result{ + Fields: mysql.BaseInnoDBTableSizesFields, + Rows: innodbTableSizesRows, }) indexRows := make([][]sqltypes.Value, 0, 4) diff --git a/go/vt/vttablet/endtoend/misc_test.go b/go/vt/vttablet/endtoend/misc_test.go index 29bbba56873..68f6f4b1af6 100644 --- a/go/vt/vttablet/endtoend/misc_test.go +++ b/go/vt/vttablet/endtoend/misc_test.go @@ -35,10 +35,15 @@ import ( "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/callerid" + "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/log" querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vtenv" "vitess.io/vitess/go/vt/vttablet/endtoend/framework" + "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) func TestSimpleRead(t *testing.T) { @@ -890,6 +895,11 @@ func TestShowTablesWithSizes(t *testing.T) { require.NoError(t, err) defer conn.Close() + if query := conn.BaseShowTablesWithSizes(); query == "" { + // Happens in MySQL 8.0 where we use BaseShowInnodbTableSizes, instead. + t.Skip("BaseShowTablesWithSizes is empty in this version of MySQL") + } + setupQueries := []string{ `drop view if exists show_tables_with_sizes_v1`, `drop table if exists show_tables_with_sizes_t1`, @@ -897,12 +907,14 @@ func TestShowTablesWithSizes(t *testing.T) { `create table show_tables_with_sizes_t1 (id int primary key)`, `create view show_tables_with_sizes_v1 as select * from show_tables_with_sizes_t1`, `CREATE TABLE show_tables_with_sizes_employees (id INT NOT NULL, store_id INT) PARTITION BY HASH(store_id) PARTITIONS 4`, + `create table show_tables_with_sizes_fts (id int primary key, name text, fulltext key name_fts (name))`, } defer func() { _, _ = conn.ExecuteFetch(`drop view if exists show_tables_with_sizes_v1`, 1, false) _, _ = conn.ExecuteFetch(`drop table if exists show_tables_with_sizes_t1`, 1, false) _, _ = conn.ExecuteFetch(`drop table if exists show_tables_with_sizes_employees`, 1, false) + _, _ = conn.ExecuteFetch(`drop table if exists show_tables_with_sizes_fts`, 1, false) }() for _, query := range setupQueries { _, err := conn.ExecuteFetch(query, 1, false) @@ -913,6 +925,7 @@ func TestShowTablesWithSizes(t *testing.T) { "show_tables_with_sizes_t1", "show_tables_with_sizes_v1", "show_tables_with_sizes_employees", + "show_tables_with_sizes_fts", } actualTables := []string{} @@ -933,7 +946,7 @@ func TestShowTablesWithSizes(t *testing.T) { assert.True(t, row[2].IsIntegral()) createTime, err := row[2].ToCastInt64() assert.NoError(t, err) - assert.Greater(t, createTime, int64(0)) + assert.Positive(t, createTime) // TABLE_COMMENT assert.Equal(t, "", row[3].ToString()) @@ -941,12 +954,12 @@ func TestShowTablesWithSizes(t *testing.T) { assert.True(t, row[4].IsDecimal()) fileSize, err := row[4].ToCastInt64() assert.NoError(t, err) - assert.Greater(t, fileSize, int64(0)) + assert.Positive(t, fileSize) assert.True(t, row[4].IsDecimal()) allocatedSize, err := row[5].ToCastInt64() assert.NoError(t, err) - assert.Greater(t, allocatedSize, int64(0)) + assert.Positive(t, allocatedSize) actualTables = append(actualTables, tableName) } else if tableName == "show_tables_with_sizes_v1" { @@ -956,7 +969,7 @@ func TestShowTablesWithSizes(t *testing.T) { assert.True(t, row[2].IsIntegral()) createTime, err := row[2].ToCastInt64() assert.NoError(t, err) - assert.Greater(t, createTime, int64(0)) + assert.Positive(t, createTime) // TABLE_COMMENT assert.Equal(t, "VIEW", row[3].ToString()) @@ -972,7 +985,30 @@ func TestShowTablesWithSizes(t *testing.T) { assert.True(t, row[2].IsIntegral()) createTime, err := row[2].ToCastInt64() assert.NoError(t, err) - assert.Greater(t, createTime, int64(0)) + assert.Positive(t, createTime) + + // TABLE_COMMENT + assert.Equal(t, "", row[3].ToString()) + + assert.True(t, row[4].IsDecimal()) + fileSize, err := row[4].ToCastInt64() + assert.NoError(t, err) + assert.Positive(t, fileSize) + + assert.True(t, row[5].IsDecimal()) + allocatedSize, err := row[5].ToCastInt64() + assert.NoError(t, err) + assert.Positive(t, allocatedSize) + + actualTables = append(actualTables, tableName) + } else if tableName == "show_tables_with_sizes_fts" { + // TABLE_TYPE + assert.Equal(t, "BASE TABLE", row[1].ToString()) + + assert.True(t, row[2].IsIntegral()) + createTime, err := row[2].ToCastInt64() + assert.NoError(t, err) + assert.Positive(t, createTime) // TABLE_COMMENT assert.Equal(t, "", row[3].ToString()) @@ -980,12 +1016,12 @@ func TestShowTablesWithSizes(t *testing.T) { assert.True(t, row[4].IsDecimal()) fileSize, err := row[4].ToCastInt64() assert.NoError(t, err) - assert.Greater(t, fileSize, int64(0)) + assert.Positive(t, fileSize) assert.True(t, row[5].IsDecimal()) allocatedSize, err := row[5].ToCastInt64() assert.NoError(t, err) - assert.Greater(t, allocatedSize, int64(0)) + assert.Positive(t, allocatedSize) actualTables = append(actualTables, tableName) } @@ -995,6 +1031,137 @@ func TestShowTablesWithSizes(t *testing.T) { assert.ElementsMatch(t, expectedTables, actualTables) } +func newTestSchemaEngine(connParams *mysql.ConnParams) *schema.Engine { + cfg := tabletenv.NewDefaultConfig() + cfg.DB = dbconfigs.NewTestDBConfigs(*connParams, *connParams, connParams.DbName) + env := tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "EngineTest") + se := schema.NewEngine(env) + se.InitDBConfig(dbconfigs.New(connParams)) + return se +} + +func TestEngineReload(t *testing.T) { + ctx := context.Background() + conn, err := mysql.Connect(ctx, &connParams) + require.NoError(t, err) + defer conn.Close() + t.Run("validate innodb size query", func(t *testing.T) { + q := conn.BaseShowInnodbTableSizes() + require.NotEmpty(t, q) + }) + t.Run("validate conn schema", func(t *testing.T) { + rs, err := conn.ExecuteFetch(`select database() as d`, 1, true) + require.NoError(t, err) + row := rs.Named().Row() + require.NotNil(t, row) + database := row.AsString("d", "") + require.Equal(t, connParams.DbName, database) + }) + + defer func() { + _, _ = conn.ExecuteFetch(`drop view if exists view_simple`, 1, false) + _, _ = conn.ExecuteFetch(`drop view if exists view_simple2`, 1, false) + _, _ = conn.ExecuteFetch(`drop view if exists view_simple3`, 1, false) + _, _ = conn.ExecuteFetch(`drop table if exists tbl_simple`, 1, false) + _, _ = conn.ExecuteFetch(`drop table if exists tbl_part`, 1, false) + _, _ = conn.ExecuteFetch(`drop table if exists tbl_fts`, 1, false) + }() + + engine := newTestSchemaEngine(&connParams) + require.NotNil(t, engine) + err = engine.Open() + require.NoError(t, err) + defer engine.Close() + + t.Run("schema", func(t *testing.T) { + setupQueries := []string{ + `drop view if exists view_simple`, + `drop view if exists view_simple2`, + `drop table if exists tbl_simple`, + `drop table if exists tbl_part`, + `drop table if exists tbl_fts`, + `create table tbl_simple (id int primary key)`, + `create view view_simple as select * from tbl_simple`, + `create view view_simple2 as select * from tbl_simple`, + `create table tbl_part (id INT NOT NULL, store_id INT) PARTITION BY HASH(store_id) PARTITIONS 4`, + `create table tbl_fts (id int primary key, name text, fulltext key name_fts (name))`, + } + + for _, query := range setupQueries { + _, err := conn.ExecuteFetch(query, 1, false) + require.NoError(t, err) + } + + expectedTables := []string{ + "tbl_simple", + "tbl_part", + "tbl_fts", + "view_simple", + "view_simple2", + } + err := engine.Reload(ctx) + require.NoError(t, err) + + schema := engine.GetSchema() + require.NotEmpty(t, schema) + for _, expectTable := range expectedTables { + t.Run(expectTable, func(t *testing.T) { + tbl := engine.GetTable(sqlparser.NewIdentifierCS(expectTable)) + require.NotNil(t, tbl) + + switch expectTable { + case "view_simple", "view_simple2": + assert.Zero(t, tbl.FileSize) + assert.Zero(t, tbl.AllocatedSize) + default: + assert.NotZero(t, tbl.FileSize) + assert.NotZero(t, tbl.AllocatedSize) + } + }) + } + }) + t.Run("schema changes", func(t *testing.T) { + setupQueries := []string{ + `alter view view_simple as select *, 2 from tbl_simple`, + `drop view view_simple2`, + `create view view_simple3 as select * from tbl_simple`, + } + + for _, query := range setupQueries { + _, err := conn.ExecuteFetch(query, 1, false) + require.NoError(t, err) + } + + expectedTables := []string{ + "tbl_simple", + "tbl_part", + "tbl_fts", + "view_simple", + "view_simple3", + } + err := engine.Reload(ctx) + require.NoError(t, err) + + schema := engine.GetSchema() + require.NotEmpty(t, schema) + for _, expectTable := range expectedTables { + t.Run(expectTable, func(t *testing.T) { + tbl := engine.GetTable(sqlparser.NewIdentifierCS(expectTable)) + require.NotNil(t, tbl) + + switch expectTable { + case "view_simple", "view_simple2", "view_simple3": + assert.Zero(t, tbl.FileSize) + assert.Zero(t, tbl.AllocatedSize) + default: + assert.NotZero(t, tbl.FileSize) + assert.NotZero(t, tbl.AllocatedSize) + } + }) + } + }) +} + // TestTuple tests that bind variables having tuple values work with vttablet. func TestTuple(t *testing.T) { client := framework.NewClient() diff --git a/go/vt/vttablet/tabletserver/connpool/dbconn.go b/go/vt/vttablet/tabletserver/connpool/dbconn.go index 4f3d5fe893d..fcadc3d4075 100644 --- a/go/vt/vttablet/tabletserver/connpool/dbconn.go +++ b/go/vt/vttablet/tabletserver/connpool/dbconn.go @@ -563,6 +563,11 @@ func (dbc *Conn) BaseShowTablesWithSizes() string { return dbc.conn.BaseShowTablesWithSizes() } +// BaseShowInnodbTableSizes returns a query that shows innodb-internal FULLTEXT index tables and their sizes +func (dbc *Conn) BaseShowInnodbTableSizes() string { + return dbc.conn.BaseShowInnodbTableSizes() +} + func (dbc *Conn) ConnCheck(ctx context.Context) error { if err := dbc.conn.ConnCheck(); err != nil { return dbc.Reconnect(ctx) diff --git a/go/vt/vttablet/tabletserver/health_streamer_test.go b/go/vt/vttablet/tabletserver/health_streamer_test.go index 95517880339..5cf434e8fa2 100644 --- a/go/vt/vttablet/tabletserver/health_streamer_test.go +++ b/go/vt/vttablet/tabletserver/health_streamer_test.go @@ -39,6 +39,8 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" ) +const baseInnoDBTableSizesPattern = `(?s).*SELECT.*its\.space = it\.space.*SUM\(its\.file_size\).*` + func TestHealthStreamerClosed(t *testing.T) { cfg := newConfig(nil) env := tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "ReplTrackerTest") @@ -276,6 +278,13 @@ func TestReloadSchema(t *testing.T) { // Update the query pattern for the query that schema.Engine uses to get the tables so that it runs a reload again. // If we don't change the t.create_time to a value greater than before, then the schema engine doesn't reload the database. + db.AddQueryPattern(baseInnoDBTableSizesPattern, + sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "it.name | normal_tables_sum_file_size | normal_tables_sum_allocated_size", + "varchar|int64|int64", + ), + )) db.AddQueryPattern("SELECT .* information_schema.innodb_tablespaces .*", sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -285,7 +294,6 @@ func TestReloadSchema(t *testing.T) { "product|BASE TABLE|1684735967||114688|114688", "users|BASE TABLE|1684735967||114688|114688", )) - db.AddQuery(mysql.BaseShowTables, sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -351,6 +359,14 @@ func TestReloadView(t *testing.T) { db.AddQuery("commit", &sqltypes.Result{}) db.AddQuery("rollback", &sqltypes.Result{}) // Add the query pattern for the query that schema.Engine uses to get the tables. + // InnoDBTableSizes query + db.AddQueryPattern(baseInnoDBTableSizesPattern, + sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "it.name | normal_tables_sum_file_size | normal_tables_sum_allocated_size", + "varchar|int64|int64", + ), + )) db.AddQueryPattern("SELECT .* information_schema.innodb_tablespaces .*", sqltypes.MakeTestResult( sqltypes.MakeTestFields( @@ -417,7 +433,7 @@ func TestReloadView(t *testing.T) { expGetViewDefinitionsQuery string viewDefinitionsOutput *sqltypes.Result - expClearQuery string + expClearQuery []string expInsertQuery []string expViewsChanged []string }{ @@ -433,7 +449,10 @@ func TestReloadView(t *testing.T) { expViewsChanged: []string{"view_a", "view_b"}, expGetViewDefinitionsQuery: "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('view_a', 'view_b')", expCreateStmtQuery: []string{"show create table view_a", "show create table view_b"}, - expClearQuery: "delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('view_a', 'view_b')", + expClearQuery: []string{ + "delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('view_a', 'view_b')", + "delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('view_b', 'view_a')", + }, expInsertQuery: []string{ "insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_a', 'create_view_a', 'def_a')", "insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_b', 'create_view_b', 'def_b')", @@ -450,7 +469,7 @@ func TestReloadView(t *testing.T) { expViewsChanged: []string{"view_b"}, expGetViewDefinitionsQuery: "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('view_b')", expCreateStmtQuery: []string{"show create table view_b"}, - expClearQuery: "delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('view_b')", + expClearQuery: []string{"delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('view_b')"}, expInsertQuery: []string{ "insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_b', 'create_view_mod_b', 'def_mod_b')", }, @@ -467,7 +486,14 @@ func TestReloadView(t *testing.T) { expViewsChanged: []string{"view_a", "view_b", "view_c"}, expGetViewDefinitionsQuery: "select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('view_b', 'view_c', 'view_a')", expCreateStmtQuery: []string{"show create table view_a", "show create table view_c"}, - expClearQuery: "delete from _vt.views where table_schema = database() and table_name in ('view_b', 'view_c', 'view_a')", + expClearQuery: []string{ + "delete from _vt.views where table_schema = database() and table_name in ('view_a', 'view_b', 'view_c')", + "delete from _vt.views where table_schema = database() and table_name in ('view_a', 'view_c', 'view_b')", + "delete from _vt.views where table_schema = database() and table_name in ('view_b', 'view_a', 'view_c')", + "delete from _vt.views where table_schema = database() and table_name in ('view_b', 'view_c', 'view_a')", + "delete from _vt.views where table_schema = database() and table_name in ('view_c', 'view_a', 'view_b')", + "delete from _vt.views where table_schema = database() and table_name in ('view_c', 'view_b', 'view_a')", + }, expInsertQuery: []string{ "insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_a', 'create_view_mod_a', 'def_mod_a')", "insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'view_c', 'create_view_c', 'def_c')", @@ -486,8 +512,9 @@ func TestReloadView(t *testing.T) { for idx := range tcases[0].expInsertQuery { db.AddQuery(tcases[0].expInsertQuery[idx], &sqltypes.Result{}) } - db.AddQuery(tcases[0].expClearQuery, &sqltypes.Result{}) - + for _, query := range tcases[0].expClearQuery { + db.AddQuery(query, &sqltypes.Result{}) + } var tcCount atomic.Int32 ch := make(chan struct{}) @@ -519,7 +546,9 @@ func TestReloadView(t *testing.T) { for i := range tcases[idx].expInsertQuery { db.AddQuery(tcases[idx].expInsertQuery[i], &sqltypes.Result{}) } - db.AddQuery(tcases[idx].expClearQuery, &sqltypes.Result{}) + for _, query := range tcases[idx].expClearQuery { + db.AddQuery(query, &sqltypes.Result{}) + } db.AddQueryPattern("SELECT .* information_schema.innodb_tablespaces .*", tcases[idx].showTablesWithSizesOutput) db.AddQueryPattern(".*SELECT table_name, view_definition.*views.*", tcases[idx].detectViewChangeOutput) case <-time.After(10 * time.Second): diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index afb28080167..09b3a91ffed 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -31,6 +31,7 @@ import ( "vitess.io/vitess/go/acl" "vitess.io/vitess/go/constants/sidecar" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations/charset" "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/sqltypes" @@ -424,10 +425,55 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error { return err } + var innodbTablesStats map[string]*Table + if includeStats { + if innodbTableSizesQuery := conn.Conn.BaseShowInnodbTableSizes(); innodbTableSizesQuery != "" { + // Since the InnoDB table size query is available to us on this MySQL version, we should use it. + // We therefore don't want to query for table sizes in getTableData() + includeStats = false + + innodbResults, err := conn.Conn.Exec(ctx, innodbTableSizesQuery, maxTableCount, false) + if err != nil { + return vterrors.Wrapf(err, "in Engine.reload(), reading innodb tables") + } + innodbTablesStats = make(map[string]*Table, len(innodbResults.Rows)) + for _, row := range innodbResults.Rows { + innodbTableName := row[0].ToString() // In the form of encoded `schema/table` + fileSize, _ := row[1].ToCastUint64() + allocatedSize, _ := row[2].ToCastUint64() + + if _, ok := innodbTablesStats[innodbTableName]; !ok { + innodbTablesStats[innodbTableName] = &Table{} + } + // There could be multiple appearances of the same table in the result set: + // A table that has FULLTEXT indexes will appear once for the table itself, + // with total size of row data, and once for the aggregates size of all + // FULLTEXT indexes. We aggregate the sizes of all appearances of the same table. + table := innodbTablesStats[innodbTableName] + table.FileSize += fileSize + table.AllocatedSize += allocatedSize + + if originalTableName, _, found := strings.Cut(innodbTableName, "#p#"); found { + // innodbTableName is encoded any special characters are turned into some @0-f0-f0-f value. + // Therefore this "#p#" here is a clear indication that we are looking at a partitioned table. + // We turn `my@002ddb/tbl_part#p#p0` into `my@002ddb/tbl_part` + // and aggregate the total partition sizes. + if _, ok := innodbTablesStats[originalTableName]; !ok { + innodbTablesStats[originalTableName] = &Table{} + originalTable := innodbTablesStats[originalTableName] + originalTable.FileSize += fileSize + originalTable.AllocatedSize += allocatedSize + } + } + } + // See testing in TestEngineReload + } + } tableData, err := getTableData(ctx, conn.Conn, includeStats) if err != nil { return vterrors.Wrapf(err, "in Engine.reload(), reading tables") } + // On the primary tablet, we also check the data we have stored in our schema tables to see what all needs reloading. shouldUseDatabase := se.isServingPrimary && se.schemaCopy @@ -462,8 +508,14 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error { changedTables := make(map[string]*Table) // created and altered contain the names of created and altered tables for broadcast. var created, altered []*Table + databaseName := se.cp.DBName() for _, row := range tableData.Rows { tableName := row[0].ToString() + var innodbTable *Table + if innodbTablesStats != nil { + innodbTableName := fmt.Sprintf("%s/%s", charset.TablenameToFilename(databaseName), charset.TablenameToFilename(tableName)) + innodbTable = innodbTablesStats[innodbTableName] + } curTables[tableName] = true createTime, _ := row[2].ToCastInt64() var fileSize, allocatedSize uint64 @@ -474,6 +526,9 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error { // publish the size metrics se.tableFileSizeGauge.Set(tableName, int64(fileSize)) se.tableAllocatedSizeGauge.Set(tableName, int64(allocatedSize)) + } else if innodbTable != nil { + se.tableFileSizeGauge.Set(tableName, int64(innodbTable.FileSize)) + se.tableAllocatedSizeGauge.Set(tableName, int64(innodbTable.AllocatedSize)) } // Table schemas are cached by tabletserver. For each table we cache `information_schema.tables.create_time` (`tbl.CreateTime`). @@ -501,6 +556,9 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error { if includeStats { tbl.FileSize = fileSize tbl.AllocatedSize = allocatedSize + } else if innodbTable != nil { + tbl.FileSize = innodbTable.FileSize + tbl.AllocatedSize = innodbTable.AllocatedSize } continue } @@ -520,6 +578,9 @@ func (se *Engine) reload(ctx context.Context, includeStats bool) error { if includeStats { table.FileSize = fileSize table.AllocatedSize = allocatedSize + } else if innodbTable != nil { + table.FileSize = innodbTable.FileSize + table.AllocatedSize = innodbTable.AllocatedSize } table.CreateTime = createTime changedTables[tableName] = table diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index caaf505779d..3faa7a10554 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -54,11 +54,15 @@ import ( ) const baseShowTablesWithSizesPattern = `SELECT t\.table_name.*SUM\(i\.file_size\).*` +const baseInnoDBTableSizesPattern = `(?s).*SELECT.*its\.space = it\.space.*SUM\(its\.file_size\).*` var mustMatch = utils.MustMatchFn(".Mutex") -func TestOpenAndReload(t *testing.T) { - db := fakesqldb.New(t) +// TestOpenAndReloadLegacy +// +// Runs with 5.7 env +func TestOpenAndReloadLegacy(t *testing.T) { + db := fakesqldb.NewWithEnv(t, vtenv.NewLegacyTestEnv()) defer db.Close() schematest.AddDefaultQueries(db) @@ -87,7 +91,7 @@ func TestOpenAndReload(t *testing.T) { )) firstReadRowsValue := 12 AddFakeInnoDBReadRowsResult(db, firstReadRowsValue) - se := newEngine(10*time.Second, 10*time.Second, 0, db) + se := newEngine(10*time.Second, 10*time.Second, 0, db, vtenv.NewLegacyTestEnv()) se.Open() defer se.Close() @@ -292,23 +296,279 @@ func TestOpenAndReload(t *testing.T) { assert.Equal(t, want, se.GetSchema()) } -func TestReloadWithSwappedTables(t *testing.T) { +func TestOpenAndReload(t *testing.T) { db := fakesqldb.New(t) defer db.Close() schematest.AddDefaultQueries(db) db.RejectQueryPattern(baseShowTablesWithSizesPattern, "Opening schema engine should query tables without size information") + db.RejectQueryPattern(baseInnoDBTableSizesPattern, "Opening schema engine should query tables without size information") + + db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + RowsAffected: 0, + InsertID: 0, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_03", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + SessionStateChanges: "", + StatusFlags: 0, + }) + + // advance to one second after the default 1427325875. + db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "t", + "int64"), + "1427325876", + )) + firstReadRowsValue := 12 + AddFakeInnoDBReadRowsResult(db, firstReadRowsValue) + se := newEngine(10*time.Second, 10*time.Second, 0, db, nil) + se.Open() + defer se.Close() + + want := initialSchema() + mustMatch(t, want, se.GetSchema()) + assert.Equal(t, int64(0), se.tableFileSizeGauge.Counts()["msg"]) + assert.Equal(t, int64(0), se.tableAllocatedSizeGauge.Counts()["msg"]) + + t.Run("EnsureConnectionAndDB", func(t *testing.T) { + // Verify that none of the following configurations run any schema change detection queries - + // 1. REPLICA serving + // 2. REPLICA non-serving + // 3. PRIMARY serving + err := se.EnsureConnectionAndDB(topodatapb.TabletType_REPLICA, true) + require.NoError(t, err) + err = se.EnsureConnectionAndDB(topodatapb.TabletType_PRIMARY, false) + require.NoError(t, err) + err = se.EnsureConnectionAndDB(topodatapb.TabletType_REPLICA, false) + require.NoError(t, err) + }) + + // Advance time some more. + db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "t", + "int64"), + "1427325877", + )) + assert.EqualValues(t, firstReadRowsValue, se.innoDbReadRowsCounter.Get()) + // Modify test_table_03 + // Add test_table_04 + // Drop msg db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ Fields: mysql.BaseShowTablesFields, RowsAffected: 0, InsertID: 0, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + { + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_03")), // table_name + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), // table_type + sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325877")), // unix_timestamp(t.create_time) + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")), // table_comment + sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size + sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size + }, + mysql.BaseShowTablesRow("test_table_04", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + }, + SessionStateChanges: "", + StatusFlags: 0, + }) + // Modify test_table_03 + // Add test_table_04 + // Drop msg + db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{ + Fields: mysql.BaseInnoDBTableSizesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_01"), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_02"), + { + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("fakesqldb/test_table_03")), // table_name + sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size + sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size + }, + mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_04"), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "seq"), + }, + }) + db.RejectQueryPattern(baseShowTablesWithSizesPattern, "Opening schema engine should query tables without size information") + + db.MockQueriesForTable("test_table_03", &sqltypes.Result{ + Fields: []*querypb.Field{{ + Name: "pk1", + Type: sqltypes.Int32, + }, { + Name: "pk2", + Type: sqltypes.Int32, + }, { + Name: "val", + Type: sqltypes.Int32, + }}, + }) + + db.MockQueriesForTable("test_table_04", &sqltypes.Result{ + Fields: []*querypb.Field{{ + Name: "pk", + Type: sqltypes.Int32, + }}, + }) + + db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{ + Fields: mysql.ShowPrimaryFields, + Rows: [][]sqltypes.Value{ + mysql.ShowPrimaryRow("test_table_01", "pk"), + mysql.ShowPrimaryRow("test_table_02", "pk"), + mysql.ShowPrimaryRow("test_table_03", "pk1"), + mysql.ShowPrimaryRow("test_table_03", "pk2"), + mysql.ShowPrimaryRow("test_table_04", "pk"), + mysql.ShowPrimaryRow("seq", "id"), + }, + }) + secondReadRowsValue := 123 + AddFakeInnoDBReadRowsResult(db, secondReadRowsValue) + + firstTime := true + notifier := func(full map[string]*Table, created, altered, dropped []*Table, _ bool) { + if firstTime { + firstTime = false + createTables := extractNamesFromTablesList(created) + sort.Strings(createTables) + assert.Equal(t, []string{"dual", "msg", "seq", "test_table_01", "test_table_02", "test_table_03"}, createTables) + assert.Equal(t, []*Table(nil), altered) + assert.Equal(t, []*Table(nil), dropped) + } else { + assert.Equal(t, []string{"test_table_04"}, extractNamesFromTablesList(created)) + assert.Equal(t, []string{"test_table_03"}, extractNamesFromTablesList(altered)) + assert.Equal(t, []string{"msg"}, extractNamesFromTablesList(dropped)) + } + } + se.RegisterNotifier("test", notifier, true) + err := se.Reload(context.Background()) + require.NoError(t, err) + + assert.EqualValues(t, secondReadRowsValue, se.innoDbReadRowsCounter.Get()) + + want["seq"].FileSize = 100 + want["seq"].AllocatedSize = 150 + + want["test_table_01"].FileSize = 100 + want["test_table_01"].AllocatedSize = 150 + + want["test_table_02"].FileSize = 100 + want["test_table_02"].AllocatedSize = 150 + + want["test_table_03"] = &Table{ + Name: sqlparser.NewIdentifierCS("test_table_03"), + Fields: []*querypb.Field{{ + Name: "pk1", + Type: sqltypes.Int32, + }, { + Name: "pk2", + Type: sqltypes.Int32, + }, { + Name: "val", + Type: sqltypes.Int32, + }}, + PKColumns: []int{0, 1}, + CreateTime: 1427325877, + FileSize: 128, + AllocatedSize: 256, + } + want["test_table_04"] = &Table{ + Name: sqlparser.NewIdentifierCS("test_table_04"), + Fields: []*querypb.Field{{ + Name: "pk", + Type: sqltypes.Int32, + }}, + PKColumns: []int{0}, + CreateTime: 1427325875, + FileSize: 100, + AllocatedSize: 150, + } + delete(want, "msg") + assert.Equal(t, want, se.GetSchema()) + assert.Equal(t, int64(0), se.tableAllocatedSizeGauge.Counts()["msg"]) + assert.Equal(t, int64(0), se.tableFileSizeGauge.Counts()["msg"]) + + // ReloadAt tests + pos1, err := replication.DecodePosition("MariaDB/0-41983-20") + require.NoError(t, err) + pos2, err := replication.DecodePosition("MariaDB/0-41983-40") + require.NoError(t, err) + se.UnregisterNotifier("test") + + err = se.ReloadAt(context.Background(), replication.Position{}) + require.NoError(t, err) + assert.Equal(t, want, se.GetSchema()) + + err = se.ReloadAt(context.Background(), pos1) + require.NoError(t, err) + assert.Equal(t, want, se.GetSchema()) + + db.AddQueryPattern(baseShowTablesWithSizesPattern, &sqltypes.Result{ + Fields: mysql.BaseShowTablesWithSizesFields, Rows: [][]sqltypes.Value{ mysql.BaseShowTablesWithSizesRow("test_table_01", false, ""), mysql.BaseShowTablesWithSizesRow("test_table_02", false, ""), - mysql.BaseShowTablesWithSizesRow("test_table_03", false, ""), + mysql.BaseShowTablesWithSizesRow("test_table_04", false, ""), mysql.BaseShowTablesWithSizesRow("seq", false, "vitess_sequence"), - mysql.BaseShowTablesWithSizesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + }) + + db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_04", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + }, + }) + + db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{ + Fields: mysql.ShowPrimaryFields, + Rows: [][]sqltypes.Value{ + mysql.ShowPrimaryRow("test_table_01", "pk"), + mysql.ShowPrimaryRow("test_table_02", "pk"), + mysql.ShowPrimaryRow("test_table_04", "pk"), + mysql.ShowPrimaryRow("seq", "id"), + }, + }) + err = se.ReloadAt(context.Background(), pos1) + require.NoError(t, err) + assert.Equal(t, want, se.GetSchema()) + + delete(want, "test_table_03") + err = se.ReloadAt(context.Background(), pos2) + require.NoError(t, err) + assert.Equal(t, want, se.GetSchema()) +} + +func TestReloadWithSwappedTables(t *testing.T) { + db := fakesqldb.New(t) + defer db.Close() + schematest.AddDefaultQueries(db) + + db.RejectQueryPattern(baseShowTablesWithSizesPattern, "Opening schema engine should query tables without size information") + db.RejectQueryPattern(baseInnoDBTableSizesPattern, "Opening schema engine should query tables without size information") + + db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + RowsAffected: 0, + InsertID: 0, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_03", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), }, SessionStateChanges: "", StatusFlags: 0, @@ -316,7 +576,7 @@ func TestReloadWithSwappedTables(t *testing.T) { firstReadRowsValue := 12 AddFakeInnoDBReadRowsResult(db, firstReadRowsValue) - se := newEngine(10*time.Second, 10*time.Second, 0, db) + se := newEngine(10*time.Second, 10*time.Second, 0, db, nil) se.Open() defer se.Close() want := initialSchema() @@ -329,23 +589,42 @@ func TestReloadWithSwappedTables(t *testing.T) { "int64"), "1427325876", )) - db.AddQueryPattern(baseShowTablesWithSizesPattern, &sqltypes.Result{ - Fields: mysql.BaseShowTablesWithSizesFields, + db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{ + Fields: mysql.BaseInnoDBTableSizesFields, Rows: [][]sqltypes.Value{ - mysql.BaseShowTablesWithSizesRow("test_table_01", false, ""), - mysql.BaseShowTablesWithSizesRow("test_table_02", false, ""), - mysql.BaseShowTablesWithSizesRow("test_table_03", false, ""), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_01"), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_02"), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_03"), { - sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_04")), - sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), - sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325877")), // unix_timestamp(create_time) - sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")), - sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size - sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("fakesqldb/test_table_04")), // table_name + sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size + sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size }, - mysql.BaseShowTablesWithSizesRow("seq", false, "vitess_sequence"), - mysql.BaseShowTablesWithSizesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "seq"), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "msg"), + }, + }) + db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + RowsAffected: 0, + InsertID: 0, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_03", false, ""), + { + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_04")), // table_name + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), // table_type + sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325877")), // unix_timestamp(t.create_time) + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")), // table_comment + sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size + sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size + }, + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), }, + SessionStateChanges: "", + StatusFlags: 0, }) db.MockQueriesForTable("test_table_04", &sqltypes.Result{ Fields: []*querypb.Field{{ @@ -403,11 +682,28 @@ func TestReloadWithSwappedTables(t *testing.T) { "int64"), "1427325877", )) - db.AddQueryPattern(baseShowTablesWithSizesPattern, &sqltypes.Result{ - Fields: mysql.BaseShowTablesWithSizesFields, + db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{ + Fields: mysql.BaseInnoDBTableSizesFields, Rows: [][]sqltypes.Value{ - mysql.BaseShowTablesWithSizesRow("test_table_01", false, ""), - mysql.BaseShowTablesWithSizesRow("test_table_02", false, ""), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_01"), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_02"), + { + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("fakesqldb/test_table_03")), // table_name + sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size + sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size + }, + mysql.BaseInnoDBTableSizesRow("fakesqldb", "test_table_04"), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "seq"), + mysql.BaseInnoDBTableSizesRow("fakesqldb", "msg"), + }, + }) + db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + RowsAffected: 0, + InsertID: 0, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), { sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_03")), sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), @@ -416,10 +712,12 @@ func TestReloadWithSwappedTables(t *testing.T) { sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size }, - mysql.BaseShowTablesWithSizesRow("test_table_04", false, ""), - mysql.BaseShowTablesWithSizesRow("seq", false, "vitess_sequence"), - mysql.BaseShowTablesWithSizesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + mysql.BaseShowTablesRow("test_table_04", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), }, + SessionStateChanges: "", + StatusFlags: 0, }) db.MockQueriesForTable("test_table_03", &sqltypes.Result{ Fields: []*querypb.Field{{ @@ -482,7 +780,7 @@ func TestOpenFailedDueToExecErr(t *testing.T) { schematest.AddDefaultQueries(db) want := "injected error" db.AddRejectedQuery(mysql.BaseShowTables, errors.New(want)) - se := newEngine(1*time.Second, 1*time.Second, 0, db) + se := newEngine(1*time.Second, 1*time.Second, 0, db, nil) err := se.Open() if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("se.Open: %v, want %s", err, want) @@ -513,7 +811,7 @@ func TestOpenFailedDueToLoadTableErr(t *testing.T) { db.AddRejectedQuery("SELECT * FROM `fakesqldb`.`test_view` WHERE 1 != 1", sqlerror.NewSQLErrorFromError(errors.New("The user specified as a definer ('root'@'%') does not exist (errno 1449) (sqlstate HY000)"))) AddFakeInnoDBReadRowsResult(db, 0) - se := newEngine(1*time.Second, 1*time.Second, 0, db) + se := newEngine(1*time.Second, 1*time.Second, 0, db, nil) err := se.Open() // failed load should return an error because of test_table assert.ErrorContains(t, err, "Row count exceeded") @@ -548,7 +846,7 @@ func TestOpenNoErrorDueToInvalidViews(t *testing.T) { db.AddRejectedQuery("SELECT `col1`, `col2` FROM `fakesqldb`.`bar_view` WHERE 1 != 1", sqlerror.NewSQLError(sqlerror.ERWrongFieldWithGroup, sqlerror.SSClientError, "random error for table bar_view")) AddFakeInnoDBReadRowsResult(db, 0) - se := newEngine(1*time.Second, 1*time.Second, 0, db) + se := newEngine(1*time.Second, 1*time.Second, 0, db, nil) err := se.Open() require.NoError(t, err) @@ -564,7 +862,7 @@ func TestExportVars(t *testing.T) { db := fakesqldb.New(t) defer db.Close() schematest.AddDefaultQueries(db) - se := newEngine(1*time.Second, 1*time.Second, 0, db) + se := newEngine(1*time.Second, 1*time.Second, 0, db, nil) se.Open() defer se.Close() expvar.Do(func(kv expvar.KeyValue) { @@ -576,7 +874,7 @@ func TestStatsURL(t *testing.T) { db := fakesqldb.New(t) defer db.Close() schematest.AddDefaultQueries(db) - se := newEngine(1*time.Second, 1*time.Second, 0, db) + se := newEngine(1*time.Second, 1*time.Second, 0, db, nil) se.Open() defer se.Close() @@ -606,7 +904,7 @@ func TestSchemaEngineCloseTickRace(t *testing.T) { }) AddFakeInnoDBReadRowsResult(db, 12) // Start the engine with a small reload tick - se := newEngine(100*time.Millisecond, 1*time.Second, 0, db) + se := newEngine(100*time.Millisecond, 1*time.Second, 0, db, nil) err := se.Open() require.NoError(t, err) @@ -633,7 +931,7 @@ func TestSchemaEngineCloseTickRace(t *testing.T) { } } -func newEngine(reloadTime time.Duration, idleTimeout time.Duration, schemaMaxAgeSeconds int64, db *fakesqldb.DB) *Engine { +func newEngine(reloadTime time.Duration, idleTimeout time.Duration, schemaMaxAgeSeconds int64, db *fakesqldb.DB, env *vtenv.Environment) *Engine { cfg := tabletenv.NewDefaultConfig() cfg.SchemaReloadInterval = reloadTime cfg.OltpReadPool.IdleTimeout = idleTimeout @@ -642,7 +940,10 @@ func newEngine(reloadTime time.Duration, idleTimeout time.Duration, schemaMaxAge cfg.SchemaVersionMaxAgeSeconds = schemaMaxAgeSeconds dbConfigs := newDBConfigs(db) cfg.DB = dbConfigs - se := NewEngine(tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "SchemaTest")) + if env == nil { + env = vtenv.NewTestEnv() + } + se := NewEngine(tabletenv.NewEnv(env, cfg, "SchemaTest")) se.InitDBConfig(dbConfigs.DbaWithDB()) return se } @@ -1032,7 +1333,6 @@ func TestEngineGetTableData(t *testing.T) { name string expectedQueries map[string]*sqltypes.Result queriesToReject map[string]error - includeStats bool expectedError string }{ { @@ -1040,13 +1340,6 @@ func TestEngineGetTableData(t *testing.T) { expectedQueries: map[string]*sqltypes.Result{ conn.BaseShowTables(): {}, }, - includeStats: false, - }, { - name: "Success with include stats", - expectedQueries: map[string]*sqltypes.Result{ - conn.BaseShowTablesWithSizes(): {}, - }, - includeStats: true, }, { name: "Error in query", queriesToReject: map[string]error{ @@ -1068,7 +1361,7 @@ func TestEngineGetTableData(t *testing.T) { defer db.DeleteRejectedQuery(query) } - _, err = getTableData(context.Background(), conn, tt.includeStats) + _, err = getTableData(context.Background(), conn, false) if tt.expectedError != "" { require.ErrorContains(t, err, tt.expectedError) return @@ -1195,194 +1488,204 @@ func TestEngineGetDroppedTables(t *testing.T) { // TestEngineReload tests the entire functioning of engine.Reload testing all the queries that we end up running against MySQL // while simulating the responses and verifies the final list of created, altered and dropped tables. func TestEngineReload(t *testing.T) { - db := fakesqldb.New(t) - cfg := tabletenv.NewDefaultConfig() - cfg.DB = newDBConfigs(db) - cfg.SignalWhenSchemaChange = true - env := tabletenv.NewEnv(vtenv.NewTestEnv(), nil, "TestEngineReload") - conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil, env) - require.NoError(t, err) - - se := newEngine(10*time.Second, 10*time.Second, 0, db) - se.conns.Open(se.cp, se.cp, se.cp) - se.isOpen = true - se.notifiers = make(map[string]notifier) - se.MakePrimary(true) + envs := []*vtenv.Environment{ + vtenv.NewTestEnv(), + vtenv.NewLegacyTestEnv(), + } + for _, venv := range envs { + t.Run(venv.MySQLVersion(), func(t *testing.T) { + db := fakesqldb.NewWithEnv(t, venv) + cfg := tabletenv.NewDefaultConfig() + cfg.DB = newDBConfigs(db) + cfg.SignalWhenSchemaChange = true + + env := tabletenv.NewEnv(venv, nil, "TestEngineReload") + conn, err := connpool.NewConn(context.Background(), dbconfigs.New(db.ConnParams()), nil, nil, env) + require.NoError(t, err) - // If we have to skip the meta check, then there is nothing to do - se.SkipMetaCheck = true - err = se.reload(context.Background(), false) - require.NoError(t, err) + se := newEngine(10*time.Second, 10*time.Second, 0, db, venv) + se.conns.Open(se.cp, se.cp, se.cp) + se.isOpen = true + se.notifiers = make(map[string]notifier) + se.MakePrimary(true) - se.SkipMetaCheck = false - se.lastChange = 987654321 + // If we have to skip the meta check, then there is nothing to do + se.SkipMetaCheck = true + err = se.reload(context.Background(), false) + require.NoError(t, err) - // Initial tables in the schema engine - se.tables = map[string]*Table{ - "t1": { - Name: sqlparser.NewIdentifierCS("t1"), - Type: NoType, - CreateTime: 123456789, - }, - "t2": { - Name: sqlparser.NewIdentifierCS("t2"), - Type: NoType, - CreateTime: 123456789, - }, - "t4": { - Name: sqlparser.NewIdentifierCS("t4"), - Type: NoType, - CreateTime: 123456789, - }, - "v1": { - Name: sqlparser.NewIdentifierCS("v1"), - Type: View, - CreateTime: 123456789, - }, - "v2": { - Name: sqlparser.NewIdentifierCS("v2"), - Type: View, - CreateTime: 123456789, - }, - "v4": { - Name: sqlparser.NewIdentifierCS("v4"), - Type: View, - CreateTime: 123456789, - }, - } - // MySQL unix timestamp query. - db.AddQuery("SELECT UNIX_TIMESTAMP()", sqltypes.MakeTestResult(sqltypes.MakeTestFields("UNIX_TIMESTAMP", "int64"), "987654326")) - // Table t2 is updated, T2 is created and t4 is deleted. - // View v2 is updated, V2 is created and v4 is deleted. - db.AddQuery(conn.BaseShowTables(), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|table_type|unix_timestamp(create_time)|table_comment", - "varchar|varchar|int64|varchar"), - "t1|BASE_TABLE|123456789|", - "t2|BASE_TABLE|123456790|", - "T2|BASE_TABLE|123456789|", - "v1|VIEW|123456789|", - "v2|VIEW|123456789|", - "V2|VIEW|123456789|", - )) + se.SkipMetaCheck = false + se.lastChange = 987654321 - // Detecting view changes. - // According to the database, v2, V2, v4, and v5 require updating. - db.AddQuery(fmt.Sprintf(detectViewChange, sidecar.GetIdentifier()), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"), - "v2", - "V2", - "v4", - "v5", - )) + // Initial tables in the schema engine + se.tables = map[string]*Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Type: NoType, + CreateTime: 123456789, + }, + "t2": { + Name: sqlparser.NewIdentifierCS("t2"), + Type: NoType, + CreateTime: 123456789, + }, + "t4": { + Name: sqlparser.NewIdentifierCS("t4"), + Type: NoType, + CreateTime: 123456789, + }, + "v1": { + Name: sqlparser.NewIdentifierCS("v1"), + Type: View, + CreateTime: 123456789, + }, + "v2": { + Name: sqlparser.NewIdentifierCS("v2"), + Type: View, + CreateTime: 123456789, + }, + "v4": { + Name: sqlparser.NewIdentifierCS("v4"), + Type: View, + CreateTime: 123456789, + }, + } + // MySQL unix timestamp query. + db.AddQuery("SELECT UNIX_TIMESTAMP()", sqltypes.MakeTestResult(sqltypes.MakeTestFields("UNIX_TIMESTAMP", "int64"), "987654326")) + // Table t2 is updated, T2 is created and t4 is deleted. + // View v2 is updated, V2 is created and v4 is deleted. + db.AddQuery(conn.BaseShowTables(), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|table_type|unix_timestamp(create_time)|table_comment", + "varchar|varchar|int64|varchar"), + "t1|BASE_TABLE|123456789|", + "t2|BASE_TABLE|123456790|", + "T2|BASE_TABLE|123456789|", + "v1|VIEW|123456789|", + "v2|VIEW|123456789|", + "V2|VIEW|123456789|", + )) + // Detecting view changes. + // According to the database, v2, V2, v4, and v5 require updating. + db.AddQuery(fmt.Sprintf(detectViewChange, sidecar.GetIdentifier()), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"), + "v2", + "V2", + "v4", + "v5", + )) - // Finding mismatches in the tables. - // t5 exists in the database. - db.AddQuery("SELECT TABLE_NAME, CREATE_TIME FROM _vt.`tables`", sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|create_time", "varchar|int64"), - "t1|123456789", - "t2|123456789", - "t4|123456789", - "t5|123456789", - )) + // Finding mismatches in the tables. + // t5 exists in the database. + db.AddQuery("SELECT TABLE_NAME, CREATE_TIME FROM _vt.`tables`", sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|create_time", "varchar|int64"), + "t1|123456789", + "t2|123456789", + "t4|123456789", + "t5|123456789", + )) - // Read Innodb_rows_read. - db.AddQuery(mysql.ShowRowsRead, sqltypes.MakeTestResult(sqltypes.MakeTestFields("Variable_name|Value", "varchar|int64"), - "Innodb_rows_read|35")) + // Read Innodb_rows_read. + db.AddQuery(mysql.ShowRowsRead, sqltypes.MakeTestResult(sqltypes.MakeTestFields("Variable_name|Value", "varchar|int64"), + "Innodb_rows_read|35")) - // Queries to load the tables' information. - for _, tableName := range []string{"t2", "T2", "v2", "V2"} { - db.AddQuery(fmt.Sprintf(`SELECT COLUMN_NAME as column_name + // Queries to load the tables' information. + for _, tableName := range []string{"t2", "T2", "v2", "V2"} { + db.AddQuery(fmt.Sprintf(`SELECT COLUMN_NAME as column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = 'fakesqldb' AND TABLE_NAME = '%s' ORDER BY ORDINAL_POSITION`, tableName), - sqltypes.MakeTestResult(sqltypes.MakeTestFields("column_name", "varchar"), - "col1")) - db.AddQuery(fmt.Sprintf("SELECT `col1` FROM `fakesqldb`.`%v` WHERE 1 != 1", tableName), sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1", "varchar"))) - } - - // Primary key information. - db.AddQuery(mysql.BaseShowPrimary, sqltypes.MakeTestResult(mysql.ShowPrimaryFields, - "t1|col1", - "t2|col1", - "T2|col1", - )) - - // Queries for reloading the tables' information. - { - for _, tableName := range []string{"t2", "T2"} { - db.AddQuery(fmt.Sprintf(`show create table %s`, tableName), - sqltypes.MakeTestResult(sqltypes.MakeTestFields("Table | Create Table", "varchar|varchar"), - fmt.Sprintf("%v|create_table_%v", tableName, tableName))) - } - db.AddQuery("begin", &sqltypes.Result{}) - db.AddQuery("commit", &sqltypes.Result{}) - db.AddQuery("rollback", &sqltypes.Result{}) - // We are adding both the variants of the delete statements that we can see in the test, since the deleted tables are initially stored as a map, the order is not defined. - db.AddQuery("delete from _vt.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('t5', 't4', 'T2', 't2')", &sqltypes.Result{}) - db.AddQuery("delete from _vt.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('t4', 't5', 'T2', 't2')", &sqltypes.Result{}) - db.AddQuery("insert into _vt.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 't2', 'create_table_t2', 123456790)", &sqltypes.Result{}) - db.AddQuery("insert into _vt.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 'T2', 'create_table_T2', 123456789)", &sqltypes.Result{}) - } + sqltypes.MakeTestResult(sqltypes.MakeTestFields("column_name", "varchar"), + "col1")) + db.AddQuery(fmt.Sprintf("SELECT `col1` FROM `fakesqldb`.`%v` WHERE 1 != 1", tableName), sqltypes.MakeTestResult(sqltypes.MakeTestFields("col1", "varchar"))) + } - // Queries for reloading the views' information. - { - for _, tableName := range []string{"v2", "V2"} { - db.AddQuery(fmt.Sprintf(`show create table %s`, tableName), - sqltypes.MakeTestResult(sqltypes.MakeTestFields(" View | Create View | character_set_client | collation_connection", "varchar|varchar|varchar|varchar"), - fmt.Sprintf("%v|create_table_%v|utf8mb4|utf8mb4_0900_ai_ci", tableName, tableName))) - } - // We are adding both the variants of the select statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined. - db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v4', 'v5', 'V2', 'v2')", - sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"), - "v2|select_v2", - "V2|select_V2", - )) - db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v5', 'v4', 'V2', 'v2')", - sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"), - "v2|select_v2", - "V2|select_V2", + // Primary key information. + db.AddQuery(mysql.BaseShowPrimary, sqltypes.MakeTestResult(mysql.ShowPrimaryFields, + "t1|col1", + "t2|col1", + "T2|col1", )) - // We are adding both the variants of the delete statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined. - db.AddQuery("delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('v4', 'v5', 'V2', 'v2')", &sqltypes.Result{}) - db.AddQuery("delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('v5', 'v4', 'V2', 'v2')", &sqltypes.Result{}) - db.AddQuery("insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'v2', 'create_table_v2', 'select_v2')", &sqltypes.Result{}) - db.AddQuery("insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'V2', 'create_table_V2', 'select_V2')", &sqltypes.Result{}) - } + // Queries for reloading the tables' information. + { + for _, tableName := range []string{"t2", "T2"} { + db.AddQuery(fmt.Sprintf(`show create table %s`, tableName), + sqltypes.MakeTestResult(sqltypes.MakeTestFields("Table | Create Table", "varchar|varchar"), + fmt.Sprintf("%v|create_table_%v", tableName, tableName))) + } + db.AddQuery("begin", &sqltypes.Result{}) + db.AddQuery("commit", &sqltypes.Result{}) + db.AddQuery("rollback", &sqltypes.Result{}) + // We are adding both the variants of the delete statements that we can see in the test, since the deleted tables are initially stored as a map, the order is not defined. + db.AddQuery("delete from _vt.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('t5', 't4', 'T2', 't2')", &sqltypes.Result{}) + db.AddQuery("delete from _vt.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('t4', 't5', 'T2', 't2')", &sqltypes.Result{}) + db.AddQuery("insert into _vt.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 't2', 'create_table_t2', 123456790)", &sqltypes.Result{}) + db.AddQuery("insert into _vt.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), 'T2', 'create_table_T2', 123456789)", &sqltypes.Result{}) + } - // adding query pattern for udfs - udfQueryPattern := "SELECT name.*" - db.AddQueryPattern(udfQueryPattern, &sqltypes.Result{}) + // Queries for reloading the views' information. + { + for _, tableName := range []string{"v2", "V2"} { + db.AddQuery(fmt.Sprintf(`show create table %s`, tableName), + sqltypes.MakeTestResult(sqltypes.MakeTestFields(" View | Create View | character_set_client | collation_connection", "varchar|varchar|varchar|varchar"), + fmt.Sprintf("%v|create_table_%v|utf8mb4|utf8mb4_0900_ai_ci", tableName, tableName))) + } + // We are adding both the variants of the select statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined. + db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v4', 'v5', 'V2', 'v2')", + sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"), + "v2|select_v2", + "V2|select_V2", + )) + db.AddQuery("select table_name, view_definition from information_schema.views where table_schema = database() and table_name in ('v5', 'v4', 'V2', 'v2')", + sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|view_definition", "varchar|varchar"), + "v2|select_v2", + "V2|select_V2", + )) + + // We are adding both the variants of the delete statements that we can see in the test, since the deleted views are initially stored as a map, the order is not defined. + db.AddQuery("delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('v4', 'v5', 'V2', 'v2')", &sqltypes.Result{}) + db.AddQuery("delete from _vt.views where TABLE_SCHEMA = database() and TABLE_NAME in ('v5', 'v4', 'V2', 'v2')", &sqltypes.Result{}) + db.AddQuery("insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'v2', 'create_table_v2', 'select_v2')", &sqltypes.Result{}) + db.AddQuery("insert into _vt.views(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, VIEW_DEFINITION) values (database(), 'V2', 'create_table_V2', 'select_V2')", &sqltypes.Result{}) + } - // Verify the list of created, altered and dropped tables seen. - se.RegisterNotifier("test", func(full map[string]*Table, created, altered, dropped []*Table, _ bool) { - require.ElementsMatch(t, extractNamesFromTablesList(created), []string{"T2", "V2"}) - require.ElementsMatch(t, extractNamesFromTablesList(altered), []string{"t2", "v2"}) - require.ElementsMatch(t, extractNamesFromTablesList(dropped), []string{"t4", "v4", "t5", "v5"}) - }, false) + // adding query pattern for udfs + udfQueryPattern := "SELECT name.*mysql.func.*" + db.AddQueryPattern(udfQueryPattern, &sqltypes.Result{}) - // Run the reload. - err = se.reload(context.Background(), false) - require.NoError(t, err) - require.NoError(t, db.LastError()) - require.Zero(t, se.throttledLogger.GetLastLogTime()) + // Verify the list of created, altered and dropped tables seen. + se.RegisterNotifier("test", func(full map[string]*Table, created, altered, dropped []*Table, _ bool) { + require.ElementsMatch(t, extractNamesFromTablesList(created), []string{"T2", "V2"}) + require.ElementsMatch(t, extractNamesFromTablesList(altered), []string{"t2", "v2"}) + require.ElementsMatch(t, extractNamesFromTablesList(dropped), []string{"t4", "v4", "t5", "v5"}) + }, false) - // Now if we remove the query pattern for udfs, schema engine shouldn't fail. - // Instead we should see a log message with the error. - db.RemoveQueryPattern(udfQueryPattern) - se.UnregisterNotifier("test") - err = se.reload(context.Background(), false) - require.NoError(t, err) - // Check for the udf error being logged. The last log time should be less than a second. - require.Less(t, time.Since(se.throttledLogger.GetLastLogTime()), 1*time.Second) + // Run the reload. + err = se.reload(context.Background(), false) + require.NoError(t, err) + require.NoError(t, db.LastError()) + require.Zero(t, se.throttledLogger.GetLastLogTime()) + + // Now if we remove the query pattern for udfs, schema engine shouldn't fail. + // Instead we should see a log message with the error. + db.RemoveQueryPattern(udfQueryPattern) + se.UnregisterNotifier("test") + err = se.reload(context.Background(), false) + require.NoError(t, err) + // Check for the udf error being logged. The last log time should be less than a second. + require.Less(t, time.Since(se.throttledLogger.GetLastLogTime()), 1*time.Second) + }) + } } -// TestEngineReload tests the vreplication specific GetTableForPos function to ensure +// TestGetTableForPosLegacy tests the vreplication specific GetTableForPos function to ensure // that it conforms to the intended/expected behavior in various scenarios. // This more specifically tests the behavior of the function when the historian is // disabled or otherwise unable to get a table schema for the given position. When it // CAN, that is tested indepenently in the historian tests. -func TestGetTableForPos(t *testing.T) { +// +// Runs with 5.7 env +func TestGetTableForPosLegacy(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - fakedb := fakesqldb.New(t) + fakedb := fakesqldb.NewWithEnv(t, vtenv.NewLegacyTestEnv()) cfg := tabletenv.NewDefaultConfig() cfg.DB = newDBConfigs(fakedb) table := sqlparser.NewIdentifierCS("t1") @@ -1400,7 +1703,7 @@ func TestGetTableForPos(t *testing.T) { } // Don't do any automatic / TTL based cache refreshes. - se := newEngine(1*time.Hour, 1*time.Hour, 0, fakedb) + se := newEngine(1*time.Hour, 1*time.Hour, 0, fakedb, vtenv.NewLegacyTestEnv()) se.conns.Open(se.cp, se.cp, se.cp) se.isOpen = true se.notifiers = make(map[string]notifier) @@ -1562,3 +1865,197 @@ func TestGetTableForPos(t *testing.T) { }) } } + +// TestGetTableForPos tests the vreplication specific GetTableForPos function to ensure +// that it conforms to the intended/expected behavior in various scenarios. +// This more specifically tests the behavior of the function when the historian is +// disabled or otherwise unable to get a table schema for the given position. When it +// CAN, that is tested indepenently in the historian tests. +func TestGetTableForPos(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fakedb := fakesqldb.New(t) + cfg := tabletenv.NewDefaultConfig() + cfg.DB = newDBConfigs(fakedb) + table := sqlparser.NewIdentifierCS("t1") + column := "col1" + tableSchema := fmt.Sprintf("create table %s (%s varchar(50), primary key(col1))", table.String(), column) + tableMt := &binlogdatapb.MinimalTable{ + Name: table.String(), + Fields: []*querypb.Field{ + { + Name: column, + Type: sqltypes.VarChar, + }, + }, + PKColumns: []int64{0}, // First column: col1 + } + + // Don't do any automatic / TTL based cache refreshes. + se := newEngine(1*time.Hour, 1*time.Hour, 0, fakedb, nil) + se.conns.Open(se.cp, se.cp, se.cp) + se.isOpen = true + se.notifiers = make(map[string]notifier) + se.MakePrimary(true) + se.historian.enabled = false + + addExpectedReloadQueries := func(db *fakesqldb.DB) { + db.AddQuery("SELECT UNIX_TIMESTAMP()", sqltypes.MakeTestResult(sqltypes.MakeTestFields( + "UNIX_TIMESTAMP()", + "int64"), + fmt.Sprintf("%d", time.Now().Unix()), + )) + db.AddQuery(fmt.Sprintf(detectViewChange, sidecar.GetIdentifier()), sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name", "varchar"))) + db.AddQuery(fmt.Sprintf(readTableCreateTimes, sidecar.GetIdentifier()), + sqltypes.MakeTestResult(sqltypes.MakeTestFields("table_name|create_time", "varchar|int64"))) + db.AddQuery(fmt.Sprintf(detectUdfChange, sidecar.GetIdentifier()), &sqltypes.Result{}) + db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{ + Fields: mysql.BaseInnoDBTableSizesFields, + Rows: [][]sqltypes.Value{ + { + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("fakesqldb/"+table.String())), // table_name + sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size + sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size + }, + }, + }) + db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + { + sqltypes.MakeTrusted(sqltypes.VarChar, []byte(table.String())), // table_name + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), // table_type + sqltypes.MakeTrusted(sqltypes.Int64, []byte(fmt.Sprintf("%d", time.Now().Unix()-1000))), // unix_timestamp(t.create_time) + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")), // table_comment + }, + }, + SessionStateChanges: "", + StatusFlags: 0, + }) + db.RejectQueryPattern(baseShowTablesWithSizesPattern, "we should expect to get sizes by InnoDBTableSizes") + db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{ + Fields: mysql.ShowPrimaryFields, + Rows: [][]sqltypes.Value{ + mysql.ShowPrimaryRow(table.String(), column), + }, + }) + db.AddQueryPattern(fmt.Sprintf(mysql.GetColumnNamesQueryPatternForTable, table.String()), + sqltypes.MakeTestResult(sqltypes.MakeTestFields("column_name", "varchar"), column)) + db.AddQuery(fmt.Sprintf("SELECT `%s` FROM `fakesqldb`.`%v` WHERE 1 != 1", column, table.String()), + sqltypes.MakeTestResult(sqltypes.MakeTestFields(column, "varchar"))) + db.AddQuery(fmt.Sprintf(`show create table %s`, table.String()), + sqltypes.MakeTestResult(sqltypes.MakeTestFields("Table|Create Table", "varchar|varchar"), table.String(), tableSchema)) + db.AddQuery("begin", &sqltypes.Result{}) + db.AddQuery(fmt.Sprintf("delete from %s.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('%s')", + sidecar.GetIdentifier(), table.String()), &sqltypes.Result{}) + db.AddQuery(fmt.Sprintf("insert into %s.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), '%s', '%s', %d)", + sidecar.GetIdentifier(), table.String(), tableSchema, time.Now().Unix()), &sqltypes.Result{RowsAffected: 1}) + db.AddQuery("rollback", &sqltypes.Result{}) + } + + type testcase struct { + name string + initialCacheState map[string]*Table + expectedQueriesFunc func(db *fakesqldb.DB) + expectFunc func() + } + tests := []testcase{ + { + name: "GetTableForPos with cache uninitialized", + initialCacheState: make(map[string]*Table), // empty + expectedQueriesFunc: func(db *fakesqldb.DB) { + // We do a reload to initialize the cache. + addExpectedReloadQueries(db) + }, + expectFunc: func() { + tbl, err := se.GetTableForPos(ctx, table, "") + require.NoError(t, err) + require.Equal(t, tableMt, tbl) + }, + }, + { + name: "GetTableForPos with cache uninitialized, table not found", + initialCacheState: make(map[string]*Table), // empty + expectedQueriesFunc: func(db *fakesqldb.DB) { + // We do a reload to initialize the cache and in doing so get the missing table. + addExpectedReloadQueries(db) + }, + expectFunc: func() { + tbl, err := se.GetTableForPos(ctx, sqlparser.NewIdentifierCS("nobueno"), "") + require.EqualError(t, err, "table nobueno not found in vttablet schema") + require.Nil(t, tbl) + }, + }, + { + name: "GetTableForPos with cache initialized, table not found", + initialCacheState: map[string]*Table{"t2": {Name: sqlparser.NewIdentifierCS("t2")}}, + expectedQueriesFunc: func(db *fakesqldb.DB) { + // We do a reload to try and get this missing table and any other recently created ones. + addExpectedReloadQueries(db) + }, + expectFunc: func() { + tbl, err := se.GetTableForPos(ctx, table, "") + require.NoError(t, err) + require.Equal(t, tableMt, tbl) + }, + }, + { + name: "GetTableForPos with cache initialized, table found", + initialCacheState: map[string]*Table{table.String(): {Name: table}}, + expectedQueriesFunc: func(db *fakesqldb.DB) { + // We only reload the column and PK info for the table in our cache. A new column + // called col2 has been added to the table schema and it is the new PK. + newTableSchema := fmt.Sprintf("create table %s (%s varchar(50), col2 varchar(50), primary key(col2))", table.String(), column) + db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{ + Fields: mysql.ShowPrimaryFields, + Rows: [][]sqltypes.Value{ + mysql.ShowPrimaryRow(table.String(), "col2"), + }, + }) + db.AddQueryPattern(fmt.Sprintf(mysql.GetColumnNamesQueryPatternForTable, table.String()), + sqltypes.MakeTestResult(sqltypes.MakeTestFields("column_name", "varchar"), column, "col2")) + db.AddQuery(fmt.Sprintf("SELECT `%s`, `%s` FROM `fakesqldb`.`%v` WHERE 1 != 1", + column, "col2", table.String()), sqltypes.MakeTestResult(sqltypes.MakeTestFields(fmt.Sprintf("%s|%s", column, "col2"), "varchar|varchar"))) + db.AddQuery(fmt.Sprintf(`show create table %s`, table.String()), + sqltypes.MakeTestResult(sqltypes.MakeTestFields("Table|Create Table", "varchar|varchar"), table.String(), newTableSchema)) + db.AddQuery("begin", &sqltypes.Result{}) + db.AddQuery(fmt.Sprintf("delete from %s.`tables` where TABLE_SCHEMA = database() and TABLE_NAME in ('%s')", + sidecar.GetIdentifier(), table.String()), &sqltypes.Result{}) + db.AddQuery(fmt.Sprintf("insert into %s.`tables`(TABLE_SCHEMA, TABLE_NAME, CREATE_STATEMENT, CREATE_TIME) values (database(), '%s', '%s', %d)", + sidecar.GetIdentifier(), table.String(), newTableSchema, time.Now().Unix()), &sqltypes.Result{}) + db.AddQuery("rollback", &sqltypes.Result{}) + }, + expectFunc: func() { + tbl, err := se.GetTableForPos(ctx, table, "MySQL56/1497ddb0-7cb9-11ed-a1eb-0242ac120002:1-891") + require.NoError(t, err) + require.NotNil(t, tbl) + require.Equal(t, &binlogdatapb.MinimalTable{ + Name: table.String(), + Fields: []*querypb.Field{ + { + Name: column, + Type: sqltypes.VarChar, + }, + { + Name: "col2", + Type: sqltypes.VarChar, + }, + }, + PKColumns: []int64{1}, // Second column: col2 + }, tbl) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fakedb.DeleteAllQueries() + AddFakeInnoDBReadRowsResult(fakedb, int(rand.Int32N(1000000))) + tc.expectedQueriesFunc(fakedb) + se.tables = tc.initialCacheState + tc.expectFunc() + fakedb.VerifyAllExecutedOrFail() + require.NoError(t, fakedb.LastError()) + }) + } +} diff --git a/go/vt/vttablet/tabletserver/schema/main_test.go b/go/vt/vttablet/tabletserver/schema/main_test.go index 7eaca5f18e5..8b705b9bfbd 100644 --- a/go/vt/vttablet/tabletserver/schema/main_test.go +++ b/go/vt/vttablet/tabletserver/schema/main_test.go @@ -34,11 +34,11 @@ func getTestSchemaEngine(t *testing.T, schemaMaxAgeSeconds int64) (*Engine, *fak "int64"), "1427325876", )) - db.AddQueryPattern(baseShowTablesWithSizesPattern, &sqltypes.Result{}) + db.AddQueryPattern(baseInnoDBTableSizesPattern, &sqltypes.Result{}) db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{}) db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{}) AddFakeInnoDBReadRowsResult(db, 1) - se := newEngine(10*time.Second, 10*time.Second, schemaMaxAgeSeconds, db) + se := newEngine(10*time.Second, 10*time.Second, schemaMaxAgeSeconds, db, nil) require.NoError(t, se.Open()) cancel := func() { defer db.Close()