From 0a65c3a49ace424e2627c318c28902b6fa57d08e Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 30 Nov 2023 00:04:10 +0800 Subject: [PATCH 1/8] add openapi loader config --- dm/config/task_converters.go | 33 ++++++++++++- dm/openapi/gen.server.go | 92 +++++++++++++++++++----------------- dm/openapi/gen.types.go | 86 +++++++++++++++++++++++++++++++++ dm/openapi/spec/dm.yaml | 56 ++++++++++++++++++++++ 4 files changed, 222 insertions(+), 45 deletions(-) diff --git a/dm/config/task_converters.go b/dm/config/task_converters.go index 261d1ccdb34..5384a523edc 100644 --- a/dm/config/task_converters.go +++ b/dm/config/task_converters.go @@ -199,6 +199,15 @@ func OpenAPITaskToSubTaskConfigs(task *openapi.Task, toDBCfg *dbconfig.DBConfig, subTaskCfg.MydumperConfig = DefaultMydumperConfig() subTaskCfg.LoaderConfig = DefaultLoaderConfig() if fullCfg := task.SourceConfig.FullMigrateConf; fullCfg != nil { + if fullCfg.Analyze != nil { + subTaskCfg.LoaderConfig.Analyze = PhysicalPostOpLevel(*fullCfg.Analyze) + } + if fullCfg.Checksum != nil { + subTaskCfg.LoaderConfig.ChecksumPhysical = PhysicalPostOpLevel(*fullCfg.Checksum) + } + if fullCfg.CompressKvPairs != nil { + subTaskCfg.CompressKVPairs = *fullCfg.CompressKvPairs + } if fullCfg.Consistency != nil { subTaskCfg.MydumperConfig.ExtraArgs = fmt.Sprintf("--consistency %s", *fullCfg.Consistency) } @@ -211,7 +220,29 @@ func OpenAPITaskToSubTaskConfigs(task *openapi.Task, toDBCfg *dbconfig.DBConfig, if fullCfg.DataDir != nil { subTaskCfg.LoaderConfig.Dir = *fullCfg.DataDir } - subTaskCfg.LoaderConfig.OnDuplicateLogical = LogicalDuplicateResolveType(task.OnDuplicate) + if fullCfg.DiskQuota != nil { + if err := subTaskCfg.LoaderConfig.DiskQuotaPhysical.UnmarshalText([]byte(*fullCfg.DiskQuota)); err != nil { + return nil, err + } + } + if fullCfg.ImportMode != nil { + subTaskCfg.LoaderConfig.ImportMode = LoadMode(*fullCfg.ImportMode) + } + if fullCfg.OnDuplicateLogical != nil { + subTaskCfg.LoaderConfig.OnDuplicateLogical = LogicalDuplicateResolveType(*fullCfg.OnDuplicateLogical) + } + if fullCfg.OnDuplicatePhysical != nil { + subTaskCfg.LoaderConfig.OnDuplicatePhysical = PhysicalDuplicateResolveType(*fullCfg.OnDuplicatePhysical) + } + if fullCfg.PdAddr != nil { + subTaskCfg.LoaderConfig.PDAddr = *fullCfg.PdAddr + } + if fullCfg.RangeConcurrency != nil { + subTaskCfg.LoaderConfig.RangeConcurrency = *fullCfg.RangeConcurrency + } + if fullCfg.SortingDir != nil { + subTaskCfg.LoaderConfig.SortingDirPhysical = *fullCfg.SortingDir + } } // set incremental config subTaskCfg.SyncerConfig = DefaultSyncerConfig() diff --git a/dm/openapi/gen.server.go b/dm/openapi/gen.server.go index 4060295efc7..12964ec809d 100644 --- a/dm/openapi/gen.server.go +++ b/dm/openapi/gen.server.go @@ -1308,50 +1308,54 @@ var swaggerSpec = []string{ "FUuTs+RKZgxQY8bDi6WVUjUV0w1l08j9brBXuvT6BAr4WoaSRcLHzVoF5AVNDDfFeZJIREjIUIqIrmWG", "iaqPrQQIJslQR7ICoUd7NoSvib9zV5oM7bZfDt3uOi0RSCkgOTEHUBQnyAm6RknL9hilq6y9I5SSPxd+", "vkcf18bUSAuiNBmiew0Mpia8XdGXQSEQU7U02kb6gfENr+D63xOmYtn+EwbnDvyWJ4nhd6lMfBeXrdyF", - "5MRSviQXcceFUcIxF4iEjtNHpTOJYDQBhRrFxPiF6kBRl19RJhV4rC6PlbMByHnOJK/W9yYX1EUCOZ2n", - "DkhQJqPpCLO2GdqbFusHxoC0ZtYDArFiCEb16rfDph5UBNMvSPqFlBj31+lT49Q78/zIObV+o3dqHwec", - "kZBtxgGWEvIwgDS0wQKKsF5oO2/X59lzSZd4xSjB/y6XUnMA9AcKc/WTlIevOSQCq6XcxXVZMpB8TUTu", - "TMP6HRu3t1OJjLrh06KZ0ZiVz9Z74m/eEMWRneVI+S5qKM29wRLmjaFLuBO9Zr0GwE1wGov5TIY/4il9", - "ys54h18NDncqH6ud6GtE39UKs4M4nO0fHUz2X4TPJ/M5ej6BR88OJkfhbPHiMHr2Mj6YHc8nz2eH88P9", - "g/Hs2eHzw+ggtIa/OHi2P9mfHUSL/cOjKDqIjueT+fOZs0VJvZ7NajmiHlSFhb43M1on0KEzXbGdM4iO", - "UwHf5te8Xg8oE4YSKG1Hd+GyVJ2l0xKaPe7z5JrW8lZ7ZBvP09S59QjAS+QmRoPdWouT+7IlNhzebShy", - "toV3eiFolqlsRlWB9Zu56OOMd5y+tr9oUDv1gtpHM7aLzwfmIBrWUz1UExT861AZ8vGwM0feWWsxkC/t", - "mN2TzxmDG5xEoYycTKKiHowvJr/eM0vfOnP1Ze9FVS7SDsIGwCqcsHaeF1rmwmcnhMcOV9zzkJsRUcR1", - "ibjJGhUY88a2zO9IwYEL+CxygzzDm+o4YtcOklZpo26aPqkKme1UxNylUGVLVRzOuo2SJt5dR2km5cN7", - "fkuvEbthWGyWBirf0t62MKuUf/TfwqrW7Qfdd08yhjhRLXr4VTtf1lEJ4rwMWarT/u5bhQKrJnXqrqZR", - "ycMQce4Bd7O6wvZc4zY1XEDpq3kP2hBsuBrSiz9yb69G55yuo9uOcMNfEtPe6GpF7x0sc9mKg8J6CWrK", - "dHhXI7G+g+c7lPD0Fe002kw+/EVsb6PErd7EvlWpHyGVcXJCQ0fC7uQd+JAh8urjGTj58EaqXJaMjkd9", - "Pf4m0nhOtEuLKTEt/3R8EVPF4lgoxFsLFIdCx6MjSUB1WpIhAjM8Oh4dqJ+kxhcrBe0UZnh6PZ+afhLT", - "YnrjL5Wtns4itdarj2f1dknq0FRrVjXf/mxmMn5F4TnMdKpYovEvrgtzKj+qsyeruzGTonrDLGpFpjaR", - "52kK2Xp0LHEAZWMmElPA83AFIAe1bk0CLrnVSWn0WZWw+rDXyqdJACWGr2m0fjDc232fWkibZcFCrnv7", - "hPchVzSrbcWek/C34xY/6gNvPpQlqy5Xj8OYjq5aXWQZjw4fEIxWpzbH0tqcdwiG1YC3MFybbMz0m/5D", - "RYS3Wv8lSPuBjp36EMcJJkiT7b0+bcoggynSu/zP1vGXBV4Rk6v2FVCsRoUhGFkwjGw1rg8OXflNf5/r", - "zy3GOXT44U9sR6mma6Od8qCNLByGgRJWtWB7HAlztHzbMQmz2kBvJGFmY6bfjBe2kYQZ73GAhNng+SXM", - "guHHlrB6U+/OjYzSvQI4p2S9ReKEhv918eG9R5TqYMm5ynuHbXaLaAjUchVUEQ0bEBkftQOcv12+Ox8E", - "jhzYA85K6ANyHzg6yOtXPVXjxD5mlvJV3D9TN5nLKx2Kp7/miK0tpsZiFZQjHEzsLjy5HTs+7rAGDImc", - "6dYyur5lYrpKFFcjXCDUmilsAsPn7WpfR69Kh6TYF36ToqNsgw+aQyp+KGJ8FaNx3/7bzce35Ww7+ptv", - "7nDPHwyeMify5O2cbswHIImKmi4ICLqxd9214W0dMP1mnSz0W7kT9bBkik6dsEzoQrX3yQn+mtdvqfsN", - "Xv2gY5DB894SbCuMmOr7ZjQrIIEJN610ij4JKqFjyilcqkPNcU+dsQOGV/MBgH08NR5iQ3aRVx7Hpm3T", - "nnTos7KD8aGTFw3lqQCx+mhK2750MURfGmdneOLzduyeK41/W0+ESnBvvw9rPDE9ZLJY8L62bRrpz4So", - "JLjf7TEfE9ktFu2LGZ6cbdFEfoBNrRpldOyp/mbHzy3d5paWbuh9d1SFZJsJ66eiX96PaU5c3z+6NfZk", - "VzVD1bAszolueVlcAnsYBttAcfzg7OX44tGucpdRUltnrrIVTwdvVb1ef1zWave7He4GP21OUxxQa9O5", - "OS9ZXwMeEGLrpoZDkrVbYB1/S6DtBrj1Ro47ckBVdD3Sxau+5OxQ9ph+039UGbwBzKJqvp8er4w7Cnw9", - "y1e4D1zeWf+7VS6tdwjYLSbV9c9359Gyy8kQDVa2AXs61rDz4syjnAU1PuO0I+xjf2va/vb6/T0swSDh", - "sfkAtt+9ujTDfvRcY7uc9c/iYhWMUKoqCqD+PoOuFejhLn3E06eZiq/Y9TKQ5HnIrx7z9Nvcm1qsi2Zq", - "uu2Ua83i2VCDVbb56lrVIR/NZZvt5cYbpactm7llVdv6WKGDCRWRE9P27uko2hKqit11Nf2Q4/1L3Tdo", - "e4f79nWB73m07/py1w6d85ffrarvcFOdTcPim/q8xzyaj+9vc//rn/f3sgCONQ9jDjDJcqF7PRtdqvve", - "F1jprqfVl/N1z3TKwDUOEbhGjMOtMlEDpd1ho0tVIKWoTEzjWNPensYANr8Z0CLq3gDOK+6ODTOpxe2w", - "R6hn3XHVXl7Ou5eOv6xu9m1D1s2dru+n3n0APFF9XtvZTYRrqpvM9Cj3MzXokfa9eUd1czbY3xI8u6Of", - "Teugu7PFN9Wyb5MavgZ3bBQd210DHWFxCcvAoNjXbnCn6+b8N6ubCnywsdydbZr9cIq9ba+7ttxbIFfd", - "sf656TtTmjZ031v6+25a+6lyRFextYIBXSMCcKy65gOeL4qwj5W9in6WW/si/QFmYmf44hFypd9DOzWC", - "yENfZ7yOomr/7veVVD9lBthqFfX9EoyzHz3BWFZXD0wwWibLcz5X9OAr+msOSQfV+nbynVFkj14c4Txj", - "0V2iTXfika/o4dfhM+pG0t0TqjG/Pv6ZeJtbdu5kXJ3V2dUVkESmG635gdFcmLtouHax+O5SObiWrKwi", - "e72WtH5ForudoP8gQvmzuq2Lv90lbvfm4g1L3spit58s/bMIb2dlyVmJ98CiJN9bJGjDlMQiQReC5aHI", - "2U+ZemoyNfZ3tPWRvOCAwTR3f7tq99P3NcnjFotvmpz5KSE/JWT+fYKlOvPtfrDUKYb+LFmZnvkpihsv", - "/qMI4sOnKK2kYFMO/1y12FriNjSb3V6rgL11LuUnyX+wzHfrU+y7eh9Xf+/8bsnnYTeLrA8r7qCyL1ua", - "73pt/Y5eYjLXKjT3bMadNOtVXvrb/D+c7tJo777qoplfc6mPj7DrYkfrzefXNN+LaAoxUa3nR5LUZgK3", - "Lhj1dbuPaDi4xb3paT/9muPwaqI08ESXpU6qrmA1HTNyeWYK7e1CdYPFahKlFjxq2TY0RRfYclzxw+3n", - "2/8LAAD//xNoTUdjtwAA", + "5MRSviQXtRNokMBk/W+XcFJ18sVoomuveJ7KKbPVmuMQJgCnRdK51NiGcbUGld6D/DOO63xvPWvRoVjo", + "iUBD04whzidX15MMYsa7wTKjwdU1UKPd8DlWIRxzgUi47py/sGOYGMdcnejq+jfKpAWN1e29cjYAOc+Z", + "VBZ14cgFdcEhp/MUYgnK4FIVDbT9gL1psX5gLHh7Zsyvgq85LVJ+NVcG8yugninwHftZrvRi9tY1u14+", + "ECuGYFQvbjxsmjklD/oFuTshJSa6cYZMGgafW1HtjB6nrEApdC1+TOhSIiblz+BYZ8TqeQtDA4cLw/mR", + "E0UDUT+KtrkIChD6uLB4QyoZFSzlOmu3QgSkCIlyAAKM3nC1sWZul5z6/TzrFKQc1eluBiVtt4GDX9kQ", + "SpCKQkne3FfzqH2cFwW9zVgyZycSJk19UG5soTZ8Bb0W4upNYL3ZqaLsJAhV1XYe9aAfluqhV5D3pvIV", + "dyWRz/adkZBtZvss98tj+iRTBQsowvoVg3m7Mtmei69JuGKU4H+XS6k5APoDhZqLpCfwNYdEYLWUu6w4", + "SwZKdBORXrH20bB+u9Ad51XOgrrb2KKZ8RWraLW31sm8IYpiBSuE9F1RUz7rBkuYN4Yu4T7iMus1AG6C", + "01jM5yz7cz1lNN2Z6eFXgxM9VXTZPuJo5B2rFWYHcTjbPzqY7L8In0/mc/R8Ao+eHUyOwtnixWH07GV8", + "MDueT57PDueH+wfj2bPD54fRQWgNf3HwbH+yPzuIFvuHR1F0EB3PJ/PnM2dzpnolr9VsST2oSqp9b2a0", + "TqBDt47ayulrx3mob/Nr8b4HlAlDCZROW/eVDWnNy3AtNHvcF8M244RbHYtuPE9T59ZzH14iNzEaHNBb", + "nNyXJ7bh8G5DcVpVWOkLQbNMRQRV7elv5oqjM9PjzDL4y6V1OkNQ+1DaTm7wgdnXhkOnHqoJCv51qAz5", + "eFi1Be+sMhvIl3a20pPJHoMbnEQhZFGRoq2nIReTX+95PtmqNvGdW4qqUK6dfhoAq3DC2lkpYZkLn50Q", + "Hjtccc9DbkZEEdeXY0y+vMCYN7ZlfkcKDlzAZ5Eb5BneTsyRtesgaZUw76bpk6oN3E4t4F1K9LZUv+as", + "WCtp4t11lGZSPryVK/QasRuGxWYJ8PIt7W0Ls0r5R//902rdftB9N8RjiBPVnIxftU8KOmrgnNfAS3Xa", + "33ewUGDVpE7d1TQqeRgizj3gblZR3Z5r3KaGCyh9KflBWyEOV0N68UfuatjoGdZVtNIRbviLAdsbXa3o", + "vX1qrplyUFgvQU2BIu9qodhXcnOH4sW+csVGg92Hb0HhbRG71R4UtyorKqQyTk5o6Ehbn7wDHzJEXn08", + "Aycf3kiVy5LR8aivu+lEGs+JdmkxJabZqY4vYqpYHAuFeGuB4jj8eHQkCagSdxkiMMOj49GB+klqfLFS", + "0E5hhqfX86nppDMtpjf+Utnk7ixSa736eFZvFKfKRbRmVfPtz2a6OWZ55QZmZf5v+i+uSxIrP6qzG7W7", + "JZ2iesMsakWmNpHnaQrZenQscQBlSzoSU8DzcAUgB7U+dQIuudVDbvRZFe/7sNfKp0kAJYavabR+MNzb", + "He9aSJtlwUKue/uE9yFXNKttxZ6T8LfjFj/qUh8+lCWr/n6Pw5iOfoJdZBmPDh8QjFaPSsfS2px3CIbV", + "erwwXJtszPSb/kNFhLda/yVI+4GOnfoQxwkmSJPtvT5nzyCDKdK7/M/Wwb8FXhGTq8Y9UKxGhSEYWTCM", + "bDWuSyZc+U1/h//PLcY5dPjhT2xHqaZro5H8oI0sHIaBElY1n3wcCXM0u9wxCbMa4G8kYWZjpt+MF7aR", + "hBnvcYCE2eD5JcyC4ceWsPrnDDo3Mkr3CuCckvUWiRMa/tfFh/ceUaqDJecqb1y32S2iIVDLVVBFNGxA", + "ZHzUDnD+dvnufBA4cmAPOCuhS4N84Oggr1/1VC1j+5hZyldx81b1cCgvsyme/pojtraYGotVUI5wMLG7", + "5O527PiszRowJHKmm2rpyr6J6adTXApzgVBrI7MJDJ+3q30dXXodkmK3OkiKXtoNPmgOqfihiPFVjMZ9", + "+29/dmFbzrbjyw6bO9zzB4OnzIk8eTunW5ICSKKimhUCgm7sXXdteFsHTL9ZJwv9Vu5EPSyZolMnLBO6", + "UI3NcoK/5vX+HH6DVz/oGGTwvPej2wojpvqmLc0KSGDCTROxokOMSuiYcgqX6lBz3FNn7IDh1XwAYB9P", + "jYfYkF3klcexadu0Jx36rOzdfujkRUN5KkCsPhfVti9dDNGXxtkZnvi8HbvnSuPf1hOhEtzb78MaT0wP", + "mSwWvK9tm0b6A0kqCe53e8xnlHaLRftihidnWzSRH2BTqxZBHXuqv1b0c0u3uaWlG3rfHVUh2WbC+qno", + "FPpjmhPXl99ujT3ZVc1QtWqMc6Kb/RbXXx+GwTZQHD84ezm+9bar3GWU1NaZq2xC1sFbVZfrH5e12p2+", + "h7vBT5vTFAfUGhRvzkvWd9AHhNi6neuQZO0WWMffDG27AW69he2OHFAV/d508aovOTuUPabf9B9VBm8A", + "s6ia76fHK+OOAl/P8hXuA5d31v9ulUvrvVF2i0l1/fPdebTs7zREg5UNEJ+ONey8OPMoZ0GND9jtCPvY", + "X9mvmnk9hIclGCQ8Np/+97tXl2bYj55rbJez/llcrIIRSlVFAdRfptG1Aj3cpY94+jRT8f3OXgaSPA/5", + "1WOefpt7U4t10UZSN9xzrVk8G2qwygaHXas65KO5bLOx5nij9LRlM7esalufaXUwoSJyYhp+Ph1FW0JV", + "sbuuph9yvH+pO6Zt73Dfvi7wPY/2Xd8s3KFz/vKLffUdbqqzaUjJNWJF5W7X9uuB29z/ApQeFsCx5mHM", + "ASZZLnSXe6NL9Rc/Cqx0v2fIr0wnJ/21CMrANQ4RuEaMw60yUQOl3WGjS1UgpahMTMts82EPGgPY/FpK", + "i6h7AzivuDs2zKQWt8MeoZ51x1V7eTnvXjr+srrZtw1ZN3e6vp969wHwRPV5bWc3Ea6paTbTrdzP1KBH", + "2vfmHdXN2WB/S/Dsjn423azuzhbfVLPSTWr4GtyxUXRs90t1hMUlLAODYl+j1Z2um/PfrG4q8MHGcne2", + "afbDKfa2ve7acm+BXHXH+uem70xp2tB9b+nvu2ntp8oRXcXWCgZ0jQjAsfpeCOD5ogj7WNmr6Ge5tS/S", + "H2AmdoYvHiFX+j20UyOIPPR1xusoqvbvfl9J9VNmgK1WUd8vwTj70ROMZXX1wASjZbI853NFD76iv+aQ", + "dFCtbyffGUX26MURzjMW3R/f9GUf+Yoefh0+o26h3z2hGvPr45+Jt7ll507G1VmdXV0BSWS60ZofGM2F", + "uYuGaxeL7y6Vg2vJyiqy12tJ61ckutsJ+g8ilD+r27r4213idm8u3rDkrSx2+8nSP4vwdlaWnJV4DyxK", + "8r1FgjZMSSwSdCFYHoqc/ZSppyZTY39HWx/JCw4YTHP3V/t2P31fkzxusfimyZmfEvJTQubfJ1iqM9/u", + "B0udYujPkpXpmZ+iuPHiP4ogPnyK0koKNuXwz1WLrSVuQ7PZ7bUK2FvnciHH/ICZ7xLvXb+Pqzb5jsnn", + "YTeLrE/K7qCyL1ua73pt/Y5eYjLXKjT3bMadNOtVXjT7IXWXRnv3VRfN/JpLfXyEXRc7Wm8+v6b5XkRT", + "iIlqPT+SpDYTuHXBqK/bfUTDwS3uTU/76dcch1cTpYEnuix1UnUFq+mYkcszU2hvF6obLFaTKLXgUcu2", + "oSm6wJbjih9uP9/+XwAAAP//rLmSJF28AAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/dm/openapi/gen.types.go b/dm/openapi/gen.types.go index 7867255bb6d..a1ac719d35d 100644 --- a/dm/openapi/gen.types.go +++ b/dm/openapi/gen.types.go @@ -33,6 +33,47 @@ const ( TaskTaskModeIncremental TaskTaskMode = "incremental" ) +// Defines values for TaskFullMigrateConfAnalyze. +const ( + TaskFullMigrateConfAnalyzeOff TaskFullMigrateConfAnalyze = "off" + + TaskFullMigrateConfAnalyzeOptional TaskFullMigrateConfAnalyze = "optional" + + TaskFullMigrateConfAnalyzeRequired TaskFullMigrateConfAnalyze = "required" +) + +// Defines values for TaskFullMigrateConfChecksum. +const ( + TaskFullMigrateConfChecksumOff TaskFullMigrateConfChecksum = "off" + + TaskFullMigrateConfChecksumOptional TaskFullMigrateConfChecksum = "optional" + + TaskFullMigrateConfChecksumRequired TaskFullMigrateConfChecksum = "required" +) + +// Defines values for TaskFullMigrateConfImportMode. +const ( + TaskFullMigrateConfImportModeLogical TaskFullMigrateConfImportMode = "logical" + + TaskFullMigrateConfImportModePhysical TaskFullMigrateConfImportMode = "physical" +) + +// Defines values for TaskFullMigrateConfOnDuplicateLogical. +const ( + TaskFullMigrateConfOnDuplicateLogicalError TaskFullMigrateConfOnDuplicateLogical = "error" + + TaskFullMigrateConfOnDuplicateLogicalIgnore TaskFullMigrateConfOnDuplicateLogical = "ignore" + + TaskFullMigrateConfOnDuplicateLogicalReplace TaskFullMigrateConfOnDuplicateLogical = "replace" +) + +// Defines values for TaskFullMigrateConfOnDuplicatePhysical. +const ( + TaskFullMigrateConfOnDuplicatePhysicalManual TaskFullMigrateConfOnDuplicatePhysical = "manual" + + TaskFullMigrateConfOnDuplicatePhysicalNone TaskFullMigrateConfOnDuplicatePhysical = "none" +) + // Defines values for TaskStage. const ( TaskStageFinished TaskStage = "Finished" @@ -539,19 +580,64 @@ type TaskBinLogFilterRule struct { // configuration of full migrate tasks type TaskFullMigrateConf struct { + // to control checksum of physical import + Analyze *TaskFullMigrateConfAnalyze `json:"analyze,omitempty"` + + // to control checksum of physical import + Checksum *TaskFullMigrateConfChecksum `json:"checksum,omitempty"` + + // to control compress kv pairs of physical import + CompressKvPairs *string `json:"compress-kv-pairs,omitempty"` + // to control the way in which data is exported for consistency assurance Consistency *string `json:"consistency,omitempty"` // storage dir name DataDir *string `json:"data_dir,omitempty"` + // disk quota for physical import + DiskQuota *string `json:"disk_quota,omitempty"` + // full export of concurrent ExportThreads *int `json:"export_threads,omitempty"` + // to control import mode of full import + ImportMode *TaskFullMigrateConfImportMode `json:"import_mode,omitempty"` + // full import of concurrent ImportThreads *int `json:"import_threads,omitempty"` + + // to control the duplication resolution when meet duplicate rows for logical import + OnDuplicateLogical *TaskFullMigrateConfOnDuplicateLogical `json:"on_duplicate_logical,omitempty"` + + // to control the duplication resolution when meet duplicate rows for physical import + OnDuplicatePhysical *TaskFullMigrateConfOnDuplicatePhysical `json:"on_duplicate_physical,omitempty"` + + // address of pd + PdAddr *string `json:"pd_addr,omitempty"` + + // to control range concurrency of physical import + RangeConcurrency *int `json:"range_concurrency,omitempty"` + + // sorting dir name for physical import + SortingDir *string `json:"sorting_dir,omitempty"` } +// to control checksum of physical import +type TaskFullMigrateConfAnalyze string + +// to control checksum of physical import +type TaskFullMigrateConfChecksum string + +// to control import mode of full import +type TaskFullMigrateConfImportMode string + +// to control the duplication resolution when meet duplicate rows for logical import +type TaskFullMigrateConfOnDuplicateLogical string + +// to control the duplication resolution when meet duplicate rows for physical import +type TaskFullMigrateConfOnDuplicatePhysical string + // configuration of incremental tasks type TaskIncrMigrateConf struct { // incremental synchronization of batch execution sql quantities diff --git a/dm/openapi/spec/dm.yaml b/dm/openapi/spec/dm.yaml index f3ef606a92d..9881b1538a2 100644 --- a/dm/openapi/spec/dm.yaml +++ b/dm/openapi/spec/dm.yaml @@ -1685,6 +1685,62 @@ components: type: string example: "auto" description: "to control the way in which data is exported for consistency assurance" + import_mode: + type: string + example: "logical" + description: "to control import mode of full import" + enum: + - "logical" + - "physical" + sorting_dir: + type: string + example: "./sort_dir" + description: "sorting dir name for physical import" + disk_quota: + type: string + example: "80G" + description: "disk quota for physical import" + checksum: + type: string + example: "optional" + description: "to control checksum of physical import" + enum: + - "required" + - "optional" + - "off" + analyze: + type: string + example: "optional" + description: "to control checksum of physical import" + enum: + - "required" + - "optional" + - "off" + range_concurrency: + type: integer + description: "to control range concurrency of physical import" + default: 0 + compress-kv-pairs: + type: string + description: "to control compress kv pairs of physical import" + pd_addr: + type: string + description: "address of pd" + on_duplicate_logical: + type: string + example: "replace" + description: "to control the duplication resolution when meet duplicate rows for logical import" + enum: + - "replace" + - "error" + - "ignore" + on_duplicate_physical: + type: string + example: "none" + description: "to control the duplication resolution when meet duplicate rows for physical import" + enum: + - "none" + - "manual" TaskIncrMigrateConf: description: configuration of incremental tasks type: object From db927c8aad10fedebe83899e2de656ca6d5e2e34 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 5 Dec 2023 17:13:44 +0800 Subject: [PATCH 2/8] update --- dm/config/task.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/dm/config/task.go b/dm/config/task.go index 4c5d09a0391..6ab35e5d2d3 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -345,9 +345,6 @@ func (m *LoaderConfig) adjust() error { m.PoolSize = defaultPoolSize } - if m.OnDuplicateLogical == "" && m.OnDuplicate != "" { - m.OnDuplicateLogical = m.OnDuplicate - } if m.OnDuplicateLogical == "" { m.OnDuplicateLogical = OnDuplicateReplace } From 10da0c4c0ec9a418effd5682531a5d81366e876e Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 5 Dec 2023 17:57:56 +0800 Subject: [PATCH 3/8] update --- dm/openapi/fixtures/task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/openapi/fixtures/task.go b/dm/openapi/fixtures/task.go index c146b385722..f87f07c59b0 100644 --- a/dm/openapi/fixtures/task.go +++ b/dm/openapi/fixtures/task.go @@ -26,7 +26,7 @@ var ( "enhance_online_schema_change": true, "meta_schema": "dm_meta", "name": "test", - "on_duplicate": "error", + "on_duplicate": "replace", "source_config": { "full_migrate_conf": { "data_dir": "./exported_data", From 0d933a7ffd278363da829793a5f5c6090785485c Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Tue, 5 Dec 2023 21:35:34 +0800 Subject: [PATCH 4/8] update --- dm/openapi/fixtures/task.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dm/openapi/fixtures/task.go b/dm/openapi/fixtures/task.go index f87f07c59b0..9c238658801 100644 --- a/dm/openapi/fixtures/task.go +++ b/dm/openapi/fixtures/task.go @@ -63,7 +63,7 @@ var ( "enhance_online_schema_change": true, "meta_schema": "dm_meta", "name": "a5fb4a7540d343fa853c55ade2d08e6d03681d9e05d6240c0", - "on_duplicate": "error", + "on_duplicate": "replace", "source_config": { "full_migrate_conf": { "data_dir": "./exported_data", @@ -103,7 +103,7 @@ var ( "enhance_online_schema_change": true, "meta_schema": "dm_meta", "name": "test", - "on_duplicate": "error", + "on_duplicate": "replace", "shard_mode": "optimistic", "strict_optimistic_shard_mode": true, "source_config": { From d161bf9d09f50e12defa3221ea62de71516baaff Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 13 Dec 2023 17:26:11 +0800 Subject: [PATCH 5/8] add it --- dm/tests/lightning_mode/conf/dm-master.toml | 1 + dm/tests/lightning_mode/run.sh | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dm/tests/lightning_mode/conf/dm-master.toml b/dm/tests/lightning_mode/conf/dm-master.toml index 53a294e7d07..c0484e89f4c 100644 --- a/dm/tests/lightning_mode/conf/dm-master.toml +++ b/dm/tests/lightning_mode/conf/dm-master.toml @@ -4,3 +4,4 @@ advertise-addr = "127.0.0.1:8261" rpc-timeout = "30s" auto-compaction-retention = "3s" +openapi = true diff --git a/dm/tests/lightning_mode/run.sh b/dm/tests/lightning_mode/run.sh index d414ef7937b..bce55d6ebc4 100755 --- a/dm/tests/lightning_mode/run.sh +++ b/dm/tests/lightning_mode/run.sh @@ -87,8 +87,10 @@ function run() { "stop-task test" \ "\"result\": true" 3 run_sql_tidb "drop database if exists lightning_mode;" + run_sql_tidb "drop database if exists dm_meta;" - dmctl_start_task "$cur/conf/dm-task.yaml" "--remove-meta" + task_conf=`cat $cur/conf/task.json` + curl -X POST http://127.0.0.1:8261/api/v1/tasks -H "Content-Type: application/json" -d "$task_conf" # use sync_diff_inspector to check full dump loader check_sync_diff $WORK_DIR $cur/conf/diff_config.toml From ff336bd04405d3ecca82a3dba108d8e66746ece0 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 13 Dec 2023 17:58:02 +0800 Subject: [PATCH 6/8] update --- dm/tests/lightning_mode/conf/task.json | 87 ++++++++++++++++++++++++++ dm/tests/lightning_mode/run.sh | 2 +- 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 dm/tests/lightning_mode/conf/task.json diff --git a/dm/tests/lightning_mode/conf/task.json b/dm/tests/lightning_mode/conf/task.json new file mode 100644 index 00000000000..de284f8fe7a --- /dev/null +++ b/dm/tests/lightning_mode/conf/task.json @@ -0,0 +1,87 @@ +{ + "task": { + "name": "test", + "task_mode": "all", + "shard_mode": "pessimistic", + "meta_schema": "dm-meta", + "enhance_online_schema_change": true, + "on_duplicate": "error", + "ignore_checking_items": [ + "version" + ], + "target_config": { + "host": "127.0.0.1", + "port": 4000, + "user": "root", + "password": "" + }, + "table_migrate_rule": [ + { + "source": { + "source_name": "mysql-replica-01", + "schema": "lightning_mode", + "table": "*" + }, + "target": { + "schema": "lightning_mode", + "table": "*" + }, + "binlog_filter_rule": [ + "rule-1" + ] + }, + { + "source": { + "source_name": "mysql-replica-02", + "schema": "lightning_mode", + "table": "" + }, + "target": { + "schema": "lightning_mode", + "table": "" + }, + "binlog_filter_rule": [ + "rule-1" + ] + } + ], + "source_config": { + "full_migrate_conf": { + "export_threads": 4, + "import_threads": 16, + "data_dir": "./exported_data", + "consistency": "auto", + "import_mode": "physical", + "sorting_dir": "./sort_dir", + "disk_quota": "80G", + "checksum": "required", + "analyze": "optional", + "range_concurrency": 0, + "compress-kv-pairs": "", + "pd_addr": "", + "on_duplicate_logical": "error", + "on_duplicate_physical": "manual" + }, + "incr_migrate_conf": { + "repl_threads": 16, + "repl_batch": 100 + }, + "source_conf": [ + { + "source_name": "mysql-replica-01" + }, + { + "source_name": "mysql-replica-02" + } + ] + }, + "binlog_filter_rule": { + "rule-1": { + "ignore_event": [ + "truncate table", + "drop table" + ] + } + } + } +} \ No newline at end of file diff --git a/dm/tests/lightning_mode/run.sh b/dm/tests/lightning_mode/run.sh index bce55d6ebc4..ad92df72cf8 100755 --- a/dm/tests/lightning_mode/run.sh +++ b/dm/tests/lightning_mode/run.sh @@ -89,7 +89,7 @@ function run() { run_sql_tidb "drop database if exists lightning_mode;" run_sql_tidb "drop database if exists dm_meta;" - task_conf=`cat $cur/conf/task.json` + task_conf=$(cat $cur/conf/task.json) curl -X POST http://127.0.0.1:8261/api/v1/tasks -H "Content-Type: application/json" -d "$task_conf" # use sync_diff_inspector to check full dump loader From 353c7a5e700422dab5c4345f5afde80c06ae14fa Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Wed, 13 Dec 2023 20:30:19 +0800 Subject: [PATCH 7/8] update --- dm/tests/lightning_mode/conf/task.json | 22 ++++------------------ dm/tests/lightning_mode/run.sh | 4 ++++ 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/dm/tests/lightning_mode/conf/task.json b/dm/tests/lightning_mode/conf/task.json index de284f8fe7a..6f96b00a720 100644 --- a/dm/tests/lightning_mode/conf/task.json +++ b/dm/tests/lightning_mode/conf/task.json @@ -3,7 +3,7 @@ "name": "test", "task_mode": "all", "shard_mode": "pessimistic", - "meta_schema": "dm-meta", + "meta_schema": "dm_meta", "enhance_online_schema_change": true, "on_duplicate": "error", "ignore_checking_items": [ @@ -20,29 +20,15 @@ "source": { "source_name": "mysql-replica-01", "schema": "lightning_mode", - "table": "*" - }, - "target": { - "schema": "lightning_mode", - "table": "*" - }, - "binlog_filter_rule": [ - "rule-1" - ] + "table": "" + } }, { "source": { "source_name": "mysql-replica-02", "schema": "lightning_mode", "table": "" - }, - "target": { - "schema": "lightning_mode", - "table": "" - }, - "binlog_filter_rule": [ - "rule-1" - ] + } } ], "source_config": { diff --git a/dm/tests/lightning_mode/run.sh b/dm/tests/lightning_mode/run.sh index ad92df72cf8..dd27e7ac0cf 100755 --- a/dm/tests/lightning_mode/run.sh +++ b/dm/tests/lightning_mode/run.sh @@ -88,9 +88,13 @@ function run() { "\"result\": true" 3 run_sql_tidb "drop database if exists lightning_mode;" run_sql_tidb "drop database if exists dm_meta;" + run_sql_tidb "drop database if exists lightning_metadata;" task_conf=$(cat $cur/conf/task.json) + # create task curl -X POST http://127.0.0.1:8261/api/v1/tasks -H "Content-Type: application/json" -d "$task_conf" + # start task + curl -X POST http://127.0.0.1:8261/api/v1/tasks/test/start -H "Content-Type: application/json" # use sync_diff_inspector to check full dump loader check_sync_diff $WORK_DIR $cur/conf/diff_config.toml From e39561c089fbfb5f8f74af8f798f5df6beae18d9 Mon Sep 17 00:00:00 2001 From: gmhdbjd Date: Thu, 14 Dec 2023 12:02:16 +0800 Subject: [PATCH 8/8] update --- dm/tests/lightning_mode/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dm/tests/lightning_mode/run.sh b/dm/tests/lightning_mode/run.sh index dd27e7ac0cf..b70b46cc9ff 100755 --- a/dm/tests/lightning_mode/run.sh +++ b/dm/tests/lightning_mode/run.sh @@ -94,7 +94,7 @@ function run() { # create task curl -X POST http://127.0.0.1:8261/api/v1/tasks -H "Content-Type: application/json" -d "$task_conf" # start task - curl -X POST http://127.0.0.1:8261/api/v1/tasks/test/start -H "Content-Type: application/json" + curl -X POST http://127.0.0.1:8261/api/v1/tasks/test/start # use sync_diff_inspector to check full dump loader check_sync_diff $WORK_DIR $cur/conf/diff_config.toml