Merge pull request #97 from zilliztech/feature_cmd_support_collection

Feature support batchCollection & DynamicFields migration

cmd/root.go

@@ -9,6 +9,7 @@ import (
 
 var (
 	configFile string
+	collection string
 )
 
 var RootCmd = &cobra.Command{
@@ -22,7 +23,7 @@ var RootCmd = &cobra.Command{
 
 func Execute() {
 	RootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "config YAML file of server")
-
+	RootCmd.PersistentFlags().StringVarP(&collection, "table", "t", "", "migration source ES index or Milvus collection")
 	RootCmd.Execute()
 }
 

cmd/start.go

@@ -29,7 +29,7 @@ var startCmd = &cobra.Command{
 				return
 			}
 		}()
-		err := starter.Start(ctx, configFile, jobId)
+		err := starter.Start(ctx, configFile, collection, jobId)
 		if err != nil {
 			log.Error("[start migration error]", zap.Error(err))
 			return

core/common/constant.go

@@ -61,3 +61,4 @@ var LOAD_CHECK_BACKLOG_INTERVAL = time.Second * 10    //second
 const SUB_FILE_SIZE = 1024 * 1024 * 300
 
 const UPSERT = "upsert"
+const MILVUS_META_FD = "$meta"

core/data/process_info.go

@@ -107,6 +107,12 @@ func calcDumpProc(p *ProcessHandler) int {
 }
 
 func calcByInsertDataProc(p *ProcessHandler) int {
+	//fmt.Printf("xxxxxxxx: (%d) ", p.LoadTotalSize)
+	//数据量很少情况下，count milvus collection可能会返回为0
+	if p.LoadTotalSize <= 0 {
+		return One_Percent
+	}
+
 	loadSize := p.LoadFinishSize.Load()
 	if loadSize == 0 {
 		return Half_Percent

core/transform/milvus2x/convert/milvus2x_convert.go

@@ -18,6 +18,11 @@ func ToMilvusParam(ctx context.Context, collCfg *milvus2xtype.CollectionCfg, mil
 	if err != nil {
 		return nil, err
 	}
+	//当source是开启动态列表，并且 target也打开动态列属性，则动态列需要迁移(DynamicField=true)
+	collCfg.DynamicField = srcCollEntity.Schema.EnableDynamicField && !collCfg.MilvusCfg.CloseDynamicField
+
+	log.Info("milvus2x source collection_schema", zap.Bool("DynamicFieldStatus", collCfg.DynamicField),
+		zap.String("Collection", collCfg.Collection))
 
 	fields, err := ToMilvusFields(srcCollEntity, collCfg)
 	if err != nil {

core/type/milvus2xtype/mlv2x_type.go

@@ -14,6 +14,8 @@ type CollectionCfg struct {
 	Rows       int64                 `json:"rows"`
 	Fields     []FieldCfg            `json:"fields"`
 	MilvusCfg  *milvustype.MilvusCfg `json:"milvus"`
+
+	DynamicField bool //source collection Dynamic Field status, if it open, will sync $meta field data to target collection
 }
 
 type FieldCfg struct {

go.mod

@@ -14,10 +14,12 @@ require (
 	github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
 	github.com/matoous/go-nanoid/v2 v2.0.0
 	github.com/milvus-io/milvus-sdk-go v1.1.1
-	github.com/milvus-io/milvus-sdk-go/v2 v2.4.1
+	github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240819094545-542b5a0158c1
 	github.com/minio/minio-go/v7 v7.0.66
+	github.com/olivere/elastic/v7 v7.0.32
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/satori/go.uuid v1.2.0
 	github.com/shopspring/decimal v1.3.1
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.18.2
@@ -90,7 +92,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-sqlite3 v1.14.19 // indirect
-	github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 // indirect
+	github.com/milvus-io/milvus-proto/go-api/v2 v2.4.6 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -124,7 +126,7 @@ require (
 	golang.org/x/tools v0.16.1 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -47,6 +47,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/congqixia/milvus-sdk-go/v2 v2.0.0-20240819072515-04d118f1639d h1:JkYXo7c1r/ncfAqt32clf196c1mVr3aucCOIVTVjKDs=
+github.com/congqixia/milvus-sdk-go/v2 v2.0.0-20240819072515-04d118f1639d/go.mod h1:kQ7SuDWugDhCqEHIvokitjVO9CyATIFa+h1dHEk86Zc=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -68,6 +70,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -181,24 +185,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
 github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430025921-135167be0694 h1:iub0yx8peGNtnb9n11iuWNmhIhIXw3xfZooIDcrfeU8=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430025921-135167be0694/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4 h1:HtNGcUb52ojnl+zDAZMmbHyVaTdBjzuCnnBHpb675TU=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 h1:KUSaWVePVlHMIluAXf2qmNffI1CMlGFLLiP+4iy9014=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.4.6 h1:rSkwp5Mg/7KBSUqXcrPBUgTQGZNdvYWEKB+rHo9YJtk=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.4.6/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
 github.com/milvus-io/milvus-sdk-go v1.1.1 h1:QseeGBb92T4ny5jSscaWJ+toG74p5zE0kaxFyeskWSE=
 github.com/milvus-io/milvus-sdk-go v1.1.1/go.mod h1:jyYc5vzEQS9V+ZrSL2h/oeKcYiixvKBV/TrxXbLu01w=
-github.com/milvus-io/milvus-sdk-go/v2 v2.3.4 h1:WeZ/QCwpcZVOiaVScuqoKhjuv3DaEAx+jM6U5PJhK+E=
-github.com/milvus-io/milvus-sdk-go/v2 v2.3.4/go.mod h1:ubhpNcq6Y25PNl2JabqIlH64yGHAEeo3Y7tgQHXQwnU=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240528112137-8309bdf288fe h1:M8Fp0khgp7gdVne0BfHgy/BqN4a3TxoTV2Iq4SPYWOY=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240528112137-8309bdf288fe/go.mod h1:6ckCQ8h5iFncZcaIpGE8T8sqliwaw2Eu9UzHaCURMA4=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240530061138-979f77e0e810 h1:FVJnBYfQCQKcGJOQbKeYL2BYcp+dv7PRj1u200cAKWQ=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240530061138-979f77e0e810/go.mod h1:6ckCQ8h5iFncZcaIpGE8T8sqliwaw2Eu9UzHaCURMA4=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240530093539-f4779551f949 h1:l9++xnkflq2NFvFZ9nQe+VzEFgo1og8mMQ4kUuPUqPc=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.1-0.20240530093539-f4779551f949/go.mod h1:6ckCQ8h5iFncZcaIpGE8T8sqliwaw2Eu9UzHaCURMA4=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.1 h1:KhqjmaJE4mSxj1a88XtkGaqgH4duGiHs1sjnvSXkwE0=
-github.com/milvus-io/milvus-sdk-go/v2 v2.4.1/go.mod h1:7SJxshlnVhNLksS73tLPtHYY9DiX7lyL43Rv41HCPCw=
+github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240819094545-542b5a0158c1 h1:tmPS7djDZ0VrXmgp/UbWKDFwsBuIVTjF+p4jF4Pd3KI=
+github.com/milvus-io/milvus-sdk-go/v2 v2.4.2-0.20240819094545-542b5a0158c1/go.mod h1:kQ7SuDWugDhCqEHIvokitjVO9CyATIFa+h1dHEk86Zc=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
@@ -213,6 +205,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
 github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
@@ -242,6 +236,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke
 github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -445,8 +441,8 @@ google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQ
 google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

server/function.go

@@ -121,11 +121,11 @@ func handleStart(c *gin.Context) (interface{}, error) {
 	}()
 
 	if req.Async {
-		go starter.Start(log.NewContextWithRequestId(c.Request.Context()), "", jobId)
+		go starter.Start(log.NewContextWithRequestId(c.Request.Context()), "", "", jobId)
 		return param.NewJobResponse(jobId), nil
 	}
 
-	return param.NewJobResponse(jobId), starter.Start(c.Request.Context(), "", jobId)
+	return param.NewJobResponse(jobId), starter.Start(c.Request.Context(), "", "", jobId)
 }
 
 func handlePanic(_any any, jobId string) {

starter/starter.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/zilliztech/milvus-migration/core/cleaner"
 	"github.com/zilliztech/milvus-migration/core/common"
+	"github.com/zilliztech/milvus-migration/core/config"
 	"github.com/zilliztech/milvus-migration/core/dumper"
 	"github.com/zilliztech/milvus-migration/core/gstore"
 	"github.com/zilliztech/milvus-migration/core/loader"
@@ -79,7 +80,11 @@ func Load(ctx context.Context, configFile string, param *param.LoadParam, jobId
 	return nil
 }
 
-func Start(ctx context.Context, configFile string, jobId string) error {
+func Start(ctx context.Context, configFile string, collection string, jobId string) error {
+
+	if collection != "" {
+		fmt.Printf("Migration CmdParam Collection: %s Start..", collection)
+	}
 
 	start := time.Now()
 
@@ -93,6 +98,10 @@ func Start(ctx context.Context, configFile string, jobId string) error {
 		return err
 	}
 
+	if collection != "" {
+		replaceCollectionName(migrCfg, collection)
+	}
+
 	if migrCfg.DumperWorkCfg.WorkMode == string(common.Elasticsearch) {
 		//record: es dump will split many small json file task
 		gstore.InitFileTask(jobId)
@@ -118,11 +127,27 @@ func Start(ctx context.Context, configFile string, jobId string) error {
 	}
 	log.LL(ctx).Info("[Cleaner] clean file success!")
 
+	if collection != "" {
+		fmt.Printf("Migration CmdParam Collection: %s Done!", collection)
+	}
+
 	fmt.Printf("Migration Success! Job %s cost=[%f]\n", jobId, time.Since(start).Seconds())
 	printStartJobMessage(jobId)
 	return nil
 }
 
+func replaceCollectionName(migrCfg *config.MigrationConfig, collection string) {
+	if migrCfg.MetaConfig.Milvus2xMeta != nil {
+		migrCfg.MetaConfig.Milvus2xMeta.CollCfgs[0].Collection = collection
+		if migrCfg.MetaConfig.Milvus2xMeta.CollCfgs[0].MilvusCfg != nil {
+			migrCfg.MetaConfig.Milvus2xMeta.CollCfgs[0].MilvusCfg.Collection = collection
+		}
+	}
+	if migrCfg.MetaConfig.EsMeta != nil {
+		migrCfg.MetaConfig.EsMeta.IdxCfgs[0].MilvusCfg.Collection = collection
+	}
+}
+
 func printStartJobMessage(jobId string) {
 	jobInfo, _ := gstore.GetJobInfo(jobId)
 	val, _ := json.Marshal(&jobInfo)

storage/milvus2x/milvus2_3_ver.go

@@ -52,10 +52,15 @@ func (milvus23 *Milvus23VerClient) InitIterator(ctx context.Context, collCfg *mi
 		}
 		fieldNames = append(fieldNames, fieldCfg.Name)
 	}
+	if collCfg.DynamicField {
+		fieldNames = append(fieldNames, common.MILVUS_META_FD) //把source 动态列也查出来
+	}
+
 	log.Info("start iterator milvus collection", zap.Any("migration fieldName", fieldNames))
 	log.Info("start iterator milvus collection", zap.Any("migration milvusCfg", collCfg.MilvusCfg))
 	log.Info("start iterator milvus collection", zap.Any("migration fields", collCfg.Fields))
 	iteratorParam := client.NewQueryIteratorOption(collCfg.Collection).WithBatchSize(batchSize).WithExpr(common.EMPTY).WithOutputFields(fieldNames...)
+	//iteratorParam := client.NewQueryIteratorOption(collCfg.Collection).WithBatchSize(batchSize).WithExpr(common.EMPTY).WithOutputFields("*")
 	iterator, err := milvus23._milvus.QueryIterator(ctx, iteratorParam)
 	if err != nil {
 		return err
@@ -80,9 +85,19 @@ func (milvus23 *Milvus23VerClient) IterateNext(ctx context.Context) (*Milvus2xDa
 	}
 	columns := make([]entity.Column, 0, len(rs))
 	for _, col := range rs {
-		columns = append(columns, col)
+		if col.Name() == common.MILVUS_META_FD {
+			data := col.(*entity.ColumnJSONBytes).Data()
+			dynamicCol := entity.NewColumnJSONBytes(common.EMPTY, data).WithIsDynamic(true)
+			columns = append(columns, dynamicCol)
+			log.Info("[Milvus2x] iterateNext data ======> $meta")
+		} else {
+			columns = append(columns, col)
+		}
 		if common.DEBUG {
 			log.Info("[Milvus2x] iterateNext data ======>", zap.String("colName", col.Name()), zap.Any("colLen", col.Len()))
+			log.Info("[Milvus2x] iterateNext data ======>", zap.String("colName", col.Name()), zap.Any("FieldData", col.FieldData().FieldName))
+			log.Info("[Milvus2x] iterateNext data ======>", zap.String("colName", col.Name()), zap.Any("IsDynamic", col.FieldData().IsDynamic))
+			log.Info("[Milvus2x] iterateNext data ======>", zap.String("colName", col.Name()), zap.Any("FieldDataVal", col.FieldData().String()))
 		}
 	}
 	if common.DEBUG {

testfiles/milvus2x/batch_collection_migration.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+collections=("collection1" "collection2" "collection3")
+
+for collection in "${collections[@]}"; do
+    echo "BatchMigration==> $collection"
+    ./milvus-migration start -t="$collection" -c=/{YourConfigPath}/migration.yml
+done
+
+# how to execute?
+#1. chmod +x batch_collection_migration.sh
+#2. ./batch_collection_migration.sh