chore: refine datamodel scope (#409)

Because - pollute different datamodel usage with different scope This commit - refine datamodel definition and usage scope
instill-ai · Aug 29, 2023 · b33070b · b33070b
1 parent 34bd491
commit b33070b
Show file tree

Hide file tree

Showing 22 changed files with 611 additions and 566 deletions.
diff --git a/go.mod b/go.mod
@@ -15,10 +15,11 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0
 	github.com/iancoleman/strcase v0.2.0
-	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20230828011941-3dfc20ad4f44
+	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20230829050804-7cbee906e52d
 	github.com/instill-ai/usage-client v0.2.4-alpha
 	github.com/instill-ai/x v0.3.0-alpha
 	github.com/knadh/koanf v1.4.4
+	github.com/mennanov/fieldmask-utils v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/pkg/errors v0.9.1
 	github.com/santhosh-tekuri/jsonschema/v5 v5.1.1

diff --git a/go.sum b/go.sum
@@ -1306,8 +1306,8 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20230828011941-3dfc20ad4f44 h1:VD2AkxSxpc205rchw9deT3sjlauF60KEp+sAaIE7lwM=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20230828011941-3dfc20ad4f44/go.mod h1:qsq5ecnA1xi2rLnVQFo/9xksA7I7wQu8c7rqM5xbIrQ=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20230829050804-7cbee906e52d h1:LpzcaqduRTxGjOQgncCgLQxpH7TAsLmpbxROdVDJxDM=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20230829050804-7cbee906e52d/go.mod h1:qsq5ecnA1xi2rLnVQFo/9xksA7I7wQu8c7rqM5xbIrQ=
 github.com/instill-ai/usage-client v0.2.4-alpha h1:mYXd62eZsmGKBlzwMcdEgTBgn8zlbagYUHro6+p50c8=
 github.com/instill-ai/usage-client v0.2.4-alpha/go.mod h1:BMxgyr02sqH6SeITXSV4M1ewwvfklzXIc5yzIqaN0c8=
 github.com/instill-ai/x v0.3.0-alpha h1:z9fedROOG2dVHhswBfVwU/hzHuq8/JKSUON7inF+FH8=
@@ -1502,6 +1502,8 @@ github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOj
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
+github.com/mennanov/fieldmask-utils v1.1.0 h1:OznpB7hfv+1LVkPsYyVm9HptTk984cxOCFXlwao1nk0=
+github.com/mennanov/fieldmask-utils v1.1.0/go.mod h1:QKL2T2dcS9lo+Z+jbDRUUVHU5oZPhi7YsEpiwIrDYys=
 github.com/microsoft/go-mssqldb v0.17.0 h1:Fto83dMZPnYv1Zwx5vHHxpNraeEaUlQ/hhHLgZiaenE=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
@@ -2633,6 +2635,7 @@ google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP
 google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
 google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
 google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20220531173845-685668d2de03/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To=
 google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=

diff --git a/integration-test/grpc.js b/integration-test/grpc.js
@@ -70,7 +70,7 @@ export default () => {
   // Query Model API
   queryModel.GetUserModel()
   queryModel.ListUserModels()
-  queryModel.LookupUserModel()
+  queryModel.LookupModel()
 
   // Publish Model API
   publishModel.PublishUnPublishUserModel()
@@ -90,7 +90,7 @@ export function teardown() {
   group("Model API: Delete all models created by this test", () => {
     for (const model of client.invoke('model.model.v1alpha.ModelPublicService/ListModels', {}, {}).message.models) {
       check(client.invoke('model.model.v1alpha.ModelPublicService/DeleteUserModel', {
-        name: "users/instill-ai/" + model.name
+        name: model.name
       }), {
         'DeleteModel model status is OK': (r) => r && r.status === grpc.StatusOK,
       });

diff --git a/integration-test/grpc_create_model.js b/integration-test/grpc_create_model.js
@@ -88,7 +88,6 @@ export function CreateUserModel() {
       sleep(1)
       currentTime = new Date().getTime();
     }
-
     check(client.invoke('model.model.v1alpha.ModelPublicService/TriggerUserModel', {
       name: `${constant.namespace}/models/${model_id}`,
       task_inputs: [{

diff --git a/integration-test/grpc_query_model.js b/integration-test/grpc_query_model.js
@@ -181,7 +181,7 @@ export function ListUserModels() {
   });
 };
 
-export function LookupUserModel() {
+export function LookupModel() {
   // LookUpModel check
   group("Model API: LookUpModel", () => {
     client.connect(constant.gRPCPublicHost, {
@@ -222,8 +222,8 @@ export function LookupUserModel() {
     let res = client.invoke('model.model.v1alpha.ModelPublicService/GetUserModel', {
       name: `${constant.namespace}/models/${model_id}`
     }, {})
-    check(client.invoke('model.model.v1alpha.ModelPublicService/LookUpUserModel', {
-      permalink: "users/instill-ai/models/" + res.message.model.uid
+    check(client.invoke('model.model.v1alpha.ModelPublicService/LookUpModel', {
+      permalink: "models/" + res.message.model.uid
     }, {}), {
       "LookUpModel response status": (r) => r.status === grpc.StatusOK,
       "LookUpModel response model.name": (r) => r.message.model.name === `${constant.namespace}/models/${model_id}`,
@@ -238,8 +238,8 @@ export function LookupUserModel() {
       "LookUpModel response model.update_time": (r) => r.message.model.updateTime !== undefined,
     });
 
-    check(client.invoke('model.model.v1alpha.ModelPublicService/LookUpUserModel', {
-      permalink: "users/instill-ai/models/" + randomString(10)
+    check(client.invoke('model.model.v1alpha.ModelPublicService/LookUpModel', {
+      permalink: "models/" + randomString(10)
     }, {}), {
       'LookUpModel non-existed model status not found': (r) => r && r.status === grpc.StatusNotFound,
     });

diff --git a/integration-test/proto/model/model/v1alpha/model.proto b/integration-test/proto/model/model/v1alpha/model.proto
@@ -171,6 +171,23 @@ message ListModelsResponse {
   int64 total_size = 3;
 }
 
+// LookUpModelRequest represents a request to query a model via permalink
+message LookUpModelRequest {
+  // Permalink of a model. For example:
+  // Format: models/{uid}
+  string permalink = 1 [(google.api.field_behavior) = REQUIRED];
+  // Model view (default is VIEW_BASIC)
+  // VIEW_UNSPECIFIED/VIEW_BASIC: omit `Model.configuration`
+  // VIEW_FULL: show full information
+  optional View view = 2 [(google.api.field_behavior) = OPTIONAL];
+}
+
+// LookUpModelResponse represents a response for a model
+message LookUpModelResponse {
+  // A model resource
+  Model model = 1;
+}
+
 // CreateUserModelRequest represents a request to create a model
 message CreateUserModelRequest {
   // The model to be created
@@ -299,23 +316,6 @@ message DeleteUserModelRequest {
 // DeleteUserModelResponse represents an empty response
 message DeleteUserModelResponse {}
 
-// LookUpUserModelRequest represents a request to query a model via permalink
-message LookUpUserModelRequest {
-  // Permalink of a model. For example:
-  // Format: users/{user}/models/{uid}
-  string permalink = 1 [(google.api.field_behavior) = REQUIRED];
-  // Model view (default is VIEW_BASIC)
-  // VIEW_UNSPECIFIED/VIEW_BASIC: omit `Model.configuration`
-  // VIEW_FULL: show full information
-  optional View view = 2 [(google.api.field_behavior) = OPTIONAL];
-}
-
-// LookUpUserModelResponse represents a response for a model
-message LookUpUserModelResponse {
-  // A model resource
-  Model model = 1;
-}
-
 // RenameUserModelRequest represents a request to rename a model
 message RenameUserModelRequest {
   // Resource name for the model to be renamed.
@@ -610,7 +610,7 @@ message TestUserModelBinaryFileUploadResponse {
   repeated TaskOutput task_outputs = 2 [(google.api.field_behavior) = REQUIRED];
 }
 
-// GetModelOperationRequest represents a request to query a model operation
+// GerModelOperationRequest represents a request to query a model operation
 message GetModelOperationRequest {
   // The name of the operation resource.
   string name = 1 [(google.api.field_behavior) = REQUIRED];

diff --git a/integration-test/proto/model/model/v1alpha/model_public_service.proto b/integration-test/proto/model/model/v1alpha/model_public_service.proto
@@ -54,6 +54,13 @@ service ModelPublicService {
     option (google.api.http) = {get: "/v1alpha/models"};
   }
 
+  // LookUpUodel method receives a LookUpModelRequest message and returns a
+  // LookUpModelResponse
+  rpc LookUpModel(LookUpModelRequest) returns (LookUpModelResponse) {
+    option (google.api.http) = {get: "/v1alpha/{permalink=models/*}/lookUp"};
+    option (google.api.method_signature) = "permalink";
+  }
+
   // LisUsertModels method receives a ListUserModelsRequest message and returns a
   // ListUserModelsResponse
   rpc ListUserModels(ListUserModelsRequest) returns (ListUserModelsResponse) {
@@ -104,13 +111,6 @@ service ModelPublicService {
     option (google.api.method_signature) = "name";
   }
 
- // LookUpUserModel method receives a LookUpUserModelRequest message and returns a
-  // LookUpUserModelResponse
-  rpc LookUpUserModel(LookUpUserModelRequest) returns (LookUpUserModelResponse) {
-    option (google.api.http) = {get: "/v1alpha/{permalink=users/*/models/*}/lookUp"};
-    option (google.api.method_signature) = "permalink";
-  }
-
   // RenameUserModel method rename a model
   rpc RenameUserModel(RenameUserModelRequest) returns (RenameUserModelResponse) {
     option (google.api.http) = {

diff --git a/integration-test/rest_create_model_with_jwt.js b/integration-test/rest_create_model_with_jwt.js
@@ -84,11 +84,9 @@ export function CreateModelFromLocal() {
       }
 
       // clean up
-      let s = http.request("DELETE", `${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${model_id}`, null, {
+      check(http.request("DELETE", `${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${model_id}`, null, {
         headers: genHeaderwithJwtSub(`application/json`, uuidv4()),
-      })
-      console.log(s)
-      check(s, {
+      }), {
         [`[with random "jwt-sub" header] DELETE /v1alpha/models/${model_id} status 401`]: (r) =>
           r.status === 401
       });

diff --git a/integration-test/rest_query_model.js b/integration-test/rest_query_model.js
@@ -54,9 +54,6 @@ export function GetModel() {
         sleep(1)
         currentTime = new Date().getTime();
       }
-      console.log(http.get(`${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${model_id}`, {
-        headers: genHeader(`application/json`),
-      }))
       check(http.get(`${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${model_id}`, {
         headers: genHeader(`application/json`),
       }), {
@@ -424,7 +421,7 @@ export function LookupModel() {
       let modelRes = http.get(`${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${model_id}`, {
         headers: genHeader(`application/json`),
       })
-      check(http.get(`${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${modelRes.json().model.uid}/lookUp`, {
+      check(http.get(`${constant.apiPublicHost}/v1alpha/models/${modelRes.json().model.uid}/lookUp`, {
         headers: genHeader(`application/json`),
       }), {
         [`GET /v1alpha/models/${modelRes.json().model.uid}/lookUp task cls response status`]: (r) =>
@@ -455,7 +452,7 @@ export function LookupModel() {
           r.json().model.update_time !== undefined,
       });
 
-      check(http.get(`${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${modelRes.json().model.uid}/lookUp?view=VIEW_FULL`, {
+      check(http.get(`${constant.apiPublicHost}/v1alpha/models/${modelRes.json().model.uid}/lookUp?view=VIEW_FULL`, {
         headers: genHeader(`application/json`),
       }), {
         [`GET /v1alpha/models/${modelRes.json().model.uid}/lookUp task cls response status`]: (r) =>

diff --git a/integration-test/rest_query_model_with_jwt.js b/integration-test/rest_query_model_with_jwt.js
@@ -200,28 +200,28 @@ export function LookupModel() {
     let modelUid = resp.json().model.uid
 
     group(`Model Backend API: Look up model [with "jwt-sub" header]`, function () {
-      check(http.get(`${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${modelUid}/lookUp`, {
+      check(http.get(`${constant.apiPublicHost}/v1alpha/models/${modelUid}/lookUp`, {
         headers: genHeaderwithJwtSub(`application/json`, uuidv4()),
       }), {
         [`[with random "jwt-sub" header] GET /v1alpha/models/${modelUid}/lookUp task cls response status 401`]: (r) =>
           r.status === 401,
       });
 
-      check(http.get(`${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${modelUid}/lookUp?view=VIEW_FULL`, {
+      check(http.get(`${constant.apiPublicHost}/v1alpha/models/${modelUid}/lookUp?view=VIEW_FULL`, {
         headers: genHeaderwithJwtSub(`application/json`, uuidv4()),
       }), {
         [`[with random "jwt-sub" header] GET /v1alpha/models/${modelUid}/lookUp?view=VIEW_FULL task cls response status 401`]: (r) =>
           r.status === 401,
       });
 
-      check(http.get(`${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${modelUid}/lookUp`, {
+      check(http.get(`${constant.apiPublicHost}/v1alpha/models/${modelUid}/lookUp`, {
         headers: genHeaderwithJwtSub(`application/json`, userUid),
       }), {
         [`[with default "jwt-sub" header] GET /v1alpha/models/${modelUid}/lookUp task cls response status 200`]: (r) =>
           r.status === 200,
       });
 
-      check(http.get(`${constant.apiPublicHost}/v1alpha/${constant.namespace}/models/${modelUid}/lookUp?view=VIEW_FULL`, {
+      check(http.get(`${constant.apiPublicHost}/v1alpha/models/${modelUid}/lookUp?view=VIEW_FULL`, {
         headers: genHeaderwithJwtSub(`application/json`, userUid),
       }), {
         [`[with default "jwt-sub" header] GET /v1alpha/models/${modelUid}/lookUp?view=VIEW_FULL task cls response status 200`]: (r) =>

diff --git a/internal/resource/resource.go b/internal/resource/resource.go
@@ -64,11 +64,12 @@ func GetRscNameID(path string) (string, error) {
 
 // GetRscPermalinkUID returns the resource UID given a resource permalink
 func GetRscPermalinkUID(path string) (uuid.UUID, error) {
-	uid := path[strings.LastIndex(path, "/")+1:]
-	if uid == "" {
+	splits := strings.Split(path, "/")
+	if len(splits) < 2 {
 		return uuid.Nil, fmt.Errorf("error when extract resource id from resource permalink '%s'", path)
 	}
-	return uuid.FromStringOrNil(uid), nil
+
+	return uuid.FromStringOrNil(splits[1]), nil
 }
 
 func UserUidToUserPermalink(userUid uuid.UUID) string {

diff --git a/pkg/handler/fieldannotation.go b/pkg/handler/fieldannotation.go
@@ -0,0 +1,7 @@
+package handler
+
+// immutableFields are Protobuf message fields with IMMUTABLE field_behavior annotation
+var immutableFields = []string{"id", "model_definition", "configuration"}
+
+// outputOnlyFields are Protobuf message fields with OUTPUT_ONLY field_behavior annotation
+var outputOnlyFields = []string{"name", "uid", "visibility", "owner", "create_time", "update_time"}