feat: add resource management support for maxcompute (#282)

* feat: add initial resource mapping for maxcompute table * feat: improve validation, add table spec * tests: add test for table_spec and properties * feat: add code to create table on maxcompute * feat: initial setup of maxcompute store * feat: add maxcompute client and register maxcompute store * fix: change table handler initialization parameter type * fix: fix error on missing partition * fix: fix parameter type interface and handling error creation properly * fix: change partition column handling * test: add test for maxcompute client and table implementation * feat: implement checking existing table schema * chore: sort imports # Conflicts: # ext/store/maxcompute/maxcompute.go # ext/store/maxcompute/table.go # ext/store/maxcompute/table_test.go * feat: add support for views in maxcompute * feat: connect support for views * feat: implement standard table updation and improve view table * feat: add clustering support for table creation and update view * feat: implement alter table column with json type * test: add cluster table and view test * feat: implement schema format temporary for resource urn * chore: fix lint errors * chore: restructure file * chore: update golangci-lint action * feat: incorporate version 2 changes for maxcompute * update name to be used in create and update * fix: fix failure for decode * fix test and list issues * fix: handle name for view updation * test: fix the error on maxcompute test handle form * test: fix the error on maxcompute view handle form mock --------- Co-authored-by: Frederick Wijayadi Susilo <[email protected]> Co-authored-by: Frederick Wijayadi Susilo <[email protected]>
goto · Nov 6, 2024 · fffa8ad · fffa8ad
1 parent b307b9b
commit fffa8ad
Show file tree

Hide file tree

Showing 54 changed files with 3,234 additions and 81 deletions.
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
@@ -16,10 +16,10 @@ jobs:
         with:
           fetch-depth: 0
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6
         with:
           skip-go-installation: true
-          version: v1.55.2
+          version: v1.61.0
   unit-test:
     runs-on: ubuntu-latest
     steps:

diff --git a/.golangci.yml b/.golangci.yml
@@ -16,8 +16,6 @@ linters:
     - durationcheck
     - errname
     - errorlint
-    - execinquery
-    - exportloopref
     - forbidigo
     - gci
     - ginkgolinter
@@ -29,7 +27,6 @@ linters:
     - gofumpt
     - goheader
     - goimports
-    - gomnd
     - gomodguard
     - goprintffuncname
     - gosec
@@ -42,6 +39,7 @@ linters:
     - makezero
     - mirror
     - misspell
+    - mnd
     - nakedret
     - nilerr
     - nilnil
@@ -98,7 +96,7 @@ linters-settings:
   goconst:
     ignore-tests: true
     numbers: true
-  gomnd:
+  mnd:
     ignored-numbers: # Why we have a big range of file permissions
       - '0o600'
       - '0o644'
@@ -115,6 +113,7 @@ linters-settings:
   gosec:
     excludes:
       - G101
+      - G115
   gosimple:
     checks:
       - "all"
@@ -153,7 +152,7 @@ issues:
     - path: tests
       linters:
         - gocritic
-        - gomnd
+        - mnd
         - unparam
         - testpackage
 severity:

diff --git a/Makefile b/Makefile
@@ -58,7 +58,7 @@ lint:
 
 install: ## install required dependencies
 	@echo "> installing dependencies"
-	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61.0
 	go install google.golang.org/protobuf/cmd/[email protected]
 	go install github.com/bufbuild/buf/cmd/[email protected]
 	go install google.golang.org/grpc/cmd/[email protected]

diff --git a/client/cmd/extension/run.go b/client/cmd/extension/run.go
@@ -17,7 +17,7 @@ func generateRunCommands(reservedCommandNames []string) []*cobra.Command {
 				Short: fmt.Sprintf("Execute [%s/%s@%s] extension",
 					owner.Name, project.Name, project.ActiveTagName,
 				),
-				RunE: func(cmd *cobra.Command, args []string) error {
+				RunE: func(_ *cobra.Command, args []string) error {
 					manager, err := getExtensionManager(true, reservedCommandNames...)
 					if err != nil {
 						return err

diff --git a/client/cmd/job/inspect.go b/client/cmd/job/inspect.go
@@ -264,7 +264,7 @@ func (e *inspectCommand) displayBasicInfoSection(basicInfoSection *pb.JobInspect
 	e.printLogs(basicInfoSection.Notice)
 }
 
-func (e *inspectCommand) processJobInspectResponse(resp *pb.JobInspectResponse) error {
+func (e *inspectCommand) processJobInspectResponse(resp *pb.JobInspectResponse) error { //nolint:unparam
 	e.displayBasicInfoSection(resp.BasicInfo)
 	e.displayUpstreamSection(resp.Upstreams)
 	e.displayDownstreamSection(resp.Downstreams)

diff --git a/client/cmd/playground/window/v1v2/model.go b/client/cmd/playground/window/v1v2/model.go
@@ -93,7 +93,7 @@ func (m *model) generateWindowResultView() string {
 	table := tablewriter.NewWriter(buff)
 	table.SetHeader([]string{"Version", "Start Time", "End Time"})
 	table.Append(m.generateWindowTableRowView(1))
-	table.Append(m.generateWindowTableRowView(2)) //nolint: gomnd
+	table.Append(m.generateWindowTableRowView(2)) //nolint: mnd
 	table.Render()
 	return buff.String()
 }

diff --git a/client/cmd/replay/cancel.go b/client/cmd/replay/cancel.go
@@ -37,7 +37,7 @@ func CancelCommand() *cobra.Command {
 		Short:   "Cancel replay using replay ID",
 		Long:    "This operation takes 1 argument, replayID [required] \nwhich UUID format ",
 		Example: "optimus replay cancel <replay_id>",
-		Args: func(cmd *cobra.Command, args []string) error {
+		Args: func(_ *cobra.Command, args []string) error {
 			if len(args) < 1 {
 				return errors.New("replayID is required")
 			}

diff --git a/client/cmd/replay/create.go b/client/cmd/replay/create.go
@@ -59,11 +59,11 @@ func CreateCommand() *cobra.Command {
 			"second is start time[required] of\nreplay, third is end time[optional] of replay. \nDate ranges are inclusive. " +
 			"Supported date formats are RFC3339 and \nsimple date YYYY-MM-DD",
 		Example: "optimus replay create <job_name> <2023-01-01T02:30:00Z00:00> [2023-01-02T02:30:00Z00:00]\noptimus replay create <job_name> <2023-01-01> [2023-01-02]",
-		Args: func(cmd *cobra.Command, args []string) error {
+		Args: func(_ *cobra.Command, args []string) error {
 			if len(args) < 1 {
 				return errors.New("job name is required")
 			}
-			if len(args) < 2 { //nolint: gomnd
+			if len(args) < 2 { //nolint: mnd
 				return errors.New("replay start time is required")
 			}
 			return nil
@@ -117,7 +117,7 @@ func (r *createCommand) RunE(_ *cobra.Command, args []string) error {
 	jobName := args[0]
 	startTime := args[1]
 	endTime := args[1]
-	if len(args) >= 3 { //nolint: gomnd
+	if len(args) >= 3 { //nolint: mnd
 		endTime = args[2]
 	}
 

diff --git a/client/cmd/replay/status.go b/client/cmd/replay/status.go
@@ -39,7 +39,7 @@ func StatusCommand() *cobra.Command {
 		Short:   "Get replay detailed status by replay ID",
 		Long:    "This operation takes 1 argument, replayID [required] \nwhich UUID format ",
 		Example: "optimus replay status <replay_id>",
-		Args: func(cmd *cobra.Command, args []string) error {
+		Args: func(_ *cobra.Command, args []string) error {
 			if len(args) < 1 {
 				return errors.New("replayID is required")
 			}

diff --git a/client/cmd/secret/secret.go b/client/cmd/secret/secret.go
@@ -46,7 +46,7 @@ func getSecretName(args []string) (string, error) {
 func getSecretValue(args []string, filePath string, encoded bool) (string, error) {
 	var secretValue string
 	if filePath == "" {
-		if len(args) < 2 { //nolint: gomnd
+		if len(args) < 2 { //nolint: mnd
 			return "", errors.New("secret value is required")
 		}
 		secretValue = args[1]

diff --git a/client/extension/internal/install_test.go b/client/extension/internal/install_test.go
@@ -47,7 +47,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 
 	i.Run("should return error if error encountered during extracting remote metadata", func() {
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return nil, errors.New("extraction failed")
 			},
 		}
@@ -70,7 +70,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 
 	i.Run("should return error if no parser could recognize remote path", func() {
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return nil, model.ErrUnrecognizedRemotePath
 			},
 		}
@@ -93,7 +93,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 
 	i.Run("should return error if error loading manifest", func() {
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return &model.Metadata{}, nil
 			},
 		}
@@ -122,7 +122,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -153,7 +153,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -190,7 +190,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProjectName:  "optimus-extension-valor",
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -248,7 +248,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			TagName:      "",
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -305,7 +305,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			TagName:      "",
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -359,7 +359,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -398,7 +398,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -439,7 +439,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -481,7 +481,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -525,7 +525,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}

diff --git a/client/extension/manager_test.go b/client/extension/manager_test.go
@@ -43,7 +43,7 @@ func (m *ManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}

diff --git a/client/extension/provider/github/client_test.go b/client/extension/provider/github/client_test.go
@@ -51,7 +51,7 @@ func (c *ClientTestSuite) TestDownloadRelease() {
 	c.Run("should return nil and error if response status is not ok", func() {
 		testPath := "/gojek/optimus-extension-valor"
 
-		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			w.Header().Add("Content-Type", "application/json")
 			w.WriteHeader(http.StatusNotFound)
 		})
@@ -69,7 +69,7 @@ func (c *ClientTestSuite) TestDownloadRelease() {
 		testPath := "/gojek/optimus-extension-valor"
 		message := "invalid-content"
 
-		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			content := []byte(message)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -90,7 +90,7 @@ func (c *ClientTestSuite) TestDownloadRelease() {
 		testPath := "/gojek/optimus-extension-valor"
 		release := &github.Release{}
 
-		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -129,7 +129,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		server := httptest.NewServer(router)
 		defer server.Close()
 
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content := []byte(message)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -152,7 +152,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		defer server.Close()
 
 		releaseAPIPath := server.URL + testReleasePath
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -182,7 +182,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		defer server.Close()
 
 		releaseAPIPath := server.URL + testReleasePath
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -212,7 +212,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		defer server.Close()
 
 		releaseAPIPath := server.URL + testReleasePath
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -222,7 +222,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		testAssetPath := release.Assets[0].BrowserDownloadURL
 		assetAPIPath := server.URL + testAssetPath
 		release.Assets[0].BrowserDownloadURL = assetAPIPath
-		router.HandleFunc(testAssetPath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testAssetPath, func(w http.ResponseWriter, _ *http.Request) {
 			w.Header().Add("Content-Type", "application/json")
 			w.WriteHeader(http.StatusNotFound)
 		})
@@ -250,7 +250,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		defer server.Close()
 
 		releaseAPIPath := server.URL + testReleasePath
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -260,7 +260,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		testAssetPath := release.Assets[0].BrowserDownloadURL
 		assetAPIPath := server.URL + testAssetPath
 		release.Assets[0].BrowserDownloadURL = assetAPIPath
-		router.HandleFunc(testAssetPath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testAssetPath, func(w http.ResponseWriter, _ *http.Request) {
 			content := []byte(assetPayload)
 
 			w.Header().Add("Content-Type", "application/json")

diff --git a/core/resource/resource.go b/core/resource/resource.go
@@ -78,6 +78,7 @@ func NameFrom(name string) (Name, error) {
 	return Name(cleaned), nil
 }
 
+// Deprecated: Sections is deprecated. move to warehouse
 func (n Name) Sections() []string {
 	return strings.Split(n.String(), nameSectionSeparator)
 }

diff --git a/core/resource/store.go b/core/resource/store.go
@@ -3,7 +3,8 @@ package resource
 import "github.com/goto/optimus/internal/errors"
 
 const (
-	Bigquery Store = "bigquery"
+	Bigquery   Store = "bigquery"
+	MaxCompute Store = "maxcompute"
 )
 
 // Store represents the type of datasource, resource corresponds to
@@ -17,6 +18,8 @@ func FromStringToStore(name string) (Store, error) {
 	switch name {
 	case string(Bigquery):
 		return Bigquery, nil
+	case string(MaxCompute):
+		return MaxCompute, nil
 	default:
 		return "", errors.InvalidArgument(EntityResource, "unknown store "+name)
 	}