feat: add resource management support for maxcompute (#282)

* feat: add initial resource mapping for maxcompute table

* feat: improve validation, add table spec

* tests: add test for table_spec and properties

* feat: add code to create table on maxcompute

* feat: initial setup of maxcompute store

* feat: add maxcompute client and register maxcompute store

* fix: change table handler initialization parameter type

* fix: fix error on missing partition

* fix: fix parameter type interface and handling error creation properly

* fix: change partition column handling

* test: add test for maxcompute client and table implementation

* feat: implement checking existing table schema

* chore: sort imports

# Conflicts:
#	ext/store/maxcompute/maxcompute.go
#	ext/store/maxcompute/table.go
#	ext/store/maxcompute/table_test.go

* feat: add support for views in maxcompute

* feat: connect support for views

* feat: implement standard table updation and improve view table

* feat: add clustering support for table creation and update view

* feat: implement alter table column with json type

* test: add cluster table and view test

* feat: implement schema format temporary for resource urn

* chore: fix lint errors

* chore: restructure file

* chore: update golangci-lint action

* feat: incorporate version 2 changes for maxcompute

* update name to be used in create and update

* fix: fix failure for decode

* fix test and list issues

* fix: handle name for view updation

* test: fix the error on maxcompute test handle form

* test: fix the error on maxcompute view handle form mock

---------

Co-authored-by: Frederick Wijayadi Susilo <[email protected]>
Co-authored-by: Frederick Wijayadi Susilo <[email protected]>

.github/workflows/verify.yml

@@ -16,10 +16,10 @@ jobs:
         with:
           fetch-depth: 0
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6
         with:
           skip-go-installation: true
-          version: v1.55.2
+          version: v1.61.0
   unit-test:
     runs-on: ubuntu-latest
     steps:

.golangci.yml

@@ -16,8 +16,6 @@ linters:
     - durationcheck
     - errname
     - errorlint
-    - execinquery
-    - exportloopref
     - forbidigo
     - gci
     - ginkgolinter
@@ -29,7 +27,6 @@ linters:
     - gofumpt
     - goheader
     - goimports
-    - gomnd
     - gomodguard
     - goprintffuncname
     - gosec
@@ -42,6 +39,7 @@ linters:
     - makezero
     - mirror
     - misspell
+    - mnd
     - nakedret
     - nilerr
     - nilnil
@@ -98,7 +96,7 @@ linters-settings:
   goconst:
     ignore-tests: true
     numbers: true
-  gomnd:
+  mnd:
     ignored-numbers: # Why we have a big range of file permissions
       - '0o600'
       - '0o644'
@@ -115,6 +113,7 @@ linters-settings:
   gosec:
     excludes:
       - G101
+      - G115
   gosimple:
     checks:
       - "all"
@@ -153,7 +152,7 @@ issues:
     - path: tests
       linters:
         - gocritic
-        - gomnd
+        - mnd
         - unparam
         - testpackage
 severity:

Makefile

@@ -58,7 +58,7 @@ lint:
 
 install: ## install required dependencies
 	@echo "> installing dependencies"
-	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61.0
 	go install google.golang.org/protobuf/cmd/[email protected]
 	go install github.com/bufbuild/buf/cmd/[email protected]
 	go install google.golang.org/grpc/cmd/[email protected]

client/cmd/extension/run.go

@@ -17,7 +17,7 @@ func generateRunCommands(reservedCommandNames []string) []*cobra.Command {
 				Short: fmt.Sprintf("Execute [%s/%s@%s] extension",
 					owner.Name, project.Name, project.ActiveTagName,
 				),
-				RunE: func(cmd *cobra.Command, args []string) error {
+				RunE: func(_ *cobra.Command, args []string) error {
 					manager, err := getExtensionManager(true, reservedCommandNames...)
 					if err != nil {
 						return err

client/cmd/job/inspect.go

@@ -264,7 +264,7 @@ func (e *inspectCommand) displayBasicInfoSection(basicInfoSection *pb.JobInspect
 	e.printLogs(basicInfoSection.Notice)
 }
 
-func (e *inspectCommand) processJobInspectResponse(resp *pb.JobInspectResponse) error {
+func (e *inspectCommand) processJobInspectResponse(resp *pb.JobInspectResponse) error { //nolint:unparam
 	e.displayBasicInfoSection(resp.BasicInfo)
 	e.displayUpstreamSection(resp.Upstreams)
 	e.displayDownstreamSection(resp.Downstreams)

client/cmd/playground/window/v1v2/model.go

@@ -93,7 +93,7 @@ func (m *model) generateWindowResultView() string {
 	table := tablewriter.NewWriter(buff)
 	table.SetHeader([]string{"Version", "Start Time", "End Time"})
 	table.Append(m.generateWindowTableRowView(1))
-	table.Append(m.generateWindowTableRowView(2)) //nolint: gomnd
+	table.Append(m.generateWindowTableRowView(2)) //nolint: mnd
 	table.Render()
 	return buff.String()
 }

client/cmd/replay/cancel.go

@@ -37,7 +37,7 @@ func CancelCommand() *cobra.Command {
 		Short:   "Cancel replay using replay ID",
 		Long:    "This operation takes 1 argument, replayID [required] \nwhich UUID format ",
 		Example: "optimus replay cancel <replay_id>",
-		Args: func(cmd *cobra.Command, args []string) error {
+		Args: func(_ *cobra.Command, args []string) error {
 			if len(args) < 1 {
 				return errors.New("replayID is required")
 			}

client/cmd/replay/create.go

@@ -59,11 +59,11 @@ func CreateCommand() *cobra.Command {
 			"second is start time[required] of\nreplay, third is end time[optional] of replay. \nDate ranges are inclusive. " +
 			"Supported date formats are RFC3339 and \nsimple date YYYY-MM-DD",
 		Example: "optimus replay create <job_name> <2023-01-01T02:30:00Z00:00> [2023-01-02T02:30:00Z00:00]\noptimus replay create <job_name> <2023-01-01> [2023-01-02]",
-		Args: func(cmd *cobra.Command, args []string) error {
+		Args: func(_ *cobra.Command, args []string) error {
 			if len(args) < 1 {
 				return errors.New("job name is required")
 			}
-			if len(args) < 2 { //nolint: gomnd
+			if len(args) < 2 { //nolint: mnd
 				return errors.New("replay start time is required")
 			}
 			return nil
@@ -117,7 +117,7 @@ func (r *createCommand) RunE(_ *cobra.Command, args []string) error {
 	jobName := args[0]
 	startTime := args[1]
 	endTime := args[1]
-	if len(args) >= 3 { //nolint: gomnd
+	if len(args) >= 3 { //nolint: mnd
 		endTime = args[2]
 	}
 

client/cmd/replay/status.go

@@ -39,7 +39,7 @@ func StatusCommand() *cobra.Command {
 		Short:   "Get replay detailed status by replay ID",
 		Long:    "This operation takes 1 argument, replayID [required] \nwhich UUID format ",
 		Example: "optimus replay status <replay_id>",
-		Args: func(cmd *cobra.Command, args []string) error {
+		Args: func(_ *cobra.Command, args []string) error {
 			if len(args) < 1 {
 				return errors.New("replayID is required")
 			}

client/cmd/secret/secret.go

@@ -46,7 +46,7 @@ func getSecretName(args []string) (string, error) {
 func getSecretValue(args []string, filePath string, encoded bool) (string, error) {
 	var secretValue string
 	if filePath == "" {
-		if len(args) < 2 { //nolint: gomnd
+		if len(args) < 2 { //nolint: mnd
 			return "", errors.New("secret value is required")
 		}
 		secretValue = args[1]

client/extension/internal/install_test.go

@@ -47,7 +47,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 
 	i.Run("should return error if error encountered during extracting remote metadata", func() {
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return nil, errors.New("extraction failed")
 			},
 		}
@@ -70,7 +70,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 
 	i.Run("should return error if no parser could recognize remote path", func() {
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return nil, model.ErrUnrecognizedRemotePath
 			},
 		}
@@ -93,7 +93,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 
 	i.Run("should return error if error loading manifest", func() {
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return &model.Metadata{}, nil
 			},
 		}
@@ -122,7 +122,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -153,7 +153,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -190,7 +190,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProjectName:  "optimus-extension-valor",
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -248,7 +248,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			TagName:      "",
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -305,7 +305,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			TagName:      "",
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -359,7 +359,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -398,7 +398,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -439,7 +439,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -481,7 +481,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}
@@ -525,7 +525,7 @@ func (i *InstallManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}

client/extension/manager_test.go

@@ -43,7 +43,7 @@ func (m *ManagerTestSuite) TestInstall() {
 			ProviderName: provider,
 		}
 		factory.ParseRegistry = []model.Parser{
-			func(remotePath string) (*model.Metadata, error) {
+			func(_ string) (*model.Metadata, error) {
 				return metadata, nil
 			},
 		}

client/extension/provider/github/client_test.go

@@ -51,7 +51,7 @@ func (c *ClientTestSuite) TestDownloadRelease() {
 	c.Run("should return nil and error if response status is not ok", func() {
 		testPath := "/gojek/optimus-extension-valor"
 
-		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			w.Header().Add("Content-Type", "application/json")
 			w.WriteHeader(http.StatusNotFound)
 		})
@@ -69,7 +69,7 @@ func (c *ClientTestSuite) TestDownloadRelease() {
 		testPath := "/gojek/optimus-extension-valor"
 		message := "invalid-content"
 
-		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			content := []byte(message)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -90,7 +90,7 @@ func (c *ClientTestSuite) TestDownloadRelease() {
 		testPath := "/gojek/optimus-extension-valor"
 		release := &github.Release{}
 
-		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -129,7 +129,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		server := httptest.NewServer(router)
 		defer server.Close()
 
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content := []byte(message)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -152,7 +152,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		defer server.Close()
 
 		releaseAPIPath := server.URL + testReleasePath
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -182,7 +182,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		defer server.Close()
 
 		releaseAPIPath := server.URL + testReleasePath
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -212,7 +212,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		defer server.Close()
 
 		releaseAPIPath := server.URL + testReleasePath
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -222,7 +222,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		testAssetPath := release.Assets[0].BrowserDownloadURL
 		assetAPIPath := server.URL + testAssetPath
 		release.Assets[0].BrowserDownloadURL = assetAPIPath
-		router.HandleFunc(testAssetPath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testAssetPath, func(w http.ResponseWriter, _ *http.Request) {
 			w.Header().Add("Content-Type", "application/json")
 			w.WriteHeader(http.StatusNotFound)
 		})
@@ -250,7 +250,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		defer server.Close()
 
 		releaseAPIPath := server.URL + testReleasePath
-		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testReleasePath, func(w http.ResponseWriter, _ *http.Request) {
 			content, _ := json.Marshal(release)
 
 			w.Header().Add("Content-Type", "application/json")
@@ -260,7 +260,7 @@ func (c *ClientTestSuite) TestDownloadAsset() {
 		testAssetPath := release.Assets[0].BrowserDownloadURL
 		assetAPIPath := server.URL + testAssetPath
 		release.Assets[0].BrowserDownloadURL = assetAPIPath
-		router.HandleFunc(testAssetPath, func(w http.ResponseWriter, r *http.Request) {
+		router.HandleFunc(testAssetPath, func(w http.ResponseWriter, _ *http.Request) {
 			content := []byte(assetPayload)
 
 			w.Header().Add("Content-Type", "application/json")

core/resource/resource.go

@@ -78,6 +78,7 @@ func NameFrom(name string) (Name, error) {
 	return Name(cleaned), nil
 }
 
+// Deprecated: Sections is deprecated. move to warehouse
 func (n Name) Sections() []string {
 	return strings.Split(n.String(), nameSectionSeparator)
 }

core/resource/store.go

@@ -3,7 +3,8 @@ package resource
 import "github.com/goto/optimus/internal/errors"
 
 const (
-	Bigquery Store = "bigquery"
+	Bigquery   Store = "bigquery"
+	MaxCompute Store = "maxcompute"
 )
 
 // Store represents the type of datasource, resource corresponds to
@@ -17,6 +18,8 @@ func FromStringToStore(name string) (Store, error) {
 	switch name {
 	case string(Bigquery):
 		return Bigquery, nil
+	case string(MaxCompute):
+		return MaxCompute, nil
 	default:
 		return "", errors.InvalidArgument(EntityResource, "unknown store "+name)
 	}