feat(bigquery): adds sql query for VIEW and MATERIALIZED VIEW in asset for BQ

.goreleaser.yml

@@ -2,13 +2,20 @@ project_name: meteor
 
 release:
   prerelease: auto
-  draft: true
 
 before:
   hooks:
     - go mod tidy
     - make clean
 
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - "^docs:"
+      - "^test:"
+      - "^build:"
+
 builds:
   - main: ./main.go
     id: "linux"
@@ -59,14 +66,6 @@ checksum:
 snapshot:
   name_template: "{{ .Tag }}-next"
 
-changelog:
-  sort: asc
-  filters:
-    exclude:
-      - "^docs:"
-      - "^test:"
-      - "^build:"
-
 dockers:
   - goos: linux
     goarch: amd64
@@ -78,6 +77,24 @@ dockers:
       - "docker.io/raystack/{{.ProjectName}}:{{ .Version }}"
       - "docker.io/raystack/{{.ProjectName}}:{{ .Version }}-amd64"
 
+nfpms:
+  - maintainer: Raystack
+    description: "Metadata collection tool."
+    homepage: https://github.com/raystack/meteor
+    license: Apache 2.0
+    formats:
+      - deb
+      - rpm
+      - apk
+
+scoops:
+  - homepage: "https://github.com/raystack/meteor"
+    description: "Metadata collection tool."
+    license: Apache 2.0
+    bucket:
+      owner: raystack
+      name: scoop-bucket
+
 brews:
   - name: meteor
     homepage: "https://github.com/raystack/meteor"
@@ -87,7 +104,6 @@ brews:
       name: homebrew-tap
     license: "Apache 2.0"
     folder: Formula
-
     dependencies:
       - name: git
     install: |-

cmd/gen.go

@@ -31,7 +31,7 @@ func GenCmd() *cobra.Command {
 			$ meteor gen my-template.yaml -o ./output-dir -d ./data.yaml
 		`),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			templatePath := args[0]

cmd/info.go

@@ -17,7 +17,7 @@ func InfoCmd() *cobra.Command {
 		Use:   "info <command>",
 		Short: "Display plugin information",
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 	}
 	cmd.AddCommand(InfoSinkCmd())
@@ -41,7 +41,7 @@ func InfoSinkCmd() *cobra.Command {
 		`),
 		Args: cobra.MaximumNArgs(1),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			var sinks []string
@@ -86,7 +86,7 @@ func InfoExtCmd() *cobra.Command {
 		`),
 		Args: cobra.MaximumNArgs(1),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			var extractors []string
@@ -130,7 +130,7 @@ func InfoProccCmd() *cobra.Command {
 		`),
 		Args: cobra.MaximumNArgs(1),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			var processors []string

cmd/lint.go

@@ -45,7 +45,7 @@ func LintCmd() *cobra.Command {
 			$ meteor lint .
 		`),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cfg, err := config.Load("./meteor.yaml")

cmd/list.go

@@ -19,7 +19,7 @@ func ListCmd() *cobra.Command {
 		Use:   "list <command>",
 		Short: "List available plugins",
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 	}
 	cmd.AddCommand(ListExtCmd())
@@ -49,7 +49,7 @@ func ListExtCmd() *cobra.Command {
 			$ meteor list e
 		`),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			extractors := registry.Extractors.List()
@@ -89,7 +89,7 @@ func ListSinksCmd() *cobra.Command {
 			$ meteor list s
 		`),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			sinks := registry.Sinks.List()
@@ -128,7 +128,7 @@ func ListProccCmd() *cobra.Command {
 			$ meteor list p
 		`),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			processors := registry.Processors.List()

cmd/new.go

@@ -18,7 +18,7 @@ func NewCmd() *cobra.Command {
 		Use:   "new",
 		Short: "Bootstrap new recipes",
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 	}
 
@@ -58,7 +58,7 @@ func NewRecipeCmd() *cobra.Command {
 			$ meteor new recipe sample -e bigquery -s compass > recipe.yaml
 		`),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			var sinkList []string

cmd/root.go

@@ -20,7 +20,7 @@ func New() *cobra.Command {
 			$ meteor gen recipe --extractor=date --sink console
 		`),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 			"help:learn": heredoc.Doc(`
 				Use 'meteor <command> <subcommand> --help' for more information about a command.
 				Read the manual at https://raystack.github.io/meteor/

cmd/run.go

@@ -55,7 +55,7 @@ func RunCmd() *cobra.Command {
 		`),
 		Args: cobra.ExactArgs(1),
 		Annotations: map[string]string{
-			"group:core": "true",
+			"group": "core",
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cfg, err := config.Load(configFile)

plugins/extractors/bigquery/bigquery.go

@@ -415,44 +415,47 @@ func (e *Extractor) buildAsset(ctx context.Context, t *bigquery.Table, md *bigqu
 		}
 	}
 
-	table, err := anypb.New(&v1beta2.Table{
-		Columns:       e.buildColumns(ctx, md.Schema, md),
-		PreviewFields: previewFields,
-		PreviewRows:   previewRows,
-		Profile:       tableProfile,
-		Attributes: utils.TryParseMapToProto(map[string]interface{}{
-			"full_qualified_name": tableFQN,
-			"dataset":             t.DatasetID,
-			"project":             t.ProjectID,
-			"type":                string(md.Type),
-			"partition_data":      partitionData,
-			"clustering_fields":   clusteringFields,
-		}),
-		CreateTime: timestamppb.New(md.CreationTime),
-		UpdateTime: timestamppb.New(md.LastModifiedTime),
-	})
-	if err != nil {
-		e.logger.Warn("error creating Any struct", "error", err)
-	}
-
 	asset := &v1beta2.Asset{
 		Urn:         tableURN,
 		Name:        t.TableID,
 		Type:        "table",
 		Description: md.Description,
 		Service:     "bigquery",
-		Data:        table,
-		Labels:      md.Labels,
+		// Data:        table,
+		Labels: md.Labels,
+	}
+	attributesData := map[string]interface{}{
+		"full_qualified_name": tableFQN,
+		"dataset":             t.DatasetID,
+		"project":             t.ProjectID,
+		"type":                string(md.Type),
+		"partition_data":      partitionData,
+		"clustering_fields":   clusteringFields,
 	}
 
-	if e.config.BuildViewLineage && (md.Type == bigquery.ViewTable || md.Type == bigquery.MaterializedView) {
+	if md.Type == bigquery.ViewTable || md.Type == bigquery.MaterializedView {
 		query := getViewQuery(md)
-		upstreamResources := getUpstreamResources(query)
-		asset.Lineage = &v1beta2.Lineage{
-			Upstreams: upstreamResources,
+		attributesData["sql"] = query
+		if e.config.BuildViewLineage {
+			upstreamResources := getUpstreamResources(query)
+			asset.Lineage = &v1beta2.Lineage{
+				Upstreams: upstreamResources,
+			}
 		}
 	}
-
+	table, err := anypb.New(&v1beta2.Table{
+		Columns:       e.buildColumns(ctx, md.Schema, md),
+		PreviewFields: previewFields,
+		PreviewRows:   previewRows,
+		Profile:       tableProfile,
+		Attributes:    utils.TryParseMapToProto(attributesData),
+		CreateTime:    timestamppb.New(md.CreationTime),
+		UpdateTime:    timestamppb.New(md.LastModifiedTime),
+	})
+	if err != nil {
+		e.logger.Warn("error creating Any struct", "error", err)
+	}
+	asset.Data = table
 	return asset, nil
 }