From 4c182109bfd0acf19f29d6ea3201faf9f4053ef2 Mon Sep 17 00:00:00 2001 From: Rishab Jain <68556660+Chief-Rishab@users.noreply.github.com> Date: Wed, 22 Mar 2023 23:05:21 +0530 Subject: [PATCH] chore: add struct tags and update docs (#477) --- plugins/extractors/bigquery/bigquery.go | 26 +++++++++---------- plugins/extractors/bigtable/bigtable.go | 2 +- plugins/extractors/caramlstore/caramlstore.go | 6 ++--- plugins/extractors/cassandra/cassandra.go | 8 +++--- plugins/extractors/clickhouse/clickhouse.go | 2 +- plugins/extractors/couchdb/couchdb.go | 8 +++--- plugins/extractors/elastic/elastic.go | 6 ++--- plugins/extractors/gcs/gcs.go | 6 ++--- plugins/extractors/github/github.go | 8 +++--- plugins/extractors/grafana/grafana.go | 4 +-- plugins/extractors/gsuite/gsuite.go | 4 +-- plugins/extractors/kafka/kafka.go | 4 +-- plugins/extractors/mariadb/mariadb.go | 8 +++--- plugins/extractors/merlin/merlin.go | 8 +++--- plugins/extractors/metabase/metabase.go | 10 +++---- plugins/extractors/mongodb/mongodb.go | 2 +- plugins/extractors/mssql/mssql.go | 2 +- plugins/extractors/mysql/README.md | 1 - plugins/extractors/mysql/mysql.go | 2 +- plugins/extractors/optimus/optimus.go | 4 +-- plugins/extractors/oracle/oracle.go | 2 +- plugins/extractors/postgres/postgres.go | 4 +-- plugins/extractors/presto/presto.go | 4 +-- plugins/extractors/redash/redash.go | 4 +-- plugins/extractors/redshift/redshift.go | 10 +++---- plugins/extractors/snowflake/snowflake.go | 2 +- plugins/extractors/superset/superset.go | 8 +++--- plugins/extractors/tableau/README.md | 1 - plugins/extractors/tableau/tableau.go | 15 +++++------ plugins/sinks/compass/README.md | 4 +-- plugins/sinks/compass/sink.go | 6 ++--- test/e2e/mysql_kafka.yml | 1 - 32 files changed, 89 insertions(+), 93 deletions(-) diff --git a/plugins/extractors/bigquery/bigquery.go b/plugins/extractors/bigquery/bigquery.go index 4626272fb..ecaaf9ee2 100644 --- a/plugins/extractors/bigquery/bigquery.go +++ b/plugins/extractors/bigquery/bigquery.go @@ -34,25 +34,25 @@ var summary string // Config holds the set of configuration for the bigquery extractor type Config struct { - ProjectID string `mapstructure:"project_id" validate:"required"` + ProjectID string `json:"project_id" yaml:"project_id" mapstructure:"project_id" validate:"required"` // ServiceAccountBase64 takes precedence over ServiceAccountJSON field - ServiceAccountBase64 string `mapstructure:"service_account_base64"` - ServiceAccountJSON string `mapstructure:"service_account_json"` - MaxPageSize int `mapstructure:"max_page_size"` - TablePattern string `mapstructure:"table_pattern"` - Exclude Exclude `mapstructure:"exclude"` - IncludeColumnProfile bool `mapstructure:"include_column_profile"` - MaxPreviewRows int `mapstructure:"max_preview_rows" default:"30"` - IsCollectTableUsage bool `mapstructure:"collect_table_usage" default:"false"` - UsagePeriodInDay int64 `mapstructure:"usage_period_in_day" default:"7"` - UsageProjectIDs []string `mapstructure:"usage_project_ids"` + ServiceAccountBase64 string `json:"service_account_base64" yaml:"service_account_base64" mapstructure:"service_account_base64"` + ServiceAccountJSON string `json:"service_account_json" yaml:"service_account_json" mapstructure:"service_account_json"` + MaxPageSize int `json:"max_page_size" yaml:"max_page_size" mapstructure:"max_page_size"` + TablePattern string `json:"table_pattern" yaml:"table_pattern" mapstructure:"table_pattern"` + Exclude Exclude `json:"exclude" yaml:"exclude" mapstructure:"exclude"` + IncludeColumnProfile bool `json:"include_column_profile" yaml:"include_column_profile" mapstructure:"include_column_profile"` + MaxPreviewRows int `json:"max_preview_rows" yaml:"max_preview_rows" mapstructure:"max_preview_rows" default:"30"` + IsCollectTableUsage bool `json:"collect_table_usage" yaml:"collect_table_usage" mapstructure:"collect_table_usage" default:"false"` + UsagePeriodInDay int64 `json:"usage_period_in_day" yaml:"usage_period_in_day" mapstructure:"usage_period_in_day" default:"7"` + UsageProjectIDs []string `json:"usage_project_ids" yaml:"usage_project_ids" mapstructure:"usage_project_ids"` } type Exclude struct { // list of datasetIDs - Datasets []string `mapstructure:"datasets"` + Datasets []string `json:"datasets" yaml:"datasets" mapstructure:"datasets"` // list of tableNames in format - datasetID.tableID - Tables []string `mapstructure:"tables"` + Tables []string `json:"tables" yaml:"tables" mapstructure:"tables"` } const ( diff --git a/plugins/extractors/bigtable/bigtable.go b/plugins/extractors/bigtable/bigtable.go index 87266848c..d6af4a993 100644 --- a/plugins/extractors/bigtable/bigtable.go +++ b/plugins/extractors/bigtable/bigtable.go @@ -27,7 +27,7 @@ const ( // Config holds the configurations for the bigtable extractor type Config struct { - ProjectID string `mapstructure:"project_id" validate:"required"` + ProjectID string `json:"project_id" yaml:"project_id" mapstructure:"project_id" validate:"required"` } var info = plugins.Info{ diff --git a/plugins/extractors/caramlstore/caramlstore.go b/plugins/extractors/caramlstore/caramlstore.go index 0c11326e9..5bacef7bf 100644 --- a/plugins/extractors/caramlstore/caramlstore.go +++ b/plugins/extractors/caramlstore/caramlstore.go @@ -29,9 +29,9 @@ var summary string // Config holds the set of configuration for the CaraML Store extractor type Config struct { - URL string `mapstructure:"url" validate:"required"` - MaxSizeInMB int `mapstructure:"max_size_in_mb"` - RequestTimeout time.Duration `mapstructure:"request_timeout" validate:"min=1ms" default:"10s"` + URL string `json:"url" yaml:"url" mapstructure:"url" validate:"required"` + MaxSizeInMB int `json:"max_size_in_mb" yaml:"max_size_in_mb" mapstructure:"max_size_in_mb"` + RequestTimeout time.Duration `json:"request_timeout" yaml:"request_timeout" mapstructure:"request_timeout" validate:"min=1ms" default:"10s"` } var sampleConfig = `url: caraml-store.com:80` diff --git a/plugins/extractors/cassandra/cassandra.go b/plugins/extractors/cassandra/cassandra.go index c10f5bca5..205071032 100644 --- a/plugins/extractors/cassandra/cassandra.go +++ b/plugins/extractors/cassandra/cassandra.go @@ -37,10 +37,10 @@ const ( // Config holds the set of configuration for the cassandra extractor type Config struct { - UserID string `mapstructure:"user_id" validate:"required"` - Password string `mapstructure:"password" validate:"required"` - Host string `mapstructure:"host" validate:"required"` - Port int `mapstructure:"port" validate:"required"` + UserID string `json:"user_id" yaml:"user_id" mapstructure:"user_id" validate:"required"` + Password string `json:"password" yaml:"password" mapstructure:"password" validate:"required"` + Host string `json:"host" yaml:"host" mapstructure:"host" validate:"required"` + Port int `json:"port" yaml:"port" mapstructure:"port" validate:"required"` } var sampleConfig = ` diff --git a/plugins/extractors/clickhouse/clickhouse.go b/plugins/extractors/clickhouse/clickhouse.go index 62e9284cc..7592d78db 100644 --- a/plugins/extractors/clickhouse/clickhouse.go +++ b/plugins/extractors/clickhouse/clickhouse.go @@ -22,7 +22,7 @@ var summary string // Config holds the connection URL for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` } var sampleConfig = `connection_url: "tcp://localhost:3306?username=admin&password=pass123&debug=true"` diff --git a/plugins/extractors/couchdb/couchdb.go b/plugins/extractors/couchdb/couchdb.go index ef1b23d6e..2a0bbc565 100644 --- a/plugins/extractors/couchdb/couchdb.go +++ b/plugins/extractors/couchdb/couchdb.go @@ -27,7 +27,7 @@ var defaultDBList = []string{ // Config holds the connection URL for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` } var sampleConfig = `connection_url: http://admin:pass123@localhost:3306/` @@ -139,10 +139,10 @@ func (e *Extractor) processTable(ctx context.Context, dbName string, docID strin // push table to channel e.emit(models.NewRecord(&v1beta2.Asset{ Urn: models.NewURN("couchdb", e.UrnScope, "table", fmt.Sprintf("%s.%s", dbName, docID)), - Name: docID, - Type: "table", + Name: docID, + Type: "table", Service: "couchdb", - Data: table, + Data: table, })) return diff --git a/plugins/extractors/elastic/elastic.go b/plugins/extractors/elastic/elastic.go index 841a976b0..6727adb72 100644 --- a/plugins/extractors/elastic/elastic.go +++ b/plugins/extractors/elastic/elastic.go @@ -22,9 +22,9 @@ import ( var summary string type Config struct { - User string `mapstructure:"user"` - Password string `mapstructure:"password"` - Host string `mapstructure:"host" validate:"required"` + User string `json:"user" yaml:"user" mapstructure:"user"` + Password string `json:"password" yaml:"password" mapstructure:"password"` + Host string `json:"host" yaml:"host" mapstructure:"host" validate:"required"` } var sampleConfig = ` diff --git a/plugins/extractors/gcs/gcs.go b/plugins/extractors/gcs/gcs.go index f0dc77e73..f3366931a 100644 --- a/plugins/extractors/gcs/gcs.go +++ b/plugins/extractors/gcs/gcs.go @@ -25,9 +25,9 @@ var summary string // Config holds the set of configuration for the extractor type Config struct { - ProjectID string `mapstructure:"project_id" validate:"required"` - ServiceAccountJSON string `mapstructure:"service_account_json"` - ExtractBlob bool `mapstructure:"extract_blob"` + ProjectID string `json:"project_id" yaml:"project_id" mapstructure:"project_id" validate:"required"` + ServiceAccountJSON string `json:"service_account_json" yaml:"service_account_json" mapstructure:"service_account_json"` + ExtractBlob bool `json:"extract_blob" yaml:"extract_blob" mapstructure:"extract_blob"` } var sampleConfig = ` diff --git a/plugins/extractors/github/github.go b/plugins/extractors/github/github.go index ff294ba19..c3f468824 100644 --- a/plugins/extractors/github/github.go +++ b/plugins/extractors/github/github.go @@ -21,8 +21,8 @@ var summary string // Config holds the set of configuration for the extractor type Config struct { - Org string `mapstructure:"org" validate:"required"` - Token string `mapstructure:"token" validate:"required"` + Org string `json:"org" yaml:"org" mapstructure:"org" validate:"required"` + Token string `json:"token" yaml:"token" mapstructure:"token" validate:"required"` } var sampleConfig = ` @@ -96,8 +96,8 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error) emit(models.NewRecord(&v1beta2.Asset{ Urn: models.NewURN("github", e.UrnScope, "user", usr.GetNodeID()), Service: "github", - Type: "user", - Data: u, + Type: "user", + Data: u, })) } diff --git a/plugins/extractors/grafana/grafana.go b/plugins/extractors/grafana/grafana.go index 2c7d6b421..c1cd1360f 100644 --- a/plugins/extractors/grafana/grafana.go +++ b/plugins/extractors/grafana/grafana.go @@ -21,8 +21,8 @@ var summary string // Config holds the set of configuration for the grafana extractor type Config struct { - BaseURL string `mapstructure:"base_url" validate:"required"` - APIKey string `mapstructure:"api_key" validate:"required"` + BaseURL string `json:"base_url" yaml:"base_url" mapstructure:"base_url" validate:"required"` + APIKey string `json:"api_key" yaml:"api_key" mapstructure:"api_key" validate:"required"` } var sampleConfig = ` diff --git a/plugins/extractors/gsuite/gsuite.go b/plugins/extractors/gsuite/gsuite.go index c99b3c94d..8fa7e776d 100644 --- a/plugins/extractors/gsuite/gsuite.go +++ b/plugins/extractors/gsuite/gsuite.go @@ -23,8 +23,8 @@ import ( var summary string type Config struct { - ServiceAccountJSON string `mapstructure:"service_account_json" validate:"required"` - UserEmail string `mapstructure:"user_email" validate:"required"` + ServiceAccountJSON string `json:"service_account_json" yaml:"service_account_json" mapstructure:"service_account_json" validate:"required"` + UserEmail string `json:"user_email" yaml:"user_email" mapstructure:"user_email" validate:"required"` } var sampleConfig = ` diff --git a/plugins/extractors/kafka/kafka.go b/plugins/extractors/kafka/kafka.go index a294cbd0c..01fd66b44 100644 --- a/plugins/extractors/kafka/kafka.go +++ b/plugins/extractors/kafka/kafka.go @@ -30,8 +30,8 @@ var defaultTopics = map[string]byte{ // Config holds the set of configuration for the kafka extractor type Config struct { - Broker string `mapstructure:"broker" validate:"required"` - Auth AuthConfig `mapstructure:"auth_config"` + Broker string `json:"broker" yaml:"broker" mapstructure:"broker" validate:"required"` + Auth AuthConfig `json:"auth_config" yaml:"auth_config" mapstructure:"auth_config"` } type AuthConfig struct { diff --git a/plugins/extractors/mariadb/mariadb.go b/plugins/extractors/mariadb/mariadb.go index 4db88f7bf..98f8b6957 100644 --- a/plugins/extractors/mariadb/mariadb.go +++ b/plugins/extractors/mariadb/mariadb.go @@ -31,7 +31,7 @@ var defaultDBList = []string{ // Config holds the connection URL for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` } var sampleConfig = `connection_url: "admin:pass123@tcp(localhost:3306)/"` @@ -139,10 +139,10 @@ func (e *Extractor) processTable(database string, tableName string) (err error) // push table to channel e.emit(models.NewRecord(&v1beta2.Asset{ Urn: models.NewURN("mariadb", e.UrnScope, "table", fmt.Sprintf("%s.%s", database, tableName)), - Name: tableName, - Type: "table", + Name: tableName, + Type: "table", Service: "mariadb", - Data: data, + Data: data, })) return } diff --git a/plugins/extractors/merlin/merlin.go b/plugins/extractors/merlin/merlin.go index ae92af45b..bf1b74e63 100644 --- a/plugins/extractors/merlin/merlin.go +++ b/plugins/extractors/merlin/merlin.go @@ -33,10 +33,10 @@ var summary string // Config holds the set of configuration for the Merlin extractor. type Config struct { - URL string `mapstructure:"url" validate:"required"` - ServiceAccountBase64 string `mapstructure:"service_account_base64"` - RequestTimeout time.Duration `mapstructure:"request_timeout" validate:"min=1ms" default:"10s"` - WorkerCount int `mapstructure:"worker_count" validate:"min=1" default:"5"` + URL string `json:"url" yaml:"url" mapstructure:"url" validate:"required"` + ServiceAccountBase64 string `json:"service_account_base64" yaml:"service_account_base64" mapstructure:"service_account_base64"` + RequestTimeout time.Duration `json:"request_timeout" yaml:"request_timeout" mapstructure:"request_timeout" validate:"min=1ms" default:"10s"` + WorkerCount int `json:"worker_count" yaml:"worker_count" mapstructure:"worker_count" validate:"min=1" default:"5"` } var sampleConfig = heredoc.Doc(` diff --git a/plugins/extractors/metabase/metabase.go b/plugins/extractors/metabase/metabase.go index 8b6ce0243..96607d702 100644 --- a/plugins/extractors/metabase/metabase.go +++ b/plugins/extractors/metabase/metabase.go @@ -34,11 +34,11 @@ var info = plugins.Info{ // Config holds the set of configuration for the metabase extractor type Config struct { - Host string `mapstructure:"host" validate:"required"` - InstanceLabel string `mapstructure:"instance_label" validate:"required"` - Username string `mapstructure:"username" validate:"required_without=SessionID"` - Password string `mapstructure:"password"` - SessionID string `mapstructure:"session_id"` + Host string `json:"host" yaml:"host" mapstructure:"host" validate:"required"` + InstanceLabel string `json:"instance_label" yaml:"instance_label" mapstructure:"instance_label" validate:"required"` + Username string `json:"username" yaml:"username" mapstructure:"username" validate:"required_without=SessionID"` + Password string `json:"password" yaml:"password" mapstructure:"password"` + SessionID string `json:"session_id" yaml:"session_id" mapstructure:"session_id"` } // Extractor manages the extraction of data diff --git a/plugins/extractors/mongodb/mongodb.go b/plugins/extractors/mongodb/mongodb.go index c850c5fcb..8bbbc88f1 100644 --- a/plugins/extractors/mongodb/mongodb.go +++ b/plugins/extractors/mongodb/mongodb.go @@ -31,7 +31,7 @@ var defaultCollections = []string{ // Config holds the connection URL for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` } var sampleConfig = `connection_url: "mongodb://admin:pass123@localhost:3306"` diff --git a/plugins/extractors/mssql/mssql.go b/plugins/extractors/mssql/mssql.go index 885b05c60..7cc4d661c 100644 --- a/plugins/extractors/mssql/mssql.go +++ b/plugins/extractors/mssql/mssql.go @@ -33,7 +33,7 @@ var defaultDBList = []string{ // Config holds the connection URL for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` } var sampleConfig = `connection_url: "sqlserver://admin:pass123@localhost:3306/"` diff --git a/plugins/extractors/mysql/README.md b/plugins/extractors/mysql/README.md index 63d85c6f9..ae6c78328 100644 --- a/plugins/extractors/mysql/README.md +++ b/plugins/extractors/mysql/README.md @@ -7,7 +7,6 @@ source: name: mysql config: connection_url: admin:pass123@tcp(localhost:3306)/ - identifier: my-mysql ``` ## Inputs diff --git a/plugins/extractors/mysql/mysql.go b/plugins/extractors/mysql/mysql.go index 34ddf1352..65a0a040f 100644 --- a/plugins/extractors/mysql/mysql.go +++ b/plugins/extractors/mysql/mysql.go @@ -32,7 +32,7 @@ var defaultDBList = []string{ // Config holds the connection URL for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` } var sampleConfig = `connection_url: "admin:pass123@tcp(localhost:3306)/"` diff --git a/plugins/extractors/optimus/optimus.go b/plugins/extractors/optimus/optimus.go index 3181b8804..d00128fc5 100644 --- a/plugins/extractors/optimus/optimus.go +++ b/plugins/extractors/optimus/optimus.go @@ -22,8 +22,8 @@ var summary string // Config holds the set of configuration for the bigquery extractor type Config struct { - Host string `mapstructure:"host" validate:"required"` - MaxSizeInMB int `mapstructure:"max_size_in_mb"` + Host string `json:"host" yaml:"host" mapstructure:"host" validate:"required"` + MaxSizeInMB int `json:"max_size_in_mb" yaml:"max_size_in_mb" mapstructure:"max_size_in_mb"` } var sampleConfig = `host: optimus.com:80` diff --git a/plugins/extractors/oracle/oracle.go b/plugins/extractors/oracle/oracle.go index 84e2141d9..011f086c0 100644 --- a/plugins/extractors/oracle/oracle.go +++ b/plugins/extractors/oracle/oracle.go @@ -23,7 +23,7 @@ var summary string // Config holds the set of configuration options for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` } var sampleConfig = `connection_url: oracle://username:passwd@localhost:1521/xe` diff --git a/plugins/extractors/postgres/postgres.go b/plugins/extractors/postgres/postgres.go index bb3059a12..364616798 100644 --- a/plugins/extractors/postgres/postgres.go +++ b/plugins/extractors/postgres/postgres.go @@ -31,8 +31,8 @@ var defaultDBList = []string{"information_schema", "root", "postgres"} // Config holds the set of configuration options for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` - Exclude string `mapstructure:"exclude"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` + Exclude string `json:"exclude" yaml:"exclude" mapstructure:"exclude"` } var sampleConfig = ` diff --git a/plugins/extractors/presto/presto.go b/plugins/extractors/presto/presto.go index 0a8caaa9b..acad5a4d3 100644 --- a/plugins/extractors/presto/presto.go +++ b/plugins/extractors/presto/presto.go @@ -25,8 +25,8 @@ var summary string // Config holds the set of configuration options for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` - Exclude string `mapstructure:"exclude_catalog"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` + Exclude string `json:"exclude_catalog" yaml:"exclude_catalog" mapstructure:"exclude_catalog"` } var sampleConfig = ` diff --git a/plugins/extractors/redash/redash.go b/plugins/extractors/redash/redash.go index 4a9cb9604..72342c96c 100644 --- a/plugins/extractors/redash/redash.go +++ b/plugins/extractors/redash/redash.go @@ -25,8 +25,8 @@ var summary string // Config holds the set of configuration for the redash extractor type Config struct { - BaseURL string `mapstructure:"base_url" validate:"required"` - ApiKey string `mapstructure:"api_key" validate:"required"` + BaseURL string `json:"base_url" yaml:"base_url" mapstructure:"base_url" validate:"required"` + ApiKey string `json:"api_key" yaml:"api_key" mapstructure:"api_key" validate:"required"` } var sampleConfig = ` diff --git a/plugins/extractors/redshift/redshift.go b/plugins/extractors/redshift/redshift.go index 87cd793c9..18178db76 100644 --- a/plugins/extractors/redshift/redshift.go +++ b/plugins/extractors/redshift/redshift.go @@ -22,11 +22,11 @@ var summary string // Config holds the set of configuration for the metabase extractor type Config struct { - ClusterID string `mapstructure:"cluster_id" validate:"required"` - DBName string `mapstructure:"db_name" validate:"required"` - DBUser string `mapstructure:"db_user" validate:"required"` - AWSRegion string `mapstructure:"aws_region" validate:"required"` - Exclude string `mapstructure:"exclude"` + ClusterID string `json:"cluster_id" yaml:"cluster_id" mapstructure:"cluster_id" validate:"required"` + DBName string `json:"db_name" yaml:"db_name" mapstructure:"db_name" validate:"required"` + DBUser string `json:"db_user" yaml:"db_user" mapstructure:"db_user" validate:"required"` + AWSRegion string `json:"aws_region" yaml:"aws_region" mapstructure:"aws_region" validate:"required"` + Exclude string `json:"exclude" yaml:"exclude" mapstructure:"exclude"` } var sampleConfig = ` diff --git a/plugins/extractors/snowflake/snowflake.go b/plugins/extractors/snowflake/snowflake.go index 2becbff37..987899afe 100644 --- a/plugins/extractors/snowflake/snowflake.go +++ b/plugins/extractors/snowflake/snowflake.go @@ -23,7 +23,7 @@ var summary string // Config holds the connection URL for the extractor type Config struct { - ConnectionURL string `mapstructure:"connection_url" validate:"required"` + ConnectionURL string `json:"connection_url" yaml:"connection_url" mapstructure:"connection_url" validate:"required"` } var sampleConfig = `connection_url: "user:password@my_organization-my_account/mydb"` diff --git a/plugins/extractors/superset/superset.go b/plugins/extractors/superset/superset.go index b8661d5e7..0d3822f93 100644 --- a/plugins/extractors/superset/superset.go +++ b/plugins/extractors/superset/superset.go @@ -25,10 +25,10 @@ var summary string // Config holds the set of configuration for the superset extractor type Config struct { - Username string `mapstructure:"username" validate:"required"` - Password string `mapstructure:"password" validate:"required"` - Host string `mapstructure:"host" validate:"required"` - Provider string `mapstructure:"provider" validate:"required"` + Username string `json:"username" yaml:"username" mapstructure:"username" validate:"required"` + Password string `json:"password" yaml:"password" mapstructure:"password" validate:"required"` + Host string `json:"host" yaml:"host" mapstructure:"host" validate:"required"` + Provider string `json:"provider" yaml:"provider" mapstructure:"provider" validate:"required"` } var sampleConfig = ` diff --git a/plugins/extractors/tableau/README.md b/plugins/extractors/tableau/README.md index abfdd5df3..8c365c995 100644 --- a/plugins/extractors/tableau/README.md +++ b/plugins/extractors/tableau/README.md @@ -12,7 +12,6 @@ source: config: host: http://server.tableau.com version: 3.12 - identifier: my-tableau username: meteor_user password: xxxxxxxxxx ``` diff --git a/plugins/extractors/tableau/tableau.go b/plugins/extractors/tableau/tableau.go index 6a82ab0e4..b1e61c14b 100644 --- a/plugins/extractors/tableau/tableau.go +++ b/plugins/extractors/tableau/tableau.go @@ -22,7 +22,6 @@ var summary string var sampleConfig = ` host: https://server.tableau.com version: "3.12" -identifier: my-tableau username: meteor_user password: xxxxxxxxxx sitename: testdev550928 @@ -37,13 +36,13 @@ var info = plugins.Info{ // Config that holds a set of configuration for tableau extractor type Config struct { - Host string `mapstructure:"host" validate:"required"` - Version string `mapstructure:"version" validate:"required"` // float as string - Username string `mapstructure:"username"` - Password string `mapstructure:"password" validate:"required_with=Username"` - AuthToken string `mapstructure:"auth_token" validate:"required_without=Username"` - SiteID string `mapstructure:"site_id" validate:"required_without=Username"` - Sitename string `mapstructure:"sitename"` + Host string `json:"host" yaml:"host" mapstructure:"host" validate:"required"` + Version string `json:"version" yaml:"version" mapstructure:"version" validate:"required"` // float as string + Username string `json:"username" yaml:"username" mapstructure:"username"` + Password string `json:"password" yaml:"password" mapstructure:"password" validate:"required_with=Username"` + AuthToken string `json:"auth_token" yaml:"auth_token" mapstructure:"auth_token" validate:"required_without=Username"` + SiteID string `json:"site_id" yaml:"site_id" mapstructure:"site_id" validate:"required_without=Username"` + Sitename string `json:"sitename" yaml:"sitename" mapstructure:"sitename"` } // Extractor manages the extraction of data diff --git a/plugins/sinks/compass/README.md b/plugins/sinks/compass/README.md index cc14fe5f0..05ea7c8b7 100644 --- a/plugins/sinks/compass/README.md +++ b/plugins/sinks/compass/README.md @@ -13,8 +13,8 @@ sinks: compass-User-Email: meteor@odpf.io Header-1: value11,value12 labels: - myCustom: $properties.attributes.myCustomField - sampleLabel: $properties.labels.sampleLabelField + myCustom: $attributes.myCustomField + sampleLabel: $labels.sampleLabelField ``` ## Contributing diff --git a/plugins/sinks/compass/sink.go b/plugins/sinks/compass/sink.go index ac49652a9..5d3cab4d8 100644 --- a/plugins/sinks/compass/sink.go +++ b/plugins/sinks/compass/sink.go @@ -26,9 +26,9 @@ import ( var summary string type Config struct { - Host string `mapstructure:"host" validate:"required"` - Headers map[string]string `mapstructure:"headers"` - Labels map[string]string `mapstructure:"labels"` + Host string `json:"host" yaml:"host" mapstructure:"host" validate:"required"` + Headers map[string]string `json:"headers" yaml:"headers" mapstructure:"headers"` + Labels map[string]string `json:"labels" yaml:"labels" mapstructure:"labels"` } var info = plugins.Info{ diff --git a/test/e2e/mysql_kafka.yml b/test/e2e/mysql_kafka.yml index e83a9e67c..26fc6a31b 100644 --- a/test/e2e/mysql_kafka.yml +++ b/test/e2e/mysql_kafka.yml @@ -4,7 +4,6 @@ source: name: mysql config: connection_url: root:admin@tcp(127.0.0.1:3306)/ - identifier: test_db sinks: - name: kafka config: