feat: add Instance_label to extractors (#354)

* feat(postgres): add instance label through config for URN component * feat(mysql): add instance label through config for URN component * fix: missing build tag, instance_label to config * feat(mssql): add instance label through config for URN component * fix: missing build tag * feat(tableau): add instance label through config for URN component * refactor: rename to identifier * refactor(mysql): update README
raystack · Jun 7, 2022 · b94d77f · b94d77f
1 parent f373bb2
commit b94d77f
Show file tree

Hide file tree

Showing 13 changed files with 142 additions and 119 deletions.
diff --git a/plugins/extractors/mssql/README.md b/plugins/extractors/mssql/README.md
@@ -7,19 +7,21 @@ source:
   name: mssql
   config:
     connection_url: sqlserver://admin:pass123@localhost:3306/
+    identifier: my-mssql
 ```
 
 ## Inputs
 
 | Key | Value | Example | Description |    |
 | :-- | :---- | :------ | :---------- | :- |
+| `identifier` | `string` | `my-mssql` | Instance alias, the value will be used as part of the urn component | *required* |
 | `connection_url` | `string` | `sqlserver://admin:pass123@localhost:3306/` | URL to access the mssql server | *required* |
 
 ## Outputs
 
 | Field | Sample Value |
 | :---- | :---- |
-| `resource.urn` | `my_database.my_table` |
+| `resource.urn` | `mssql::my-mssql/my_database/my_table` |
 | `resource.name` | `my_table` |
 | `resource.service` | `mssql` |
 | `description` | `table description` |

diff --git a/plugins/extractors/mssql/mssql.go b/plugins/extractors/mssql/mssql.go
@@ -36,10 +36,12 @@ var defaultDBList = []string{
 // Config holds the connection URL for the extractor
 type Config struct {
 	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	Identifier    string `mapstructure:"identifier" validate:"required"`
 }
 
 var sampleConfig = `
-connection_url: "sqlserver://admin:pass123@localhost:3306/"`
+connection_url: "sqlserver://admin:pass123@localhost:3306/"
+identifier: my-mssql`
 
 // Extractor manages the extraction of data from the database
 type Extractor struct {
@@ -132,7 +134,7 @@ func (e *Extractor) processTable(database string, tableName string) (err error)
 	// push table to channel
 	e.emit(models.NewRecord(&assetsv1beta1.Table{
 		Resource: &commonv1beta1.Resource{
-			Urn:  fmt.Sprintf("%s.%s", database, tableName),
+			Urn:  models.TableURN("mssql", e.config.Identifier, database, tableName),
 			Name: tableName,
 			Type: "table",
 		},

diff --git a/plugins/extractors/mssql/mssql_test.go b/plugins/extractors/mssql/mssql_test.go
@@ -96,6 +96,7 @@ func TestExtract(t *testing.T) {
 
 		err := extr.Init(ctx, map[string]interface{}{
 			"connection_url": fmt.Sprintf("sqlserver://%s:%s@%s/", user, pass, host),
+			"identifier":     "my-mssql",
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -146,7 +147,7 @@ func getExpected() []models.Record {
 	return []models.Record{
 		models.NewRecord(&assetsv1beta1.Table{
 			Resource: &commonv1beta1.Resource{
-				Urn:  "mockdata_meteor_metadata_test.applicant",
+				Urn:  "mssql::my-mssql/mockdata_meteor_metadata_test/applicant",
 				Name: "applicant",
 				Type: "table",
 			},
@@ -175,7 +176,7 @@ func getExpected() []models.Record {
 		}),
 		models.NewRecord(&assetsv1beta1.Table{
 			Resource: &commonv1beta1.Resource{
-				Urn:  "mockdata_meteor_metadata_test.jobs",
+				Urn:  "mssql::my-mssql/mockdata_meteor_metadata_test/jobs",
 				Name: "jobs",
 				Type: "table",
 			},

diff --git a/plugins/extractors/mysql/README.md b/plugins/extractors/mysql/README.md
@@ -7,19 +7,21 @@ source:
   name: mysql
   config:
     connection_url: admin:pass123@tcp(localhost:3306)/
+    identifier: my-mysql
 ```
 
 ## Inputs
 
 | Key | Value | Example | Description |    |
 | :-- | :---- | :------ | :---------- | :- |
 | `connection_url` | `string` | `admin:pass123@tcp(localhost:3306)/` | URL to access the mysql server | *required* |
+| `identifier` | `string` | `my-mysql` | Instance alias, the value will be used as part of the urn component | *required* |
 
 ## Outputs
 
 | Field | Sample Value |
 | :---- | :---- |
-| `resource.urn` | `my_database.my_table` |
+| `resource.urn` | `mysql::my-mysql/my_database/my_table` |
 | `resource.name` | `my_table` |
 | `resource.service` | `mysql` |
 | `description` | `table description` |

diff --git a/plugins/extractors/mysql/mysql.go b/plugins/extractors/mysql/mysql.go
@@ -35,9 +35,12 @@ var defaultDBList = []string{
 // Config holds the connection URL for the extractor
 type Config struct {
 	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
+	Identifier    string `mapstructure:"identifier" validate:"required"`
 }
 
-var sampleConfig = `connection_url: "admin:pass123@tcp(localhost:3306)/"`
+var sampleConfig = `
+connection_url: "admin:pass123@tcp(localhost:3306)/"
+identifier: "my-mysql"`
 
 // Extractor manages the extraction of data from MySQL
 type Extractor struct {
@@ -143,7 +146,7 @@ func (e *Extractor) processTable(database string, tableName string) (err error)
 	// push table to channel
 	e.emit(models.NewRecord(&assetsv1beta1.Table{
 		Resource: &commonv1beta1.Resource{
-			Urn:  fmt.Sprintf("%s.%s", database, tableName),
+			Urn:  models.TableURN("mysql", e.config.Identifier, database, tableName),
 			Name: tableName,
 			Type: "table",
 		},

diff --git a/plugins/extractors/mysql/mysql_test.go b/plugins/extractors/mysql/mysql_test.go
@@ -96,6 +96,7 @@ func TestExtract(t *testing.T) {
 
 		err := extr.Init(ctx, map[string]interface{}{
 			"connection_url": fmt.Sprintf("%s:%s@tcp(%s)/", user, pass, host),
+			"identifier":     "my-mysql",
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -152,7 +153,7 @@ func getExpected() []models.Record {
 	return []models.Record{
 		models.NewRecord(&assetsv1beta1.Table{
 			Resource: &commonv1beta1.Resource{
-				Urn:  "mockdata_meteor_metadata_test.applicant",
+				Urn:  "mysql::my-mysql/mockdata_meteor_metadata_test/applicant",
 				Name: "applicant",
 				Type: "table",
 			},
@@ -184,7 +185,7 @@ func getExpected() []models.Record {
 		}),
 		models.NewRecord(&assetsv1beta1.Table{
 			Resource: &commonv1beta1.Resource{
-				Urn:  "mockdata_meteor_metadata_test.jobs",
+				Urn:  "mysql::my-mysql/mockdata_meteor_metadata_test/jobs",
 				Name: "jobs",
 				Type: "table",
 			},

diff --git a/plugins/extractors/postgres/README.md b/plugins/extractors/postgres/README.md
@@ -14,14 +14,15 @@ source:
 
 | Key | Value | Example | Description |    |
 | :-- | :---- | :------ | :---------- | :- |
+| `identifier` | `string` | `my-postgres` | Instance alias, the value will be used as part of the urn component | *required* |
 | `connection_url` | `string` | `postgres://admin:pass123@localhost:3306/testDB?sslmode=disable` | URL to access the postgres server | *required* |
 | `exclude` | `string` | `primaryDB,secondaryDB` | This is a comma separated db list | *optional* |
 
 ## Outputs
 
 | Field | Sample Value |
 | :---- | :---- |
-| `resource.urn` | `postgres::localhost:3306/my_database/my_table` |
+| `resource.urn` | `postgres::my-postgres/my_database/my_table` |
 | `resource.name` | `my_table` |
 | `resource.service` | `postgres` |
 | `description` | `table description` |

diff --git a/plugins/extractors/postgres/postgres.go b/plugins/extractors/postgres/postgres.go
@@ -34,11 +34,13 @@ var defaultDBList = []string{"information_schema", "root", "postgres"}
 type Config struct {
 	ConnectionURL string `mapstructure:"connection_url" validate:"required"`
 	Exclude       string `mapstructure:"exclude"`
+	Identifier    string `mapstructure:"identifier" validate:"required"`
 }
 
 var sampleConfig = `
 connection_url: "postgres://admin:pass123@localhost:3306/postgres?sslmode=disable"
-exclude: testDB,secondaryDB`
+exclude: testDB,secondaryDB
+identifier: my-postgres`
 
 // Extractor manages the extraction of data from the extractor
 type Extractor struct {
@@ -171,7 +173,7 @@ func (e *Extractor) getTableMetadata(db *sql.DB, dbName string, tableName string
 
 	result = &assetsv1beta1.Table{
 		Resource: &commonv1beta1.Resource{
-			Urn:     models.TableURN("postgres", e.host, dbName, tableName),
+			Urn:     models.TableURN("postgres", e.config.Identifier, dbName, tableName),
 			Name:    tableName,
 			Service: "postgres",
 			Type:    "table",

diff --git a/plugins/extractors/postgres/postgres_test.go b/plugins/extractors/postgres/postgres_test.go
@@ -94,6 +94,7 @@ func TestExtract(t *testing.T) {
 
 		err := extr.Init(ctx, map[string]interface{}{
 			"connection_url": fmt.Sprintf("postgres://%s:%s@%s/postgres?sslmode=disable", user, pass, host),
+			"identifier":     "my-postgres",
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -148,7 +149,7 @@ func getExpected() []models.Record {
 	return []models.Record{
 		models.NewRecord(&assetsv1beta1.Table{
 			Resource: &commonv1beta1.Resource{
-				Urn:     "postgres::localhost:5438/test_db/article",
+				Urn:     "postgres::my-postgres/test_db/article",
 				Name:    "article",
 				Service: "postgres",
 				Type:    "table",
@@ -182,7 +183,7 @@ func getExpected() []models.Record {
 		}),
 		models.NewRecord(&assetsv1beta1.Table{
 			Resource: &commonv1beta1.Resource{
-				Urn:     "postgres::localhost:5438/test_db/post",
+				Urn:     "postgres::my-postgres/test_db/post",
 				Name:    "post",
 				Service: "postgres",
 				Type:    "table",

diff --git a/plugins/extractors/tableau/README.md b/plugins/extractors/tableau/README.md
@@ -1,14 +1,18 @@
 # tableau
 
 ## Usage
+
 ### Note
+
 To use tableau extractor, you need to enable [metadata service api](https://help.tableau.com/current/api/metadata_api/en-us/)
+
 ```yaml
 source:
   type: tableau
   config:
     host: http://server.tableau.com
     version: 3.12
+    identifier: my-tableau
     username: meteor_user
     password: xxxxxxxxxx
 ```
@@ -21,6 +25,7 @@ source:
 | :-- | :---- | :------ | :---------- | :- |
 | `host` | `string` | `https://server.tableau.com`         | The host at which tableau is running | *required* |
 | `version` | `string` | `3.12`     | The version of [Tableau REST API](https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_concepts_versions.htm), tested with 3.12 | *required* |
+| `identifier` | `string` | `my-tableau` | Instance alias, the value will be used as part of the urn component | *required* |
 | `username` | `string` | `meteor_user` | Username/email to access the tableau | *required* |
 | `password` | `string` | `xxxxxxxxxx` | Password for the tableau | *required* |
 | `sitename` | `string` | `testdev550928` | The name of your tableau site, it will point to the default one if you leave it empty | *not required* |
@@ -29,7 +34,7 @@ source:
 
 | Field | Sample Value |
 | :---- | :---- |
-| `resource.urn` | `tableau::{host}/workbook/{workbook_id}` |
+| `resource.urn` | `tableau::{identifier}/workbook/{workbook_id}` |
 | `resource.name` | `workbook_name` |
 | `resource.service` | `tableau` |
 | `resource.description` | `a description of the dashboard` |
@@ -39,9 +44,9 @@ source:
 
 | Field | Sample Value |
 | :---- | :---- |
-| `urn` | `tableau::{host}/sheet/{sheet_id}`             |
+| `urn` | `tableau::{identifier}/sheet/{sheet_id}`             |
 | `source` | `tableau` |
-| `dashboard_urn` | `tableau::{host}/workbook/{workbook_id}` |
+| `dashboard_urn` | `tableau::{identifier}/workbook/{workbook_id}` |
 | `dashboard_source` | `tableau` |
 
 ## Contributing

diff --git a/plugins/extractors/tableau/tableau.go b/plugins/extractors/tableau/tableau.go
@@ -24,18 +24,20 @@ var summary string
 var sampleConfig = `
 host: https://server.tableau.com
 version: 3.12
+identifier: my-tableau
 username: meteor_user
 password: xxxxxxxxxx
 sitename: testdev550928
 `
 
 // Config that holds a set of configuration for tableau extractor
 type Config struct {
-	Host     string `mapstructure:"host" validate:"required"`
-	Version  string `mapstructure:"version" validate:"required"` // float as string
-	Username string `mapstructure:"username" validate:"required"`
-	Password string `mapstructure:"password" validate:"required"`
-	Sitename string `mapstructure:"sitename"`
+	Host       string `mapstructure:"host" validate:"required"`
+	Version    string `mapstructure:"version" validate:"required"` // float as string
+	Identifier string `mapstructure:"identifier" validate:"required"`
+	Username   string `mapstructure:"username" validate:"required"`
+	Password   string `mapstructure:"password" validate:"required"`
+	Sitename   string `mapstructure:"sitename"`
 }
 
 // Extractor manages the extraction of data
@@ -129,7 +131,7 @@ func (e *Extractor) Extract(ctx context.Context, emit plugins.Emit) (err error)
 
 func (e *Extractor) buildDashboard(wb *Workbook) (data *assetsv1beta1.Dashboard, err error) {
 	lineages := e.buildLineage(wb.UpstreamTables)
-	dashboardURN := models.DashboardURN("tableau", e.config.Host, fmt.Sprintf("workbook/%s", wb.ID))
+	dashboardURN := models.DashboardURN("tableau", e.config.Identifier, fmt.Sprintf("workbook/%s", wb.ID))
 	data = &assetsv1beta1.Dashboard{
 		Resource: &commonv1beta1.Resource{
 			Urn:         dashboardURN,
@@ -170,7 +172,7 @@ func (e *Extractor) buildDashboard(wb *Workbook) (data *assetsv1beta1.Dashboard,
 
 func (e *Extractor) buildCharts(dashboardURN string, wb *Workbook, lineages *facetsv1beta1.Lineage) (charts []*assetsv1beta1.Chart) {
 	for _, sh := range wb.Sheets {
-		chartURN := models.DashboardURN("tableau", e.config.Host, fmt.Sprintf("sheet/%s", sh.ID))
+		chartURN := models.DashboardURN("tableau", e.config.Identifier, fmt.Sprintf("sheet/%s", sh.ID))
 		charts = append(charts, &assetsv1beta1.Chart{
 			Urn:          chartURN,
 			Name:         sh.Name,

diff --git a/plugins/extractors/tableau/tableau_test.go b/plugins/extractors/tableau/tableau_test.go
@@ -52,11 +52,12 @@ func TestExtract(t *testing.T) {
 				Transport: r,
 			}))
 		err = extr.Init(ctx, map[string]interface{}{
-			"host":     host,
-			"version":  version,
-			"sitename": sitename,
-			"username": username,
-			"password": password,
+			"host":       host,
+			"version":    version,
+			"identifier": "my-tableau",
+			"sitename":   sitename,
+			"username":   username,
+			"password":   password,
 		})
 		if err != nil {
 			t.Fatal(err)