Skip to content

Commit

Permalink
refactor: rename columbus sink to compass sink (#341)
Browse files Browse the repository at this point in the history
  • Loading branch information
scortier authored Apr 21, 2022
1 parent 23eca16 commit 7a38fdc
Show file tree
Hide file tree
Showing 14 changed files with 98 additions and 98 deletions.
2 changes: 1 addition & 1 deletion cmd/info.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func InfoSinkCmd() *cobra.Command {
The list of supported sinks is available via the 'meteor list sinks' command.`),
Example: heredoc.Doc(`
$ meteor info sink console
$ meteor info sink columbus
$ meteor info sink compass
`),
Args: cobra.MaximumNArgs(1),
Annotations: map[string]string{
Expand Down
4 changes: 2 additions & 2 deletions cmd/new.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ func NewRecipeCmd() *cobra.Command {
$ meteor new recipe sample -e bigquery -s console
# generate recipe with multiple sinks
$ meteor new recipe sample -e bigquery -s columbus,kafka -p enrich
$ meteor new recipe sample -e bigquery -s compass,kafka -p enrich
# store recipe to a file
$ meteor new recipe sample -e bigquery -s columbus > recipe.yaml
$ meteor new recipe sample -e bigquery -s compass > recipe.yaml
`),
Annotations: map[string]string{
"group:core": "true",
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/guides/2_manage_recipes.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ After making the necessary changes to the source, and sinks as per your local se
$ meteor new recipe sample -e <name-of-extractor> -s <single-or-multiple-sinks> -p <name-of-processors>

# command to generate recipe with multiple sinks
$ meteor new recipe sample -e bigquery -s columbus,kafka
$ meteor new recipe sample -e bigquery -s compass,kafka

# for the tour you can use a single console sink
# extracor(-e) as postgres, sink(-s) and enrich processor(-p)
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/guides/introduction.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,6 @@ The tour takes you through how you can use meteor as an end user and is a great
- You are required to have atleast one Database, Dashboard or any other Data source and their proper credentials ready.
For e.g. Postgres, MySQL, MSSQL, MongoDB, GitHub, etc.

- For sink we will be using console, if you are planning to use columbus or Kafka please make necessary changes accordingly.
- For sink we will be using console, if you are planning to use compass or Kafka please make necessary changes accordingly.

Once done with the pre-requisites Get started with [Installation](./0_installation.md).
4 changes: 2 additions & 2 deletions docs/docs/reference/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,11 @@ You can create a sample recipe usin the gen command.
$ meteor gen recipe sample -e bigquery -s console

# generate recipe with multiple sinks
$ meteor gen recipe sample -e bigquery -s columbus,kafka
$ meteor gen recipe sample -e bigquery -s compass,kafka

# extracor(-e) as postgres, multiple sinks(-s) and enrich processor(-p)
# save the generated recipe to a recipe.yaml
meteor gen recipe sample -e postgres -s columbus,kafka -p enrich > recipe.yaml
meteor gen recipe sample -e postgres -s compass,kafka -p enrich > recipe.yaml
```

## Linting recipes
Expand Down
16 changes: 8 additions & 8 deletions docs/docs/reference/sinks.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,20 @@ sinks:
- name: console
```
## Columbus
## Compass
`columbus`
`compass`

Upload metadata to a given `type` in [Columbus](https://github.com/odpf/meteor/tree/cb12c3ecf8904cf3f4ce365ca8981ccd132f35d0/docs/reference/github.com/odpf/columbus/README.md). Request will be send via HTTP to given host.
Upload metadata to a given `type` in [Compass](https://github.com/odpf/meteor/tree/cb12c3ecf8904cf3f4ce365ca8981ccd132f35d0/docs/reference/github.com/odpf/compass/README.md). Request will be send via HTTP to given host.

### Sample usage of columbus sink
### Sample usage of compass sink

```yaml
sinks:
- name: columbus
- name: compass
config:
host: https://columbus.com
type: sample-columbus-type
host: https://compass.com
type: sample-compass-type
mapping:
new_fieldname: "json_field_name"
id: "resource.urn"
Expand All @@ -35,4 +35,4 @@ sinks:

_**Notes**_

Columbus' Type requires certain fields to be sent, hence why `mapping` config is needed to map value from any of our metadata models to any field name when sending to Columbus. Supports getting value from nested fields.
Compass' Type requires certain fields to be sent, hence why `mapping` config is needed to map value from any of our metadata models to any field name when sending to Compass. Supports getting value from nested fields.
2 changes: 1 addition & 1 deletion docs/src/pages/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ export default function Home() {
content: (
<div>
Meteor supports sink plugins to send metadata to a variety of
third party APIs and catalog services, including Columbus, HTTP, BigQuery,
third party APIs and catalog services, including Compass, HTTP, BigQuery,
Kafka, and many others.
</div>
),
Expand Down
4 changes: 2 additions & 2 deletions docs/static/assets/overview.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
22 changes: 0 additions & 22 deletions plugins/sinks/columbus/README.md

This file was deleted.

22 changes: 22 additions & 0 deletions plugins/sinks/compass/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Compass

Compass is a search and discovery engine built for querying application deployments, datasets and meta resources. It can also optionally track data flow relationships between these resources and allow the user to view a representation of the data flow graph.

## Usage

```yaml
sinks:
name: compass
config:
host: https://compass.com
headers:
compass-User-Email: [email protected]
Header-1: value11,value12
labels:
myCustom: $properties.attributes.myCustomField
sampleLabel: $properties.labels.sampleLabelField
```
## Contributing
Refer to the contribution guidelines for information on contributing to this module.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package columbus
package compass

type RequestPayload struct {
Asset Asset `json:"asset"`
Expand Down
28 changes: 14 additions & 14 deletions plugins/sinks/columbus/sink.go → plugins/sinks/compass/sink.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package columbus
package compass

import (
"bytes"
Expand Down Expand Up @@ -27,11 +27,11 @@ type Config struct {
}

var sampleConfig = `
# The hostname of the columbus service
host: https://columbus.com
# Additional HTTP headers send to columbus, multiple headers value are separated by a comma
# The hostname of the compass service
host: https://compass.com
# Additional HTTP headers send to compass, multiple headers value are separated by a comma
headers:
Columbus-User-Email: [email protected]
Compass-User-Email: [email protected]
X-Other-Header: value1, value2
# The labels to pass as payload label of the patch api
labels:
Expand All @@ -56,7 +56,7 @@ func New(c httpClient, logger log.Logger) plugins.Syncer {

func (s *Sink) Info() plugins.Info {
return plugins.Info{
Description: "Send metadata to columbus http service",
Description: "Send metadata to compass http service",
SampleConfig: sampleConfig,
Summary: summary,
Tags: []string{"http", "sink"},
Expand All @@ -78,17 +78,17 @@ func (s *Sink) Init(ctx context.Context, configMap map[string]interface{}) (err
func (s *Sink) Sink(ctx context.Context, batch []models.Record) (err error) {
for _, record := range batch {
metadata := record.Data()
s.logger.Info("sinking record to columbus", "record", metadata.GetResource().Urn)
s.logger.Info("sinking record to compass", "record", metadata.GetResource().Urn)

columbusPayload, err := s.buildColumbusPayload(metadata)
compassPayload, err := s.buildCompassPayload(metadata)
if err != nil {
return errors.Wrap(err, "failed to build columbus payload")
return errors.Wrap(err, "failed to build compass payload")
}
if err = s.send(columbusPayload); err != nil {
if err = s.send(compassPayload); err != nil {
return errors.Wrap(err, "error sending data")
}

s.logger.Info("successfully sinked record to columbus", "record", metadata.GetResource().Urn)
s.logger.Info("successfully sinked record to compass", "record", metadata.GetResource().Urn)
}

return
Expand Down Expand Up @@ -129,7 +129,7 @@ func (s *Sink) send(record RequestPayload) (err error) {
if err != nil {
return
}
err = fmt.Errorf("columbus returns %d: %v", res.StatusCode, string(bodyBytes))
err = fmt.Errorf("compass returns %d: %v", res.StatusCode, string(bodyBytes))

switch code := res.StatusCode; {
case code >= 500:
Expand All @@ -139,7 +139,7 @@ func (s *Sink) send(record RequestPayload) (err error) {
}
}

func (s *Sink) buildColumbusPayload(metadata models.Metadata) (RequestPayload, error) {
func (s *Sink) buildCompassPayload(metadata models.Metadata) (RequestPayload, error) {
labels, err := s.buildLabels(metadata)
if err != nil {
return RequestPayload{}, errors.Wrap(err, "failed to build labels")
Expand Down Expand Up @@ -299,7 +299,7 @@ func (s *Sink) getLabelValueFromProperties(field1 string, field2 string, metadat
}

func init() {
if err := registry.Sinks.Register("columbus", func() plugins.Syncer {
if err := registry.Sinks.Register("compass", func() plugins.Syncer {
return New(&http.Client{}, plugins.GetLog())
}); err != nil {
panic(err)
Expand Down
Loading

0 comments on commit 7a38fdc

Please sign in to comment.