test: track coverage for plugins (#443)

.github/workflows/test.yml

@@ -1,5 +1,9 @@
 name: Test
-on: [push, pull_request]
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
 
 jobs:
   test:
@@ -16,9 +20,58 @@ jobs:
         run: sudo apt-get install build-essential
       - name: Run Test
         run: make test
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage
+          path: coverage.out
+
+  plugins-test:
+    runs-on: ubuntu-latest
+    if: |
+      github.ref == 'refs/heads/main' || 
+      github.event_name == 'pull_request' ||
+      github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+          cache: true
+      - name: Install dependencies
+        run: sudo apt-get install build-essential
+      - name: Run Test
+        run: make test-plugins
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-plugins
+          path: coverage-plugins.out
+
+  coverage:
+    runs-on: ubuntu-latest
+    needs: [test, plugins-test]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+          cache: true
+      - name: Download coverage
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage
+      - name: Download plugins coverage
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage-plugins
       - name: Install goveralls and send coverage
         env:
           COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           go install github.com/mattn/[email protected]
-          goveralls -coverprofile=coverage.out -service=github
+          goveralls -coverprofile=coverage.out,coverage-plugins.out -service=github

Makefile

@@ -21,15 +21,17 @@ copy-config:
 test:
 	go test ./... -coverprofile=coverage.out
 
-test-coverage: test
-	go tool cover -html=coverage.out
-
 test-e2e:
 	go test ./test/e2e -tags=integration -count=1
 
 test-plugins:
 	@echo " > Testing plugins with tag 'plugins'"
-	go test ./plugins... -tags=plugins -count=1
+	go test ./plugins... -tags=plugins -coverprofile=coverage-plugins.out -parallel=1
+
+test-coverage: # test test-plugins
+	cp coverage.out coverage-all.out
+	tail -n +2 coverage-plugins.out >> coverage-all.out
+	go tool cover -html=coverage-all.out
 
 generate-proto: ## regenerate protos
 	@echo " > cloning protobuf from odpf/proton"

plugins/base_extractor_test.go

@@ -1,3 +1,6 @@
+//go:build plugins
+// +build plugins
+
 package plugins_test
 
 import (

plugins/extractors/bigquery/auditlog/auditlog_test.go

@@ -1,3 +1,6 @@
+//go:build plugins
+// +build plugins
+
 package auditlog
 
 import (

plugins/extractors/bigquery/auditlog/logdata_test.go

@@ -1,3 +1,6 @@
+//go:build plugins
+// +build plugins
+
 package auditlog
 
 import (

plugins/extractors/bigquery/auditlog/stats_test.go

@@ -1,3 +1,6 @@
+//go:build plugins
+// +build plugins
+
 package auditlog
 
 import (

plugins/extractors/bigquery/auditlog/testdata.go → plugins/extractors/bigquery/auditlog/testdata_test.go

@@ -1,3 +1,6 @@
+//go:build plugins
+// +build plugins
+
 package auditlog
 
 import (

plugins/extractors/bigquery/sqlparser/parser_test.go

@@ -1,3 +1,6 @@
+//go:build plugins
+// +build plugins
+
 package sqlparser
 
 import (
@@ -208,3 +211,60 @@ func TestComplexQueries(t *testing.T) {
 		})
 	}
 }
+
+const testDataSQL1 = `
+SELECT
+COUNT(DISTINCT user_id) AS purchasers_count
+FROM
+-- PLEASE REPLACE WITH YOUR TABLE NAME.
+` + "`" + "YOUR_TABLE.events_*" + "`" + `
+WHERE
+event_name IN ('in_app_purchase', 'purchase')
+-- PLEASE REPLACE WITH YOUR DESIRED DATE RANGE
+AND _TABLE_SUFFIX BETWEEN '20180501' AND '20240131'
+AND ` + "`" + "_TABLE_SUFFIX" + "`" + ` BETWEEN '1' AND '2';`
+
+const testDataSQL2 = `
+USING (
+    SELECT 
+    t.column_1,
+    t.column_2,
+    CAST(count(distinct(id)) AS BIGNUMERIC) as total,
+    TIMESTAMP("2021-10-29", "UTC") as window_start_time,
+    CAST("2021-11-22T02:01:06Z" AS TIMESTAMP) as event_timestamp
+    FROM` + "`" + `a-dataset.a-specific_table-2021` + "`" + `, 
+    UNNEST(trial) as t 
+    WHERE 
+        t.column_type = 'tester' AND
+        t.param_id = "280481a2-2384-4b81-aa3e-214ac60b31db" AND 
+        event_timestamp >= TIMESTAMP("2021-10-29", "UTC") AND
+        event_timestamp < TIMESTAMP("2021-11-22T02:01:06Z")
+    GROUP BY t.column_1, t.column_2
+    ) source 
+    ON
+        target.column_1 = source.column_1
+        and target.variant_name = source.variant_name
+        and DATE(target.event_timestamp) = DATE(source.event_timestamp)
+    WHEN matched then update set 
+        target.total = source.total,
+        target.event_timestamp = source.event_timestamp
+    WHEN NOT MATCHED then insert(param1,param2,param3,param4,param5_ext) values(
+        source.param1,
+        source.param2,
+        source.param3,
+        source.param4,
+        source.param5_ext
+    )`
+
+const testDataSQL3 = `
+SELECT
+  COUNT(DISTINCT user_id) AS acquired_users_count
+FROM
+  -- PLEASE REPLACE WITH YOUR TABLE NAME.
+  ` + "`" + "YOUR_TABLE.events_*" + "`" + `
+WHERE
+  traffic_source.source = 'google'
+  AND traffic_source.medium = 'cpc'
+  AND traffic_source.name = 'VTA-Test-Android'
+  -- PLEASE REPLACE YOUR DESIRED DATE RANGE.
+  AND _TABLE_SUFFIX BETWEEN '20180521' AND '20240131';`

plugins/extractors/elastic/elastic_test.go

@@ -240,7 +240,7 @@ func getExpectedVal(t *testing.T) []models.Record {
 
 func matchRecords(t *testing.T, expected, actual []models.Record) {
 	if actual[0].Data().Name == "index2" {
-		//swap index order
+		// swap index order
 		temp := actual[0]
 		actual[0] = actual[1]
 		actual[1] = temp

plugins/extractors/gsuite/gsuite_test.go

@@ -1,3 +1,6 @@
+//go:build plugins
+// +build plugins
+
 package gsuite_test
 
 import (

plugins/extractors/redash/redash_test.go

@@ -1,3 +1,6 @@
+//go:build plugins
+// +build plugins
+
 package redash_test
 
 import (