From 64956143669a19c5403a15e8697cceb4e8c65b1a Mon Sep 17 00:00:00 2001 From: Suhas Karanth Date: Mon, 7 Nov 2022 20:56:41 +0530 Subject: [PATCH] ci: enable cache to speed up jobs (#433) - Bump Go version {1.17 => 1.18}. - Ensure to fetch the latest version in 'build' and 'release' workflows to pull in the most up-to-date release so that security fixes can be included. - Bump the golangci-lint version {v1.42.1 => v1.50}. This is required to be able to lint source code with Go 1.18. Fails with 'undeclared name' errors otherwise. - Do not run 'go mod tidy' as this can modify the mod file. Changes to dependencies should be explicitly committed where required. - Modify release workflow as per https://goreleaser.com/ci/actions/#workflow. - Bump action versions: - actions/setup-go@{v2 => v3}. Enable the 'cache' option for caching dependency files and build outputs. - actions/checkout@{v2 => v3}. - golangci/golangci-lint-action@{v2 => v3}. Caching is enabled by default. Bump golangci-lint version {v1.42.1 => v1.50}, needed for Go 1.18. - Fix linting issues due to deprecation of ioutil package. Migrate to io and os packages instead. - Consistent formatting: - Checkout code first in all jobs across workflows with the name 'Checkout code'. - Setup Go after 'Checkout code' with the name 'Set up Go'. --- .github/workflows/build.yml | 16 +++++++++++----- .github/workflows/build_dev.yml | 2 +- .github/workflows/docs.yml | 3 ++- .github/workflows/lint.yml | 16 +++++++++------- .github/workflows/plugin_test.yml | 11 ++++++----- .github/workflows/release.yml | 16 ++++++++++------ .github/workflows/test.yml | 13 +++++++------ plugins/extractors/redash/redash.go | 4 ++-- plugins/extractors/superset/superset.go | 6 +++--- plugins/extractors/superset/superset_test.go | 3 +-- plugins/extractors/tableau/client.go | 4 ++-- plugins/extractors/tableau/client_test.go | 6 +++--- plugins/sinks/compass/sink.go | 4 ++-- plugins/sinks/compass/sink_test.go | 6 +++--- plugins/sinks/http/http.go | 4 ++-- plugins/sinks/stencil/stencil.go | 4 ++-- plugins/sinks/stencil/stencil_test.go | 8 ++++---- recipe/generator_test.go | 7 +++---- test/utils/logger.go | 4 ++-- 19 files changed, 75 insertions(+), 62 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 077b87802..275efb710 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,15 +10,21 @@ jobs: build: runs-on: ubuntu-latest steps: + # See https://goreleaser.com/ci/actions/#fetch-depthness - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - run: git fetch --force --tags + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.18' + cache: true + check-latest: true - name: Get release tag id: get_version uses: battila7/get-version-action@v2 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '1.17' - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2.6.1 with: diff --git a/.github/workflows/build_dev.yml b/.github/workflows/build_dev.yml index 6c46b68b4..8e6069627 100644 --- a/.github/workflows/build_dev.yml +++ b/.github/workflows/build_dev.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Login to DockerHub uses: docker/login-action@v1 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 058488922..51544a831 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,8 @@ jobs: documentation: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - name: Checkout code + uses: actions/checkout@v3 - uses: actions/setup-node@v2 - name: Installation uses: bahmutov/npm-install@v1 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 96dcfc4f9..19e168580 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,15 +10,17 @@ jobs: golangci: runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v2 - with: - go-version: '1.17' - - uses: actions/checkout@v2 + - name: Checkout code + uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.18' + # See https://github.com/golangci/golangci-lint-action#how-to-use - name: golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3 with: - skip-go-installation: true - version: v1.42.1 + version: v1.50 args: --timeout=5m diff --git a/.github/workflows/plugin_test.yml b/.github/workflows/plugin_test.yml index e0a35c706..d70b37ce6 100644 --- a/.github/workflows/plugin_test.yml +++ b/.github/workflows/plugin_test.yml @@ -11,13 +11,14 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: '1.18' + cache: true - name: Install dependencies run: sudo apt-get install build-essential - - name: Install packages - run: go mod tidy - name: Run Test run: make test-plugins diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a90bd4ff..f90df0901 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,14 +10,18 @@ jobs: release: runs-on: ubuntu-latest steps: - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: '^1.17' - - name: Checkout - uses: actions/checkout@v2 + # See https://goreleaser.com/ci/actions/#fetch-depthness + - name: Checkout code + uses: actions/checkout@v3 with: fetch-depth: 0 + - run: git fetch --force --tags + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '^1.18' + cache: true + check-latest: true - name: Login to DockerHub uses: docker/login-action@v1 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38f50dde7..9220c5f81 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,19 +5,20 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: '1.18' + cache: true - name: Install dependencies run: sudo apt-get install build-essential - - name: Install packages - run: go mod tidy - name: Run Test run: make test - name: Install goveralls and send coverage env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - go get github.com/mattn/goveralls + go install github.com/mattn/goveralls@v0.0.11 goveralls -coverprofile=coverage.out -service=github diff --git a/plugins/extractors/redash/redash.go b/plugins/extractors/redash/redash.go index a1fcf1a05..4a9cb9604 100644 --- a/plugins/extractors/redash/redash.go +++ b/plugins/extractors/redash/redash.go @@ -6,7 +6,7 @@ import ( _ "embed" // used to print the embedded assets "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "time" @@ -160,7 +160,7 @@ func (e *Extractor) makeRequest(method, url string, payload interface{}, data in if res.StatusCode < 200 || res.StatusCode >= 300 { return fmt.Errorf("response failed with status code %d: %w", res.StatusCode, err) } - b, err := ioutil.ReadAll(res.Body) + b, err := io.ReadAll(res.Body) if err != nil { return fmt.Errorf("failed to read response body: %w", err) } diff --git a/plugins/extractors/superset/superset.go b/plugins/extractors/superset/superset.go index 64bba531e..b8661d5e7 100644 --- a/plugins/extractors/superset/superset.go +++ b/plugins/extractors/superset/superset.go @@ -6,7 +6,7 @@ import ( _ "embed" // used to print the embedded assets "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "time" @@ -167,7 +167,7 @@ func (e *Extractor) getChartsList(id int) (charts []*v1beta2.Chart, err error) { return tempCharts, nil } -//getAccessToken authenticate and get a JWT access token +// getAccessToken authenticate and get a JWT access token func (e *Extractor) getAccessToken() (accessToken string, err error) { payload := map[string]interface{}{ "username": e.config.Username, @@ -220,7 +220,7 @@ func (e *Extractor) makeRequest(method, url string, payload interface{}, data in if res.StatusCode < 200 || res.StatusCode >= 300 { return errors.Wrapf(err, "response failed with status code: %d", res.StatusCode) } - b, err := ioutil.ReadAll(res.Body) + b, err := io.ReadAll(res.Body) if err != nil { return errors.Wrap(err, "failed to read response body") } diff --git a/plugins/extractors/superset/superset_test.go b/plugins/extractors/superset/superset_test.go index 8ea730795..2040d5ee9 100644 --- a/plugins/extractors/superset/superset_test.go +++ b/plugins/extractors/superset/superset_test.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "log" "net/http" "os" @@ -253,7 +252,7 @@ func makeRequest(method, url string, payload interface{}, data interface{}) (err bodyString := string(bodyBytes) return fmt.Errorf("response failed with status code: %d and body: %s", res.StatusCode, bodyString) } - b, err := ioutil.ReadAll(res.Body) + b, err := io.ReadAll(res.Body) if err != nil { return fmt.Errorf("failed to read response body: %w", err) } diff --git a/plugins/extractors/tableau/client.go b/plugins/extractors/tableau/client.go index 0a3cc388c..4dc382b6c 100644 --- a/plugins/extractors/tableau/client.go +++ b/plugins/extractors/tableau/client.go @@ -5,7 +5,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "time" @@ -243,7 +243,7 @@ func (c *client) makeRequest(method, url string, payload interface{}, data inter return fmt.Errorf("getting %d status code", res.StatusCode) } - bytes, err := ioutil.ReadAll(res.Body) + bytes, err := io.ReadAll(res.Body) if err != nil { return errors.Wrap(err, "failed to read response body") } diff --git a/plugins/extractors/tableau/client_test.go b/plugins/extractors/tableau/client_test.go index aa3abc8e8..ef138b813 100644 --- a/plugins/extractors/tableau/client_test.go +++ b/plugins/extractors/tableau/client_test.go @@ -6,8 +6,8 @@ package tableau import ( "context" "encoding/json" - "io/ioutil" "net/http" + "os" "testing" "github.com/dnaeon/go-vcr/v2/recorder" @@ -105,7 +105,7 @@ func TestGetWorkbooksByProjectName(t *testing.T) { } func testDataGetWorkbooksByProjectName(t *testing.T) (wbs []*Workbook, err error) { - byteString, err := ioutil.ReadFile("testdata/workbooks_by_project_response.json") + byteString, err := os.ReadFile("testdata/workbooks_by_project_response.json") if err != nil { t.Fatal(err) } @@ -120,7 +120,7 @@ func testDataGetWorkbooksByProjectName(t *testing.T) (wbs []*Workbook, err error } func testDataGetAllProjects(t *testing.T) (ps []*Project, err error) { - byteString, err := ioutil.ReadFile("testdata/projects_response.json") + byteString, err := os.ReadFile("testdata/projects_response.json") if err != nil { t.Fatal(err) } diff --git a/plugins/sinks/compass/sink.go b/plugins/sinks/compass/sink.go index ca17b411f..bea9b255c 100644 --- a/plugins/sinks/compass/sink.go +++ b/plugins/sinks/compass/sink.go @@ -6,7 +6,7 @@ import ( _ "embed" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strings" @@ -125,7 +125,7 @@ func (s *Sink) send(record RequestPayload) (err error) { } var bodyBytes []byte - bodyBytes, err = ioutil.ReadAll(res.Body) + bodyBytes, err = io.ReadAll(res.Body) if err != nil { return } diff --git a/plugins/sinks/compass/sink_test.go b/plugins/sinks/compass/sink_test.go index 91dbd4a98..7e5ffb99c 100644 --- a/plugins/sinks/compass/sink_test.go +++ b/plugins/sinks/compass/sink_test.go @@ -9,7 +9,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "strings" "testing" @@ -573,7 +573,7 @@ func (m *mockHTTPClient) Do(req *http.Request) (res *http.Response, err error) { Request: req, Header: make(http.Header), ContentLength: int64(len(m.ResponseJSON)), - Body: ioutil.NopCloser(bytes.NewBufferString(m.ResponseJSON)), + Body: io.NopCloser(bytes.NewBufferString(m.ResponseJSON)), } return @@ -597,7 +597,7 @@ func (m *mockHTTPClient) Assert(t *testing.T) { var bodyBytes = []byte("") if m.req.Body != nil { var err error - bodyBytes, err = ioutil.ReadAll(m.req.Body) + bodyBytes, err = io.ReadAll(m.req.Body) if err != nil { t.Error(err) } diff --git a/plugins/sinks/http/http.go b/plugins/sinks/http/http.go index f01f443ae..85490ef33 100644 --- a/plugins/sinks/http/http.go +++ b/plugins/sinks/http/http.go @@ -6,7 +6,7 @@ import ( _ "embed" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strings" @@ -114,7 +114,7 @@ func (s *Sink) send(payloadBytes []byte) (err error) { } var bodyBytes []byte - bodyBytes, err = ioutil.ReadAll(res.Body) + bodyBytes, err = io.ReadAll(res.Body) if err != nil { return } diff --git a/plugins/sinks/stencil/stencil.go b/plugins/sinks/stencil/stencil.go index 9c7827a78..5419e34f3 100644 --- a/plugins/sinks/stencil/stencil.go +++ b/plugins/sinks/stencil/stencil.go @@ -6,7 +6,7 @@ import ( _ "embed" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strings" @@ -173,7 +173,7 @@ func (s *Sink) send(tableURN string, record interface{}) (err error) { } var bodyBytes []byte - bodyBytes, err = ioutil.ReadAll(res.Body) + bodyBytes, err = io.ReadAll(res.Body) if err != nil { return } diff --git a/plugins/sinks/stencil/stencil_test.go b/plugins/sinks/stencil/stencil_test.go index 69a74c394..3c7906136 100644 --- a/plugins/sinks/stencil/stencil_test.go +++ b/plugins/sinks/stencil/stencil_test.go @@ -9,14 +9,14 @@ import ( "encoding/json" "errors" "fmt" - "github.com/stretchr/testify/require" - "io/ioutil" + "io" "net/http" "testing" "github.com/odpf/meteor/models" v1beta2 "github.com/odpf/meteor/models/odpf/assets/v1beta2" "github.com/odpf/meteor/plugins/sinks/stencil" + "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/anypb" "github.com/odpf/meteor/plugins" @@ -621,7 +621,7 @@ func (m *mockHTTPClient) Do(req *http.Request) (res *http.Response, err error) { Request: req, Header: make(http.Header), ContentLength: int64(len(m.ResponseJSON)), - Body: ioutil.NopCloser(bytes.NewBufferString(m.ResponseJSON)), + Body: io.NopCloser(bytes.NewBufferString(m.ResponseJSON)), } return @@ -640,7 +640,7 @@ func (m *mockHTTPClient) Assert(t *testing.T) { var bodyBytes = []byte("") if m.req.Body != nil { var err error - bodyBytes, err = ioutil.ReadAll(m.req.Body) + bodyBytes, err = io.ReadAll(m.req.Body) if err != nil { t.Error(err) } diff --git a/recipe/generator_test.go b/recipe/generator_test.go index 6896adac4..35e0735ae 100644 --- a/recipe/generator_test.go +++ b/recipe/generator_test.go @@ -2,7 +2,6 @@ package recipe_test import ( "fmt" - "io/ioutil" "os" "path" "testing" @@ -17,7 +16,7 @@ func TestFromTemplate(t *testing.T) { t.Run("should throw error for invalid template path", func(t *testing.T) { templatePath := "./testdata/template.yaml" outputDir := "./test/temp" - bytes, err := ioutil.ReadFile("./testdata/generator/data-3.yaml") + bytes, err := os.ReadFile("./testdata/generator/data-3.yaml") if err != nil { fmt.Println(fmt.Errorf("error reading data: %w", err)) return @@ -42,7 +41,7 @@ func TestFromTemplate(t *testing.T) { outputDir := "./testdata/generator/temp" t.Run("when recipe has a name", func(t *testing.T) { - bytes, err := ioutil.ReadFile("./testdata/generator/data-1-2.yaml") + bytes, err := os.ReadFile("./testdata/generator/data-1-2.yaml") if err != nil { fmt.Println(fmt.Errorf("error reading data: %w", err)) return @@ -75,7 +74,7 @@ func TestFromTemplate(t *testing.T) { }) t.Run("when recipe does not have a name", func(t *testing.T) { - bytes, err := ioutil.ReadFile("./testdata/generator/data-3.yaml") + bytes, err := os.ReadFile("./testdata/generator/data-3.yaml") if err != nil { fmt.Println(fmt.Errorf("error reading data: %w", err)) return diff --git a/test/utils/logger.go b/test/utils/logger.go index 602b3896e..5b9d2f026 100644 --- a/test/utils/logger.go +++ b/test/utils/logger.go @@ -1,10 +1,10 @@ package utils import ( - "io/ioutil" + "io" "github.com/odpf/salt/log" ) // Logger set with writer -var Logger log.Logger = log.NewLogrus(log.LogrusWithWriter(ioutil.Discard)) +var Logger log.Logger = log.NewLogrus(log.LogrusWithWriter(io.Discard))