From 7215e34686683d2b82dda9c34bb7c52f38fa9c1f Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 29 Nov 2023 22:29:40 -0300 Subject: [PATCH] enable BigQuery Default Credentials [nt] --- database/database_bigquery.go | 14 +++++++++----- scripts/check_parquet.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/database/database_bigquery.go b/database/database_bigquery.go index 09502cd..fa977d8 100755 --- a/database/database_bigquery.go +++ b/database/database_bigquery.go @@ -7,7 +7,6 @@ import ( "encoding/base64" "fmt" "io" - "io/ioutil" "math/big" "os" "path" @@ -90,6 +89,7 @@ func (conn *BigQueryConn) Init() error { func (conn *BigQueryConn) getNewClient(timeOut ...int) (client *bigquery.Client, err error) { var authOption option.ClientOption var credJsonBody string + var useDefault bool to := 15 if len(timeOut) > 0 { @@ -101,7 +101,7 @@ func (conn *BigQueryConn) getNewClient(timeOut ...int) (client *bigquery.Client, authOption = option.WithCredentialsJSON([]byte(val)) } else if val := conn.GetProp("GC_KEY_FILE"); val != "" { authOption = option.WithCredentialsFile(val) - b, err := ioutil.ReadFile(val) + b, err := os.ReadFile(val) if err != nil { return client, g.Error(err, "could not read google cloud key file") } @@ -110,14 +110,13 @@ func (conn *BigQueryConn) getNewClient(timeOut ...int) (client *bigquery.Client, authOption = option.WithAPIKey(val) } else if val := conn.GetProp("GOOGLE_APPLICATION_CREDENTIALS"); val != "" { authOption = option.WithCredentialsFile(val) - b, err := ioutil.ReadFile(val) + b, err := os.ReadFile(val) if err != nil { return client, g.Error(err, "could not read google cloud key file") } credJsonBody = string(b) } else { - err = g.Error("no Google credentials provided") - return + useDefault = true } if conn.ProjectID == "" && credJsonBody != "" { @@ -128,6 +127,11 @@ func (conn *BigQueryConn) getNewClient(timeOut ...int) (client *bigquery.Client, ctx, cancel := context.WithTimeout(conn.BaseConn.Context().Ctx, time.Duration(to)*time.Second) defer cancel() + + if useDefault { + g.Debug("no BigQuery Google credentials provided, using Application Default Credentials") + return bigquery.NewClient(ctx, conn.ProjectID) + } return bigquery.NewClient(ctx, conn.ProjectID, authOption) } diff --git a/scripts/check_parquet.py b/scripts/check_parquet.py index 3351d6b..0ba97c8 100644 --- a/scripts/check_parquet.py +++ b/scripts/check_parquet.py @@ -1,6 +1,6 @@ import pyarrow.parquet as pq -file = pq.ParquetFile("filesys/test/test_write/parquet.test") +file = pq.ParquetFile("filesys/test/test1/parquet/test1.1.parquet") print(f'num_row_groups: {file.num_row_groups}') print(f'metadata: {file.metadata}')