#147 Refactoring: prepare BucketFS speedup (#151)

.project-keeper.yml

@@ -26,11 +26,10 @@ sources:
     path: registry-upload/package.json
   - type: npm
     path: pkg/parameterValidator/package.json
-version: 0.5.2
+version: 0.5.3
 linkReplacements:
   - "Unknown|https://github.com/DATA-DOG/go-sqlmock/blob/master/LICENSE"
   - "https://github.com/swagger-api/swagger-core/modules/swagger-annotations|https://github.com/swagger-api/swagger-core/tree/master/modules/swagger-annotations"
   - "jquery.org/license|https://jquery.org/license/"
   - "https://github.com/swagger-api/swagger-codegen/modules/swagger-codegen-maven-plugin|https://github.com/swagger-api/swagger-codegen/tree/master/modules/swagger-codegen-maven-plugin"
 excludes:
-  - "E-PK-CORE-18: Outdated content: '.github/workflows/project-keeper-verify.yml'"

doc/changes/changes_0.5.3.md

@@ -0,0 +1,27 @@
+# Extension Manager 0.5.3, released 2023-??-??
+
+Code name: Speedup listing extensions
+
+## Summary
+
+This release speeds up listing extensions, especially when there are many files in BucketFS.
+
+**Notes:** Starting with this release EM is tested against Exasol version 8 instead of 7.1. This means that integration tests using `extension-manager-integration-test-java` will also need to run with Exasol 8.
+
+## Bugfix
+
+* #147: Improved speed of listing available extensions
+
+## Dependency Updates
+
+### Extension-manager
+
+#### Compile Dependency Updates
+
+* Updated `github.com/exasol/exasol-driver-go:v1.0.2` to `v1.0.3`
+
+### Extension Integration Tests Library
+
+#### Compile Dependency Updates
+
+* Updated `com.exasol:extension-manager-client-java:0.5.2` to `0.5.3`

go.mod

@@ -11,7 +11,7 @@ require (
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
-	github.com/exasol/exasol-driver-go v1.0.2
+	github.com/exasol/exasol-driver-go v1.0.3
 	github.com/exasol/exasol-test-setup-abstraction-server/go-client v0.3.3
 	github.com/go-chi/chi/v5 v5.0.10
 	github.com/kinbiko/jsonassert v1.1.1
@@ -49,7 +49,7 @@ require (
 	github.com/stretchr/objx v0.5.1 // indirect
 	github.com/swaggo/files v1.0.1 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/net v0.16.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect

pkg/backend/ExasolSqlClient_test.go

@@ -27,6 +27,7 @@ func (suite *ExasolSqlClientUTestSuite) SetupTest() {
 	}
 	suite.db = db
 	suite.dbMock = mock
+	suite.dbMock.MatchExpectationsInOrder(true)
 }
 
 func (suite *ExasolSqlClientUTestSuite) createClient() SimpleSQLClient {
@@ -104,12 +105,19 @@ func (suite *ExasolSqlClientUTestSuite) TestQueryFails() {
 /* [utest -> dsn~extension-context-sql-client~1]. */
 func (suite *ExasolSqlClientUTestSuite) TestQuerySucceeds() {
 	client := suite.createClient()
-	suite.dbMock.ExpectQuery("query").WillReturnRows(sqlmock.NewRows([]string{"col1", "col2"}).AddRow(1, "a").AddRow(2, "b")).RowsWillBeClosed()
+	suite.dbMock.ExpectQuery("query").WillReturnRows(
+		sqlmock.NewRowsWithColumnDefinition(
+			sqlmock.NewColumn("col1").OfType("type1", "sample"),
+			sqlmock.NewColumn("col2").OfType("type2", "sample"),
+		).
+			AddRow(1, "a").
+			AddRow(2, "b")).
+		RowsWillBeClosed()
 	result, err := client.Query("query")
 	suite.NoError(err)
 	suite.NotNil(result)
 	suite.Equal(&QueryResult{
-		Columns: []Column{{Name: "col1"}, {Name: "col2"}},
+		Columns: []Column{{Name: "col1", TypeName: "type1"}, {Name: "col2", TypeName: "type2"}},
 		Rows:    []Row{{int64(1), "a"}, {int64(2), "b"}}}, result)
 }
 

pkg/extensionAPI/context/bucketfs.go

@@ -1,11 +1,9 @@
 package context
 
 import (
-	"context"
-	"database/sql"
 	"fmt"
 
-	"github.com/exasol/extension-manager/pkg/extensionController/bfs"
+	"github.com/exasol/extension-manager/pkg/extensionController/transaction"
 )
 
 // BucketFsContext allows extensions to interact with BucketFS.
@@ -16,16 +14,22 @@ type BucketFsContext interface {
 }
 
 type bucketFsContextImpl struct {
-	bucketFsClient bfs.BucketFsAPI
-	context        context.Context
-	db             *sql.DB
+	txCtx *transaction.TransactionContext
 }
 
 /* [impl -> dsn~resolving-files-in-bucketfs~1]. */
 func (b *bucketFsContextImpl) ResolvePath(fileName string) string {
-	path, err := b.bucketFsClient.FindAbsolutePath(b.context, b.db, fileName)
+	path, err := b.resolvePath(fileName)
 	if err != nil {
 		reportError(fmt.Errorf("failed to find absolute path for file %q: %w", fileName, err))
 	}
 	return path
 }
+
+func (b *bucketFsContextImpl) resolvePath(fileName string) (string, error) {
+	bfsClient, err := b.txCtx.GetBucketFsClient()
+	if err != nil {
+		return "", err
+	}
+	return bfsClient.FindAbsolutePath(fileName)
+}

pkg/extensionAPI/context/bucketfs_mock.go

@@ -0,0 +1,27 @@
+package context
+
+import (
+	"github.com/stretchr/testify/mock"
+)
+
+type BucketFsContextMock struct {
+	mock.Mock
+}
+
+func CreateBucketFsContextMock() *BucketFsContextMock {
+	//nolint:exhaustruct // Empty struct is OK for Mock
+	return &BucketFsContextMock{}
+}
+
+func (mock *BucketFsContextMock) SimulateResolvePath(fileName string, result string) {
+	mock.On("ResolvePath", fileName).Return(result)
+}
+
+func (mock *BucketFsContextMock) SimulateResolvePathPanics(fileName string, panicMessage string) {
+	mock.On("ResolvePath", fileName).Panic(panicMessage)
+}
+
+func (mock *BucketFsContextMock) ResolvePath(fileName string) string {
+	mockArgs := mock.Called(fileName)
+	return mockArgs.String(0)
+}

pkg/extensionAPI/context/context.go

@@ -3,27 +3,22 @@ package context
 import (
 	"github.com/exasol/extension-manager/pkg/backend"
 	"github.com/exasol/extension-manager/pkg/extensionAPI/exaMetadata"
-	"github.com/exasol/extension-manager/pkg/extensionController/bfs"
 	"github.com/exasol/extension-manager/pkg/extensionController/transaction"
 )
 
-func CreateContext(txCtx *transaction.TransactionContext, extensionSchemaName string, bucketFsBasePath string) *ExtensionContext {
+func CreateContext(txCtx *transaction.TransactionContext, extensionSchemaName string) *ExtensionContext {
 	var sqlClient backend.SimpleSQLClient = backend.NewSqlClient(txCtx.GetContext(), txCtx.GetTransaction())
-	var bucketFsClient bfs.BucketFsAPI = bfs.CreateBucketFsAPI(bucketFsBasePath)
 	var metadataReader exaMetadata.ExaMetadataReader = exaMetadata.CreateExaMetaDataReader()
-	return CreateContextWithClient(extensionSchemaName, txCtx, sqlClient, bucketFsClient, metadataReader)
+	var bfsContext BucketFsContext = &bucketFsContextImpl{txCtx: txCtx}
+	return CreateContextWithClient(extensionSchemaName, txCtx, sqlClient, bfsContext, metadataReader)
 }
 
 func CreateContextWithClient(extensionSchemaName string, txCtx *transaction.TransactionContext,
-	client backend.SimpleSQLClient, bucketFsClient bfs.BucketFsAPI, metadataReader exaMetadata.ExaMetadataReader) *ExtensionContext {
+	client backend.SimpleSQLClient, bucketFsContext BucketFsContext, metadataReader exaMetadata.ExaMetadataReader) *ExtensionContext {
 	return &ExtensionContext{
 		ExtensionSchemaName: extensionSchemaName,
 		SqlClient:           &contextSqlClient{client},
-		BucketFs: &bucketFsContextImpl{
-			bucketFsClient: bucketFsClient,
-			context:        txCtx.GetContext(),
-			db:             txCtx.GetDBConnection(),
-		},
+		BucketFs:            bucketFsContext,
 		Metadata: &metadataContextImpl{
 			transaction:    txCtx.GetTransaction(),
 			schemaName:     extensionSchemaName,

pkg/extensionAPI/context/context_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/DATA-DOG/go-sqlmock"
 	"github.com/exasol/extension-manager/pkg/backend"
 	"github.com/exasol/extension-manager/pkg/extensionAPI/exaMetadata"
-	"github.com/exasol/extension-manager/pkg/extensionController/bfs"
 	"github.com/exasol/extension-manager/pkg/extensionController/transaction"
 	"github.com/stretchr/testify/suite"
 )
@@ -18,7 +17,7 @@ type ContextSuite struct {
 	suite.Suite
 	db                 *sql.DB
 	dbMock             sqlmock.Sqlmock
-	bucketFSMock       *bfs.BucketFsMock
+	bucketFSMock       *BucketFsContextMock
 	metadataReaderMock *exaMetadata.ExaMetaDataReaderMock
 }
 
@@ -27,14 +26,15 @@ func TestContextSuite(t *testing.T) {
 }
 
 const EXTENSION_SCHEMA = "EXT_SCHEMA"
+const BUCKETFS_BASE_PATH = "bucketfs-base-path"
 
 func (suite *ContextSuite) SetupTest() {
 	db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
 	suite.NoError(err)
 	suite.db = db
 	suite.dbMock = mock
 	suite.dbMock.MatchExpectationsInOrder(true)
-	suite.bucketFSMock = bfs.CreateBucketFsMock()
+	suite.bucketFSMock = CreateBucketFsContextMock()
 	suite.metadataReaderMock = exaMetadata.CreateExaMetaDataReaderMock(EXTENSION_SCHEMA)
 }
 
@@ -55,10 +55,16 @@ func (suite *ContextSuite) TestCreate() {
 /* [utest -> dsn~extension-context-sql-client~1]. */
 func (suite *ContextSuite) TestSqlClientQuerySuccess() {
 	ctx := suite.createContext()
-	suite.dbMock.ExpectQuery("select 1").WillReturnRows(sqlmock.NewRows([]string{"col1", "col2"}).AddRow(1, "a").AddRow(2, "b")).RowsWillBeClosed()
+	suite.dbMock.ExpectQuery("select 1").WillReturnRows(sqlmock.NewRowsWithColumnDefinition(
+		sqlmock.NewColumn("col1").OfType("type1", "sample"),
+		sqlmock.NewColumn("col2").OfType("type2", "sample"),
+	).
+		AddRow(1, "a").
+		AddRow(2, "b")).
+		RowsWillBeClosed()
 	result := ctx.SqlClient.Query("select 1")
 	suite.Equal(backend.QueryResult{
-		Columns: []backend.Column{{Name: "col1"}, {Name: "col2"}},
+		Columns: []backend.Column{{Name: "col1", TypeName: "type1"}, {Name: "col2", TypeName: "type2"}},
 		Rows:    []backend.Row{{int64(1), "a"}, {int64(2), "b"}}}, result)
 }
 
@@ -90,14 +96,14 @@ func (suite *ContextSuite) TestSqlClientExecuteFailure() {
 /* [utest -> dsn~extension-context-bucketfs~1]. */
 func (suite *ContextSuite) TestBucketFsResolvePath() {
 	ctx := suite.createContextWithClients()
-	suite.bucketFSMock.SimulateAbsolutePath("file.txt", "/absolute/path/file.txt")
+	suite.bucketFSMock.SimulateResolvePath("file.txt", "/absolute/path/file.txt")
 	suite.Equal("/absolute/path/file.txt", ctx.BucketFs.ResolvePath("file.txt"))
 }
 
 func (suite *ContextSuite) TestBucketFsResolvePathError() {
 	ctx := suite.createContextWithClients()
-	suite.bucketFSMock.SimulateAbsolutePathError("file.txt", fmt.Errorf("mock error"))
-	suite.PanicsWithError("failed to find absolute path for file \"file.txt\": mock error", func() {
+	suite.bucketFSMock.SimulateResolvePathPanics("file.txt", "mock error")
+	suite.PanicsWithValue("mock error", func() {
 		ctx.BucketFs.ResolvePath("file.txt")
 	})
 }
@@ -125,14 +131,14 @@ func (suite *ContextSuite) TestMetadataGetScriptByNameFails() {
 
 func (suite *ContextSuite) createContext() *ExtensionContext {
 	suite.dbMock.ExpectBegin()
-	txCtx, err := transaction.BeginTransaction(context.Background(), suite.db)
+	txCtx, err := transaction.BeginTransaction(context.Background(), suite.db, BUCKETFS_BASE_PATH)
 	suite.NoError(err)
-	return CreateContext(txCtx, "EXT_SCHEMA", "/bucketfs/base/path/")
+	return CreateContext(txCtx, "EXT_SCHEMA")
 }
 
 func (suite *ContextSuite) createContextWithClients() *ExtensionContext {
 	suite.dbMock.ExpectBegin()
-	txCtx, err := transaction.BeginTransaction(context.Background(), suite.db)
+	txCtx, err := transaction.BeginTransaction(context.Background(), suite.db, BUCKETFS_BASE_PATH)
 	suite.NoError(err)
 	return CreateContextWithClient("EXT_SCHEMA", txCtx, nil, suite.bucketFSMock, suite.metadataReaderMock)
 }

pkg/extensionAPI/errorHandlingExtension_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/exasol/extension-manager/pkg/backend"
 	"github.com/exasol/extension-manager/pkg/extensionAPI/context"
 	"github.com/exasol/extension-manager/pkg/extensionAPI/exaMetadata"
-	"github.com/exasol/extension-manager/pkg/extensionController/bfs"
 	"github.com/exasol/extension-manager/pkg/extensionController/transaction"
 	"github.com/stretchr/testify/suite"
 )
@@ -30,7 +29,7 @@ func (suite *ErrorHandlingExtensionSuite) SetupSuite() {
 		Category:                "category",
 		Description:             "desc",
 		InstallableVersions:     []rawJsExtensionVersion{{Name: "v1", Deprecated: true, Latest: false}, {Name: "v2", Deprecated: false, Latest: true}},
-		BucketFsUploads:         []BucketFsUpload{{Name: "uploadName"}},
+		BucketFsUploads:         []BucketFsUpload{{Name: "uploadName", DownloadURL: "url", LicenseURL: "license", FileSize: 123, BucketFsFilename: "filename"}},
 		GetParameterDefinitions: nil,
 		Install:                 nil,
 		Uninstall:               nil,
@@ -50,22 +49,26 @@ func (suite *ErrorHandlingExtensionSuite) TestProperties() {
 		Name:                "name",
 		Description:         "desc",
 		InstallableVersions: []JsExtensionVersion{{Name: "v1", Deprecated: true, Latest: false}, {Name: "v2", Deprecated: false, Latest: true}},
-		BucketFsUploads:     []BucketFsUpload{{Name: "uploadName"}},
+		BucketFsUploads:     []BucketFsUpload{{Name: "uploadName", DownloadURL: "url", LicenseURL: "license", FileSize: 123, BucketFsFilename: "filename"}},
 		extension:           suite.rawExtension,
 		vm:                  suite.extension.vm},
 		suite.extension)
 }
 
 const EXTENSION_SCHEMA = "extension_schema"
 
-func createMockContextWithClients(sqlClient backend.SimpleSQLClient, bucketFsClient bfs.BucketFsAPI, metadataReader exaMetadata.ExaMetadataReader) *context.ExtensionContext {
+func createMockContextWithClients(
+	sqlClient backend.SimpleSQLClient,
+	bucketFsContext context.BucketFsContext,
+	metadataReader exaMetadata.ExaMetadataReader,
+) *context.ExtensionContext {
 	txCtx := &transaction.TransactionContext{}
-	return context.CreateContextWithClient(EXTENSION_SCHEMA, txCtx, sqlClient, bucketFsClient, metadataReader)
+	return context.CreateContextWithClient(EXTENSION_SCHEMA, txCtx, sqlClient, bucketFsContext, metadataReader)
 }
 
 func createMockContext() *context.ExtensionContext {
 	var sqlClientMock backend.SimpleSQLClient = backend.CreateSimpleSqlClientMock()
-	var bucketFsClientMock bfs.BucketFsAPI = bfs.CreateBucketFsMock()
+	var bucketFsClientMock context.BucketFsContext = context.CreateBucketFsContextMock()
 	var metadataReader exaMetadata.ExaMetadataReader = exaMetadata.CreateExaMetaDataReaderMock(EXTENSION_SCHEMA)
 	return createMockContextWithClients(sqlClientMock, bucketFsClientMock, metadataReader)
 }