#116 Make BucketFS base path configurable (#117) (#121)

Co-authored-by: Christoph Kuhnke <[email protected]>

.github/workflows/ci-build.yml

@@ -61,9 +61,7 @@ jobs:
       - name: Install dependencies
         run: |
           go get -t ./...
-
-      - name: Revert go.sum modification by previous "go get"
-        run: |
+          echo Revert go.sum modification by previous "go get"
           go mod tidy
 
       - name: Check generated code
@@ -73,6 +71,8 @@ jobs:
             echo "Project files did change:\n"
             git --no-pager diff
             exit 1
+          else
+            echo "Generated code is up-to-date"
           fi
 
       - name: Build

.project-keeper.yml

@@ -24,7 +24,7 @@ sources:
     path: registry/package.json
   - type: npm
     path: pkg/parameterValidator/package.json
-version: 0.4.1
+version: 0.5.0
 linkReplacements:
   - "https://github.com/exasol/exasol-test-setup-abstraction-server/blob/go-client/v0.2.3/go-client/LICENSE|https://github.com/exasol/exasol-test-setup-abstraction-server/blob/main/LICENSE"
   - "Unknown|https://github.com/DATA-DOG/go-sqlmock/blob/master/LICENSE"

.vscode/settings.json

@@ -1,31 +1,26 @@
 {
-    "go.testTimeout": "240s",
-    "go.testFlags": [
-        "-p",
-        "1",
-        "-v"
-    ],
-    "go.coverOnSingleTest": true,
-    "go.coverOnSingleTestFile": true,
-    "go.coverOnTestPackage": true,
-    "go.lintOnSave": "workspace",
-    "go.lintTool": "golangci-lint",
-    //"go.lintFlags": ["--enable-all"],
-    "sonarlint.connectedMode.project": {
-        "connectionId": "exasol",
-        "projectKey": "com.exasol:extension-manager"
-    },
-    "editor.formatOnSave": true,
-    "editor.codeActionsOnSave": {
-        "source.organizeImports": true,
-        "source.generate.finalModifiers": true
-    },
-    "java.saveActions.organizeImports": true,
-    "java.sources.organizeImports.starThreshold": 3,
-    "java.sources.organizeImports.staticStarThreshold": 3,
-    "java.test.config": {
-        "vmArgs": [
-            "-Djava.util.logging.config.file=src/test/resources/logging.properties"
-        ]
-    },
-}
+  "go.testTimeout": "240s",
+  "go.testFlags": ["-p", "1", "-v"],
+  "go.coverOnSingleTest": true,
+  "go.coverOnSingleTestFile": true,
+  "go.coverOnTestPackage": true,
+  "go.lintOnSave": "workspace",
+  "go.lintTool": "golangci-lint",
+  "sonarlint.connectedMode.project": {
+    "connectionId": "exasol",
+    "projectKey": "com.exasol:extension-manager"
+  },
+  "editor.formatOnSave": true,
+  "editor.codeActionsOnSave": {
+    "source.organizeImports": true,
+    "source.generate.finalModifiers": true
+  },
+  "java.saveActions.organizeImports": true,
+  "java.sources.organizeImports.starThreshold": 3,
+  "java.sources.organizeImports.staticStarThreshold": 3,
+  "java.test.config": {
+    "vmArgs": [
+      "-Djava.util.logging.config.file=src/test/resources/logging.properties"
+    ]
+  }
+}

cmd/main.go

@@ -31,7 +31,11 @@ func main() {
 
 func startServer(pathToExtensionFolder string, serverAddress string) {
 	log.Printf("Starting extension manager with extension folder %q", pathToExtensionFolder)
-	restApi := restAPI.Create(extensionController.Create(pathToExtensionFolder, restAPI.EXTENSION_SCHEMA_NAME), serverAddress)
+	controller := extensionController.CreateWithConfig(extensionController.ExtensionManagerConfig{
+		ExtensionRegistryURL: pathToExtensionFolder,
+		ExtensionSchema:      restAPI.EXTENSION_SCHEMA_NAME,
+		BucketFSBasePath:     "/buckets/bfsdefault/default/"})
+	restApi := restAPI.Create(controller, serverAddress)
 	restApi.Serve()
 }
 
@@ -62,7 +66,7 @@ func generateOpenAPIJson() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	err = restAPI.AddPublicEndpoints(api, restAPI.ExtensionManagerConfig{})
+	err = restAPI.AddPublicEndpoints(api, extensionController.ExtensionManagerConfig{})
 	if err != nil {
 		return nil, err
 	}

doc/changes/changes_0.4.1.md → doc/changes/changes_0.5.0.md

@@ -1,9 +1,11 @@
-# Extension Manager 0.4.1, released 2023-??-??
+# Extension Manager 0.5.0, released 2023-??-??
 
 Code name:
 
 ## Summary
 
+This release allows configuring the BucketFS base path where EM expects extension files to be located. EM searches this path recursively, so files are also found in subdirectories.
+
 This release improves error handling when using extensions not implementing all functions required by EM. EM now returns a helpful error message instead of failing with a `nil`-pointer error.
 
 A common scenario for an extension not implementing a required function is when the extension had been built using an older version of EM's extension interface.
@@ -15,15 +17,39 @@ A common scenario for an extension not implementing a required function is when
 
 ## Dependency Updates
 
+### Extension-manager
+
+#### Compile Dependency Updates
+
+* Updated `github.com/dop251/goja:v0.0.0-20230626124041-ba8a63e79201` to `v0.0.0-20230707174833-636fdf960de1`
+* Updated `github.com/go-chi/chi/v5:v5.0.8` to `v5.0.10`
+
+#### Test Dependency Updates
+
+* Updated `golang.org/x/mod:v0.11.0` to `v0.12.0`
+
 ### Extension Manager Java Client
 
+#### Compile Dependency Updates
+
+* Updated `io.swagger.core.v3:swagger-annotations:2.2.14` to `2.2.15`
+* Updated `org.glassfish.jersey.core:jersey-client:2.39.1` to `2.40`
+* Updated `org.glassfish.jersey.inject:jersey-hk2:2.39.1` to `2.40`
+* Updated `org.glassfish.jersey.media:jersey-media-json-jackson:2.39.1` to `2.40`
+* Updated `org.glassfish.jersey.media:jersey-media-multipart:2.39.1` to `2.40`
+
+#### Test Dependency Updates
+
+* Updated `nl.jqno.equalsverifier:equalsverifier:3.14.3` to `3.15`
+
 #### Plugin Dependency Updates
 
 * Updated `com.exasol:error-code-crawler-maven-plugin:1.2.3` to `1.3.0`
 * Updated `com.exasol:project-keeper-maven-plugin:2.9.7` to `2.9.9`
 * Updated `org.apache.maven.plugins:maven-gpg-plugin:3.0.1` to `3.1.0`
 * Updated `org.apache.maven.plugins:maven-surefire-plugin:3.0.0` to `3.1.2`
 * Updated `org.basepom.maven:duplicate-finder-maven-plugin:1.5.1` to `2.0.1`
+* Updated `org.codehaus.mojo:build-helper-maven-plugin:3.3.0` to `3.4.0`
 * Updated `org.codehaus.mojo:flatten-maven-plugin:1.4.1` to `1.5.0`
 * Updated `org.codehaus.mojo:versions-maven-plugin:2.15.0` to `2.16.0`
 * Updated `org.jacoco:jacoco-maven-plugin:0.8.9` to `0.8.10`
@@ -32,7 +58,13 @@ A common scenario for an extension not implementing a required function is when
 
 #### Compile Dependency Updates
 
-* Updated `com.exasol:extension-manager-client-java:0.4.0` to `0.4.1`
+* Updated `com.exasol:exasol-test-setup-abstraction-java:2.0.1` to `2.0.2`
+* Updated `com.exasol:extension-manager-client-java:0.4.0` to `0.5.0`
+* Removed `io.netty:netty-handler:4.1.94.Final`
+
+#### Test Dependency Updates
+
+* Updated `com.exasol:udf-debugging-java:0.6.8` to `0.6.10`
 
 #### Plugin Dependency Updates
 

doc/design.md

@@ -137,17 +137,17 @@ An extension might consist of JDBC driver, artifacts, configuration and database
 
 In the initial version when managing extensions EM requires the following components to be available in BucketFS and does not actively manage them:
 * JDBC driver
-* artifacts in BucketFS, e.g. jar files
+* Artifacts in BucketFS, e.g. jar files
 * 3rd party libraries
 
 Instead, for managing extensions EM only considers the following database objects:
-* ADAPTER SCRIPT
-* SET SCRIPT
-* CONNECTION
-* VIRTUAL SCHEMA including mapping
+* `ADAPTER SCRIPT`
+* `SET SCRIPT`
+* `CONNECTION`
+* `VIRTUAL SCHEMA` including mapping
 
 Note: In the following cases a separate artifact in BucketFS is not required, as extension definition can inline these contents into the SQL statements of either adapter script or set script:
-* VIRTUAL SCHEMA mapping, json content
+* JSON content of virtual schema EDML mapping, see [EDML user guide](https://github.com/exasol/virtual-schema-common-document/blob/main/doc/user_guide/edml_user_guide.md)
 * Lua scripts
 
 When creating the required database objects EM will read all required arguments, configurations and credentials from the [extension definition](#extension-definitions). The extension definition can also define [parameters](#parameters-for-extension-configuration). EM will evaluate the parameter definitions and then ask the user to enter a value for each of the parameters. See also an example of the [installation process](#installation-process).
@@ -159,6 +159,30 @@ Covers:
 
 Needs: impl, utest
 
+#### Configurable BucketFS Path
+`dsn~configure-bucketfs-path~1`
+
+EM allows configuring the BucketFS path where extensions artifacts like JAR files are located.
+
+Rationale:
+
+As described in [`dsn~extension-components~1`](#components-of-an-extension) an extension may require files in BucketFS. The path to the BucketFS location depends on the Exasol database deployment.
+
+Needs: impl, utest, itest
+
+#### Resolving Files in BucketFS
+`dsn~resolving-files-in-bucketfs~1`
+
+EM resolves filenames to absolute paths in BucketFS.
+
+Rationale:
+
+Extensions require files in BucketFS. The exact location in BucketFS of the files is not known when creating an extension, only the file name (e.g. `document-files-virtual-schema-dist-7.3.3-s3-2.6.2.jar`). In order to create adapter scripts, the extension must know the absolute path of the file in BucketFS (e.g. `/buckets/bfsdefault/default/document-files-virtual-schema-dist-7.3.3-s3-2.6.2.jar`). So EM must allow the extension to resolve file names to an absolute path in BucketFS.
+
+Needs: impl, utest, itest
+
+#### Resolving Artifacts in BucketFS
+
 ### Extension Definitions
 `dsn~extension-definition~1`
 
@@ -183,7 +207,7 @@ Covers:
 
 Needs: impl, utest, itest
 
-#### Storage for extension definitions
+#### Storage for Extension Definitions
 `dsn~extension-definitions-storage~1`
 
 The extension definitions are placed in a storage that is accessible from the extension-manager backend. Access from inside the database is not required.