Merge pull request #1233 from evidence-dev/databricks-db-adapter

Databricks db adapter

.changeset/green-pens-repair.md

@@ -0,0 +1,8 @@
+---
+'@evidence-dev/databricks': minor
+'@evidence-dev/db-orchestrator': minor
+'@evidence-dev/core-components': patch
+'evidence-docs': patch
+---
+
+add databricks connector

packages/core-components/src/lib/unsorted/ui/Databases/DatabaseSettingsPanel.svelte

@@ -11,6 +11,7 @@
 	import MysqlForm from './MysqlForm.svelte';
 	import SqliteForm from './SqliteForm.svelte';
 	import DuckdbForm from './DuckdbForm.svelte';
+	import DatabricksForm from './DatabricksForm.svelte';
 	import CSVForm from './CSVForm.svelte';
 	import MSSQLForm from './MSSQLForm.svelte';
 
@@ -40,6 +41,7 @@
 		{ id: 'snowflake', name: 'Snowflake', formComponent: SnowflakeForm },
 		{ id: 'sqlite', name: 'SQLite', formComponent: SqliteForm },
 		{ id: 'duckdb', name: 'DuckDB', formComponent: DuckdbForm },
+		{ id: 'databricks', name: 'Databricks', formComponent: DatabricksForm },
 		{ id: 'csv', name: 'CSV', formComponent: CSVForm },
 		{ id: 'mssql', name: 'SQL Server', formComponent: MSSQLForm }
 	];

packages/core-components/src/lib/unsorted/ui/Databases/DatabricksForm.svelte

@@ -0,0 +1,50 @@
+<script>
+	export let credentials;
+	export let existingCredentials;
+	export let disableSave;
+
+	credentials = { ...existingCredentials };
+
+	let opts = [
+		{
+			id: 'token',
+			label: 'Personal Access Token',
+			type: 'password',
+			optional: false,
+			override: false,
+			placeholder: 'dapi12345678901234567890123456789012',
+			value: credentials.token ?? ''
+		},
+		{
+			id: 'host',
+			label: 'Server Hostname',
+			type: 'text',
+			optional: false,
+			override: false,
+			placeholder: 'dbc-a1b2345c-d6e7.cloud.databricks.com',
+			value: credentials.host ?? ''
+		},
+		{
+			id: 'path',
+			label: 'HTTP Path',
+			type: 'text',
+			optional: false,
+			override: false,
+			placeholder:
+				'sql/protocolv1/o/1234567890123456/1234-567890-abcdefgh or /sql/1.0/endpoints/a1b234c5678901d2',
+			value: credentials.path ?? ''
+		},
+		{
+			id: 'port',
+			label: 'Port',
+			type: 'text',
+			optional: true,
+			override: false,
+			placeholder: '443',
+			value: credentials.port ?? '443'
+		}
+	];
+	import GenericForm from './GenericForm.svelte';
+</script>
+
+<GenericForm {opts} bind:credentials bind:disableSave />

packages/core-components/src/lib/unsorted/ui/Databases/GenericForm.svelte

@@ -66,7 +66,7 @@
 				id={opt.id}
 				name={opt.id}
 				data-test-id={opt.dataTestId ?? opt.id}
-				placeholder="password"
+				placeholder={opt.placeholder ?? 'password'}
 				bind:value={credentials[opt.id]}
 				on:keyup={handleChange}
 			/>
@@ -107,7 +107,7 @@
 				type="password"
 				id={opt.id}
 				name={opt.id}
-				placeholder="password"
+				placeholder={opt.placeholder ?? 'password'}
 				bind:value={credentials[opt.id]}
 				on:keyup={handleChange}
 			/>

packages/databricks/index.cjs

@@ -0,0 +1,113 @@
+const { getEnv } = require('@evidence-dev/db-commons');
+const { DBSQLClient, thrift } = require('@databricks/sql');
+const TTypeId = thrift.TCLIService_types.TTypeId;
+
+const envMap = {
+	host: [
+		{ key: 'EVIDENCE_DATABRICKS_HOST', deprecated: false },
+		{ key: 'DATABRICKS_HOST', deprecated: false }
+	],
+	port: [
+		{ key: 'EVIDENCE_DATABRICKS_PORT', deprecated: false },
+		{ key: 'DATABRICKS_PORT', deprecated: false }
+	],
+	path: [
+		{ key: 'EVIDENCE_DATABRICKS_PATH', deprecated: false },
+		{ key: 'DATABRICKS_PATH', deprecated: false }
+	],
+	token: [
+		{ key: 'EVIDENCE_DATABRICKS_TOKEN', deprecated: false },
+		{ key: 'DATABRICKS_TOKEN', deprecated: false }
+	]
+};
+
+function nativeTypeToEvidenceType(data) {
+	switch (data) {
+		case TTypeId.BOOLEAN_TYPE:
+			return 'boolean';
+		case TTypeId.DATE_TYPE:
+		case TTypeId.TIMESTAMP_TYPE:
+			return 'date';
+		case TTypeId.DECIMAL_TYPE:
+		case TTypeId.BIGINT_TYPE:
+		case TTypeId.FLOAT_TYPE:
+		case TTypeId.DOUBLE_TYPE:
+		case TTypeId.INT_TYPE:
+		case TTypeId.SMALLINT_TYPE:
+		case TTypeId.TINYINT_TYPE:
+			return 'number';
+		case TTypeId.STRUCT_TYPE:
+		case TTypeId.MAP_TYPE:
+		case TTypeId.ARRAY_TYPE:
+		case TTypeId.UNION_TYPE:
+		case TTypeId.USER_DEFINED_TYPE:
+		case TTypeId.NULL_TYPE:
+		case TTypeId.INTERVAL_YEAR_MONTH_TYPE:
+		case TTypeId.INTERVAL_DAY_TIME_TYPE:
+		case TTypeId.STRING_TYPE:
+		case TTypeId.CHAR_TYPE:
+		case TTypeId.VARCHAR_TYPE:
+		case TTypeId.BINARY_TYPE:
+		default:
+			return 'string';
+	}
+}
+
+/**
+ * @template {Function} T
+ * @typedef {Awaited<ReturnType<T>>} Returned
+ */
+
+/**
+ *
+ * @param {Returned<Returned<import("@databricks/sql").DBSQLSession["executeStatement"]>["getSchema"]>} schema
+ * @returns {{ name: string; evidenceType: string; typeFidelity: string; }[]}
+ */
+const mapResultsToEvidenceColumnTypes = function (schema) {
+	return schema?.columns.map((column) => {
+		let typeFidelity = 'precise';
+		let evidenceType = nativeTypeToEvidenceType(column.typeDesc.types[0]?.primitiveEntry?.type);
+		if (!evidenceType) {
+			typeFidelity = 'inferred';
+			evidenceType = 'string';
+		}
+		return { name: column.columnName, evidenceType, typeFidelity };
+	});
+};
+
+const runQuery = async (queryString, database = {}) => {
+	const credentials = {
+		authType: 'access-token',
+		clientId: 'Evidence',
+		host: database.host ?? getEnv(envMap, 'host'),
+		port: Number(database.port ?? getEnv(envMap, 'port') ?? 443),
+		path: database.path ?? getEnv(envMap, 'path'),
+		token: database.token ?? getEnv(envMap, 'token')
+	};
+
+	try {
+		const client = new DBSQLClient();
+		const connection = await client.connect(credentials);
+		const session = await connection.openSession();
+
+		const query = await session.executeStatement(queryString);
+
+		const rows = await query.fetchAll();
+		const schema = await query.getSchema();
+
+		await query.close();
+		await session.close();
+		await connection.close();
+		await client.close();
+
+		return { rows, columnTypes: mapResultsToEvidenceColumnTypes(schema) };
+	} catch (err) {
+		if (err.message) {
+			throw err.message;
+		} else {
+			throw err;
+		}
+	}
+};
+
+module.exports = runQuery;

packages/databricks/package.json

@@ -0,0 +1,19 @@
+{
+	"name": "@evidence-dev/databricks",
+	"version": "0.0.1",
+	"description": "Databricks driver for Evidence projects",
+	"main": "index.cjs",
+	"author": "evidence.dev",
+	"license": "MIT",
+	"scripts": {
+		"test": "uvu test test.js"
+	},
+	"dependencies": {
+		"@evidence-dev/db-commons": "workspace:^",
+		"@databricks/sql": "^1.5.0"
+	},
+	"devDependencies": {
+		"dotenv": "^16.0.1"
+	},
+	"type": "module"
+}

packages/databricks/test/test.js

@@ -0,0 +1,94 @@
+import { test } from 'uvu';
+import * as assert from 'uvu/assert';
+import runQuery from '../index.cjs';
+import { TypeFidelity } from '@evidence-dev/db-commons';
+
+let results;
+
+test('query runs', async () => {
+	if (process.env.DATABRICKS_TOKEN || process.env.EVIDENCE_DATABRICKS_TOKEN) {
+		try {
+			results = await runQuery(`
+				SELECT
+					cast(100 AS BIGINT) as bigint_col,
+					cast('true' AS BINARY) as binary_col,
+					cast(1 AS BOOLEAN) as boolean_col,
+					cast('1908-03-15' AS DATE) as date_col,
+					cast(100.0 AS DECIMAL(10,2)) as decimal_col,
+					cast(100.0 AS DOUBLE) as double_col,
+					cast(100.0 AS FLOAT) as float_col,
+					cast(100 AS INT) as int_col,
+					INTERVAL '-3600' MONTH as interval_col,
+					cast(null AS VOID) as null_col,
+					cast(100 AS SMALLINT) as smallint_col,
+					cast('string' AS STRING) as string_col,
+					cast('1908-03-15 00:00:00' AS TIMESTAMP) as timestamp_col,
+					cast('1908-03-15 00:00:00' AS TIMESTAMP_NTZ) as timestamp_ntz_col,
+					cast(2 AS TINYINT) as tinyint_col
+			`);
+			assert.instance(results.rows, Array);
+			assert.instance(results.columnTypes, Array);
+			assert.type(results.rows[0], 'object');
+
+			let actualColumnTypes = results.columnTypes.map((columnType) => columnType.evidenceType);
+			let actualColumnNames = results.columnTypes.map((columnType) => columnType.name);
+			let actualTypePrecisions = results.columnTypes.map((columnType) => columnType.typeFidelity);
+
+			let expectedColumnTypes = [
+				'number',
+				'string',
+				'boolean',
+				'date',
+				'number',
+				'number',
+				'number',
+				'number',
+				'string',
+				'string',
+				'number',
+				'string',
+				'date',
+				'date',
+				'number'
+			];
+			let expectedColumnNames = [
+				'bigint_col',
+				'binary_col',
+				'boolean_col',
+				'date_col',
+				'decimal_col',
+				'double_col',
+				'float_col',
+				'int_col',
+				'interval_col',
+				'null_col',
+				'smallint_col',
+				'string_col',
+				'timestamp_col',
+				'timestamp_ntz_col',
+				'tinyint_col'
+			];
+			let expectedTypePrecision = Array(15).fill(TypeFidelity.PRECISE);
+
+			assert.equal(
+				true,
+				expectedColumnTypes.length === actualColumnTypes.length &&
+					expectedColumnTypes.every((value, index) => value === actualColumnTypes[index])
+			);
+			assert.equal(
+				true,
+				expectedColumnNames.length === actualColumnNames.length &&
+					expectedColumnNames.every((value, index) => value === actualColumnNames[index])
+			);
+			assert.equal(
+				true,
+				expectedTypePrecision.length === actualTypePrecisions.length &&
+					expectedTypePrecision.every((value, index) => value === actualTypePrecisions[index])
+			);
+		} catch (e) {
+			throw Error(e);
+		}
+	}
+});
+
+test.run();

packages/db-orchestrator/package.json

@@ -12,6 +12,7 @@
 		"@evidence-dev/duckdb": "workspace:^",
 		"@evidence-dev/mssql": "workspace:^",
 		"@evidence-dev/mysql": "workspace:^",
+		"@evidence-dev/databricks": "workspace:^",
 		"@evidence-dev/postgres": "workspace:^",
 		"@evidence-dev/trino": "workspace:^",
 		"@evidence-dev/redshift": "workspace:^",