Merge pull request #2136 from evidence-dev/Release-2024-06-20

Release 2024-06-20

.changeset/afraid-parrots-hammer.md

@@ -0,0 +1,6 @@
+---
+'@evidence-dev/sdk': patch
+'@evidence-dev/universal-sql': patch
+---
+
+Add buildOptions.batchSize to connection.yaml spec

.changeset/beige-students-pretend.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/sdk': patch
+---
+
+Added specific updates to manifest support

.changeset/brave-students-joke.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/core-components': patch
+---
+
+Added url prop to Big value Component

.changeset/calm-cats-buy.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/sdk': patch
+---
+
+Added concept of locatedFiles to manifest

.changeset/gorgeous-monkeys-chew.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/sdk': patch
+---
+
+Added plugins/datasources export to SDK

.changeset/grumpy-suits-teach.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/duckdb': patch
+---
+
+Rearrange execution order; remove useless try/catch

.changeset/metal-apes-unite.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/evidence': minor
+---
+
+Switch to using sdk for datasources instead of plugin connector

.changeset/mighty-fireants-melt.md

@@ -0,0 +1,7 @@
+---
+'@evidence-dev/preprocess': patch
+'@evidence-dev/sdk': patch
+'@evidence-dev/core-components': patch
+---
+
+Fix source query triggering HMR on the page

.changeset/old-experts-travel.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/sdk': patch
+---
+
+Added legacy compatibility functionality to getEvidenceConfig

.changeset/proud-peaches-search.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/sdk': patch
+---
+
+Added queryCreated event and query time fields to Query

.changeset/real-elephants-trade.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/sdk': patch
+---
+
+Fixed issue with nested directories in sources

.changeset/shiny-lions-marry.md

@@ -0,0 +1,33 @@
+---
+'@evidence-dev/bigquery': patch
+'@evidence-dev/csv': patch
+'@evidence-dev/databricks': patch
+'@evidence-dev/duckdb': patch
+'@evidence-dev/faker-datasource': patch
+'@evidence-dev/motherduck': patch
+'@evidence-dev/mssql': patch
+'@evidence-dev/mysql': patch
+'@evidence-dev/postgres': patch
+'@evidence-dev/redshift': patch
+'@evidence-dev/snowflake': patch
+'@evidence-dev/sqlite': patch
+'@evidence-dev/trino': patch
+'@evidence-dev/evidence': patch
+'evidence-vscode': patch
+'@evidence-dev/component-utilities': patch
+'@evidence-dev/db-commons': patch
+'@evidence-dev/plugin-connector': patch
+'@evidence-dev/preprocess': patch
+'@evidence-dev/query-store': patch
+'@evidence-dev/sdk': patch
+'@evidence-dev/telemetry': patch
+'@evidence-dev/universal-sql': patch
+'@evidence-dev/core-components': patch
+'@evidence-dev/icons': patch
+'@evidence-dev/tailwind': patch
+'my-evidence-project': patch
+'@evidence-dev/components': patch
+'evidence-test-environment': patch
+---
+
+Remove db-orchestrator

.changeset/spotty-coats-tickle.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/evidence': patch
+---
+
+Fixed Custom Formatting

.changeset/strong-gifts-swim.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/components': patch
+---
+
+Added selectAllByDefault example and props to Docs

.changeset/tasty-beers-chew.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/core-components': patch
+---
+
+Added a new <QueryDebugger/> component to ease investigation into which queries have run on the page

.changeset/wicked-beers-act.md

@@ -0,0 +1,5 @@
+---
+'@evidence-dev/core-components': patch
+---
+
+Support dynamic title inputs in accordions

.github/workflows/chromatic.yml

@@ -0,0 +1,26 @@
+name: 'Chromatic'
+
+on: push
+
+jobs:
+  chromatic:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: pnpm/[email protected]
+        with:
+          version: 8.6.9
+      - uses: actions/setup-node@v3
+        with:
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm sources:core-components
+      - name: Core Components Chromatic
+        uses: chromaui/action@latest
+        with:
+          projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+          workingDir: packages/ui/core-components
+          exitZeroOnChanges: true # prevent the workflow from failing if there are visual changes

.github/workflows/ui-tests.yml → .github/workflows/playwright-tests.yml

@@ -1,4 +1,4 @@
-name: UI-Tests
+name: Playwright-Tests
 on:
   pull_request:
     branches: [main, next]

package.json

@@ -4,7 +4,6 @@
 		"@duckdb/duckdb-wasm": "1.28.0",
 		"@evidence-dev/component-utilities": "link:packages/lib/component-utilities",
 		"@evidence-dev/core-components": "link:packages/ui/core-components",
-		"@evidence-dev/db-orchestrator": "link:packages/lib/db-orchestrator",
 		"@evidence-dev/evidence": "link:packages/evidence",
 		"@evidence-dev/plugin-connector": "link:packages/lib/plugin-connector",
 		"@evidence-dev/preprocess": "link:packages/lib/preprocess",
@@ -78,7 +77,8 @@
 		"build:preprocess": "pnpm --filter ./packages/lib/preprocess build",
 		"build:plugin-connector": "pnpm --filter ./packages/lib/plugin-connector build",
 		"build:tailwind": "pnpm --filter ./packages/ui/tailwind build",
-		"build:query-store": "pnpm --filter ./packages/lib/query-store build"
+		"build:query-store": "pnpm --filter ./packages/lib/query-store build",
+		"sources:core-components": "pnpm --filter ./packages/ui/core-components sources"
 	},
 	"pnpm": {
 		"overrides": {

packages/datasources/duckdb/index.cjs

@@ -129,7 +129,6 @@ const runQuery = async (queryString, database, batchSize = 100000) => {
 	}
 
 	const mode = filename !== ':memory:' ? 'READ_ONLY' : 'READ_WRITE';
-
 	const db = await Database.create(filename, {
 		access_mode: mode,
 		custom_user_agent: 'evidence-dev'
@@ -143,7 +142,6 @@ const runQuery = async (queryString, database, batchSize = 100000) => {
 				encoding: 'utf-8'
 			});
 			await conn.exec(initScript);
-			console.log(`Ran Initialize`, initScript);
 		}
 	}
 

packages/datasources/faker/filter.mjs

@@ -22,7 +22,7 @@ export const filter = (rows, filters) => {
 	let outputRows = rows;
 	for (const filter of filters) {
 		switch (filter.type) {
-			case 'unique':
+			case 'unique': {
 				// O(n) unique function
 				/** @type {Record<string, boolean>} */
 				const uniq = {};
@@ -43,6 +43,7 @@ export const filter = (rows, filters) => {
 				outputRows = distinct;
 
 				break;
+			}
 			case 'ne':
 				outputRows = outputRows.filter((row) => {
 					// Map filter fields to their values in this row

packages/datasources/faker/index.mjs

@@ -141,7 +141,7 @@ const buildRow = async (
 		const col = schema[colName];
 		if ('type' in col) {
 			switch (col.type) {
-				case 'fk':
+				case 'fk': {
 					const targetTablePath = col.target?.tablePath;
 					if (!targetTablePath) {
 						throw new Error(
@@ -160,6 +160,7 @@ const buildRow = async (
 
 					output[colName] = targetRow[targetField];
 					break;
+				}
 				case 'id':
 					output[colName] = rowNum;
 					break;

packages/evidence/cli.js

@@ -7,7 +7,6 @@ import * as chokidar from 'chokidar';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import sade from 'sade';
-import { updateDatasourceOutputs } from '@evidence-dev/plugin-connector';
 import { logQueryEvent } from '@evidence-dev/telemetry';
 
 import { loadEnv } from 'vite';
@@ -158,11 +157,18 @@ const strictMode = function () {
 const buildHelper = function (command, args) {
 	const watchers = runFileWatcher(watchPatterns);
 	const flatArgs = flattenArguments(args);
+
 	// Run svelte kit build in the hidden directory
 	const child = spawn(command, flatArgs, {
 		shell: true,
 		cwd: '.evidence/template',
-		stdio: 'inherit'
+		stdio: 'inherit',
+		env: {
+			...process.env,
+			// used for source query HMR
+			EVIDENCE_DATA_URL_PREFIX: process.env.EVIDENCE_DATA_URL_PREFIX ?? 'static/data',
+			EVIDENCE_DATA_DIR: process.env.EVIDENCE_DATA_DIR ?? './static/data'
+		}
 	});
 	// Copy the outputs to the root of the project upon successful exit
 	child.on('exit', function (code) {
@@ -222,7 +228,13 @@ ${chalk.bold('[!] Unable to load source manifest')}
 			shell: true,
 			detached: false,
 			cwd: '.evidence/template',
-			stdio: 'inherit'
+			stdio: 'inherit',
+			env: {
+				...process.env,
+				// used for source query HMR
+				EVIDENCE_DATA_URL_PREFIX: process.env.EVIDENCE_DATA_URL_PREFIX ?? 'static/data',
+				EVIDENCE_DATA_DIR: process.env.EVIDENCE_DATA_DIR ?? './static/data'
+			}
 		});
 
 		child.on('exit', function () {
@@ -293,8 +305,7 @@ prog
 	.example('npx evidence sources --sources needful_things --queries orders,reviews')
 	.example('npx evidence sources --queries needful_things.orders,needful_things.reviews')
 	.example('npx evidence sources --sources needful_things,social_media')
-	.action(async (opts) => {
-		if (opts.debug) process.env.VITE_EVIDENCE_DEBUG = true;
+	.action(async () => {
 		if (process.argv.some((arg) => arg.includes('build:sources'))) {
 			console.log(
 				chalk.bold.red(
@@ -307,45 +318,22 @@ prog
 				)
 			);
 		}
-
-		loadEnvFile();
-		if (!opts.debug)
-			process.on('uncaughtException', (e) => {
-				console.error(e.message);
-				process.exit(1);
-			});
-		const sources = opts.sources?.split(',') ?? null;
-		const queries = opts.queries?.split(',') ?? null;
-
-		const isExampleProject = Boolean(process.env.__EXAMPLE_PROJECT);
-
-		if (!isExampleProject) {
-			const templatePath = path.join('.evidence', 'template');
-			await fs.mkdir(templatePath, { recursive: true });
-			process.chdir(templatePath);
+		if (!('EVIDENCE_DATA_DIR' in process.env)) {
+			process.env.EVIDENCE_DATA_DIR = './.evidence/template/static/data';
 		}
-
-		const dataDir = path.join('static', 'data');
-		const metaDir = path.join('.evidence-queries');
-
-		if (opts.debug) {
-			console.log('Building sources in', {
-				dataDir,
-				metaDir,
-				cwd: process.cwd(),
-				resolved: {
-					dataDir: path.resolve(dataDir),
-					metaDir: path.resolve(metaDir)
-				}
-			});
+		if (!('EVIDENCE_DATA_URL_PREFIX' in process.env)) {
+			process.env.EVIDENCE_DATA_URL_PREFIX = 'static/data';
 		}
+		loadEnvFile();
 
+		// The data directory is defined at import time (because we aren't using getters, and it is set once)
+		// So we need to import it here to give the opportunity to override it above
+		const sourcesCli = await import('@evidence-dev/sdk/plugins/datasources').then(
+			(m) => m.sourcesCli
+		);
 		logQueryEvent('build-sources-start');
-		await updateDatasourceOutputs(path.join('static', 'data'), '.evidence-queries', {
-			sources: sources ? new Set(sources) : sources,
-			queries: queries ? new Set(queries) : queries,
-			only_changed: opts.changed
-		});
+		await sourcesCli(...process.argv);
+		return;
 	});
 
 prog

packages/evidence/package.json

@@ -32,7 +32,6 @@
 	"devDependencies": {
 		"@evidence-dev/component-utilities": "workspace:*",
 		"@evidence-dev/core-components": "workspace:*",
-		"@evidence-dev/db-orchestrator": "workspace:*",
 		"@evidence-dev/tailwind": "workspace:*",
 		"@sveltejs/kit": "2.5.4",
 		"svelte": "4.2.12",
@@ -53,7 +52,6 @@
 	"peerDependencies": {
 		"@evidence-dev/component-utilities": "workspace:*",
 		"@evidence-dev/core-components": "workspace:*",
-		"@evidence-dev/db-orchestrator": "workspace:*",
 		"@evidence-dev/sdk": "workspace:*",
 		"@evidence-dev/query-store": "workspace:*",
 		"@evidence-dev/tailwind": "workspace:*",

packages/evidence/scripts/build-template.js

@@ -46,6 +46,7 @@ fsExtra.outputFileSync(
 	`import { sveltekit } from "@sveltejs/kit/vite"
 	import { evidenceVitePlugin } from "@evidence-dev/plugin-connector"
 	import { createLogger } from 'vite';
+	import { sourceQueryHmr } from '@evidence-dev/sdk/vite';
 
 	const logger = createLogger();
 	const loggerWarn = logger.warn;
@@ -72,7 +73,7 @@ fsExtra.outputFileSync(
     /** @type {import('vite').UserConfig} */
      const config = 
     {
-        plugins: [sveltekit(), evidenceVitePlugin()],
+        plugins: [sveltekit(), evidenceVitePlugin(), sourceQueryHmr()],
         optimizeDeps: {
             include: ['echarts-stat', 'echarts', 
 				// We need these to prevent HMR from doing a full page reload
@@ -93,7 +94,7 @@ fsExtra.outputFileSync(
             exclude: ['svelte-icons', '@evidence-dev/universal-sql']
         },
         ssr: {
-            external: ['@evidence-dev/db-orchestrator', '@evidence-dev/telemetry', 'blueimp-md5', '@evidence-dev/plugin-connector']
+            external: ['@evidence-dev/telemetry', 'blueimp-md5', '@evidence-dev/plugin-connector']
         },
         server: {
             fs: {

packages/extension/language-configuration.json

@@ -20,10 +20,9 @@
 		["\"", "\""]
 	],
 	"folding": {
-		"offSide": true,
 		"markers": {
-			"start": "^\\s*<!--\\s*#?region\\b.*-->",
-			"end": "^\\s*<!--\\s*#?endregion\\b.*-->"
+			"start": "^\\s*(```\\w+|{#if\\b|<s*\\b|<s*\\b|{#each\\b).*$",
+			"end": "^\\s*(```|{/if}|/>|</\\S*>|{/each})$"
 		}
 	},
 	"wordPattern": {