MESDK-87 Add delegate pattern for MessagesQuery<->LintReportQuery communication

.changeset/nine-adults-dance.md

@@ -0,0 +1,5 @@
+---
+"@inlang/sdk": patch
+---
+
+Adds delegate pattern for MessagesQuery<->LintReportQuery communication

inlang/source-code/sdk/src/api.ts

@@ -81,6 +81,14 @@ export type Subscribable<Value> = {
 	subscribe: (callback: (value: Value) => void) => void
 }
 
+export type MessageQueryDelegate = {
+	onMessageCreate: (messageId: string, message: Message) => void
+	onMessageUpdate: (messageId: string, message: Message) => void
+	onMessageDelete: (messageId: string) => void
+	onLoaded: (messages: Message[]) => void
+	onCleanup: () => void
+}
+
 export type MessageQueryApi = {
 	create: (args: { data: Message }) => boolean
 	get: ((args: { where: { id: Message["id"] } }) => Readonly<Message>) & {
@@ -101,6 +109,7 @@ export type MessageQueryApi = {
 	update: (args: { where: { id: Message["id"] }; data: Partial<Message> }) => boolean
 	upsert: (args: { where: { id: Message["id"] }; data: Message }) => void
 	delete: (args: { where: { id: Message["id"] } }) => boolean
+	setDelegate: (delegate: MessageQueryDelegate) => void
 }
 
 export type MessageLintReportsQueryApi = {

inlang/source-code/sdk/src/createMessageLintReportsQuery.ts

@@ -3,23 +3,19 @@ import type {
 	InstalledMessageLintRule,
 	MessageLintReportsQueryApi,
 	MessageQueryApi,
+	MessageQueryDelegate,
 } from "./api.js"
 import type { ProjectSettings } from "@inlang/project-settings"
 import type { resolveModules } from "./resolve-modules/index.js"
 import type { MessageLintReport, Message } from "./versionedInterfaces.js"
 import { lintSingleMessage } from "./lint/index.js"
-import { createRoot, createEffect } from "./reactivity/solid.js"
-
-import { throttle } from "throttle-debounce"
-import _debug from "debug"
-const debug = _debug("sdk:lintReports")
 
 function sleep(ms: number) {
 	return new Promise((resolve) => setTimeout(resolve, ms))
 }
 
 /**
- * Creates a reactive query API for messages.
+ * Creates a ~~reactive~~ query API for lint reports.
  */
 export function createMessageLintReportsQuery(
 	messagesQuery: MessageQueryApi,
@@ -42,72 +38,54 @@ export function createMessageLintReportsQuery(
 		}
 	}
 
-	const messages = messagesQuery.getAll() as Message[]
-
-	const trackedMessages: Map<string, () => void> = new Map()
-
-	debug(`createMessageLintReportsQuery ${rulesArray?.length} rules, ${messages.length} messages`)
-
-	// TODO: don't throttle when no debug
-	let lintMessageCount = 0
-	const throttledLogLintMessage = throttle(2000, (messageId) => {
-		debug(`lintSingleMessage: ${lintMessageCount} id: ${messageId}`)
-	})
-
-	createEffect(() => {
-		const currentMessageIds = new Set(messagesQuery.includedMessageIds())
-
-		const deletedTrackedMessages = [...trackedMessages].filter(
-			(tracked) => !currentMessageIds.has(tracked[0])
-		)
-
-		if (rulesArray) {
-			for (const messageId of currentMessageIds) {
-				if (!trackedMessages.has(messageId)) {
-					createRoot((dispose) => {
-						createEffect(() => {
-							const message = messagesQuery.get({ where: { id: messageId } })
-							if (!message) {
-								return
-							}
-							if (!trackedMessages?.has(messageId)) {
-								// initial effect execution - add dispose function
-								trackedMessages?.set(messageId, dispose)
-							}
+	const lintMessage = (message: Message, messages: Message[]) => {
+		if (!rulesArray) {
+			return
+		}
 
-							lintSingleMessage({
-								rules: rulesArray,
-								settings: settingsObject(),
-								messages: messages,
-								message: message,
-							}).then((report) => {
-								lintMessageCount++
-								throttledLogLintMessage(messageId)
-								if (report.errors.length === 0 && index.get(messageId) !== report.data) {
-									// console.log("lintSingleMessage", messageId, report.data.length)
-									index.set(messageId, report.data)
-								}
-							})
-						})
-					})
-				}
+		// TODO unhandled promise rejection (as before the refactor) but won't tackle this in this pr
+		lintSingleMessage({
+			rules: rulesArray,
+			settings: settingsObject(),
+			messages: messages,
+			message: message,
+		}).then((report) => {
+			if (report.errors.length === 0 && index.get(message.id) !== report.data) {
+				// console.log("lintSingleMessage", messageId, report.data.length)
+				index.set(message.id, report.data)
 			}
+		})
+	}
 
-			for (const deletedMessage of deletedTrackedMessages) {
-				const deletedMessageId = deletedMessage[0]
+	const messages = messagesQuery.getAll() as Message[]
+	// load report for all messages once
+	for (const message of messages) {
+		// NOTE: this potentually creates thousands of promisses we could create a promise that batches linting
+		lintMessage(message, messages)
+	}
 
-				// call dispose to cleanup the effect
-				const messageEffectDisposeFunction = trackedMessages.get(deletedMessageId)
-				if (messageEffectDisposeFunction) {
-					messageEffectDisposeFunction()
-					trackedMessages.delete(deletedMessageId)
-					// remove lint report result
-					index.delete(deletedMessageId)
-					debug(`delete lint message id: ${deletedMessageId}`)
-				}
+	const messageQueryChangeDelegate: MessageQueryDelegate = {
+		onCleanup: () => {
+			// NOTE: we could cancel all running lint rules - but results get overritten anyway
+			index.clear()
+		},
+		onLoaded: (messages: Message[]) => {
+			for (const message of messages) {
+				lintMessage(message, messages)
 			}
-		}
-	})
+		},
+		onMessageCreate: (messageId: string, message: Message) => {
+			lintMessage(message, messages)
+		},
+		onMessageUpdate: (messageId: string, message: Message) => {
+			lintMessage(message, messages)
+		},
+		onMessageDelete: (messageId: string) => {
+			index.delete(messageId)
+		},
+	}
+
+	messagesQuery.setDelegate(messageQueryChangeDelegate)
 
 	return {
 		getAll: async () => {

inlang/source-code/sdk/src/createMessagesQuery.ts

@@ -2,7 +2,7 @@ import type { Message } from "@inlang/message"
 import { ReactiveMap } from "./reactivity/map.js"
 import { createEffect } from "./reactivity/solid.js"
 import { createSubscribable } from "./loadProject.js"
-import type { InlangProject, MessageQueryApi } from "./api.js"
+import type { InlangProject, MessageQueryApi, MessageQueryDelegate } from "./api.js"
 import type { ResolvedPluginApi } from "./resolve-modules/plugins/types.js"
 import type { resolveModules } from "./resolve-modules/resolveModules.js"
 import { createNodeishFsWithWatcher } from "./createNodeishFsWithWatcher.js"
@@ -17,6 +17,10 @@ import { PluginLoadMessagesError, PluginSaveMessagesError } from "./errors.js"
 import { humanIdHash } from "./storage/human-id/human-readable-id.js"
 const debug = _debug("sdk:createMessagesQuery")
 
+function sleep(ms: number) {
+	return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
 type MessageState = {
 	messageDirtyFlags: {
 		[messageId: string]: boolean
@@ -62,6 +66,12 @@ export function createMessagesQuery({
 	// filepath for the lock folder
 	const messageLockDirPath = projectPath + "/messagelock"
 
+	let delegate: MessageQueryDelegate | undefined = undefined
+
+	const setDelegate = (newDelegate: MessageQueryDelegate) => {
+		delegate = newDelegate
+	}
+
 	// Map default alias to message
 	// Assumes that aliases are only created and deleted, not updated
 	// TODO #2346 - handle updates to aliases
@@ -98,6 +108,7 @@ export function createMessagesQuery({
 		onCleanup(() => {
 			// stop listening on fs events
 			abortController.abort()
+			delegate?.onCleanup()
 		})
 
 		const fsWithWatcher = createNodeishFsWithWatcher({
@@ -111,6 +122,7 @@ export function createMessagesQuery({
 					messageLockDirPath,
 					messageStates,
 					index,
+					delegate,
 					_settings, // NOTE we bang here - we don't expect the settings to become null during the livetime of a project
 					resolvedPluginApi
 				)
@@ -133,6 +145,7 @@ export function createMessagesQuery({
 			messageLockDirPath,
 			messageStates,
 			index,
+			delegate,
 			_settings, // NOTE we bang here - we don't expect the settings to become null during the livetime of a project
 			resolvedPluginApi
 		)
@@ -142,6 +155,7 @@ export function createMessagesQuery({
 			})
 			.then(() => {
 				onInitialMessageLoadResult()
+				delegate?.onLoaded([...index.values()])
 			})
 	})
 
@@ -165,6 +179,7 @@ export function createMessagesQuery({
 			messageLockDirPath,
 			messageStates,
 			index,
+			delegate,
 			_settings, // NOTE we bang here - we don't expect the settings to become null during the livetime of a project
 			resolvedPluginApi
 		)
@@ -181,6 +196,7 @@ export function createMessagesQuery({
 	}
 
 	return {
+		setDelegate,
 		create: ({ data }): boolean => {
 			if (index.has(data.id)) return false
 			index.set(data.id, data)
@@ -189,6 +205,7 @@ export function createMessagesQuery({
 			}
 
 			messageStates.messageDirtyFlags[data.id] = true
+			delegate?.onMessageCreate(data.id, index.get(data.id))
 			scheduleSave()
 			return true
 		},
@@ -215,6 +232,7 @@ export function createMessagesQuery({
 			if (message === undefined) return false
 			index.set(where.id, { ...message, ...data })
 			messageStates.messageDirtyFlags[where.id] = true
+			delegate?.onMessageCreate(where.id, index.get(data.id))
 			scheduleSave()
 			return true
 		},
@@ -225,10 +243,13 @@ export function createMessagesQuery({
 				if ("default" in data.alias) {
 					defaultAliasIndex.set(data.alias.default, data)
 				}
+				messageStates.messageDirtyFlags[where.id] = true
+				delegate?.onMessageCreate(data.id, index.get(data.id))
 			} else {
 				index.set(where.id, { ...message, ...data })
+				messageStates.messageDirtyFlags[where.id] = true
+				delegate?.onMessageUpdate(data.id, index.get(data.id))
 			}
-			messageStates.messageDirtyFlags[where.id] = true
 			scheduleSave()
 			return true
 		},
@@ -240,6 +261,7 @@ export function createMessagesQuery({
 			}
 			index.delete(where.id)
 			messageStates.messageDirtyFlags[where.id] = true
+			delegate?.onMessageDelete(where.id)
 			scheduleSave()
 			return true
 		},
@@ -253,6 +275,8 @@ export function createMessagesQuery({
 // - saving a message in two different languages would lead to a write in de.json first
 // - This will leads to a load of the messages and since en.json has not been saved yet the english variant in the message would get overritten with the old state again
 
+const maxMessagesPerTick = 500
+
 /**
  * Messsage that loads messages from a plugin - this method synchronizes with the saveMessage funciton.
  * If a save is in progress loading will wait until saving is done. If another load kicks in during this load it will queue the
@@ -271,6 +295,7 @@ async function loadMessagesViaPlugin(
 	lockDirPath: string,
 	messageState: MessageState,
 	messages: Map<string, Message>,
+	delegate: MessageQueryDelegate | undefined,
 	settingsValue: ProjectSettings,
 	resolvedPluginApi: ResolvedPluginApi
 ) {
@@ -298,6 +323,8 @@ async function loadMessagesViaPlugin(
 			})
 		)
 
+		let loadedMessageCount = 0
+
 		for (const loadedMessage of loadedMessages) {
 			const loadedMessageClone = structuredClone(loadedMessage)
 
@@ -339,6 +366,8 @@ async function loadMessagesViaPlugin(
 				messages.set(loadedMessageClone.id, loadedMessageClone)
 				// NOTE could use hash instead of the whole object JSON to save memory...
 				messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded
+				delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone)
+				loadedMessageCount++
 			} else {
 				// message with the given alias does not exist so far
 				loadedMessageClone.alias = {} as any
@@ -365,6 +394,15 @@ async function loadMessagesViaPlugin(
 				// we don't have to check - done before hand if (messages.has(loadedMessageClone.id)) return false
 				messages.set(loadedMessageClone.id, loadedMessageClone)
 				messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded
+				delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone)
+				loadedMessageCount++
+			}
+			if (loadedMessageCount > maxMessagesPerTick) {
+				// move loading of the next messages to the next ticks to allow solid to cleanup resources
+				// solid needs some time to settle and clean up
+				// https://github.com/solidjs-community/solid-primitives/blob/9ca76a47ffa2172770e075a90695cf933da0ff48/packages/trigger/src/index.ts#L64
+				await sleep(0)
+				loadedMessageCount = 0
 			}
 		}
 		await releaseLock(fs as NodeishFilesystem, lockDirPath, "loadMessage", lockTime)
@@ -388,7 +426,15 @@ async function loadMessagesViaPlugin(
 		messageState.sheduledLoadMessagesViaPlugin = undefined
 
 		// recall load unawaited to allow stack to pop
-		loadMessagesViaPlugin(fs, lockDirPath, messageState, messages, settingsValue, resolvedPluginApi)
+		loadMessagesViaPlugin(
+			fs,
+			lockDirPath,
+			messageState,
+			messages,
+			delegate,
+			settingsValue,
+			resolvedPluginApi
+		)
 			.then(() => {
 				// resolve the scheduled load message promise
 				executingScheduledMessages.resolve()
@@ -405,6 +451,7 @@ async function saveMessagesViaPlugin(
 	lockDirPath: string,
 	messageState: MessageState,
 	messages: Map<string, Message>,
+	delegate: MessageQueryDelegate | undefined,
 	settingsValue: ProjectSettings,
 	resolvedPluginApi: ResolvedPluginApi
 ): Promise<void> {
@@ -491,6 +538,7 @@ async function saveMessagesViaPlugin(
 					lockDirPath,
 					messageState,
 					messages,
+					delegate,
 					settingsValue,
 					resolvedPluginApi
 				)
@@ -530,7 +578,15 @@ async function saveMessagesViaPlugin(
 		const executingSheduledSaveMessages = messageState.sheduledSaveMessages
 		messageState.sheduledSaveMessages = undefined
 
-		saveMessagesViaPlugin(fs, lockDirPath, messageState, messages, settingsValue, resolvedPluginApi)
+		saveMessagesViaPlugin(
+			fs,
+			lockDirPath,
+			messageState,
+			messages,
+			delegate,
+			settingsValue,
+			resolvedPluginApi
+		)
 			.then(() => {
 				executingSheduledSaveMessages.resolve()
 			})

inlang/source-code/sdk/src/loadProject.ts

@@ -33,7 +33,6 @@ import { identifyProject } from "./telemetry/groupIdentify.js"
 import _debug from "debug"
 const debug = _debug("sdk:loadProject")
 
-
 const settingsCompiler = TypeCompiler.Compile(ProjectSettings)
 
 /**