diff --git a/.changeset/nine-adults-dance.md b/.changeset/nine-adults-dance.md new file mode 100644 index 0000000000..a4fb1676b1 --- /dev/null +++ b/.changeset/nine-adults-dance.md @@ -0,0 +1,5 @@ +--- +"@inlang/sdk": patch +--- + +Adds delegate pattern for MessagesQuery<->LintReportQuery communication diff --git a/inlang/source-code/sdk/src/api.ts b/inlang/source-code/sdk/src/api.ts index 3b46e73ef7..444e7b5267 100644 --- a/inlang/source-code/sdk/src/api.ts +++ b/inlang/source-code/sdk/src/api.ts @@ -81,6 +81,14 @@ export type Subscribable = { subscribe: (callback: (value: Value) => void) => void } +export type MessageQueryDelegate = { + onMessageCreate: (messageId: string, message: Message) => void + onMessageUpdate: (messageId: string, message: Message) => void + onMessageDelete: (messageId: string) => void + onLoaded: (messages: Message[]) => void + onCleanup: () => void +} + export type MessageQueryApi = { create: (args: { data: Message }) => boolean get: ((args: { where: { id: Message["id"] } }) => Readonly) & { @@ -101,6 +109,7 @@ export type MessageQueryApi = { update: (args: { where: { id: Message["id"] }; data: Partial }) => boolean upsert: (args: { where: { id: Message["id"] }; data: Message }) => void delete: (args: { where: { id: Message["id"] } }) => boolean + setDelegate: (delegate: MessageQueryDelegate) => void } export type MessageLintReportsQueryApi = { diff --git a/inlang/source-code/sdk/src/createMessageLintReportsQuery.ts b/inlang/source-code/sdk/src/createMessageLintReportsQuery.ts index e640a2e5f5..70f206d483 100644 --- a/inlang/source-code/sdk/src/createMessageLintReportsQuery.ts +++ b/inlang/source-code/sdk/src/createMessageLintReportsQuery.ts @@ -3,23 +3,19 @@ import type { InstalledMessageLintRule, MessageLintReportsQueryApi, MessageQueryApi, + MessageQueryDelegate, } from "./api.js" import type { ProjectSettings } from "@inlang/project-settings" import type { resolveModules } from "./resolve-modules/index.js" import type { MessageLintReport, Message } from "./versionedInterfaces.js" import { lintSingleMessage } from "./lint/index.js" -import { createRoot, createEffect } from "./reactivity/solid.js" - -import { throttle } from "throttle-debounce" -import _debug from "debug" -const debug = _debug("sdk:lintReports") function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)) } /** - * Creates a reactive query API for messages. + * Creates a ~~reactive~~ query API for lint reports. */ export function createMessageLintReportsQuery( messagesQuery: MessageQueryApi, @@ -42,72 +38,54 @@ export function createMessageLintReportsQuery( } } - const messages = messagesQuery.getAll() as Message[] - - const trackedMessages: Map void> = new Map() - - debug(`createMessageLintReportsQuery ${rulesArray?.length} rules, ${messages.length} messages`) - - // TODO: don't throttle when no debug - let lintMessageCount = 0 - const throttledLogLintMessage = throttle(2000, (messageId) => { - debug(`lintSingleMessage: ${lintMessageCount} id: ${messageId}`) - }) - - createEffect(() => { - const currentMessageIds = new Set(messagesQuery.includedMessageIds()) - - const deletedTrackedMessages = [...trackedMessages].filter( - (tracked) => !currentMessageIds.has(tracked[0]) - ) - - if (rulesArray) { - for (const messageId of currentMessageIds) { - if (!trackedMessages.has(messageId)) { - createRoot((dispose) => { - createEffect(() => { - const message = messagesQuery.get({ where: { id: messageId } }) - if (!message) { - return - } - if (!trackedMessages?.has(messageId)) { - // initial effect execution - add dispose function - trackedMessages?.set(messageId, dispose) - } + const lintMessage = (message: Message, messages: Message[]) => { + if (!rulesArray) { + return + } - lintSingleMessage({ - rules: rulesArray, - settings: settingsObject(), - messages: messages, - message: message, - }).then((report) => { - lintMessageCount++ - throttledLogLintMessage(messageId) - if (report.errors.length === 0 && index.get(messageId) !== report.data) { - // console.log("lintSingleMessage", messageId, report.data.length) - index.set(messageId, report.data) - } - }) - }) - }) - } + // TODO unhandled promise rejection (as before the refactor) but won't tackle this in this pr + lintSingleMessage({ + rules: rulesArray, + settings: settingsObject(), + messages: messages, + message: message, + }).then((report) => { + if (report.errors.length === 0 && index.get(message.id) !== report.data) { + // console.log("lintSingleMessage", messageId, report.data.length) + index.set(message.id, report.data) } + }) + } - for (const deletedMessage of deletedTrackedMessages) { - const deletedMessageId = deletedMessage[0] + const messages = messagesQuery.getAll() as Message[] + // load report for all messages once + for (const message of messages) { + // NOTE: this potentually creates thousands of promisses we could create a promise that batches linting + lintMessage(message, messages) + } - // call dispose to cleanup the effect - const messageEffectDisposeFunction = trackedMessages.get(deletedMessageId) - if (messageEffectDisposeFunction) { - messageEffectDisposeFunction() - trackedMessages.delete(deletedMessageId) - // remove lint report result - index.delete(deletedMessageId) - debug(`delete lint message id: ${deletedMessageId}`) - } + const messageQueryChangeDelegate: MessageQueryDelegate = { + onCleanup: () => { + // NOTE: we could cancel all running lint rules - but results get overritten anyway + index.clear() + }, + onLoaded: (messages: Message[]) => { + for (const message of messages) { + lintMessage(message, messages) } - } - }) + }, + onMessageCreate: (messageId: string, message: Message) => { + lintMessage(message, messages) + }, + onMessageUpdate: (messageId: string, message: Message) => { + lintMessage(message, messages) + }, + onMessageDelete: (messageId: string) => { + index.delete(messageId) + }, + } + + messagesQuery.setDelegate(messageQueryChangeDelegate) return { getAll: async () => { diff --git a/inlang/source-code/sdk/src/createMessagesQuery.ts b/inlang/source-code/sdk/src/createMessagesQuery.ts index 77c0112af5..7e3f8f12b6 100644 --- a/inlang/source-code/sdk/src/createMessagesQuery.ts +++ b/inlang/source-code/sdk/src/createMessagesQuery.ts @@ -2,7 +2,7 @@ import type { Message } from "@inlang/message" import { ReactiveMap } from "./reactivity/map.js" import { createEffect } from "./reactivity/solid.js" import { createSubscribable } from "./loadProject.js" -import type { InlangProject, MessageQueryApi } from "./api.js" +import type { InlangProject, MessageQueryApi, MessageQueryDelegate } from "./api.js" import type { ResolvedPluginApi } from "./resolve-modules/plugins/types.js" import type { resolveModules } from "./resolve-modules/resolveModules.js" import { createNodeishFsWithWatcher } from "./createNodeishFsWithWatcher.js" @@ -17,6 +17,10 @@ import { PluginLoadMessagesError, PluginSaveMessagesError } from "./errors.js" import { humanIdHash } from "./storage/human-id/human-readable-id.js" const debug = _debug("sdk:createMessagesQuery") +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + type MessageState = { messageDirtyFlags: { [messageId: string]: boolean @@ -62,6 +66,12 @@ export function createMessagesQuery({ // filepath for the lock folder const messageLockDirPath = projectPath + "/messagelock" + let delegate: MessageQueryDelegate | undefined = undefined + + const setDelegate = (newDelegate: MessageQueryDelegate) => { + delegate = newDelegate + } + // Map default alias to message // Assumes that aliases are only created and deleted, not updated // TODO #2346 - handle updates to aliases @@ -98,6 +108,7 @@ export function createMessagesQuery({ onCleanup(() => { // stop listening on fs events abortController.abort() + delegate?.onCleanup() }) const fsWithWatcher = createNodeishFsWithWatcher({ @@ -111,6 +122,7 @@ export function createMessagesQuery({ messageLockDirPath, messageStates, index, + delegate, _settings, // NOTE we bang here - we don't expect the settings to become null during the livetime of a project resolvedPluginApi ) @@ -133,6 +145,7 @@ export function createMessagesQuery({ messageLockDirPath, messageStates, index, + delegate, _settings, // NOTE we bang here - we don't expect the settings to become null during the livetime of a project resolvedPluginApi ) @@ -142,6 +155,7 @@ export function createMessagesQuery({ }) .then(() => { onInitialMessageLoadResult() + delegate?.onLoaded([...index.values()]) }) }) @@ -165,6 +179,7 @@ export function createMessagesQuery({ messageLockDirPath, messageStates, index, + delegate, _settings, // NOTE we bang here - we don't expect the settings to become null during the livetime of a project resolvedPluginApi ) @@ -181,6 +196,7 @@ export function createMessagesQuery({ } return { + setDelegate, create: ({ data }): boolean => { if (index.has(data.id)) return false index.set(data.id, data) @@ -189,6 +205,7 @@ export function createMessagesQuery({ } messageStates.messageDirtyFlags[data.id] = true + delegate?.onMessageCreate(data.id, index.get(data.id)) scheduleSave() return true }, @@ -215,6 +232,7 @@ export function createMessagesQuery({ if (message === undefined) return false index.set(where.id, { ...message, ...data }) messageStates.messageDirtyFlags[where.id] = true + delegate?.onMessageCreate(where.id, index.get(data.id)) scheduleSave() return true }, @@ -225,10 +243,13 @@ export function createMessagesQuery({ if ("default" in data.alias) { defaultAliasIndex.set(data.alias.default, data) } + messageStates.messageDirtyFlags[where.id] = true + delegate?.onMessageCreate(data.id, index.get(data.id)) } else { index.set(where.id, { ...message, ...data }) + messageStates.messageDirtyFlags[where.id] = true + delegate?.onMessageUpdate(data.id, index.get(data.id)) } - messageStates.messageDirtyFlags[where.id] = true scheduleSave() return true }, @@ -240,6 +261,7 @@ export function createMessagesQuery({ } index.delete(where.id) messageStates.messageDirtyFlags[where.id] = true + delegate?.onMessageDelete(where.id) scheduleSave() return true }, @@ -253,6 +275,8 @@ export function createMessagesQuery({ // - saving a message in two different languages would lead to a write in de.json first // - This will leads to a load of the messages and since en.json has not been saved yet the english variant in the message would get overritten with the old state again +const maxMessagesPerTick = 500 + /** * Messsage that loads messages from a plugin - this method synchronizes with the saveMessage funciton. * If a save is in progress loading will wait until saving is done. If another load kicks in during this load it will queue the @@ -271,6 +295,7 @@ async function loadMessagesViaPlugin( lockDirPath: string, messageState: MessageState, messages: Map, + delegate: MessageQueryDelegate | undefined, settingsValue: ProjectSettings, resolvedPluginApi: ResolvedPluginApi ) { @@ -298,6 +323,8 @@ async function loadMessagesViaPlugin( }) ) + let loadedMessageCount = 0 + for (const loadedMessage of loadedMessages) { const loadedMessageClone = structuredClone(loadedMessage) @@ -339,6 +366,8 @@ async function loadMessagesViaPlugin( messages.set(loadedMessageClone.id, loadedMessageClone) // NOTE could use hash instead of the whole object JSON to save memory... messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded + delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone) + loadedMessageCount++ } else { // message with the given alias does not exist so far loadedMessageClone.alias = {} as any @@ -365,6 +394,15 @@ async function loadMessagesViaPlugin( // we don't have to check - done before hand if (messages.has(loadedMessageClone.id)) return false messages.set(loadedMessageClone.id, loadedMessageClone) messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded + delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone) + loadedMessageCount++ + } + if (loadedMessageCount > maxMessagesPerTick) { + // move loading of the next messages to the next ticks to allow solid to cleanup resources + // solid needs some time to settle and clean up + // https://github.com/solidjs-community/solid-primitives/blob/9ca76a47ffa2172770e075a90695cf933da0ff48/packages/trigger/src/index.ts#L64 + await sleep(0) + loadedMessageCount = 0 } } await releaseLock(fs as NodeishFilesystem, lockDirPath, "loadMessage", lockTime) @@ -388,7 +426,15 @@ async function loadMessagesViaPlugin( messageState.sheduledLoadMessagesViaPlugin = undefined // recall load unawaited to allow stack to pop - loadMessagesViaPlugin(fs, lockDirPath, messageState, messages, settingsValue, resolvedPluginApi) + loadMessagesViaPlugin( + fs, + lockDirPath, + messageState, + messages, + delegate, + settingsValue, + resolvedPluginApi + ) .then(() => { // resolve the scheduled load message promise executingScheduledMessages.resolve() @@ -405,6 +451,7 @@ async function saveMessagesViaPlugin( lockDirPath: string, messageState: MessageState, messages: Map, + delegate: MessageQueryDelegate | undefined, settingsValue: ProjectSettings, resolvedPluginApi: ResolvedPluginApi ): Promise { @@ -491,6 +538,7 @@ async function saveMessagesViaPlugin( lockDirPath, messageState, messages, + delegate, settingsValue, resolvedPluginApi ) @@ -530,7 +578,15 @@ async function saveMessagesViaPlugin( const executingSheduledSaveMessages = messageState.sheduledSaveMessages messageState.sheduledSaveMessages = undefined - saveMessagesViaPlugin(fs, lockDirPath, messageState, messages, settingsValue, resolvedPluginApi) + saveMessagesViaPlugin( + fs, + lockDirPath, + messageState, + messages, + delegate, + settingsValue, + resolvedPluginApi + ) .then(() => { executingSheduledSaveMessages.resolve() }) diff --git a/inlang/source-code/sdk/src/loadProject.ts b/inlang/source-code/sdk/src/loadProject.ts index 7c0a3b1d9d..d756609c72 100644 --- a/inlang/source-code/sdk/src/loadProject.ts +++ b/inlang/source-code/sdk/src/loadProject.ts @@ -33,7 +33,6 @@ import { identifyProject } from "./telemetry/groupIdentify.js" import _debug from "debug" const debug = _debug("sdk:loadProject") - const settingsCompiler = TypeCompiler.Compile(ProjectSettings) /**