diff --git a/.changeset/brown-seas-hunt.md b/.changeset/brown-seas-hunt.md new file mode 100644 index 0000000000..b8e92c642f --- /dev/null +++ b/.changeset/brown-seas-hunt.md @@ -0,0 +1,6 @@ +--- +"@inlang/cli": patch +"@inlang/sdk": patch +--- + +v2 message bundle persistence - initial store implementation diff --git a/inlang/source-code/cli/src/commands/machine/translate.ts b/inlang/source-code/cli/src/commands/machine/translate.ts index a1c9c289fc..02ec921b6f 100644 --- a/inlang/source-code/cli/src/commands/machine/translate.ts +++ b/inlang/source-code/cli/src/commands/machine/translate.ts @@ -9,6 +9,7 @@ import { projectOption } from "../../utilities/globalFlags.js" import progessBar from "cli-progress" import plimit from "p-limit" import type { Result } from "@inlang/result" +import { toV1Message, fromV1Message } from "@inlang/sdk/v2" const rpcTranslateAction = process.env.MOCK_TRANSLATE_LOCAL ? mockMachineTranslateMessage @@ -18,6 +19,7 @@ export const translate = new Command() .command("translate") .requiredOption(projectOption.flags, projectOption.description) .option("-f, --force", "Force machine translation and skip the confirmation prompt.", false) + .option("-q, --quiet", "don't log every tranlation.", false) .option("--sourceLanguageTag ", "Source language tag for translation.") .option( "--targetLanguageTags ", @@ -60,6 +62,7 @@ export async function translateCommandAction(args: { project: InlangProject }) { return } const experimentalAliases = args.project.settings().experimental?.aliases + const v2Persistence = args.project.settings().experimental?.persistence const allLanguageTags = [...projectConfig.languageTags, projectConfig.sourceLanguageTag] @@ -97,9 +100,13 @@ export async function translateCommandAction(args: { project: InlangProject }) { return } - const messages = args.project.query.messages - .getAll() - .filter((message) => hasMissingTranslations(message, sourceLanguageTag, targetLanguageTags)) + const allMessages = v2Persistence + ? (await args.project.store!.messageBundles.getAll()).map(toV1Message) + : args.project.query.messages.getAll() + + const filteredMessages = allMessages.filter((message) => + hasMissingTranslations(message, sourceLanguageTag, targetLanguageTags) + ) const bar = options.nobar ? undefined @@ -111,7 +118,7 @@ export async function translateCommandAction(args: { project: InlangProject }) { progessBar.Presets.shades_grey ) - bar?.start(messages.length, 0) + bar?.start(filteredMessages.length, 0) const logs: Array<() => void> = [] @@ -132,17 +139,23 @@ export async function translateCommandAction(args: { project: InlangProject }) { translatedMessage && translatedMessage?.variants.length > toBeTranslatedMessage.variants.length ) { - args.project.query.messages.update({ - where: { id: translatedMessage.id }, - data: translatedMessage!, - }) - logs.push(() => log.info(`Machine translated message ${logId}`)) + if (v2Persistence) { + await args.project.store!.messageBundles.set({ data: fromV1Message(translatedMessage) }) + } else { + args.project.query.messages.update({ + where: { id: translatedMessage.id }, + data: translatedMessage!, + }) + } + if (!options.quiet) { + logs.push(() => log.info(`Machine translated message ${logId}`)) + } } bar?.increment() } // parallelize rpcTranslate calls with a limit of 100 concurrent calls - const limit = plimit(100) - const promises = messages.map((message) => limit(() => rpcTranslate(message))) + const limit = plimit(process.env.MOCK_TRANSLATE_LOCAL ? 100000 : 100) + const promises = filteredMessages.map((message) => limit(() => rpcTranslate(message))) await Promise.all(promises) bar?.stop() @@ -220,5 +233,6 @@ async function mockMachineTranslateMessage(args: { // console.log("mockMachineTranslateMessage translated", q, targetLanguageTag) } } + await new Promise((resolve) => setTimeout(resolve, 100)) return { data: copy } } diff --git a/inlang/source-code/sdk/load-test/.gitignore b/inlang/source-code/sdk/load-test/.gitignore index 8a360a9b6c..ba0f037db0 100644 --- a/inlang/source-code/sdk/load-test/.gitignore +++ b/inlang/source-code/sdk/load-test/.gitignore @@ -1,4 +1,5 @@ node_modules locales project.inlang/messages -project.inlang/messages.json \ No newline at end of file +project.inlang/messages.json +temp \ No newline at end of file diff --git a/inlang/source-code/sdk/load-test/README.md b/inlang/source-code/sdk/load-test/README.md index 681337c702..38b1db6ca4 100644 --- a/inlang/source-code/sdk/load-test/README.md +++ b/inlang/source-code/sdk/load-test/README.md @@ -1,12 +1,5 @@ -# inlang sdk load-test -package for volume testing - -- The test starts by generating engish messages in ./locales/en/common.json - or ./project.inlang/messages.json (depending on experimental.persistence) -- It then calls loadProject() and subscribes to events. -- It mock-translates into 36 languages using the inlang cli. -- It uses the i18next message storage plugin (unless experimental.persistence is set) -- To allow additional testing on the generated project e.g. with the ide-extension, the test calls `pnpm clean` when it starts, but not after it runs. +# package @inlang/sdk-load-test +The default `test` script runs a load-test with 1000 messages and translation enabled. For more messsages and different load-test options, use the `load-test` script. ``` USAGE: @@ -18,8 +11,17 @@ e.g. Defaults: translate: 1, subscribeToMessages: 1, subscribeToLintReports: 0, watchMode: 0 ``` +### what it does +- The test starts by generating engish messages in ./locales/en/common.json + or ./project.inlang/messages.json (depending on experimental.persistence) +- It then calls loadProject() and subscribes to events. +- It mock-translates into 36 languages using the inlang cli. +- It uses the i18next message storage plugin (unless experimental.persistence is set) +- To allow additional testing on the generated project e.g. with the ide-extension, the test calls `pnpm clean` when it starts, but not after it runs. + ### to configure additional debug logging -`export DEBUG=sdk:acquireFileLock,sdk:releaseLock,sdk:lintReports,sdk:loadProject` +`export DEBUG=sdk:*` for wildcard (most verbose) logging and to see more specific options. +e.g. `export DEBUG=sdk:lockFile` ### to translate from separate process 1. Run pnpm load-test with translate:0, watchMode:1 E.g. `pnpm load-test 100 0 1 1 1` diff --git a/inlang/source-code/sdk/load-test/load-test.ts b/inlang/source-code/sdk/load-test/load-test.ts index 66e610ceea..b30c03837a 100644 --- a/inlang/source-code/sdk/load-test/load-test.ts +++ b/inlang/source-code/sdk/load-test/load-test.ts @@ -1,8 +1,17 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable no-restricted-imports */ -/* eslint-disable no-console */ + import { findRepoRoot, openRepository } from "@lix-js/client" -import { loadProject, type Message, normalizeMessage } from "@inlang/sdk" -import { createMessage } from "../src/test-utilities/createMessage.js" +import { loadProject, type InlangProject, type Message, type ProjectSettings } from "@inlang/sdk" + +import { + createMessage, + createMessageBundle, + addSlots, + injectJSONNewlines, +} from "../src/v2/helper.js" +import { MessageBundle } from "../src/v2/types.js" + import { createEffect } from "../src/reactivity/solid.js" import { dirname, join } from "node:path" @@ -25,9 +34,7 @@ const projectPath = join(__dirname, "project.inlang") const mockServer = "http://localhost:3000" const cli = `PUBLIC_SERVER_BASE_URL=${mockServer} pnpm inlang` -const translateCommand = cli + " machine translate -f -n --project ./project.inlang" - -const isExperimentalPersistence = await checkExperimentalPersistence() +const translateCommand = cli + " machine translate -f -q -n --project ./project.inlang" export async function runLoadTest( messageCount: number = 1000, @@ -36,10 +43,14 @@ export async function runLoadTest( subscribeToLintReports: boolean = false, watchMode: boolean = false ) { + const settings = await getSettings() + // experimental persistence uses v2 types + const isV2 = !!settings.experimental?.persistence + const locales = settings.languageTags debug( "load-test start" + (watchMode ? " - watchMode on, ctrl C to exit" : "") + - (isExperimentalPersistence ? " - using experimental persistence" : "") + (isV2 ? " - using experimental persistence" : "") ) if (translate && !process.env.MOCK_TRANSLATE_LOCAL && !(await isMockRpcServerRunning())) { console.error( @@ -55,7 +66,7 @@ export async function runLoadTest( debug(`generating ${messageCount} messages`) // this is called before loadProject() to avoid reading partially written JSON - await generateMessageFile(messageCount) + await generateMessageFile(isV2, messageCount, locales) debug("opening repo and loading project") const repoRoot = await findRepoRoot({ nodeishFs: fs, path: __dirname }) @@ -73,7 +84,7 @@ export async function runLoadTest( } }) - if (subscribeToMessages) { + if (subscribeToMessages && !isV2) { debug("subscribing to messages.getAll") let countMessagesGetAllEvents = 0 @@ -90,7 +101,7 @@ export async function runLoadTest( }) } - if (subscribeToLintReports) { + if (subscribeToLintReports && !isV2) { debug("subscribing to lintReports.getAll") let lintEvents = 0 const logLintEvent = throttle(throttleLintEvents, (reports: any) => { @@ -102,9 +113,17 @@ export async function runLoadTest( }) } + if (isV2) { + await summarize("loaded", project) + } + if (translate) { debug("translating messages with inlang cli") await run(translateCommand) + if (isV2) { + await project.store?.messageBundles.reload() + await summarize("translated", project) + } } debug("load-test done - " + (watchMode ? "watching for events" : "exiting")) @@ -116,19 +135,35 @@ export async function runLoadTest( } } -async function generateMessageFile(messageCount: number) { - if (isExperimentalPersistence) { +async function summarize(action: string, project: InlangProject) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const bundles = await project.store!.messageBundles.getAll() + let bundleCount = 0 + let messageCount = 0 + bundles.map((bundle) => { + bundleCount++ + messageCount += bundle.messages.length + }) + debug(`${action}: ${bundleCount} bundles, ${messageCount / bundleCount} messages/bundle`) +} + +async function generateMessageFile(isV2: boolean, messageCount: number, locales: string[]) { + if (isV2) { const messageFile = join(__dirname, "project.inlang", "messages.json") - const messages: Message[] = [] + const messages: MessageBundle[] = [] for (let i = 1; i <= messageCount; i++) { - messages.push(createMessage(`message_key_${i}`, { en: `Generated message (${i})` })) + messages.push( + createMessageBundle({ + id: `message_key_${i}`, + messages: [createMessage({ locale: "en", text: `Generated message (${i})` })], + }) + ) } - await fs.writeFile( - messageFile, - JSON.stringify(messages.map(normalizeMessage), undefined, 2), - "utf-8" + const output = injectJSONNewlines( + JSON.stringify(messages.map((bundle) => addSlots(bundle, locales))) ) + await fs.writeFile(messageFile, output, "utf-8") } else { const messageDir = join(__dirname, "locales", "en") const messageFile = join(__dirname, "locales", "en", "common.json") @@ -142,10 +177,10 @@ async function generateMessageFile(messageCount: number) { } } -async function checkExperimentalPersistence() { +async function getSettings() { const settingsFile = join(__dirname, "project.inlang", "settings.json") const settings = JSON.parse(await fs.readFile(settingsFile, "utf-8")) - return !!settings.experimental?.persistence + return settings as ProjectSettings } async function isMockRpcServerRunning(): Promise { diff --git a/inlang/source-code/sdk/load-test/package.json b/inlang/source-code/sdk/load-test/package.json index 7c0ad9af34..559b9ee18f 100644 --- a/inlang/source-code/sdk/load-test/package.json +++ b/inlang/source-code/sdk/load-test/package.json @@ -19,7 +19,7 @@ }, "scripts": { "clean": "rm -rf ./locales ./project.inlang/messages.json", - "translate": "MOCK_TRANSLATE_LOCAL=1 PUBLIC_SERVER_BASE_URL=http://localhost:3000 pnpm inlang machine translate -n -f --project ./project.inlang", + "translate": "MOCK_TRANSLATE_LOCAL=1 pnpm inlang machine translate -n -f -q --project ./project.inlang", "test-lint": "pnpm inlang lint --project ./project.inlang", "load-test": "pnpm clean && MOCK_TRANSLATE_LOCAL=1 DEBUG=$DEBUG,load-test tsx ./main.ts", "test": "tsc --noEmit && pnpm load-test 1000 1 1 1", diff --git a/inlang/source-code/sdk/load-test/project.inlang/settings.json b/inlang/source-code/sdk/load-test/project.inlang/settings.json index f8d88c22b9..d8a6fca551 100644 --- a/inlang/source-code/sdk/load-test/project.inlang/settings.json +++ b/inlang/source-code/sdk/load-test/project.inlang/settings.json @@ -49,7 +49,5 @@ "plugin.inlang.i18next": { "pathPattern": "./locales/{languageTag}/common.json" }, - "experimental": { - "persistence": true - } + "experimental": { "persistence": true } } diff --git a/inlang/source-code/sdk/multi-project-test/multi-project.test.ts b/inlang/source-code/sdk/multi-project-test/multi-project.test.ts index 038101f984..d029392300 100644 --- a/inlang/source-code/sdk/multi-project-test/multi-project.test.ts +++ b/inlang/source-code/sdk/multi-project-test/multi-project.test.ts @@ -78,6 +78,7 @@ describe.concurrent( { timeout: 20000 } ) + // skip pending new v2 persistence with translation. it( "project4 in project4-dir", async () => { diff --git a/inlang/source-code/sdk/multi-project-test/project4-dir/messages.json.bak b/inlang/source-code/sdk/multi-project-test/project4-dir/messages.json.bak index 8e4c82d680..d96a85b261 100644 --- a/inlang/source-code/sdk/multi-project-test/project4-dir/messages.json.bak +++ b/inlang/source-code/sdk/multi-project-test/project4-dir/messages.json.bak @@ -1,50 +1,65 @@ [ { - "alias": {}, "id": "project4_message_key_1", - "selectors": [], - "variants": [ + "alias": {}, + "messages": [ { - "languageTag": "en", - "match": [], - "pattern": [ + "locale": "en", + "declarations": [], + "selectors": [], + "variants": [ { - "type": "Text", - "value": "Generated message (1)" + "match": [], + "pattern": [ + { + "type": "text", + "value": "Generated message (1)" + } + ] } ] } ] }, { - "alias": {}, "id": "project4_message_key_2", - "selectors": [], - "variants": [ + "alias": {}, + "messages": [ { - "languageTag": "en", - "match": [], - "pattern": [ + "locale": "en", + "declarations": [], + "selectors": [], + "variants": [ { - "type": "Text", - "value": "Generated message (2)" + "match": [], + "pattern": [ + { + "type": "text", + "value": "Generated message (2)" + } + ] } ] } ] }, { - "alias": {}, "id": "project4_message_key_3", - "selectors": [], - "variants": [ + "alias": {}, + "messages": [ { - "languageTag": "en", - "match": [], - "pattern": [ + "locale": "en", + "declarations": [], + "selectors": [], + "variants": [ { - "type": "Text", - "value": "Generated message (3)" + "match": [], + "pattern": [ + { + "type": "text", + "value": "Generated message (3)" + } + ] } ] } diff --git a/inlang/source-code/sdk/multi-project-test/project4-dir/messages.json.translated b/inlang/source-code/sdk/multi-project-test/project4-dir/messages.json.translated index 9dda436675..bfe665a03b 100644 --- a/inlang/source-code/sdk/multi-project-test/project4-dir/messages.json.translated +++ b/inlang/source-code/sdk/multi-project-test/project4-dir/messages.json.translated @@ -1,83 +1,37 @@ [ - { - "alias": {}, - "id": "project4_message_key_1", - "selectors": [], - "variants": [ - { - "languageTag": "de", - "match": [], - "pattern": [ - { - "type": "Text", - "value": "Mock translate local en to de: Generated message (1)" - } - ] - }, - { - "languageTag": "en", - "match": [], - "pattern": [ - { - "type": "Text", - "value": "Generated message (1)" - } - ] - } - ] - }, - { - "alias": {}, - "id": "project4_message_key_2", - "selectors": [], - "variants": [ - { - "languageTag": "de", - "match": [], - "pattern": [ - { - "type": "Text", - "value": "Mock translate local en to de: Generated message (2)" - } - ] - }, - { - "languageTag": "en", - "match": [], - "pattern": [ - { - "type": "Text", - "value": "Generated message (2)" - } - ] - } - ] - }, - { - "alias": {}, - "id": "project4_message_key_3", - "selectors": [], - "variants": [ - { - "languageTag": "de", - "match": [], - "pattern": [ - { - "type": "Text", - "value": "Mock translate local en to de: Generated message (3)" - } - ] - }, - { - "languageTag": "en", - "match": [], - "pattern": [ - { - "type": "Text", - "value": "Generated message (3)" - } - ] - } - ] - } -] \ No newline at end of file + + + +{"id":"project4_message_key_1","alias":{},"messages":[ + + + +{"locale":"en","declarations":[],"selectors":[],"variants":[{"match":[],"pattern":[{"type":"text","value":"Generated message (1)"}]}]}, + + + +{"locale":"de","declarations":[],"selectors":[],"variants":[{"match":[],"pattern":[{"type":"text","value":"Mock translate local en to de: Generated message (1)"}]}]}]}, + + + +{"id":"project4_message_key_2","alias":{},"messages":[ + + + +{"locale":"en","declarations":[],"selectors":[],"variants":[{"match":[],"pattern":[{"type":"text","value":"Generated message (2)"}]}]}, + + + +{"locale":"de","declarations":[],"selectors":[],"variants":[{"match":[],"pattern":[{"type":"text","value":"Mock translate local en to de: Generated message (2)"}]}]}]}, + + + +{"id":"project4_message_key_3","alias":{},"messages":[ + + + +{"locale":"en","declarations":[],"selectors":[],"variants":[{"match":[],"pattern":[{"type":"text","value":"Generated message (3)"}]}]}, + + + +{"locale":"de","declarations":[],"selectors":[],"variants":[{"match":[],"pattern":[{"type":"text","value":"Mock translate local en to de: Generated message (3)"}]}]}]}] \ No newline at end of file diff --git a/inlang/source-code/sdk/src/api.ts b/inlang/source-code/sdk/src/api.ts index 061896b1a3..1e63041b48 100644 --- a/inlang/source-code/sdk/src/api.ts +++ b/inlang/source-code/sdk/src/api.ts @@ -10,7 +10,7 @@ import type { MessageLintReport, } from "./versionedInterfaces.js" import type { ResolvedPluginApi } from "./resolve-modules/plugins/types.js" -import type * as V2 from "./v2/types.js" +import type { StoreApi } from "./persistence/storeApi.js" export type InstalledPlugin = { id: Plugin["id"] @@ -58,18 +58,7 @@ export type InlangProject = { } // WIP V2 message apis // use with project settings: experimental.persistence = true - messageBundles?: Query - messages?: Query - variants?: Query -} - -/** - * WIP template for async V2 crud interfaces - * E.g. `await project.messageBundles.get({ id: "..." })` - **/ -interface Query { - get: (args: unknown) => Promise - getAll: () => Promise + store?: StoreApi } // const x = {} as InlangProject diff --git a/inlang/source-code/sdk/src/createNewProject.test.ts b/inlang/source-code/sdk/src/createNewProject.test.ts index 4d909f1828..b8eb70c19b 100644 --- a/inlang/source-code/sdk/src/createNewProject.test.ts +++ b/inlang/source-code/sdk/src/createNewProject.test.ts @@ -4,10 +4,7 @@ import { mockRepo } from "@lix-js/client" import { defaultProjectSettings } from "./defaultProjectSettings.js" import { loadProject } from "./loadProject.js" import { createMessage } from "./test-utilities/createMessage.js" - -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)) -} +import { sleep } from "./test-utilities/sleep.js" describe("createNewProject", () => { it("should throw if a path does not end with .inlang", async () => { diff --git a/inlang/source-code/sdk/src/loadProject.test.ts b/inlang/source-code/sdk/src/loadProject.test.ts index 30b2f43c31..b112ba536f 100644 --- a/inlang/source-code/sdk/src/loadProject.test.ts +++ b/inlang/source-code/sdk/src/loadProject.test.ts @@ -278,11 +278,15 @@ describe("initialization", () => { expect(result.data).toBeDefined() }) + // TODO: fix this test + // https://github.com/opral/inlang-message-sdk/issues/76 + // it doesn't work because failure to open the settings file doesn't throw + // errors are returned in project.errors() it("should resolve from a windows path", async () => { const repo = await mockRepo() const fs = repo.nodeishFs - fs.mkdir("C:\\Users\\user\\project.inlang", { recursive: true }) - fs.writeFile("C:\\Users\\user\\project.inlang\\settings.json", JSON.stringify(settings)) + await fs.mkdir("C:\\Users\\user\\project.inlang", { recursive: true }) + await fs.writeFile("C:\\Users\\user\\project.inlang\\settings.json", JSON.stringify(settings)) const result = await tryCatch(() => loadProject({ diff --git a/inlang/source-code/sdk/src/loadProject.ts b/inlang/source-code/sdk/src/loadProject.ts index c7ade189e0..390c6b61b0 100644 --- a/inlang/source-code/sdk/src/loadProject.ts +++ b/inlang/source-code/sdk/src/loadProject.ts @@ -4,6 +4,8 @@ import type { InstalledMessageLintRule, InstalledPlugin, Subscribable, + MessageQueryApi, + MessageLintReportsQueryApi, } from "./api.js" import { type ImportFunction, resolveModules } from "./resolve-modules/index.js" import { TypeCompiler, ValueErrorType } from "@sinclair/typebox/compiler" @@ -30,6 +32,10 @@ import { maybeCreateFirstProjectId } from "./migrations/maybeCreateFirstProjectI import { capture } from "./telemetry/capture.js" import { identifyProject } from "./telemetry/groupIdentify.js" +import { stubMessagesQuery, stubMessageLintReportsQuery } from "./v2/stubQueryApi.js" +import type { StoreApi } from "./persistence/storeApi.js" +import { openStore } from "./persistence/store.js" + import _debug from "debug" const debug = _debug("sdk:loadProject") @@ -80,9 +86,16 @@ export async function loadProject(args: { ) const [initialized, markInitAsComplete, markInitAsFailed] = createAwaitable() + const [loadedSettings, markSettingsAsLoaded, markSettingsAsFailed] = createAwaitable() // -- settings ------------------------------------------------------------ const [settings, _setSettings] = createSignal() + let v2Persistence = false + let locales: string[] = [] + + // This effect currently has no signals + // TODO: replace createEffect with await loadSettings + // https://github.com/opral/inlang-message-sdk/issues/77 createEffect(() => { // TODO: // if (projectId) { @@ -92,12 +105,17 @@ export async function loadProject(args: { // } loadSettings({ settingsFilePath: projectPath + "/settings.json", nodeishFs }) - .then((settings) => setSettings(settings)) + .then((settings) => { + setSettings(settings) + markSettingsAsLoaded() + }) .catch((err) => { markInitAsFailed(err) + markSettingsAsFailed(err) }) }) // TODO: create FS watcher and update settings on change + // https://github.com/opral/inlang-message-sdk/issues/35 const writeSettingsToDisk = skipFirst((settings: ProjectSettings) => _writeSettingsToDisk({ nodeishFs, settings, projectPath }) @@ -106,11 +124,8 @@ export async function loadProject(args: { const setSettings = (settings: ProjectSettings): Result => { try { const validatedSettings = parseSettings(settings) - if (validatedSettings.experimental?.persistence) { - settings["plugin.sdk.persistence"] = { - pathPattern: projectPath + "/messages.json", - } - } + v2Persistence = !!validatedSettings.experimental?.persistence + locales = validatedSettings.languageTags batch(() => { // reset the resolved modules first - since they are no longer valid at that point @@ -147,40 +162,11 @@ export async function loadProject(args: { .catch((err) => markInitAsFailed(err)) }) - // -- messages ---------------------------------------------------------- + // -- installed items ---------------------------------------------------- let settingsValue: ProjectSettings - createEffect(() => (settingsValue = settings()!)) // workaround to not run effects twice (e.g. settings change + modules change) (I'm sure there exists a solid way of doing this, but I haven't found it yet) - - const [loadMessagesViaPluginError, setLoadMessagesViaPluginError] = createSignal< - Error | undefined - >() - - const [saveMessagesViaPluginError, setSaveMessagesViaPluginError] = createSignal< - Error | undefined - >() - - const messagesQuery = createMessagesQuery({ - projectPath, - nodeishFs, - settings, - resolvedModules, - onInitialMessageLoadResult: (e) => { - if (e) { - markInitAsFailed(e) - } else { - markInitAsComplete() - } - }, - onLoadMessageResult: (e) => { - setLoadMessagesViaPluginError(e) - }, - onSaveMessageResult: (e) => { - setSaveMessagesViaPluginError(e) - }, - }) - - // -- installed items ---------------------------------------------------- + // workaround to not run effects twice (e.g. settings change + modules change) (I'm sure there exists a solid way of doing this, but I haven't found it yet) + createEffect(() => (settingsValue = settings()!)) const installedMessageLintRules = () => { if (!resolvedModules()) return [] @@ -213,17 +199,69 @@ export async function loadProject(args: { })) satisfies Array } + // -- messages ---------------------------------------------------------- + + const [loadMessagesViaPluginError, setLoadMessagesViaPluginError] = createSignal< + Error | undefined + >() + + const [saveMessagesViaPluginError, setSaveMessagesViaPluginError] = createSignal< + Error | undefined + >() + + let messagesQuery: MessageQueryApi + let lintReportsQuery: MessageLintReportsQueryApi + let store: StoreApi | undefined + + // wait for seetings to load v2Persistence flag + // .catch avoids throwing here if the awaitable is rejected + // error is recorded via markInitAsFailed so no need to capture it again + await loadedSettings.catch(() => {}) + + if (v2Persistence) { + messagesQuery = stubMessagesQuery + lintReportsQuery = stubMessageLintReportsQuery + try { + store = await openStore({ projectPath, nodeishFs, locales }) + markInitAsComplete() + } catch (e) { + markInitAsFailed(e) + } + } else { + messagesQuery = createMessagesQuery({ + projectPath, + nodeishFs, + settings, + resolvedModules, + onInitialMessageLoadResult: (e) => { + if (e) { + markInitAsFailed(e) + } else { + markInitAsComplete() + } + }, + onLoadMessageResult: (e) => { + setLoadMessagesViaPluginError(e) + }, + onSaveMessageResult: (e) => { + setSaveMessagesViaPluginError(e) + }, + }) + + lintReportsQuery = createMessageLintReportsQuery( + messagesQuery, + settings as () => ProjectSettings, + installedMessageLintRules, + resolvedModules + ) + + store = undefined + } + // -- app --------------------------------------------------------------- const initializeError: Error | undefined = await initialized.catch((error) => error) - const lintReportsQuery = createMessageLintReportsQuery( - messagesQuery, - settings as () => ProjectSettings, - installedMessageLintRules, - resolvedModules - ) - /** * Utility to escape reactive tracking and avoid multiple calls to * the capture event. @@ -250,6 +288,8 @@ export async function loadProject(args: { settings: settings(), installedPluginIds: installedPlugins().map((p) => p.id), installedMessageLintRuleIds: installedMessageLintRules().map((r) => r.id), + // TODO: fix for v2Persistence + // https://github.com/opral/inlang-message-sdk/issues/78 numberOfMessages: messagesQuery.includedMessageIds().length, }, }) @@ -276,6 +316,7 @@ export async function loadProject(args: { messages: messagesQuery, messageLintReports: lintReportsQuery, }, + store, } satisfies InlangProject }) } diff --git a/inlang/source-code/sdk/src/persistence/batchedIO.test.ts b/inlang/source-code/sdk/src/persistence/batchedIO.test.ts new file mode 100644 index 0000000000..ed0d6de1c8 --- /dev/null +++ b/inlang/source-code/sdk/src/persistence/batchedIO.test.ts @@ -0,0 +1,63 @@ +import { describe, it, expect, vi } from "vitest" +import { sleep } from "../test-utilities/sleep.js" +import { batchedIO } from "./batchedIO.js" + +let locked = false + +const instrumentAquireLockStart = vi.fn() + +async function mockAquireLock() { + instrumentAquireLockStart() + let pollCount = 0 + while (locked && pollCount++ < 100) { + await sleep(10) + } + if (locked) { + throw new Error("Timeout acquiring lock") + } + await sleep(10) + locked = true + return 69 +} + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +async function mockReleaseLock(_: number) { + sleep(10) + locked = false +} + +const instrumentSaveStart = vi.fn() +const instrumentSaveEnd = vi.fn() + +async function mockSave() { + instrumentSaveStart() + await sleep(50) + instrumentSaveEnd() +} + +describe("batchedIO", () => { + it("queues 2 requests while waiting for lock and pushes 2 more to the next batch", async () => { + const save = batchedIO(mockAquireLock, mockReleaseLock, mockSave) + const p1 = save("1") + const p2 = save("2") + await sleep(5) + expect(instrumentAquireLockStart).toHaveBeenCalledTimes(1) + expect(instrumentSaveStart).not.toHaveBeenCalled() + await sleep(10) + expect(instrumentSaveStart).toHaveBeenCalled() + expect(instrumentSaveEnd).not.toHaveBeenCalled() + const p3 = save("3") + const p4 = save("4") + expect(instrumentAquireLockStart).toHaveBeenCalledTimes(2) + expect(locked).toBe(true) + await sleep(50) + expect(instrumentSaveEnd).toHaveBeenCalled() + expect(await p1).toBe("1") + expect(await p2).toBe("2") + expect(instrumentSaveStart).toHaveBeenCalledTimes(1) + expect(await p3).toBe("3") + expect(await p4).toBe("4") + expect(instrumentAquireLockStart).toHaveBeenCalledTimes(2) + expect(instrumentSaveStart).toHaveBeenCalledTimes(2) + }) +}) diff --git a/inlang/source-code/sdk/src/persistence/batchedIO.ts b/inlang/source-code/sdk/src/persistence/batchedIO.ts new file mode 100644 index 0000000000..91c0c0b3c4 --- /dev/null +++ b/inlang/source-code/sdk/src/persistence/batchedIO.ts @@ -0,0 +1,64 @@ +import _debug from "debug" +const debug = _debug("sdk:batchedIO") + +/** + * State machine to convert async save() into batched async save() + * states = idle -> acquiring -> saving -> idle + * idle = nothing queued, ready to acquire lock. + * aquiring = waiting to acquire a lock: requests go into the queue. + * saving = lock is acquired, save has begun: new requests go into the next batch. + * The next batch should not acquire the lock while current save is in progress. + * Queued requests are only resolved when the save completes. + */ +export function batchedIO( + acquireLock: () => Promise, + releaseLock: (lock: number) => Promise, + save: () => Promise +): (id: string) => Promise { + // 3-state machine + let state: "idle" | "acquiring" | "saving" = "idle" + + // Hold requests while acquiring, resolve after saving + // TODO: rejectQueued if save throws (maybe?) + // https://github.com/opral/inlang-message-sdk/issues/79 + type Queued = { + id: string + resolve: (value: string) => void + reject: (reason: any) => void + } + const queue: Queued[] = [] + + // initialize nextBatch lazily, reset after saving + let nextBatch: ((id: string) => Promise) | undefined = undefined + + // batched save function + return async (id: string) => { + if (state === "idle") { + state = "acquiring" + const lock = await acquireLock() + state = "saving" + await save() + await releaseLock(lock) + resolveQueued() + nextBatch = undefined + state = "idle" + return id + } else if (state === "acquiring") { + return new Promise((resolve, reject) => { + queue.push({ id, resolve, reject }) + }) + } else { + // state === "saving" + nextBatch = nextBatch ?? batchedIO(acquireLock, releaseLock, save) + return await nextBatch(id) + } + } + + function resolveQueued() { + debug("batched", queue.length + 1) + for (const { id, resolve } of queue) { + resolve(id) + } + queue.length = 0 + } +} diff --git a/inlang/source-code/sdk/src/persistence/filelock/acquireFileLock.ts b/inlang/source-code/sdk/src/persistence/filelock/acquireFileLock.ts index cec4ee15ab..ece83c9dee 100644 --- a/inlang/source-code/sdk/src/persistence/filelock/acquireFileLock.ts +++ b/inlang/source-code/sdk/src/persistence/filelock/acquireFileLock.ts @@ -21,16 +21,19 @@ export async function acquireFileLock( try { debug(lockOrigin + " tries to acquire a lockfile Retry Nr.: " + tryCount) await fs.mkdir(lockDirPath) - const stats = await fs.stat(lockDirPath) + // NOTE: fs.stat does not need to be atomic since mkdir would crash atomically - if we are here its save to consider the lock held by this process + const stats = await fs.stat(lockDirPath) debug(lockOrigin + " acquired a lockfile Retry Nr.: " + tryCount) + return stats.mtimeMs } catch (error: any) { if (error.code !== "EEXIST") { - // we only expect the error that the file exists already (locked by other process) + // NOTE in case we have an EEXIST - this is an expected error: the folder existed - another process already acquired the lock. Rethrow all other errors throw error } } + // land here if the lockDirPath already exists => lock is held by other process let currentLockTime: number try { diff --git a/inlang/source-code/sdk/src/persistence/filelock/releaseLock.ts b/inlang/source-code/sdk/src/persistence/filelock/releaseLock.ts index ad9f8c277c..93506a900f 100644 --- a/inlang/source-code/sdk/src/persistence/filelock/releaseLock.ts +++ b/inlang/source-code/sdk/src/persistence/filelock/releaseLock.ts @@ -11,8 +11,9 @@ export async function releaseLock( debug(lockOrigin + " releasing the lock ") try { const stats = await fs.stat(lockDirPath) + // I believe this check associates the lock with the aquirer if (stats.mtimeMs === lockTime) { - // this can be corrupt as welll since the last getStat and the current a modification could have occured :-/ + // NOTE: since we have to use a timeout for stale detection (uTimes is not exposed via mermoryfs) the check for the locktime is not sufficient and can fail in rare cases when another process accuires a lock that was identifiert as tale between call to fs.state and rmDir await fs.rmdir(lockDirPath) } } catch (statError: any) { diff --git a/inlang/source-code/sdk/src/persistence/plugin.test.ts b/inlang/source-code/sdk/src/persistence/plugin.test.ts deleted file mode 100644 index f98c2f2375..0000000000 --- a/inlang/source-code/sdk/src/persistence/plugin.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { test, expect } from "vitest" -import { createMessage, createNodeishMemoryFs } from "../test-utilities/index.js" -import { normalizeMessage } from "../storage/helper.js" -import { pluginId } from "./plugin.js" - -const mockMessages = [ - createMessage("first_message", { - en: "If this fails I will be sad", - }), - createMessage("second_message", { - en: "Let's see if this works", - de: "Mal sehen ob das funktioniert", - }), -] - -// the test ensures: -// - messages can be loaded -// - messages can be saved -// - after loading and saving messages, the state is the same as before (roundtrip) -test("roundtrip (saving/loading messages)", async () => { - const { loadMessages, saveMessages } = await import("./plugin.js") - const fs = createNodeishMemoryFs() - const projectDir = "/test/project.inlang" - const pathPattern = projectDir + "/messages.json" - const persistedMessages = JSON.stringify(mockMessages.map(normalizeMessage), undefined, 2) - - const settings = { - sourceLanguageTag: "en", - languageTags: ["en", "de"], - modules: [], - [pluginId]: { pathPattern }, - } - - await fs.mkdir(projectDir, { recursive: true }) - await fs.writeFile(pathPattern, persistedMessages) - - const firstMessageLoad = await loadMessages({ - settings, - nodeishFs: fs, - }) - - expect(firstMessageLoad).toStrictEqual(mockMessages) - - await saveMessages({ - settings, - nodeishFs: fs, - messages: firstMessageLoad, - }) - - const afterRoundtrip = await fs.readFile(pathPattern, { encoding: "utf-8" }) - - expect(afterRoundtrip).toStrictEqual(persistedMessages) - - const messagesAfterRoundtrip = await loadMessages({ - settings, - nodeishFs: fs, - }) - - expect(messagesAfterRoundtrip).toStrictEqual(firstMessageLoad) -}) diff --git a/inlang/source-code/sdk/src/persistence/plugin.ts b/inlang/source-code/sdk/src/persistence/plugin.ts deleted file mode 100644 index 7a1173008c..0000000000 --- a/inlang/source-code/sdk/src/persistence/plugin.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { ProjectSettings, Message } from "@inlang/sdk" -import { getDirname, type NodeishFilesystem } from "@lix-js/fs" -import { normalizeMessage } from "../storage/helper.js" - -import _debug from "debug" -const debug = _debug("sdk:persistence") - -export const pluginId = "plugin.sdk.persistence" - -export async function loadMessages(args: { - settings: ProjectSettings - nodeishFs: NodeishFilesystem -}) { - let result: Message[] = [] - const pathPattern = args.settings[pluginId]?.pathPattern as string - - debug("loadMessages", pathPattern) - try { - const file = await args.nodeishFs.readFile(pathPattern, { encoding: "utf-8" }) - result = JSON.parse(file) - } catch (error) { - if ((error as any)?.code !== "ENOENT") { - debug("loadMessages", error) - throw error - } - } - return result -} - -export async function saveMessages(args: { - settings: ProjectSettings - nodeishFs: NodeishFilesystem - messages: Message[] -}) { - const pathPattern = args.settings[pluginId]?.pathPattern as string - - debug("saveMessages", pathPattern) - try { - await createDirectoryIfNotExits(getDirname(pathPattern), args.nodeishFs) - await args.nodeishFs.writeFile( - pathPattern, - // 2 spaces indentation - JSON.stringify(args.messages.map(normalizeMessage), undefined, 2) - ) - } catch (error) { - debug("saveMessages", error) - } -} - -async function createDirectoryIfNotExits(path: string, nodeishFs: NodeishFilesystem) { - try { - await nodeishFs.mkdir(path, { recursive: true }) - } catch { - // assume that the directory already exists - } -} diff --git a/inlang/source-code/sdk/src/persistence/store.test.ts b/inlang/source-code/sdk/src/persistence/store.test.ts new file mode 100644 index 0000000000..126a4929c5 --- /dev/null +++ b/inlang/source-code/sdk/src/persistence/store.test.ts @@ -0,0 +1,102 @@ +import { test, expect } from "vitest" +import { createNodeishMemoryFs } from "../test-utilities/index.js" +import type { MessageBundle } from "../v2/types.js" +import { createMessageBundle, createMessage, injectJSONNewlines, addSlots } from "../v2/helper.js" +import { openStore, readJSON, writeJSON } from "./store.js" + +const locales = ["en", "de"] + +const mockMessages: MessageBundle[] = [ + createMessageBundle({ + id: "first_message", + messages: [ + createMessage({ + locale: "en", + text: "If this fails I will be sad", + }), + ], + }), + createMessageBundle({ + id: "second_message", + messages: [ + createMessage({ locale: "en", text: "Let's see if this works" }), + createMessage({ locale: "de", text: "Mal sehen ob das funktioniert" }), + ], + }), +] + +test("roundtrip readJSON/writeJSON", async () => { + const nodeishFs = createNodeishMemoryFs() + const projectPath = "/test/project.inlang" + const filePath = projectPath + "/messages.json" + const persistedMessages = injectJSONNewlines( + JSON.stringify(mockMessages.map((bundle) => addSlots(bundle, locales))) + ) + + await nodeishFs.mkdir(projectPath, { recursive: true }) + await nodeishFs.writeFile(filePath, persistedMessages) + + const firstMessageLoad = await readJSON({ + filePath, + nodeishFs: nodeishFs, + }) + + expect(firstMessageLoad).toStrictEqual(mockMessages) + + await writeJSON({ + filePath, + nodeishFs, + messages: firstMessageLoad, + locales, + }) + + const afterRoundtrip = await nodeishFs.readFile(filePath, { encoding: "utf-8" }) + + expect(afterRoundtrip).toStrictEqual(persistedMessages) + + const messagesAfterRoundtrip = await readJSON({ + filePath, + nodeishFs, + }) + + expect(messagesAfterRoundtrip).toStrictEqual(firstMessageLoad) +}) + +test("openStore does minimal CRUD on messageBundles", async () => { + const nodeishFs = createNodeishMemoryFs() + const projectPath = "/test/project.inlang" + const filePath = projectPath + "/messages.json" + const persistedMessages = JSON.stringify(mockMessages) + + await nodeishFs.mkdir(projectPath, { recursive: true }) + await nodeishFs.writeFile(filePath, persistedMessages) + + const store = await openStore({ projectPath, nodeishFs, locales }) + + const messages = await store.messageBundles.getAll() + expect(messages).toStrictEqual(mockMessages) + + const firstMessageBundle = await store.messageBundles.get({ id: "first_message" }) + expect(firstMessageBundle).toStrictEqual(mockMessages[0]) + + const modifedMessageBundle = structuredClone(firstMessageBundle) as MessageBundle + const newMessage = createMessage({ locale: "de", text: "Wenn dies schiefläuft, bin ich sauer" }) + modifedMessageBundle.messages.push(newMessage) + await store.messageBundles.set({ data: modifedMessageBundle }) + + const setMessageBundle = await store.messageBundles.get({ id: "first_message" }) + expect(setMessageBundle).toStrictEqual(modifedMessageBundle) + + // no need to sleep here + // with batchedIO, the set should await until the write is done + const messagesAfterRoundtrip = await readJSON({ + filePath, + nodeishFs, + }) + const expected = [setMessageBundle, ...mockMessages.slice(1)] + expect(messagesAfterRoundtrip).toStrictEqual(expected) + + await store.messageBundles.delete({ id: "first_message" }) + const messagesAfterDelete = await store.messageBundles.getAll() + expect(messagesAfterDelete).toStrictEqual(mockMessages.slice(1)) +}) diff --git a/inlang/source-code/sdk/src/persistence/store.ts b/inlang/source-code/sdk/src/persistence/store.ts new file mode 100644 index 0000000000..065443b412 --- /dev/null +++ b/inlang/source-code/sdk/src/persistence/store.ts @@ -0,0 +1,119 @@ +import type { MessageBundle } from "../v2/types.js" +import { addSlots, removeSlots, injectJSONNewlines } from "../v2/helper.js" +import { getDirname, type NodeishFilesystem } from "@lix-js/fs" +import { acquireFileLock } from "./filelock/acquireFileLock.js" +import { releaseLock } from "./filelock/releaseLock.js" +import { batchedIO } from "./batchedIO.js" +import type { StoreApi } from "./storeApi.js" + +import _debug from "debug" +const debug = _debug("sdk:store") + +export async function openStore(args: { + projectPath: string + nodeishFs: NodeishFilesystem + locales: string[] +}): Promise { + const nodeishFs = args.nodeishFs + const filePath = args.projectPath + "/messages.json" + const lockDirPath = args.projectPath + "/messagelock" + + // the index holds the in-memory state + // TODO: reload when file changes on disk + // https://github.com/opral/inlang-message-sdk/issues/80 + let index = await load() + + const batchedSave = batchedIO(acquireSaveLock, releaseSaveLock, save) + + return { + messageBundles: { + reload: async () => { + index.clear() + index = await load() + }, + get: async (args: { id: string }) => { + return index.get(args.id) + }, + set: async (args: { data: MessageBundle }) => { + index.set(args.data.id, args.data) + await batchedSave(args.data.id) + }, + delete: async (args: { id: string }) => { + index.delete(args.id) + await batchedSave(args.id) + }, + getAll: async () => { + return [...index.values()] + }, + }, + } + + // load and save messages from file system atomically + // using a lock file to prevent partial reads and writes + async function load() { + const lockTime = await acquireFileLock(nodeishFs, lockDirPath, "load") + const messages = await readJSON({ filePath, nodeishFs: nodeishFs }) + const index = new Map(messages.map((message) => [message.id, message])) + await releaseLock(nodeishFs, lockDirPath, "load", lockTime) + return index + } + async function acquireSaveLock() { + return await acquireFileLock(nodeishFs, lockDirPath, "save") + } + async function releaseSaveLock(lock: number) { + return await releaseLock(nodeishFs, lockDirPath, "save", lock) + } + async function save() { + await writeJSON({ + filePath, + nodeishFs: nodeishFs, + messages: [...index.values()], + locales: args.locales, + }) + } +} + +export async function readJSON(args: { filePath: string; nodeishFs: NodeishFilesystem }) { + let result: MessageBundle[] = [] + + debug("loadAll", args.filePath) + try { + const file = await args.nodeishFs.readFile(args.filePath, { encoding: "utf-8" }) + result = JSON.parse(file) + } catch (error) { + if ((error as any)?.code !== "ENOENT") { + debug("loadMessages", error) + throw error + } + } + return result.map(removeSlots) +} + +export async function writeJSON(args: { + filePath: string + nodeishFs: NodeishFilesystem + messages: MessageBundle[] + locales: string[] +}) { + debug("saveall", args.filePath) + try { + await createDirectoryIfNotExits(getDirname(args.filePath), args.nodeishFs) + const output = injectJSONNewlines( + JSON.stringify(args.messages.map((bundle) => addSlots(bundle, args.locales))) + ) + await args.nodeishFs.writeFile(args.filePath, output) + } catch (error) { + debug("saveMessages", error) + throw error + } +} + +async function createDirectoryIfNotExits(path: string, nodeishFs: NodeishFilesystem) { + try { + await nodeishFs.mkdir(path, { recursive: true }) + } catch (error: any) { + if (error.code !== "EEXIST") { + throw error + } + } +} diff --git a/inlang/source-code/sdk/src/persistence/storeApi.ts b/inlang/source-code/sdk/src/persistence/storeApi.ts new file mode 100644 index 0000000000..e3bdf681e4 --- /dev/null +++ b/inlang/source-code/sdk/src/persistence/storeApi.ts @@ -0,0 +1,19 @@ +import type * as V2 from "../v2/types.js" + +/** + * WIP async V2 store interface + * E.g. `await project.store.messageBundles.get({ id: "..." })` + **/ +export type StoreApi = { + messageBundles: Store +} + +export interface Store { + // TODO: remove reload when fs.watch can trigger auto-invalidation + reload: () => Promise + + get: (args: { id: string }) => Promise + set: (args: { data: T }) => Promise + delete: (args: { id: string }) => Promise + getAll: () => Promise +} diff --git a/inlang/source-code/sdk/src/reactivity/solid.test.ts b/inlang/source-code/sdk/src/reactivity/solid.test.ts index abd713e433..277268906f 100644 --- a/inlang/source-code/sdk/src/reactivity/solid.test.ts +++ b/inlang/source-code/sdk/src/reactivity/solid.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect } from "vitest" +import { sleep, delay } from "../test-utilities/sleep.js" import { createRoot, createSignal, @@ -9,14 +10,6 @@ import { } from "./solid.js" import { ReactiveMap } from "./map.js" -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)) -} - -function delay(v: unknown, ms: number) { - return new Promise((resolve) => setTimeout(() => resolve(v), ms)) -} - describe("vitest", () => { it("waits for the result of an async function", async () => { const rval = await delay(42, 1) diff --git a/inlang/source-code/sdk/src/resolve-modules/plugins/resolvePlugins.ts b/inlang/source-code/sdk/src/resolve-modules/plugins/resolvePlugins.ts index 53ed397a7c..3de9c55760 100644 --- a/inlang/source-code/sdk/src/resolve-modules/plugins/resolvePlugins.ts +++ b/inlang/source-code/sdk/src/resolve-modules/plugins/resolvePlugins.ts @@ -1,10 +1,6 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import type { ResolvePluginsFunction } from "./types.js" import { Plugin } from "@inlang/plugin" -import { - loadMessages as sdkLoadMessages, - saveMessages as sdkSaveMessages, -} from "../../persistence/plugin.js" import { PluginReturnedInvalidCustomApiError, PluginLoadMessagesFunctionAlreadyDefinedError, @@ -121,15 +117,10 @@ export const resolvePlugins: ResolvePluginsFunction = async (args) => { // --- LOADMESSAGE / SAVEMESSAGE NOT DEFINED --- - if (experimentalPersistence) { - debug("Override load/save for experimental persistence") - // @ts-ignore - type mismatch error - result.data.loadMessages = sdkLoadMessages - // @ts-ignore - type mismatch error - result.data.saveMessages = sdkSaveMessages - } else if ( - typeof result.data.loadMessages !== "function" || - typeof result.data.saveMessages !== "function" + if ( + !experimentalPersistence && + (typeof result.data.loadMessages !== "function" || + typeof result.data.saveMessages !== "function") ) { result.errors.push(new PluginsDoNotProvideLoadOrSaveMessagesError()) } diff --git a/inlang/source-code/sdk/src/test-utilities/sleep.ts b/inlang/source-code/sdk/src/test-utilities/sleep.ts new file mode 100644 index 0000000000..5ff4e91aa0 --- /dev/null +++ b/inlang/source-code/sdk/src/test-utilities/sleep.ts @@ -0,0 +1,11 @@ +export function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +export function delay(v: unknown, ms: number) { + return new Promise((resolve) => setTimeout(() => resolve(v), ms)) +} + +export function fail(ms: number) { + return new Promise((resolve, reject) => setTimeout(() => reject(new Error("fail")), ms)) +} diff --git a/inlang/source-code/sdk/src/v2/helper.ts b/inlang/source-code/sdk/src/v2/helper.ts index 4de79f3b32..0f26534bf6 100644 --- a/inlang/source-code/sdk/src/v2/helper.ts +++ b/inlang/source-code/sdk/src/v2/helper.ts @@ -1,4 +1,11 @@ -import { LanguageTag, MessageBundle, Message, Text } from "./types.js" +import { + LanguageTag, + MessageBundle, + MessageBundleWithSlots, + Message, + MessageSlot, + Text, +} from "./types.js" /** * create v2 MessageBundle @@ -45,3 +52,47 @@ export function toTextElement(text: string): Text { value: text, } } + +// **************************** +// WIP experimental persistence +// **************************** + +/** + * create MessageSlot for a locale (only used for persistence) + */ +export function createMessageSlot(locale: LanguageTag): MessageSlot { + return { + locale, + slot: true, + } +} + +/** + * return structuredClone with message slots for all locales not yet present + */ +export function addSlots(messageBundle: MessageBundle, locales: string[]): MessageBundleWithSlots { + const bundle = structuredClone(messageBundle) as MessageBundleWithSlots + bundle.messages = locales.map((locale) => { + return bundle.messages.find((message) => message.locale === locale) ?? createMessageSlot(locale) + }) + return bundle +} + +/** + * remove empty message slots without first creating a structured clone + */ +export function removeSlots(messageBundle: MessageBundleWithSlots) { + messageBundle.messages = messageBundle.messages.filter((message) => !("slot" in message)) + return messageBundle as MessageBundle +} + +/** + * Add newlines between bundles and messages to avoid merge conflicts + */ +export function injectJSONNewlines(json: string): string { + return json + .replace(/\{"id":"/g, '\n\n\n\n{"id":"') + .replace(/"messages":\[\{"locale":"/g, '"messages":[\n\n\n\n{"locale":"') + .replace(/\}\]\}\]\},\{"locale":"/g, '}]}]},\n\n\n\n{"locale":"') + .replace(/"slot":true\},\{"locale":/g, '"slot":true},\n\n\n\n{"locale":') +} diff --git a/inlang/source-code/sdk/src/v2/index.ts b/inlang/source-code/sdk/src/v2/index.ts index cb69b2380b..b4a1467579 100644 --- a/inlang/source-code/sdk/src/v2/index.ts +++ b/inlang/source-code/sdk/src/v2/index.ts @@ -1,2 +1,3 @@ export type * from "./types.js" export * from "./helper.js" +export * from "./shim.js" diff --git a/inlang/source-code/sdk/src/v2/shim.test.ts b/inlang/source-code/sdk/src/v2/shim.test.ts new file mode 100644 index 0000000000..9b74803ec6 --- /dev/null +++ b/inlang/source-code/sdk/src/v2/shim.test.ts @@ -0,0 +1,56 @@ +import { test, expect } from "vitest" +import * as V2 from "./types.js" +import * as V1 from "@inlang/message" +import { createMessageBundle, createMessage } from "./helper.js" +import { toV1Message, fromV1Message } from "./shim.js" +import { Value } from "@sinclair/typebox/value" + +const bundle = createMessageBundle({ + id: "hello_world", + messages: [ + createMessage({ locale: "en", text: "Hello World!" }), + createMessage({ locale: "de", text: "Hallo Welt!" }), + ], +}) + +test("toV1Message and fromV1Message", () => { + expect(Value.Check(V2.MessageBundle, bundle)).toBe(true) + + const v1Message: unknown = toV1Message(bundle) + expect(Value.Check(V1.Message, v1Message)).toBe(true) + + expect(v1Message).toEqual({ + id: "hello_world", + alias: {}, + variants: [ + { + languageTag: "en", + match: [], + pattern: [ + { + type: "Text", + value: "Hello World!", + }, + ], + }, + { + languageTag: "de", + match: [], + pattern: [ + { + type: "Text", + value: "Hallo Welt!", + }, + ], + }, + ], + selectors: [], + }) + + const v2MessageBundle: unknown = fromV1Message(v1Message as V1.Message) + expect(Value.Check(V2.MessageBundle, v2MessageBundle)).toBe(true) + + expect(v2MessageBundle).toEqual(bundle) +}) + +test.todo("with variable references", () => {}) diff --git a/inlang/source-code/sdk/src/v2/shim.ts b/inlang/source-code/sdk/src/v2/shim.ts new file mode 100644 index 0000000000..14b7ab1894 --- /dev/null +++ b/inlang/source-code/sdk/src/v2/shim.ts @@ -0,0 +1,173 @@ +/** + * Convert between v1 Message and v2 MessageBundle formats. + * Code adapted from https://github.com/opral/monorepo/pull/2655 legacy.ts + */ + +import * as V2 from "./types.js" +import * as V1 from "@inlang/message" + +/** + * @throws If the message cannot be represented in the v1 format + */ +export function toV1Message(bundle: V2.MessageBundle): V1.Message { + const variants: V1.Variant[] = [] + const selectorNames = new Set() + + for (const message of bundle.messages) { + // collect all selector names + for (const selector of message.selectors.map(toV1Expression)) { + selectorNames.add(selector.name) + } + + // collect all variants + for (const variant of message.variants) { + variants.push({ + languageTag: message.locale, + match: variant.match, + pattern: toV1Pattern(variant.pattern), + }) + } + } + + const selectors: V1.Expression[] = [...selectorNames].map((name) => ({ + type: "VariableReference", + name, + })) + + return { + id: bundle.id, + alias: bundle.alias, + variants, + selectors, + } +} + +/** + * @throws If the pattern cannot be represented in the v1 format + */ +function toV1Pattern(pattern: V2.Pattern): V1.Pattern { + return pattern.map((element) => { + switch (element.type) { + case "text": { + return { + type: "Text", + value: element.value, + } + } + + case "expression": { + return toV1Expression(element) + } + + default: { + throw new Error(`Unsupported pattern element type`) + } + } + }) +} + +function toV1Expression(expression: V2.Expression): V1.Expression { + if (expression.annotation !== undefined) + throw new Error("Cannot convert an expression with an annotation to the v1 format") + + if (expression.arg.type !== "variable") { + throw new Error("Can only convert variable references to the v1 format") + } + + return { + type: "VariableReference", + name: expression.arg.name, + } +} + +export function fromV1Message(v1Message: V1.Message): V2.MessageBundle { + const languages = dedup(v1Message.variants.map((variant) => variant.languageTag)) + + const messages: V2.Message[] = languages.map((language): V2.Message => { + //All variants that will be part of this message + const v1Variants = v1Message.variants.filter((variant) => variant.languageTag === language) + + //find all selector names + const selectorNames = new Set() + for (const v1Selector of v1Message.selectors) { + selectorNames.add(v1Selector.name) + } + const selectors: V2.Expression[] = [...selectorNames].map((name) => ({ + type: "expression", + annotation: undefined, + arg: { + type: "variable", + name: name, + }, + })) + + //The set of variables that need to be defined - Certainly includes the selectors + const variableNames = new Set(selectorNames) + const variants: V2.Variant[] = [] + for (const v1Variant of v1Variants) { + for (const element of v1Variant.pattern) { + if (element.type === "VariableReference") { + variableNames.add(element.name) + } + } + + variants.push({ + match: v1Variant.match, + pattern: fromV1Pattern(v1Variant.pattern), + }) + } + + //Create an input declaration for each variable and selector we need + const declarations: V2.Declaration[] = [...variableNames].map((name) => ({ + type: "input", + name, + value: { + type: "expression", + annotation: undefined, + arg: { + type: "variable", + name, + }, + }, + })) + + return { + locale: language, + declarations, + selectors, + variants, + } + }) + + return { + id: v1Message.id, + alias: v1Message.alias, + messages, + } +} + +function fromV1Pattern(pattern: V1.Pattern): V2.Pattern { + return pattern.map((element) => { + switch (element.type) { + case "Text": { + return { + type: "text", + value: element.value, + } + } + case "VariableReference": + return { + type: "expression", + arg: { + type: "variable", + name: element.name, + }, + } + } + }) +} + +/** + * Dedups an array by converting it to a set and back + */ +const dedup = >(arr: T): T => [...new Set(arr)] as T diff --git a/inlang/source-code/sdk/src/v2/stubQueryApi.ts b/inlang/source-code/sdk/src/v2/stubQueryApi.ts new file mode 100644 index 0000000000..1be88977a1 --- /dev/null +++ b/inlang/source-code/sdk/src/v2/stubQueryApi.ts @@ -0,0 +1,43 @@ +import type { MessageQueryApi, MessageLintReportsQueryApi } from "../api.js" + +/** + * noop implementation of the message query api for use with experimental.persistence = true. + * NOTE: If we implemented v2 shims for the old api we could use existing tests and make apps + * backwards compatible. + */ +export const stubMessagesQuery: MessageQueryApi = { + create: () => false, + // @ts-expect-error + get: subscribable(() => undefined), + // @ts-expect-error + getByDefaultAlias: subscribable(() => undefined), + // @ts-expect-error + includedMessageIds: subscribable(() => []), + // @ts-expect-error + getAll: subscribable(() => []), + update: () => false, + upsert: () => {}, + delete: () => false, + setDelegate: () => {}, +} + +export const stubMessageLintReportsQuery: MessageLintReportsQueryApi = { + // @ts-expect-error + get: subscribable(() => []), + // @ts-expect-error + getAll: settleable(subscribable(() => [])), +} + +// eslint-disable-next-line @typescript-eslint/ban-types +function subscribable(fn: Function) { + return Object.assign(fn, { + subscribe: () => {}, + }) +} + +// eslint-disable-next-line @typescript-eslint/ban-types +function settleable(fn: Function) { + return Object.assign(fn, { + settled: async () => {}, + }) +} diff --git a/inlang/source-code/sdk/src/v2/types.ts b/inlang/source-code/sdk/src/v2/types.ts index 65a1d659e2..c8b9eee057 100644 --- a/inlang/source-code/sdk/src/v2/types.ts +++ b/inlang/source-code/sdk/src/v2/types.ts @@ -140,3 +140,20 @@ export const MessageBundle = Type.Object({ alias: Type.Record(Type.String(), Type.String()), messages: Type.Array(Message), }) + +/** + * A MessageSlot is a placeholder for a message with a locale. + * This is useful to avoid merge conflicts when translations are added. + */ +export type MessageSlot = Static +export const MessageSlot = Type.Object({ + locale: LanguageTag, + slot: Type.Literal(true), +}) + +export type MessageBundleWithSlots = Static +export const MessageBundleWithSlots = Type.Object({ + id: Type.String(), + alias: Type.Record(Type.String(), Type.String()), + messages: Type.Array(Type.Union([Message, MessageSlot])), +})