Merge pull request #2801 from opral/v2-persistence

MESDK-88 Minimal v2 persistence

.changeset/brown-seas-hunt.md

@@ -0,0 +1,6 @@
+---
+"@inlang/cli": patch
+"@inlang/sdk": patch
+---
+
+v2 message bundle persistence - initial store implementation

inlang/source-code/cli/src/commands/machine/translate.ts

@@ -9,6 +9,7 @@ import { projectOption } from "../../utilities/globalFlags.js"
 import progessBar from "cli-progress"
 import plimit from "p-limit"
 import type { Result } from "@inlang/result"
+import { toV1Message, fromV1Message } from "@inlang/sdk/v2"
 
 const rpcTranslateAction = process.env.MOCK_TRANSLATE_LOCAL
 	? mockMachineTranslateMessage
@@ -18,6 +19,7 @@ export const translate = new Command()
 	.command("translate")
 	.requiredOption(projectOption.flags, projectOption.description)
 	.option("-f, --force", "Force machine translation and skip the confirmation prompt.", false)
+	.option("-q, --quiet", "don't log every tranlation.", false)
 	.option("--sourceLanguageTag <source>", "Source language tag for translation.")
 	.option(
 		"--targetLanguageTags <targets...>",
@@ -60,6 +62,7 @@ export async function translateCommandAction(args: { project: InlangProject }) {
 			return
 		}
 		const experimentalAliases = args.project.settings().experimental?.aliases
+		const v2Persistence = args.project.settings().experimental?.persistence
 
 		const allLanguageTags = [...projectConfig.languageTags, projectConfig.sourceLanguageTag]
 
@@ -97,9 +100,13 @@ export async function translateCommandAction(args: { project: InlangProject }) {
 			return
 		}
 
-		const messages = args.project.query.messages
-			.getAll()
-			.filter((message) => hasMissingTranslations(message, sourceLanguageTag, targetLanguageTags))
+		const allMessages = v2Persistence
+			? (await args.project.store!.messageBundles.getAll()).map(toV1Message)
+			: args.project.query.messages.getAll()
+
+		const filteredMessages = allMessages.filter((message) =>
+			hasMissingTranslations(message, sourceLanguageTag, targetLanguageTags)
+		)
 
 		const bar = options.nobar
 			? undefined
@@ -111,7 +118,7 @@ export async function translateCommandAction(args: { project: InlangProject }) {
 					progessBar.Presets.shades_grey
 			  )
 
-		bar?.start(messages.length, 0)
+		bar?.start(filteredMessages.length, 0)
 
 		const logs: Array<() => void> = []
 
@@ -132,17 +139,23 @@ export async function translateCommandAction(args: { project: InlangProject }) {
 				translatedMessage &&
 				translatedMessage?.variants.length > toBeTranslatedMessage.variants.length
 			) {
-				args.project.query.messages.update({
-					where: { id: translatedMessage.id },
-					data: translatedMessage!,
-				})
-				logs.push(() => log.info(`Machine translated message ${logId}`))
+				if (v2Persistence) {
+					await args.project.store!.messageBundles.set({ data: fromV1Message(translatedMessage) })
+				} else {
+					args.project.query.messages.update({
+						where: { id: translatedMessage.id },
+						data: translatedMessage!,
+					})
+				}
+				if (!options.quiet) {
+					logs.push(() => log.info(`Machine translated message ${logId}`))
+				}
 			}
 			bar?.increment()
 		}
 		// parallelize rpcTranslate calls with a limit of 100 concurrent calls
-		const limit = plimit(100)
-		const promises = messages.map((message) => limit(() => rpcTranslate(message)))
+		const limit = plimit(process.env.MOCK_TRANSLATE_LOCAL ? 100000 : 100)
+		const promises = filteredMessages.map((message) => limit(() => rpcTranslate(message)))
 		await Promise.all(promises)
 
 		bar?.stop()
@@ -220,5 +233,6 @@ async function mockMachineTranslateMessage(args: {
 			// console.log("mockMachineTranslateMessage translated", q, targetLanguageTag)
 		}
 	}
+	await new Promise((resolve) => setTimeout(resolve, 100))
 	return { data: copy }
 }

inlang/source-code/sdk/load-test/.gitignore

@@ -1,4 +1,5 @@
 node_modules
 locales
 project.inlang/messages
-project.inlang/messages.json
+project.inlang/messages.json
+temp

inlang/source-code/sdk/load-test/README.md

@@ -1,12 +1,5 @@
-# inlang sdk load-test
-package for volume testing
-
-- The test starts by generating engish messages in ./locales/en/common.json
-  or ./project.inlang/messages.json (depending on experimental.persistence)
-- It then calls loadProject() and subscribes to events.
-- It mock-translates into 36 languages using the inlang cli.
-- It uses the i18next message storage plugin (unless experimental.persistence is set)
-- To allow additional testing on the generated project e.g. with the ide-extension, the test calls `pnpm clean` when it starts, but not after it runs.
+# package @inlang/sdk-load-test
+The default `test` script runs a load-test with 1000 messages and translation enabled. For more messsages and different load-test options, use the `load-test` script.
 
 ```
 USAGE:
@@ -18,8 +11,17 @@ e.g.
 Defaults: translate: 1, subscribeToMessages: 1, subscribeToLintReports: 0, watchMode: 0
 ```
 
+### what it does
+- The test starts by generating engish messages in ./locales/en/common.json
+  or ./project.inlang/messages.json (depending on experimental.persistence)
+- It then calls loadProject() and subscribes to events.
+- It mock-translates into 36 languages using the inlang cli.
+- It uses the i18next message storage plugin (unless experimental.persistence is set)
+- To allow additional testing on the generated project e.g. with the ide-extension, the test calls `pnpm clean` when it starts, but not after it runs.
+
 ### to configure additional debug logging
-`export DEBUG=sdk:acquireFileLock,sdk:releaseLock,sdk:lintReports,sdk:loadProject`
+`export DEBUG=sdk:*` for wildcard (most verbose) logging and to see more specific options.
+e.g. `export DEBUG=sdk:lockFile`
 
 ### to translate from separate process
 1. Run pnpm load-test with translate:0, watchMode:1 E.g. `pnpm load-test 100 0 1 1 1`

inlang/source-code/sdk/load-test/load-test.ts

@@ -1,8 +1,17 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-disable no-restricted-imports */
-/* eslint-disable no-console */
+
 import { findRepoRoot, openRepository } from "@lix-js/client"
-import { loadProject, type Message, normalizeMessage } from "@inlang/sdk"
-import { createMessage } from "../src/test-utilities/createMessage.js"
+import { loadProject, type InlangProject, type Message, type ProjectSettings } from "@inlang/sdk"
+
+import {
+	createMessage,
+	createMessageBundle,
+	addSlots,
+	injectJSONNewlines,
+} from "../src/v2/helper.js"
+import { MessageBundle } from "../src/v2/types.js"
+
 import { createEffect } from "../src/reactivity/solid.js"
 
 import { dirname, join } from "node:path"
@@ -25,9 +34,7 @@ const projectPath = join(__dirname, "project.inlang")
 const mockServer = "http://localhost:3000"
 
 const cli = `PUBLIC_SERVER_BASE_URL=${mockServer} pnpm inlang`
-const translateCommand = cli + " machine translate -f -n --project ./project.inlang"
-
-const isExperimentalPersistence = await checkExperimentalPersistence()
+const translateCommand = cli + " machine translate -f -q -n --project ./project.inlang"
 
 export async function runLoadTest(
 	messageCount: number = 1000,
@@ -36,10 +43,14 @@ export async function runLoadTest(
 	subscribeToLintReports: boolean = false,
 	watchMode: boolean = false
 ) {
+	const settings = await getSettings()
+	// experimental persistence uses v2 types
+	const isV2 = !!settings.experimental?.persistence
+	const locales = settings.languageTags
 	debug(
 		"load-test start" +
 			(watchMode ? " - watchMode on, ctrl C to exit" : "") +
-			(isExperimentalPersistence ? " - using experimental persistence" : "")
+			(isV2 ? " - using experimental persistence" : "")
 	)
 	if (translate && !process.env.MOCK_TRANSLATE_LOCAL && !(await isMockRpcServerRunning())) {
 		console.error(
@@ -55,7 +66,7 @@ export async function runLoadTest(
 
 	debug(`generating ${messageCount} messages`)
 	// this is called before loadProject() to avoid reading partially written JSON
-	await generateMessageFile(messageCount)
+	await generateMessageFile(isV2, messageCount, locales)
 
 	debug("opening repo and loading project")
 	const repoRoot = await findRepoRoot({ nodeishFs: fs, path: __dirname })
@@ -73,7 +84,7 @@ export async function runLoadTest(
 		}
 	})
 
-	if (subscribeToMessages) {
+	if (subscribeToMessages && !isV2) {
 		debug("subscribing to messages.getAll")
 		let countMessagesGetAllEvents = 0
 
@@ -90,7 +101,7 @@ export async function runLoadTest(
 		})
 	}
 
-	if (subscribeToLintReports) {
+	if (subscribeToLintReports && !isV2) {
 		debug("subscribing to lintReports.getAll")
 		let lintEvents = 0
 		const logLintEvent = throttle(throttleLintEvents, (reports: any) => {
@@ -102,9 +113,17 @@ export async function runLoadTest(
 		})
 	}
 
+	if (isV2) {
+		await summarize("loaded", project)
+	}
+
 	if (translate) {
 		debug("translating messages with inlang cli")
 		await run(translateCommand)
+		if (isV2) {
+			await project.store?.messageBundles.reload()
+			await summarize("translated", project)
+		}
 	}
 
 	debug("load-test done - " + (watchMode ? "watching for events" : "exiting"))
@@ -116,19 +135,35 @@ export async function runLoadTest(
 	}
 }
 
-async function generateMessageFile(messageCount: number) {
-	if (isExperimentalPersistence) {
+async function summarize(action: string, project: InlangProject) {
+	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+	const bundles = await project.store!.messageBundles.getAll()
+	let bundleCount = 0
+	let messageCount = 0
+	bundles.map((bundle) => {
+		bundleCount++
+		messageCount += bundle.messages.length
+	})
+	debug(`${action}: ${bundleCount} bundles, ${messageCount / bundleCount} messages/bundle`)
+}
+
+async function generateMessageFile(isV2: boolean, messageCount: number, locales: string[]) {
+	if (isV2) {
 		const messageFile = join(__dirname, "project.inlang", "messages.json")
 
-		const messages: Message[] = []
+		const messages: MessageBundle[] = []
 		for (let i = 1; i <= messageCount; i++) {
-			messages.push(createMessage(`message_key_${i}`, { en: `Generated message (${i})` }))
+			messages.push(
+				createMessageBundle({
+					id: `message_key_${i}`,
+					messages: [createMessage({ locale: "en", text: `Generated message (${i})` })],
+				})
+			)
 		}
-		await fs.writeFile(
-			messageFile,
-			JSON.stringify(messages.map(normalizeMessage), undefined, 2),
-			"utf-8"
+		const output = injectJSONNewlines(
+			JSON.stringify(messages.map((bundle) => addSlots(bundle, locales)))
 		)
+		await fs.writeFile(messageFile, output, "utf-8")
 	} else {
 		const messageDir = join(__dirname, "locales", "en")
 		const messageFile = join(__dirname, "locales", "en", "common.json")
@@ -142,10 +177,10 @@ async function generateMessageFile(messageCount: number) {
 	}
 }
 
-async function checkExperimentalPersistence() {
+async function getSettings() {
 	const settingsFile = join(__dirname, "project.inlang", "settings.json")
 	const settings = JSON.parse(await fs.readFile(settingsFile, "utf-8"))
-	return !!settings.experimental?.persistence
+	return settings as ProjectSettings
 }
 
 async function isMockRpcServerRunning(): Promise<boolean> {

inlang/source-code/sdk/load-test/package.json

@@ -19,7 +19,7 @@
 	},
 	"scripts": {
 		"clean": "rm -rf ./locales ./project.inlang/messages.json",
-		"translate": "MOCK_TRANSLATE_LOCAL=1 PUBLIC_SERVER_BASE_URL=http://localhost:3000 pnpm inlang machine translate -n -f --project ./project.inlang",
+		"translate": "MOCK_TRANSLATE_LOCAL=1 pnpm inlang machine translate -n -f -q --project ./project.inlang",
 		"test-lint": "pnpm inlang lint --project ./project.inlang",
 		"load-test": "pnpm clean && MOCK_TRANSLATE_LOCAL=1 DEBUG=$DEBUG,load-test tsx ./main.ts",
 		"test": "tsc --noEmit && pnpm load-test 1000 1 1 1",

inlang/source-code/sdk/load-test/project.inlang/settings.json

@@ -49,7 +49,5 @@
 	"plugin.inlang.i18next": {
 		"pathPattern": "./locales/{languageTag}/common.json"
 	},
-	"experimental": {
-		"persistence": true
-	}
+	"experimental": { "persistence": true }
 }

inlang/source-code/sdk/multi-project-test/multi-project.test.ts

@@ -78,6 +78,7 @@ describe.concurrent(
 			{ timeout: 20000 }
 		)
 
+		// skip pending new v2 persistence with translation.
 		it(
 			"project4 in project4-dir",
 			async () => {