MESDK-88 Minimal v2 persistence

inlang/source-code/cli/src/commands/machine/translate.ts

@@ -9,6 +9,7 @@ import { projectOption } from "../../utilities/globalFlags.js"
 import progessBar from "cli-progress"
 import plimit from "p-limit"
 import type { Result } from "@inlang/result"
+import { toV1Message, fromV1Message } from "@inlang/sdk/v2"
 
 const rpcTranslateAction = process.env.MOCK_TRANSLATE_LOCAL
 	? mockMachineTranslateMessage
@@ -60,6 +61,7 @@ export async function translateCommandAction(args: { project: InlangProject }) {
 			return
 		}
 		const experimentalAliases = args.project.settings().experimental?.aliases
+		const v2Persistence = args.project.settings().experimental?.persistence
 
 		const allLanguageTags = [...projectConfig.languageTags, projectConfig.sourceLanguageTag]
 
@@ -97,9 +99,13 @@ export async function translateCommandAction(args: { project: InlangProject }) {
 			return
 		}
 
-		const messages = args.project.query.messages
-			.getAll()
-			.filter((message) => hasMissingTranslations(message, sourceLanguageTag, targetLanguageTags))
+		const allMessages = v2Persistence
+			? (await args.project.store!.messageBundles.getAll()).map(toV1Message)
+			: args.project.query.messages.getAll()
+
+		const filteredMessages = allMessages.filter((message) =>
+			hasMissingTranslations(message, sourceLanguageTag, targetLanguageTags)
+		)
 
 		const bar = options.nobar
 			? undefined
@@ -111,7 +117,7 @@ export async function translateCommandAction(args: { project: InlangProject }) {
 					progessBar.Presets.shades_grey
 			  )
 
-		bar?.start(messages.length, 0)
+		bar?.start(filteredMessages.length, 0)
 
 		const logs: Array<() => void> = []
 
@@ -132,17 +138,21 @@ export async function translateCommandAction(args: { project: InlangProject }) {
 				translatedMessage &&
 				translatedMessage?.variants.length > toBeTranslatedMessage.variants.length
 			) {
-				args.project.query.messages.update({
-					where: { id: translatedMessage.id },
-					data: translatedMessage!,
-				})
+				if (v2Persistence) {
+					await args.project.store!.messageBundles.set({ data: fromV1Message(translatedMessage) })
+				} else {
+					args.project.query.messages.update({
+						where: { id: translatedMessage.id },
+						data: translatedMessage!,
+					})
+				}
 				logs.push(() => log.info(`Machine translated message ${logId}`))
 			}
 			bar?.increment()
 		}
 		// parallelize rpcTranslate calls with a limit of 100 concurrent calls
 		const limit = plimit(100)
-		const promises = messages.map((message) => limit(() => rpcTranslate(message)))
+		const promises = filteredMessages.map((message) => limit(() => rpcTranslate(message)))
 		await Promise.all(promises)
 
 		bar?.stop()

inlang/source-code/sdk/load-test/README.md

@@ -1,12 +1,5 @@
-# inlang sdk load-test
-package for volume testing
-
-- The test starts by generating engish messages in ./locales/en/common.json
-  or ./project.inlang/messages.json (depending on experimental.persistence)
-- It then calls loadProject() and subscribes to events.
-- It mock-translates into 36 languages using the inlang cli.
-- It uses the i18next message storage plugin (unless experimental.persistence is set)
-- To allow additional testing on the generated project e.g. with the ide-extension, the test calls `pnpm clean` when it starts, but not after it runs.
+# package @inlang/sdk-load-test
+The default `test` script runs a load-test with 1000 messages and translation enabled. For more messsages and different load-test options, use the `load-test` script.
 
 ```
 USAGE:
@@ -18,8 +11,17 @@ e.g.
 Defaults: translate: 1, subscribeToMessages: 1, subscribeToLintReports: 0, watchMode: 0
 ```
 
+### what it does
+- The test starts by generating engish messages in ./locales/en/common.json
+  or ./project.inlang/messages.json (depending on experimental.persistence)
+- It then calls loadProject() and subscribes to events.
+- It mock-translates into 36 languages using the inlang cli.
+- It uses the i18next message storage plugin (unless experimental.persistence is set)
+- To allow additional testing on the generated project e.g. with the ide-extension, the test calls `pnpm clean` when it starts, but not after it runs.
+
 ### to configure additional debug logging
-`export DEBUG=sdk:acquireFileLock,sdk:releaseLock,sdk:lintReports,sdk:loadProject`
+`export DEBUG=sdk:*` for wildcard (most verbose) logging and to see more specific options.
+e.g. `export DEBUG=sdk:lockFile`
 
 ### to translate from separate process
 1. Run pnpm load-test with translate:0, watchMode:1 E.g. `pnpm load-test 100 0 1 1 1`

inlang/source-code/sdk/load-test/load-test.ts

@@ -1,8 +1,16 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-disable no-restricted-imports */
-/* eslint-disable no-console */
+
 import { findRepoRoot, openRepository } from "@lix-js/client"
-import { loadProject, type Message, normalizeMessage } from "@inlang/sdk"
-import { createMessage } from "../src/test-utilities/createMessage.js"
+import { loadProject, type Message } from "@inlang/sdk"
+
+import {
+	createMessage,
+	createMessageBundle,
+	normalizeMessageBundle,
+} from "../src/v2/createMessageBundle.js"
+import { MessageBundle } from "../src/v2/types.js"
+
 import { createEffect } from "../src/reactivity/solid.js"
 
 import { dirname, join } from "node:path"
@@ -27,19 +35,19 @@ const mockServer = "http://localhost:3000"
 const cli = `PUBLIC_SERVER_BASE_URL=${mockServer} pnpm inlang`
 const translateCommand = cli + " machine translate -f -n --project ./project.inlang"
 
-const isExperimentalPersistence = await checkExperimentalPersistence()
-
 export async function runLoadTest(
 	messageCount: number = 1000,
 	translate: boolean = true,
 	subscribeToMessages: boolean = true,
 	subscribeToLintReports: boolean = false,
 	watchMode: boolean = false
 ) {
+	// experimental persistence uses v2 types
+	const isV2 = await checkExperimentalPersistence()
 	debug(
 		"load-test start" +
 			(watchMode ? " - watchMode on, ctrl C to exit" : "") +
-			(isExperimentalPersistence ? " - using experimental persistence" : "")
+			(isV2 ? " - using experimental persistence" : "")
 	)
 	if (translate && !process.env.MOCK_TRANSLATE_LOCAL && !(await isMockRpcServerRunning())) {
 		console.error(
@@ -55,7 +63,7 @@ export async function runLoadTest(
 
 	debug(`generating ${messageCount} messages`)
 	// this is called before loadProject() to avoid reading partially written JSON
-	await generateMessageFile(messageCount)
+	await generateMessageFile(isV2, messageCount)
 
 	debug("opening repo and loading project")
 	const repoRoot = await findRepoRoot({ nodeishFs: fs, path: __dirname })
@@ -73,7 +81,7 @@ export async function runLoadTest(
 		}
 	})
 
-	if (subscribeToMessages) {
+	if (subscribeToMessages && !isV2) {
 		debug("subscribing to messages.getAll")
 		let countMessagesGetAllEvents = 0
 
@@ -90,7 +98,7 @@ export async function runLoadTest(
 		})
 	}
 
-	if (subscribeToLintReports) {
+	if (subscribeToLintReports && !isV2) {
 		debug("subscribing to lintReports.getAll")
 		let lintEvents = 0
 		const logLintEvent = throttle(throttleLintEvents, (reports: any) => {
@@ -102,6 +110,12 @@ export async function runLoadTest(
 		})
 	}
 
+	if (isV2) {
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+		const bundles = await project.store!.messageBundles.getAll()
+		debug(`loaded ${bundles.length} v2 MessageBundles`)
+	}
+
 	if (translate) {
 		debug("translating messages with inlang cli")
 		await run(translateCommand)
@@ -116,17 +130,22 @@ export async function runLoadTest(
 	}
 }
 
-async function generateMessageFile(messageCount: number) {
-	if (isExperimentalPersistence) {
+async function generateMessageFile(isV2: boolean, messageCount: number) {
+	if (isV2) {
 		const messageFile = join(__dirname, "project.inlang", "messages.json")
 
-		const messages: Message[] = []
+		const messages: MessageBundle[] = []
 		for (let i = 1; i <= messageCount; i++) {
-			messages.push(createMessage(`message_key_${i}`, { en: `Generated message (${i})` }))
+			messages.push(
+				createMessageBundle({
+					id: `message_key_${i}`,
+					messages: [createMessage({ locale: "en", text: `Generated message (${i})` })],
+				})
+			)
 		}
 		await fs.writeFile(
 			messageFile,
-			JSON.stringify(messages.map(normalizeMessage), undefined, 2),
+			JSON.stringify(messages.map(normalizeMessageBundle), undefined, 2),
 			"utf-8"
 		)
 	} else {

inlang/source-code/sdk/load-test/project.inlang/settings.json

@@ -49,7 +49,5 @@
 	"plugin.inlang.i18next": {
 		"pathPattern": "./locales/{languageTag}/common.json"
 	},
-	"experimental": {
-		"persistence": true
-	}
+	"experimental": { "persistence": true }
 }

inlang/source-code/sdk/multi-project-test/multi-project.test.ts

@@ -78,6 +78,7 @@ describe.concurrent(
 			{ timeout: 20000 }
 		)
 
+		// skip pending new v2 persistence with translation.
 		it(
 			"project4 in project4-dir",
 			async () => {

inlang/source-code/sdk/multi-project-test/project4-dir/messages.json.bak

@@ -1,50 +1,65 @@
 [
   {
-    "alias": {},
     "id": "project4_message_key_1",
-    "selectors": [],
-    "variants": [
+    "alias": {},
+    "messages": [
       {
-        "languageTag": "en",
-        "match": [],
-        "pattern": [
+        "locale": "en",
+        "declarations": [],
+        "selectors": [],
+        "variants": [
           {
-            "type": "Text",
-            "value": "Generated message (1)"
+            "match": [],
+            "pattern": [
+              {
+                "type": "text",
+                "value": "Generated message (1)"
+              }
+            ]
           }
         ]
       }
     ]
   },
   {
-    "alias": {},
     "id": "project4_message_key_2",
-    "selectors": [],
-    "variants": [
+    "alias": {},
+    "messages": [
       {
-        "languageTag": "en",
-        "match": [],
-        "pattern": [
+        "locale": "en",
+        "declarations": [],
+        "selectors": [],
+        "variants": [
           {
-            "type": "Text",
-            "value": "Generated message (2)"
+            "match": [],
+            "pattern": [
+              {
+                "type": "text",
+                "value": "Generated message (2)"
+              }
+            ]
           }
         ]
       }
     ]
   },
   {
-    "alias": {},
     "id": "project4_message_key_3",
-    "selectors": [],
-    "variants": [
+    "alias": {},
+    "messages": [
       {
-        "languageTag": "en",
-        "match": [],
-        "pattern": [
+        "locale": "en",
+        "declarations": [],
+        "selectors": [],
+        "variants": [
           {
-            "type": "Text",
-            "value": "Generated message (3)"
+            "match": [],
+            "pattern": [
+              {
+                "type": "text",
+                "value": "Generated message (3)"
+              }
+            ]
           }
         ]
       }