feat: allow splitting translations (#52)

* feat: add translation splitting

I guess you could say this is a "prototype"

* feat: add documentation

* fix: update files

* feat: add async functions

* feat: better rewrite

* fix: small changes

* fix: fix for tests

* fix: add missing properties

* fix: add comma

* fix: small changes

* fix: small changes

Changed `NestedTranslation` from `type` to `interface`.

* fix: simplify sentence

* test: add support for split translations

* fix: rephrase example

README.md

@@ -19,17 +19,23 @@ import { I18n, I18nFlavor } from "https://deno.land/x/grammy_i18n/mod.ts";
 
 ## Example
 
-Example project structure:
+Below is an example featuring both nested (`locales/en/...`) and standard (`locales/it.ftl`) file structure variants. Nested translations allow you to seperate your keys into different files (making it easier to maintain larger projects) while also letting you use the standard variant at the same time. Using a nested file structure alongside the standard variant won't break any existing translations.
 
 ```
 .
-├─ locales/
-│  ├── en.ftl
-│  ├── it.ftl
-│  └── ru.ftl
+├── locales/
+│   ├── en/
+│   │   ├── dialogues/
+│   │   │   ├── greeting.ftl
+│   │   │   └── goodbye.ftl
+│   │   └── help.ftl
+│   ├── it.ftl
+│   └── ru.ftl
 └── bot.ts
 ```
 
+By splitting translations you don't change how you retrieve the keys contained within them, so for example, a key called `greeting` which is located in either `locales/en.ftl` or `locales/en/dialogues/greeting.ftl` can be retrieved by simply using `ctx.t("greeting")`.
+
 Example bot
 [not using sessions](https://grammy.dev/plugins/i18n.html#without-sessions):
 

examples/deno.ts

@@ -38,7 +38,7 @@ bot.command("start", async (ctx) => {
   await ctx.reply(ctx.t("greeting"));
 });
 
-bot.command(["en", "de", "ku", "ckb"], async (ctx) => {
+bot.command(["en", "de", "ku", "ckb", "ru"], async (ctx) => {
   const locale = ctx.msg.text.substring(1).split(" ")[0];
   await ctx.i18n.setLocale(locale);
   await ctx.reply(ctx.t("language-set"));
@@ -62,4 +62,8 @@ bot.command("checkout", async (ctx) => {
   await ctx.reply(ctx.t("checkout"));
 });
 
+bot.command("multiline", async (ctx) => {
+  await ctx.reply(ctx.t("multiline"));
+});
+
 bot.start();

examples/locales/ckb.ftl

@@ -2,3 +2,9 @@ greeting = سڵاو، { $first_name }!
 cart = سڵاو، { $first_name }، لە سەبەتەکەتدا{ $apples } سێو هەن.
 checkout = سپاس بۆ بازاڕیکردنەکەت!
 language-set = کوردی هەڵبژێردرا!
+multiline =
+  ئەمەش نموونەی...
+  ئە
+  فرە هێڵی
+  پەیام
+  بۆ ئەوەی بزانین چۆن فۆرمات کراون!

examples/locales/de.ftl

@@ -10,3 +10,10 @@ cart = { $first_name }, es {
 checkout = Danke für deinen Einkauf!
 
 language-set = Die Sprache wurde zu Deutsch geändert!
+
+multiline =
+  Dies ist ein Beispiel für
+  eine
+  mehrzeilige
+  Nachricht,
+  um zu sehen, wie sie formatiert ist!

examples/locales/en.ftl

@@ -10,3 +10,10 @@ cart = { $first_name }, there {
 checkout = Thank you for purchasing!
 
 language-set = Language has been set to English!
+
+multiline =
+  This is an example of
+  a
+  multiline
+  message
+  to see how they are formatted!

examples/locales/ku.ftl

@@ -2,3 +2,9 @@ greeting = Silav, { $first_name }!
 cart = { $first_name }, di sepeta te de { $apples } sêv hene.
 checkout = Spas bo kirîna te!
 language-set = Kurdî hate hilbijartin!
+multiline =
+  Ev mînakek e
+  yek
+  multiline
+  agah
+  da ku bibînin ka ew çawa têne format kirin!

examples/locales/ru/cart.ftl

@@ -0,0 +1,8 @@
+cart = { $first_name }, у вас {
+  $apples ->
+    [0] нет яблок
+    [one] одно яблоко
+    *[other] { $apples } яблок
+  } в корзине.
+
+checkout = Спасибо за покупку!

examples/locales/ru/greeting.ftl

@@ -0,0 +1 @@
+greeting = Привет { $first_name }!

examples/locales/ru/language.ftl

@@ -0,0 +1 @@
+language-set = Язык был изменен на Русский!

examples/locales/ru/multiline.ftl

@@ -0,0 +1,5 @@
+multiline =
+  Это пример
+  многострочных
+  сообщений
+  чтобы увидеть, как они отформатированы!

examples/node.ts

@@ -33,7 +33,7 @@ bot.command("start", async (ctx) => {
   await ctx.reply(ctx.t("greeting"));
 });
 
-bot.command(["en", "de", "ku", "ckb"], async (ctx) => {
+bot.command(["en", "de", "ku", "ckb", "ru"], async (ctx) => {
   const locale = ctx.msg.text.substring(1).split(" ")[0];
   await ctx.i18n.setLocale(locale);
   await ctx.reply(ctx.t("language-set"));
@@ -58,4 +58,8 @@ bot.command("checkout", async (ctx) => {
   await ctx.reply(ctx.t("checkout"));
 });
 
+bot.command("multiline", async (ctx) => {
+  await ctx.reply(ctx.t("multiline"));
+});
+
 bot.start();

src/deps.ts

@@ -12,4 +12,6 @@ export {
   type MiddlewareFn,
 } from "https://lib.deno.dev/x/[email protected]/mod.ts";
 
-export { extname, resolve } from "https://deno.land/[email protected]/path/mod.ts";
+export { extname, join, SEP } from "https://deno.land/[email protected]/path/mod.ts";
+
+export { walk, walkSync } from "https://deno.land/[email protected]/fs/walk.ts";

src/i18n.ts

@@ -1,9 +1,4 @@
-import {
-  type Context,
-  type HearsContext,
-  type MiddlewareFn,
-  resolve,
-} from "./deps.ts";
+import type { Context, HearsContext, MiddlewareFn } from "./deps.ts";
 import { Fluent } from "./fluent.ts";
 import type {
   I18nConfig,
@@ -35,27 +30,21 @@ export class I18n<C extends Context = Context> {
   async loadLocalesDir(directory: string): Promise<void> {
     const localeFiles = await readLocalesDir(directory);
     await Promise.all(localeFiles.map(async (file) => {
-      const path = resolve(directory, file);
-      const locale = file.substring(0, file.lastIndexOf("."));
-
-      await this.loadLocale(locale, {
-        filePath: path,
+      await this.loadLocale(file.belongsTo, {
+        source: file.translationSource,
         bundleOptions: this.config.fluentBundleOptions,
       });
     }));
   }
 
   /**
-   * Loads locales from the specified directory and registers them in the Fluent instance.
+   * Loads locales from any existing nested file or folder within the specified directory and registers them in the Fluent instance.
    * @param directory Path to the directory to look for the translation files.
    */
   loadLocalesDirSync(directory: string): void {
     for (const file of readLocalesDirSync(directory)) {
-      const path = resolve(directory, file);
-      const locale = file.substring(0, file.lastIndexOf("."));
-
-      this.loadLocaleSync(locale, {
-        filePath: path,
+      this.loadLocaleSync(file.belongsTo, {
+        source: file.translationSource,
         bundleOptions: this.config.fluentBundleOptions,
       });
     }

src/types.ts

@@ -19,6 +19,11 @@ export type LoadLocaleOptions = FilepathOrSource & {
   bundleOptions?: FluentBundleOptions;
 };
 
+export interface NestedTranslation {
+  belongsTo: LocaleId;
+  translationSource: string;
+}
+
 export interface FluentOptions {
   warningHandler?: WarningHandler;
 }

src/utils.ts

@@ -1,23 +1,81 @@
-import { extname } from "./deps.ts";
-
-export async function readLocalesDir(path: string): Promise<string[]> {
-  const files = new Array<string>();
-  for await (const entry of Deno.readDir(path)) {
-    if (!entry.isFile) continue;
-    const extension = extname(entry.name);
-    if (extension !== ".ftl") continue;
-    files.push(entry.name);
+import { extname, join, SEP, walk, walkSync } from "./deps.ts";
+import { NestedTranslation } from "./types.ts";
+
+function throwReadFileError(path: string) {
+  throw new Error(
+    `Something went wrong while reading the "${path}" file, usually, this can be caused by the file being empty. \
+If it is, please add at least one translation key to this file (or simply just delete it) to solve this error.`,
+  );
+}
+
+export async function readLocalesDir(
+  path: string,
+): Promise<NestedTranslation[]> {
+  const files = new Array<NestedTranslation>();
+  const locales = new Set<string>();
+
+  for await (const entry of walk(path)) {
+    if (entry.isFile && extname(entry.name) === ".ftl") {
+      try {
+        const decoder = new TextDecoder("utf-8");
+        const excludeRoot = entry.path.replace(path, "");
+        const contents = await Deno.readFile(join(path, excludeRoot));
+
+        const belongsTo = excludeRoot.split(SEP)[1].split(".")[0];
+        const translationSource = decoder.decode(contents);
+
+        files.push({
+          belongsTo,
+          translationSource,
+        });
+        locales.add(belongsTo);
+      } catch {
+        throwReadFileError(entry.path);
+      }
+    }
   }
-  return files;
+
+  return Array.from(locales).map((locale) => {
+    const sameLocale = files.filter((file) => file.belongsTo === locale);
+    const sourceOnly = sameLocale.map((file) => file.translationSource);
+    return {
+      belongsTo: locale,
+      translationSource: sourceOnly.join("\n"),
+    };
+  });
 }
 
-export function readLocalesDirSync(path: string): string[] {
-  const files = new Array<string>();
-  for (const entry of Deno.readDirSync(path)) {
-    if (!entry.isFile) continue;
-    const extension = extname(entry.name);
-    if (extension !== ".ftl") continue;
-    files.push(entry.name);
+export function readLocalesDirSync(path: string): NestedTranslation[] {
+  const files = new Array<NestedTranslation>();
+  const locales = new Set<string>();
+
+  for (const entry of walkSync(path)) {
+    if (entry.isFile && extname(entry.name) === ".ftl") {
+      try {
+        const decoder = new TextDecoder("utf-8");
+        const excludeRoot = entry.path.replace(path, "");
+        const contents = Deno.readFileSync(join(path, excludeRoot));
+
+        const belongsTo = excludeRoot.split(SEP)[1].split(".")[0];
+        const translationSource = decoder.decode(contents);
+
+        files.push({
+          belongsTo,
+          translationSource,
+        });
+        locales.add(belongsTo);
+      } catch {
+        throwReadFileError(entry.path);
+      }
+    }
   }
-  return files;
+
+  return Array.from(locales).map((locale) => {
+    const sameLocale = files.filter((file) => file.belongsTo === locale);
+    const sourceOnly = sameLocale.map((file) => file.translationSource);
+    return {
+      belongsTo: locale,
+      translationSource: sourceOnly.join("\n"),
+    };
+  });
 }

tests/deps.ts

@@ -31,6 +31,8 @@ export class Chats<C extends Context> {
       can_join_groups: true,
       can_read_all_group_messages: false,
       supports_inline_queries: false,
+      can_connect_to_business: false,
+      has_main_web_app: false,
     };
 
     this.bot.api.config.use(() => {

tests/utils.ts

@@ -2,9 +2,8 @@ import { join } from "./deps.ts";
 
 export function makeTempLocalesDir() {
   const dir = Deno.makeTempDirSync();
-  Deno.writeTextFileSync(
-    join(dir, "en.ftl"),
-    `hello = Hello!
+
+  const englishTranslation = `hello = Hello!
 
 greeting = Hello, { $name }!
 
@@ -21,11 +20,9 @@ language =
   .hint = Enter a language with the command
   .invalid-locale = Invalid language
   .already-set = Language is already set!
-  .language-set = Language set successfullY!`,
-  );
-  Deno.writeTextFileSync(
-    join(dir, "ru.ftl"),
-    `hello = Здравствуйте!
+  .language-set = Language set successfullY!`;
+
+  const russianTranslation = `hello = Здравствуйте!
 
 greeting = Здравствуйте, { $name }!
 
@@ -34,7 +31,7 @@ cart = Привет { $name }, в твоей корзине {
     [0] нет яблок
     [one] {$apples} яблоко
     [few] {$apples} яблока
-   *[other] {$apples} яблок
+    *[other] {$apples} яблок
 }.
 
 checkout = Спасибо за покупку!
@@ -43,7 +40,26 @@ language =
   .hint = Отправьте язык после команды
   .invalid-locale = Неверный язык
   .already-set = Этот язык уже установлен!
-  .language-set = Язык успешно установлен!`,
-  );
+  .language-set = Язык успешно установлен!`;
+
+  function writeNestedFiles() {
+    const nestedPath = join(dir, "/ru/test/nested/");
+    const keys = russianTranslation.split(/\n\s*\n/);
+
+    Deno.mkdirSync(nestedPath, { recursive: true });
+
+    for (const key of keys) {
+      const fileName = key.split(" ")[0] + ".ftl";
+      const filePath = join(nestedPath, fileName);
+
+      Deno.writeTextFileSync(filePath, key);
+    }
+  }
+
+  // Using normal, singular translation files.
+  Deno.writeTextFileSync(join(dir, "en.ftl"), englishTranslation);
+  // Using split translation files.
+  writeNestedFiles();
+
   return dir;
 }