Merge pull request #8711 from keymanapp/feat/typescript-keymanweb-com…

…piler

feat(developer): Rewrite of KeymanWeb compiler in Typescript

common/web/types/build.sh

@@ -78,7 +78,7 @@ fi
 
 if builder_start_action test; then
   copy_schemas
-  npm test
+  npm test -- "${builder_extra_params[@]}"
   builder_finish_action success test
 fi
 

common/web/types/src/keyman-touch-layout/keyman-touch-layout-file-writer.ts

@@ -17,12 +17,19 @@ export class TouchLayoutFileWriter {
    * @param source TouchLayoutFile
    * @returns Uint8Array, the .keyman-touch-layout file
    */
-  write(source: TouchLayoutFile): Uint8Array {
-    const output = JSON.stringify(source, null, this.options?.formatted ? 2 : undefined);
+  public write(source: TouchLayoutFile): Uint8Array {
+    const output = this.toJSONString(source);
     const encoder = new TextEncoder();
     return encoder.encode(output);
   }
 
+  /**
+   * Gets the output as a JSON string
+   */
+  public toJSONString(source: TouchLayoutFile): string {
+    return JSON.stringify(source, null, this.options?.formatted ? 2 : undefined);
+  }
+
   /**
    * Compiles the touch layout file into a KeymanWeb-compatible JSON-style
    * object string. In the future, this may be optimized to remove unnecessary
@@ -31,7 +38,11 @@ export class TouchLayoutFileWriter {
    * @param source
    * @returns string
    */
-  compile(source: TouchLayoutFile): string {
+  public compile(source: TouchLayoutFile): string {
+    return this.toJSONString(this.fixup(source));
+  }
+
+  public fixup(source: TouchLayoutFile): TouchLayoutFile {
     // Deep copy the source
     source = JSON.parse(JSON.stringify(source));
 
@@ -40,13 +51,23 @@ export class TouchLayoutFileWriter {
 
     const fixupKey = (key: TouchLayoutKey | TouchLayoutSubKey) => {
       if(Object.hasOwn(key, 'pad')) (key.pad as any) = key.pad.toString();
-      if(Object.hasOwn(key, 'sp')) (key.sp as any) = key.sp.toString();
+      if(Object.hasOwn(key, 'sp')) {
+        if(key.sp == 0) {
+          delete key.sp;
+        }
+        else {
+          (key.sp as any) = key.sp.toString();
+        }
+      }
       if(Object.hasOwn(key, 'width')) (key.width as any) = key.width.toString();
+      if(Object.hasOwn(key, 'text') && key.text === '') delete key.text;
     };
 
     const fixupPlatform = (platform: TouchLayoutPlatform) => {
       for(let layer of platform.layer) {
         for(let row of layer.row) {
+          // this matches the old spec for touch layout files
+          (row.id as any) = row.id.toString();
           for(let key of row.key) {
             fixupKey(key);
             if(key.sk) {
@@ -79,6 +100,6 @@ export class TouchLayoutFileWriter {
       fixupPlatform(source.tablet);
     }
 
-    return JSON.stringify(source, null, this.options?.formatted ? 2 : undefined);
+    return source;
   }
 };

common/web/types/src/kmx/kmx-file-reader.ts

@@ -1,4 +1,4 @@
-import { KMXFile, BUILDER_COMP_KEYBOARD, KEYBOARD, STORE } from "./kmx.js";
+import { KMXFile, BUILDER_COMP_KEYBOARD, KEYBOARD, STORE, GROUP, KEY } from "./kmx.js";
 import * as r from 'restructure';
 
 export class KmxFileReader {
@@ -13,17 +13,60 @@ export class KmxFileReader {
     return this.rString.fromBuffer(source.slice(offset));
   }
 
+  private processSystemStore(store: STORE, result: KEYBOARD) {
+    switch(store.dwSystemID) {
+      case KMXFile.TSS_MNEMONIC:
+        result.isMnemonic = store.dpString == '1';
+        break;
+      case KMXFile.TSS_KEYBOARDVERSION:
+        result.keyboardVersion = store.dpString;
+        break;
+      case KMXFile.TSS_BEGIN_NEWCONTEXT:
+        if(store.dpString.length == 3 && store.dpString.charCodeAt(0) == 0xFFFF && store.dpString.charCodeAt(1) == KMXFile.CODE_USE) {
+          result.startGroup.newContext = store.dpString.charCodeAt(2) - 1;
+        }
+        else {
+          // TODO: error
+          return false;
+        }
+        break;
+      case KMXFile.TSS_BEGIN_POSTKEYSTROKE:
+        if(store.dpString.length == 3 && store.dpString.charCodeAt(0) == 0xFFFF && store.dpString.charCodeAt(1) == KMXFile.CODE_USE) {
+          result.startGroup.postKeystroke = store.dpString.charCodeAt(2) - 1;
+        }
+        else {
+          // TODO: error
+          return false;
+        }
+        break;
+    }
+    return true;
+  }
+
   public read(source: Uint8Array): KEYBOARD {
     let binaryKeyboard: BUILDER_COMP_KEYBOARD;
     let kmx = new KMXFile();
     binaryKeyboard = kmx.COMP_KEYBOARD.fromBuffer(source);
     if(binaryKeyboard.dwIdentifier != KMXFile.FILEID_COMPILED) {
+      // TODO: error
       return null;
     }
 
-    // TODO: all header fields
-
     let result = new KEYBOARD();
+    result.fileVersion = binaryKeyboard.dwFileVersion;
+    result.flags = binaryKeyboard.dwFlags;
+    result.hotkey = binaryKeyboard.dwHotKey;
+    result.startGroup = {
+      ansi: binaryKeyboard.StartGroup_ANSI == 0xFFFFFFFF ? -1 : binaryKeyboard.StartGroup_ANSI,
+      unicode: binaryKeyboard.StartGroup_Unicode == 0xFFFFFFFF ? -1 : binaryKeyboard.StartGroup_Unicode,
+      newContext: -1, //TODO
+      postKeystroke: -1 // TODO
+    }
+
+    // Informative data
+    result.keyboardVersion = '';
+    result.isMnemonic = false;
+
     let offset = binaryKeyboard.dpStoreArray;
     for(let i = 0; i < binaryKeyboard.cxStoreArray; i++) {
       let binaryStore = kmx.COMP_STORE.fromBuffer(source.slice(offset));
@@ -32,13 +75,52 @@ export class KmxFileReader {
       store.dpName = this.readString(source, binaryStore.dpName);
       store.dpString = this.readString(source, binaryStore.dpString);
       result.stores.push(store);
+
+      if(!this.processSystemStore(store, result)) {
+        return null;
+      }
+
       offset += KMXFile.COMP_STORE_SIZE;
     }
 
-    // TODO: groups
+    offset = binaryKeyboard.dpGroupArray;
+    for(let i = 0; i < binaryKeyboard.cxGroupArray; i++) {
+      let binaryGroup = kmx.COMP_GROUP.fromBuffer(source.slice(offset));
+      let group = new GROUP();
+      group.dpMatch = this.readString(source, binaryGroup.dpMatch);
+      group.dpName = this.readString(source, binaryGroup.dpName);
+      group.dpNoMatch = this.readString(source, binaryGroup.dpNoMatch);
+      group.fUsingKeys = binaryGroup.fUsingKeys;
+      group.keys = [];
+
+      let keyOffset = binaryGroup.dpKeyArray;
+      for(let j = 0; j < binaryGroup.cxKeyArray; j++) {
+        let binaryKey = kmx.COMP_KEY.fromBuffer(source.slice(keyOffset));
+        let key = new KEY();
+        key.Key = binaryKey.Key;
+        key.Line = binaryKey.Line;
+        key.ShiftFlags = binaryKey.ShiftFlags;
+        key.dpContext = this.readString(source, binaryKey.dpContext);
+        key.dpOutput = this.readString(source, binaryKey.dpOutput);
+        group.keys.push(key);
+        keyOffset += KMXFile.COMP_KEY_SIZE;
+      }
+
+      result.groups.push(group);
+      offset += KMXFile.COMP_GROUP_SIZE;
+    }
 
     // TODO: KMXPlusFile
 
+    // Validate startGroup offsets
+    let gp: keyof KEYBOARD['startGroup'];
+    for(gp in result.startGroup) {
+      if(result.startGroup[gp] < -1 || result.startGroup[gp] >= result.groups.length) {
+        // TODO: error
+        return null;
+      }
+    }
+
     return result;
   }
 };

common/web/types/src/kmx/kmx.ts

@@ -8,9 +8,28 @@ import * as r from 'restructure';
 // kmx-builder will transform these to the corresponding COMP_xxxx
 
 export class KEYBOARD {
-  //TODO: additional header fields
+  fileVersion?: number;  // dwFileVersion (TSS_FILEVERSION)
+
+  startGroup: {
+    ansi: number;           // from COMP_KEYBOARD
+    unicode: number;        // from COMP_KEYBOARD
+    newContext: number;     // from TSS_BEGIN_NEWCONTEXT store
+    postKeystroke: number;  // from TSS_BEGIN_POSTKEYSTROKE store
+  } = {ansi:-1, unicode:-1, newContext:-1, postKeystroke:-1};
+
+  flags?: number;
+  hotkey?: number;
+
+  //bitmap:
   groups: GROUP[] = [];
   stores: STORE[] = [];
+
+  // Following values are extracted from stores[] but are
+  // informative only
+
+  keyboardVersion?: string;  // version (TSS_KEYBOARDVERSION)
+  isMnemonic: boolean;        // TSS_MNEMONICLAYOUT store
+
 };
 
 export class STORE {
@@ -21,7 +40,7 @@ export class STORE {
 
 export class GROUP {
   dpName: string;
-  keys: KEY[];
+  keys: KEY[] = [];
   dpMatch: string;
   dpNoMatch: string;
   fUsingKeys: boolean;
@@ -319,6 +338,21 @@ export class KMXFile {
   public static readonly ISVIRTUALKEY   = 0x4000;    // It is a Virtual Key Sequence
   public static readonly VIRTUALCHARKEY = 0x8000;    // Keyman 6.0: Virtual Key Cap Sequence NOT YET
 
+  public static readonly MASK_MODIFIER_CHIRAL = KMXFile.LCTRLFLAG | KMXFile.RCTRLFLAG | KMXFile.LALTFLAG | KMXFile.RALTFLAG;
+  public static readonly MASK_MODIFIER_SHIFT = KMXFile.K_SHIFTFLAG;
+  public static readonly MASK_MODIFIER_NONCHIRAL = KMXFile.K_CTRLFLAG | KMXFile.K_ALTFLAG;
+
+  public static readonly MASK_STATEKEY = KMXFile.CAPITALFLAG | KMXFile.NOTCAPITALFLAG |
+                                         KMXFile.NUMLOCKFLAG | KMXFile.NOTNUMLOCKFLAG |
+                                         KMXFile.SCROLLFLAG | KMXFile.NOTSCROLLFLAG;
+  public static readonly MASK_KEYTYPE  = KMXFile.ISVIRTUALKEY | KMXFile.VIRTUALCHARKEY;
+
+  public static readonly MASK_MODIFIER = KMXFile.MASK_MODIFIER_CHIRAL | KMXFile.MASK_MODIFIER_SHIFT | KMXFile.MASK_MODIFIER_NONCHIRAL;
+
+  public static readonly MASK_KEYS = KMXFile.MASK_MODIFIER | KMXFile.MASK_STATEKEY;
+  public static readonly KMX_MASK_VALID    = KMXFile.MASK_KEYS | KMXFile.MASK_KEYTYPE;
+
+
   public static readonly K_MODIFIERFLAG    = 0x007F;
   public static readonly K_NOTMODIFIERFLAG = 0xFF00;   // I4548
 
@@ -328,12 +362,13 @@ export class KMXFile {
   public static readonly COMP_GROUP_SIZE  = 24;
   public static readonly COMP_KEY_SIZE    = 20;
 
+
+  public static readonly VERSION_MASK_MINOR = 0x00FF;
+  public static readonly VERSION_MASK_MAJOR = 0xFF00;
+
   /* In-memory representation of the keyboard */
 
-  public keyboard: KEYBOARD = {
-    groups: [],
-    stores: []
-  };
+  public keyboard: KEYBOARD = new KEYBOARD();
 
   constructor() {
 

common/web/types/src/kvk/visual-keyboard.ts

@@ -26,6 +26,7 @@ export class VisualKeyboardFont {
   name?: string;
   size?: number;
   color?: number; // unused
+  style?: string; // TODO: figure out style vs color issues
 };
 
 export { BUILDER_KVK_KEY_FLAGS as VisualKeyboardKeyFlags } from "./kvk-file.js";

developer/build.sh

@@ -29,6 +29,7 @@ builder_describe \
   test \
   ":kmcmplib=src/kmcmplib                   Compiler - .kmn compiler" \
   ":kmc-kmn=src/kmc-kmn                     Compiler - .kmn wrapper Keyboard Module" \
+  ":kmc-kmw=src/kmc-kmw                     Compiler - .kmn Javascript Keyboard Module" \
   ":kmc-ldml=src/kmc-ldml                   Compiler - LDML Keyboard Module" \
   ":kmc-model=src/kmc-model                 Compiler - Lexical Model Module" \
   ":kmc-model-info=src/kmc-model-info       Compiler - .model_info Module" \

developer/src/kmc-kmn/src/compiler/compiler.ts

@@ -26,13 +26,15 @@ export interface CompilerOptions {
   saveDebug?: boolean;
   compilerWarningsAsErrors?: boolean;
 	warnDeprecatedCode?: boolean;
+  target?: 'kmx' | 'js';
 };
 
 const baseOptions: CompilerOptions = {
   shouldAddCompilerVersion: true,
   saveDebug: true,
   compilerWarningsAsErrors: false,
-  warnDeprecatedCode: true
+  warnDeprecatedCode: true,
+  target: 'kmx'
 };
 
 /**
@@ -66,6 +68,7 @@ export class KmnCompiler {
         return false;
       }
     }
+
     return this.verifyInitialized();
   }
 

developer/src/kmc-kmw/Makefile

@@ -0,0 +1,39 @@
+#
+# Keyman Developer - kmc KMW Keyboard Compiler Makefile
+#
+
+!include ..\Defines.mak
+
+# We do configure here because parent Makefile calls this first; other
+# kmc and kmc-* makefiles don't do it
+build: configure .virtual
+    $(GIT_BASH_FOR_KEYMAN) build.sh build
+
+configure: .virtual
+    $(GIT_BASH_FOR_KEYMAN) build.sh configure
+
+clean: .virtual
+    $(GIT_BASH_FOR_KEYMAN) build.sh clean
+
+test: .virtual
+    $(GIT_BASH_FOR_KEYMAN) build.sh test
+
+# build.sh bundle must be run from shell as it requires a temp folder to be
+# passed in. See inst/download.in.mak for instantiation.
+
+publish: .virtual
+    $(GIT_BASH_FOR_KEYMAN) build.sh publish
+
+signcode:
+    @rem nothing to do
+
+wrap-symbols:
+    @rem nothing to do
+
+test-manifest:
+    @rem nothing to do
+
+install:
+    @rem nothing to do
+
+!include ..\Target.mak