Merge branch 'feature/kmc-kmw' into feat/typescript-keymanweb-compiler

HISTORY.md

@@ -1,5 +1,23 @@
 # Keyman Version History
 
+## 17.0.120 alpha 2023-06-07
+
+* chore(core): look for emcc.py, not emcc (#8934)
+* chore(linux): Move some files to keyman-config (#8917)
+* fix(linux): Fix title of reference page and links (#8939)
+* fix(android/engine): Re-enable KMManager tests (#8940)
+
+## 17.0.119 alpha 2023-06-06
+
+* chore(developer): ensure kmcmplib messages are all UTF-8 (#8927)
+* chore(linux): Fix lintian warnings (#8931)
+
+## 17.0.118 alpha 2023-06-05
+
+* chore(ios): replace fv cert (#8900)
+* fix(core): Fix compilation if hotdoc is installed (#8912)
+* feat(linux): Rename column and add tooltip (#8918)
+
 ## 17.0.117 alpha 2023-06-04
 
 * chore(developer): verify long lines compile correctly (#8915)

VERSION.md

@@ -1 +1 @@
-17.0.118
+17.0.121

android/KMEA/app/src/main/java/com/keyman/engine/KMKeyboard.java

@@ -29,6 +29,7 @@
 import android.annotation.SuppressLint;
 import android.content.Context;
 import android.content.SharedPreferences;
+import android.content.pm.ApplicationInfo;
 import android.content.pm.PackageManager;
 import android.content.res.Configuration;
 import android.graphics.Color;
@@ -223,8 +224,11 @@ public void initKMKeyboard(final Context context) {
 
     getSettings().setUseWideViewPort(true);
     getSettings().setLoadWithOverviewMode(true);
-    setWebContentsDebuggingEnabled(true);
-
+    if (0 != (context.getApplicationInfo().flags & ApplicationInfo.FLAG_DEBUGGABLE)) {
+      // Enable debugging of WebView via adb. Not used during unit tests
+      // Refer: https://developer.chrome.com/docs/devtools/remote-debugging/webviews/#configure_webviews_for_debugging
+      setWebContentsDebuggingEnabled(true);
+    }
     setWebChromeClient(new WebChromeClient() {
       public boolean onConsoleMessage(ConsoleMessage cm) {
         String msg = KMString.format("KMW JS Log: Line %d, %s:%s", cm.lineNumber(), cm.sourceId(), cm.message());

android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java

@@ -624,8 +624,11 @@ private static void initKeyboard(Context appContext, KeyboardType keyboardType)
       return;
     }
 
-    RelativeLayout.LayoutParams params = getKeyboardLayoutParams();
-    keyboard.setLayoutParams(params);
+    if (!isTestMode()) {
+      // Keyboard layout not needed in unit tests. #5125
+      RelativeLayout.LayoutParams params = getKeyboardLayoutParams();
+      keyboard.setLayoutParams(params);
+    }
     keyboard.setVerticalScrollBarEnabled(false);
     keyboard.setHorizontalScrollBarEnabled(false);
     keyboard.setWebViewClient(webViewClient);

android/KMEA/app/src/test/java/com/keyman/engine/KMManagerTest.java

@@ -29,13 +29,14 @@ public class KMManagerTest {
   private static final String OLD_KEYBOARDS_LIST = "old_keyboards_list.dat";
   ArrayList<HashMap<String, String>> dat_list;
 
-  @Before
-  public void loadOldKeyboardsList() throws FileNotFoundException {
+  // For some keyboard list tests, load an existing keyboard list.
+  // Can't use @Before because context is null before running tests.
+  public void loadOldKeyboardsList() {
     KMManager.initialize(ApplicationProvider.getApplicationContext(), KMManager.KeyboardType.KEYBOARD_TYPE_INAPP);
 
     File keyboards_dat = new File(TEST_RESOURCE_ROOT, OLD_KEYBOARDS_LIST);
     if (keyboards_dat == null || !keyboards_dat.exists()) {
-      throw new FileNotFoundException();
+      Assert.fail(TAG + ": keyboards list not found");
     }
     try {
       ObjectInputStream inputStream = new ObjectInputStream(new FileInputStream(keyboards_dat));
@@ -46,7 +47,6 @@ public void loadOldKeyboardsList() throws FileNotFoundException {
     }
   }
 
-  @Ignore("Investigate ResourcesNotFoundException")
   @Test
   public void test_getTier() {
     String versionName = "14.0.248-alpha";
@@ -109,9 +109,9 @@ public void test_getTier() {
   /*
   * This test is manually run to edit/regenerate the old keyboards list
   */
-  @Ignore
   @Test
   public void create_newKeyboardsList() {
+    loadOldKeyboardsList();
     dat_list = new ArrayList<HashMap<String, String>>();
 
     HashMap<String, String> usInfo = new HashMap<String, String>();
@@ -181,9 +181,9 @@ public void create_newKeyboardsList() {
     }
   }
 
-  @Ignore("Investigate ResourcesNotFoundException")
   @Test
   public void test_updateOldKeyboardsList() {
+    loadOldKeyboardsList();
     Assert.assertNotNull(dat_list);
 
     // Verify old keyboards list contains deprecated keyboards

core/commands.inc.sh

@@ -1,4 +1,7 @@
-#!/usr/bin/env bash
+# shellcheck shell=bash
+# no hashbang for .inc.sh
+
+. "$KEYMAN_ROOT/resources/locate_emscripten.inc.sh"
 
 # ----------------------------------------------------------------------------
 # clean
@@ -114,34 +117,6 @@ do_uninstall() {
 # utility functions
 # ----------------------------------------------------------------------------
 
-#
-# We don't want to rely on emcc being on the path, because Emscripten puts far
-# too many things onto the path (in particular for us, node).
-#
-# The following comment suggests that we don't need emcc on the path.
-# https://github.com/emscripten-core/emscripten/issues/4848#issuecomment-1097357775
-#
-# So we try and locate emcc in common locations ourselves. The search pattern
-# is:
-#
-# 1. Look for $EMSCRIPTEN_BASE (our primary emscripten variable), which should
-#    point to the folder that emcc is located in
-# 2. Look for $EMCC which should point to the emcc executable
-# 3. Look for emcc on the path
-#
-locate_emscripten() {
-  if [[ -z ${EMSCRIPTEN_BASE+x} ]]; then
-    if [[ -z ${EMCC+x} ]]; then
-      local EMCC=`which emcc`
-      [[ -z $EMCC ]] && builder_die "locate_emscripten: Could not locate emscripten (emcc) on the path or with \$EMCC or \$EMSCRIPTEN_BASE"
-    fi
-    [[ -x $EMCC ]] || builder_die "locate_emscripten: Variable EMCC ($EMCC) does not point to a valid executable emcc"
-    EMSCRIPTEN_BASE="$(dirname "$EMCC")"
-  fi
-
-  [[ -x ${EMSCRIPTEN_BASE}/emcc ]] || builder_die "locate_emscripten: Variable EMSCRIPTEN_BASE ($EMSCRIPTEN_BASE) does not point to emcc's folder"
-}
-
 build_meson_cross_file_for_wasm() {
   if [ $BUILDER_OS == win ]; then
     local R=$(cygpath -w $(echo $EMSCRIPTEN_BASE) | sed 's_\\_\\\\_g')
@@ -159,6 +134,8 @@ build_meson_cross_file_for_wasm() {
 #     This link.exe is not a linker.
 #     You may need to reorder entries to your %PATH% variable to resolve this.
 #
+# TODO: is this still required with meson 1.0?
+#
 
 cleanup_visual_studio_path() {
   local _split_path _new_path=""

core/doc/hotdoc.json

@@ -1,10 +1,14 @@
 {
     "project_name": "@project_name@",
     "project_version": "@project_version@",
-    "sitemap": "../../doc/sitemap.txt",
-    "index": "../../doc/markdown_files/index.md",
+    "sitemap": "@doc_dir@/sitemap.txt",
+    "index": "@doc_dir@/markdown_files/index.md",
     "c_sources": [
-      "../include/keyboardprocessor.h"
+      "@include_dir@/keyman/keyboardprocessor.h"
+    ],
+    "c_include_directories": [
+      "@include_dir@",
+      "@doc_dir@/../../common/include"
     ],
     "c_smart_index" : true,
     "output": ".",

core/doc/meson.build

@@ -12,18 +12,20 @@ if hotdoc.found()
   cfg = configuration_data()
   cfg.set('project_name', meson.project_name())
   cfg.set('project_version', meson.project_version())
+  cfg.set('doc_dir', meson.current_source_dir())
+  cfg.set('include_dir', meson.current_source_dir() / '../include')
   configure_file(input: 'hotdoc.json',
                  output: 'hotdoc.json',
                  configuration: cfg)
   deps = files(
-    '../include/keyman/keyboardprocessor.h.in',
+    '../include/keyman/keyboardprocessor.h',
     '../src/jsonpp.hpp',
     '../src/utfcodec.hpp'
   )
 
   docs = custom_target('docs',
     output: ['html'],
-    input: ['sitemap.txt', 'markdown_files/index.md'],
+    input: ['sitemap.txt', 'markdown_files/index.md', deps],
     command: [hotdoc, '--verbose', '--conf-file=doc/hotdoc.json', '--output=doc', 'run'],
     depend_files: deps,
     install: true,

core/include/keyman/keyboardprocessor.h

@@ -779,7 +779,7 @@ km_kbp_keyboard_get_key_list(km_kbp_keyboard const *keyboard,
                             km_kbp_keyboard_key **out);
 
 
-/*
+/**
 ```
 ### `km_kbp_keyboard_key_list_dispose`
 ##### Description:
@@ -794,28 +794,41 @@ returned by `km_kbp_keyboard_get_key_list`.
 KMN_API
 void km_kbp_keyboard_key_list_dispose(km_kbp_keyboard_key *key_list);
 
-
 /**
- * Returns the list of IMX libraries and function names that are referenced by
- * the keyboard. The matching dispose call needs to be called to free the memory.
+ * km_kbp_keyboard_get_imx_list:
+ *
+ * Returns: the list of IMX libraries and function names that are referenced by
+ * the keyboard.The matching dispose call needs to be called to free the memory.
  */
 KMN_API
-km_kbp_status km_kbp_keyboard_get_imx_list(km_kbp_keyboard const *keyboard, km_kbp_keyboard_imx** imx_list);
+km_kbp_status km_kbp_keyboard_get_imx_list(km_kbp_keyboard const *keyboard, km_kbp_keyboard_imx **imx_list);
 
 /**
+ * km_kbp_keyboard_imx_list_dispose:
+ *
  * Disposes of the IMX list
+ *
+ * Returns: --
  */
 KMN_API
 void km_kbp_keyboard_imx_list_dispose(km_kbp_keyboard_imx *imx_list);
 
 /**
+ * km_kbp_state_imx_register_callback:
+ *
  * Register the IMX callback endpoint for the client.
+ *
+ * Returns: --
  */
 KMN_API
 void km_kbp_state_imx_register_callback(km_kbp_state *state, km_kbp_keyboard_imx_platform imx_callback, void *callback_object);
 
 /**
+ *  km_kbp_state_imx_deregister_callback:
+ *
  * De-register IMX callback endpoint for the client.
+ *
+ * Returns: --
  */
 KMN_API
 void km_kbp_state_imx_deregister_callback(km_kbp_state *state);
@@ -1042,6 +1055,8 @@ enum km_kbp_tech_value {
 };
 
 /**
+ * km_kbp_event_flags:
+ *
  * Bit flags to be used with the event_flags parameter of km_kbp_process_event
  */
 enum km_kbp_event_flags {
@@ -1162,10 +1177,8 @@ km_kbp_event(
 );
 
 enum km_kbp_event_code {
-  /**
-   * A keyboard has been activated by the user. The processor may use this
-   * event, for example, to switch caps lock state or provide other UX.
-   */
+  // A keyboard has been activated by the user. The processor may use this
+  // event, for example, to switch caps lock state or provide other UX.
   KM_KBP_EVENT_KEYBOARD_ACTIVATED = 1,
   //future: KM_KBP_EVENT_KEYBOARD_DEACTIVATED = 2,
 };

developer/src/kmc-kmn/test/fixtures/invalid-keyboards/cerr_duplicate_group.kmn

@@ -0,0 +1,30 @@
+c Description: Verifies that kmcmplib picks up on duplicated groups with Unicode names,
+c and that the Unicode names are correctly reported in error messages in UTF-8
+
+store(&NAME) 'cerr_duplicate_group'
+store(&VERSION) '9.0'
+
+begin unicode > use(ខ្មែរ)
+
+group(ខ្មែរ) using keys
+
+store(c_key)           [K_K] [K_X] [SHIFT K_K] [SHIFT K_X] [K_G] \
+                       [K_C] [K_Q] [SHIFT K_C] [SHIFT K_Q] [SHIFT K_J] \
+                       [K_D] [K_Z] [SHIFT K_D] [SHIFT K_Z] [SHIFT K_N] \
+                       [K_T] [K_F] [SHIFT K_T] [SHIFT K_F] [K_N] \
+                       [K_B] [K_P] [SHIFT K_B] [SHIFT K_P] [K_M] \
+                       [K_Y] [K_R] [K_L] [K_V] [K_S] [K_H] [SHIFT K_L] [SHIFT K_G] \
+                       [RALT K_K] [RALT K_B]
+store(c_out)           U+1780 U+1781 U+1782 U+1783 U+1784 \
+    	   	   	       U+1785 U+1786 U+1787 U+1788 U+1789 \
+    	   	   	       U+178A U+178B U+178C U+178D U+178E \
+    	   	   	       U+178F U+1790 U+1791 U+1792 U+1793 \
+    	   	   	       U+1794 U+1795 U+1796 U+1797 U+1798 \
+    	   	   	       U+1799 U+179A U+179B U+179C U+179F U+17A0 U+17A1 U+17A2 \
+    	   	   	       U+179D U+179E c deprecated, but they are used in minority languages
+
++ any(c_key) > index(c_out, 1)
+
+group(ខ្មែរ) using keys
+
++ 'a' > 'oops two groups!'

developer/src/kmc-kmn/test/fixtures/invalid-keyboards/cerr_duplicate_store.kmn

@@ -0,0 +1,13 @@
+c Description: Verifies that kmcmplib picks up on duplicated stores with Unicode names,
+c and that the Unicode names are correctly reported in error messages in UTF-8
+
+store(&NAME) 'cerr_duplicate_store'
+store(&VERSION) '9.0'
+
+begin unicode > use(ខ្មែរ)
+
+group(ខ្មែរ) using keys
+
+store(ខ្មែរ) 'abc'
+store(ខ្មែរ) 'def'
+

developer/src/kmc-kmn/test/test-messages.ts

@@ -52,4 +52,18 @@ describe('CompilerMessages', function () {
     await testForMessage(this, ['invalid-keyboards', 'warn_invalid_vkey_in_kvks_file.kmn'], CompilerMessages.WARN_InvalidVkeyInKvksFile);
   });
 
+  // CERR_DuplicateGroup
+
+  it('should generate CERR_DuplicateGroup if the kmn contains two groups with the same name', async function() {
+    await testForMessage(this, ['invalid-keyboards', 'cerr_duplicate_group.kmn'], 0x302071); //TODO: consolidate messages from kmcmplib, CompilerMessages.CERR_DuplicateGroup
+    assert.equal(callbacks.messages[0].message, "A group with this name has already been defined. Group 'ខ្មែរ' declared on line 9");
+  });
+
+  // CERR_DuplicateStore
+
+  it('should generate CERR_DuplicateStore if the kmn contains two stores with the same name', async function() {
+    await testForMessage(this, ['invalid-keyboards', 'cerr_duplicate_store.kmn'], 0x302072); //TODO: consolidate messages from kmcmplib, CompilerMessages.CERR_DuplicateStore
+    assert.equal(callbacks.messages[0].message, "A store with this name has already been defined. Store 'ខ្មែរ' declared on line 11");
+  });
+
 });

developer/src/kmc-kmw/README.md

@@ -0,0 +1 @@
+# kmc-kmw

developer/src/kmcmplib/commands.inc.sh

@@ -1,4 +1,7 @@
-#!/usr/bin/env bash
+# shellcheck shell=bash
+# no hashbang for .inc.sh
+
+. "$KEYMAN_ROOT/resources/locate_emscripten.inc.sh"
 
 # ----------------------------------------------------------------------------
 # clean
@@ -115,34 +118,6 @@ do_command() {
 # utility functions
 # ----------------------------------------------------------------------------
 
-#
-# We don't want to rely on emcc being on the path, because Emscripten puts far
-# too many things onto the path (in particular for us, node).
-#
-# The following comment suggests that we don't need emcc on the path.
-# https://github.com/emscripten-core/emscripten/issues/4848#issuecomment-1097357775
-#
-# So we try and locate emcc in common locations ourselves. The search pattern
-# is:
-#
-# 1. Look for $EMSCRIPTEN_BASE (our primary emscripten variable), which should
-#    point to the folder that emcc is located in
-# 2. Look for $EMCC which should point to the emcc executable
-# 3. Look for emcc on the path
-#
-locate_emscripten() {
-  if [[ -z ${EMSCRIPTEN_BASE+x} ]]; then
-    if [[ -z ${EMCC+x} ]]; then
-      local EMCC=`which emcc`
-      [[ -z $EMCC ]] && builder_die "locate_emscripten: Could not locate emscripten (emcc) on the path or with \$EMCC or \$EMSCRIPTEN_BASE"
-    fi
-    [[ -x $EMCC ]] || builder_die "locate_emscripten: Variable EMCC ($EMCC) does not point to a valid executable emcc"
-    EMSCRIPTEN_BASE="$(dirname "$EMCC")"
-  fi
-
-  [[ -x ${EMSCRIPTEN_BASE}/emcc ]] || builder_die "locate_emscripten: Variable EMSCRIPTEN_BASE ($EMSCRIPTEN_BASE) does not point to emcc's folder"
-}
-
 build_meson_cross_file_for_wasm() {
   if [ $BUILDER_OS == win ]; then
     local R=$(cygpath -w $(echo $EMSCRIPTEN_BASE) | sed 's_\\_\\\\_g')