Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve sanity check for locales #3573

Merged
merged 1 commit into from
Sep 16, 2024
Merged

Improve sanity check for locales #3573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .eslintrc.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ module.exports = {
"getSerial": "readonly",
"getTime": "readonly",
"global": "readonly",
"globalThis": "readonly",
"HIGH": "readonly",
"I2C1": "readonly",
"Infinity": "readonly",
Expand Down
72 changes: 43 additions & 29 deletions apps/locale/locale.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,17 @@
<label><input id="customize" type="checkbox" /> Advanced: Customize the date and time formats.</label>
</div>
<p>
<span id="customize-warning"></span>
<table id="examples-short-long"></table>
<table id="examples"></table>
</p>

<p id="customize-warning"></p>

<p>Then click <button id="upload" class="btn btn-primary">Upload</button></p>

<script src="../../core/lib/customize.js"></script>
<script src="../../core/js/utils.js"></script>
<script src="sanitycheck.js"></script>
<script src="locales.js"></script>

<script>
Expand Down Expand Up @@ -103,32 +105,6 @@
return '\\x'+(n+256).toString(16).slice(-2);
}

// do some sanity checks
Object.keys(locales).forEach(function(localeName) {
var locale = locales[localeName];
if (locale.trans && !locale.trans.on) console.error(localeName+": If translations are provided, 'on' *must* be included");
if (distanceUnits[locale.distance[0]]===undefined) console.error(localeName+": Unknown distance unit "+locale.distance[0]);
if (distanceUnits[locale.distance[1]]===undefined) console.error(localeName+": Unknown distance unit "+locale.distance[1]);
if (speedUnits[locale.speed]===undefined) console.error(localeName+": Unknown speed unit "+locale.speed);
if (locale.temperature!='°C' && locale.temperature!='°F')
console.error(localeName+": Unknown temperature unit "+locale.temperature);
// Now check that codepage is ok and all chars in translation are in that codepage
const codePageName = "ISO8859-1";
if (locale.codePage) codePageName = locale.codePage;
const codePage = codePages[codePageName];
if (codePage===undefined) console.error(localeName+": Unknown codePage "+codePageName);
function checkChars(v,path) {
if ("object"==typeof v)
Object.keys(v).forEach(k=>checkChars(v[k], path+"."+k));
else if ("string"==typeof v)
for (var i=0;i<v.length;i++)
if (codePageLookup(localeName, codePage, v[i])===undefined)
console.error(` ... in ${path}[${i}]`);
}
checkChars(locale,localeName);
});


function createLocaleModule() {
console.log(`Language ${lang}`);

Expand Down Expand Up @@ -269,8 +245,6 @@
}

var date = new Date();
// TODO: This warning should have a link to an article explaining how the formats work, and how long they are allowed to be
document.getElementById("customize-warning").innerText = customizeLocale ? "⚠️ If you make the formats too long, some apps will not work!" : "";
document.getElementById("examples-short-long").innerHTML = `
<tr><td class="table_t"></td><td style="font-weight:bold">Short</td><td style="font-weight:bold">Long</td></tr>
<tr><td class="table_t">Day</td><td>${exports.dow(date,1)}</td><td>${exports.dow(date,0)}</td></tr>
Expand Down Expand Up @@ -332,27 +306,63 @@
document.querySelector("input#short-date-pattern").addEventListener("input", event => {
locale.datePattern["1"] = event.target.value;
document.querySelector("td#short-date-pattern-output").innerText = patternToOutput(event.target.value);
checkCustomLocale();
});
document.querySelector("input#long-date-pattern").addEventListener("input", event => {
locale.datePattern["0"] = event.target.value;
document.querySelector("td#long-date-pattern-output").innerText = patternToOutput(event.target.value);
checkCustomLocale();
});
document.querySelector("input#short-time-pattern").addEventListener("input", event => {
locale.timePattern["1"] = event.target.value;
document.querySelector("td#short-time-pattern-output").innerText = patternToOutput(event.target.value);
checkCustomLocale();
});
document.querySelector("input#long-time-pattern").addEventListener("input", event => {
locale.timePattern["0"] = event.target.value;
document.querySelector("td#long-time-pattern-output").innerText = patternToOutput(event.target.value);
checkCustomLocale();
});
document.querySelector("input#meridian-am").addEventListener("input", event => {
locale.ampm["0"] = event.target.value;
document.querySelector("span#meridian-am-output").innerText = event.target.value;
checkCustomLocale();
});
document.querySelector("input#meridian-pm").addEventListener("input", event => {
locale.ampm["1"] = event.target.value;
document.querySelector("span#meridian-pm-output").innerText = event.target.value;
checkCustomLocale();
});

let isCheckingLocale = false;
// Polyfill for WebKit:
const requestIdleCallback = globalThis.requestIdleCallback || ((func) => {func()});
// Check that a custom locale follows some basic standards
function checkCustomLocale(){
if(isCheckingLocale) return;
isCheckingLocale = true;
setTimeout(() => {
requestIdleCallback(() => {
isCheckingLocale = false;
const result = globalThis.checkLocale(locale, {speedUnits, distanceUnits, codePages, CODEPAGE_CONVERSIONS});
let text = "";
for(const w of [...result.errors, ...result.warnings]){
text += `⚠️ ${w.name} ${w.error}.\n`;
}
const element = document.getElementById("customize-warning");
if(text.length > 0){
text += "\nIf you upload this locale, some apps might no longer work.\nPlease try to resolve the issues before uploading."
element.classList.add("toast");
element.classList.add("toast-error");
}else{
element.classList.remove("toast");
element.classList.remove("toast-error");
}
element.innerText = text;
}, {timeout: 2000})
}, 500);
}

}
return getLocaleModule(false);
}
Expand Down Expand Up @@ -416,6 +426,10 @@
}else{
createLocaleModule();
}
const warningsElement = document.getElementById("customize-warning")
warningsElement.innerText = "";
warningsElement.classList.remove("toast");
warningsElement.classList.remove("toast-error");
}
customizeSelector.addEventListener('change', handleCustomizeChange);
function handleCustomizeChange(){
Expand Down
2 changes: 1 addition & 1 deletion apps/locale/locales.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ var locales = {
temperature: '°C',
ampm: { 0: "öö", 1: "ös" },
timePattern: { 0: "%HH:%MM:%SS", 1: "%HH:%MM" },
datePattern: { 0: "%d %w %Y %A", 1: "%d/%m/%Y" }, // 1 Mart 2020 Pazar // "01/03/2020"
datePattern: { 0: "%d %B %Y %A", 1: "%d/%m/%Y" }, // 1 Mart 2020 Pazar // "01/03/2020"
abmonth: "Oca,Sub,Mar,Nis,May,Haz,Tem,Agu,Eyl,Eki,Kas,Ara",
month: "Ocak,Subat,Mart,Nisan,Mayis,Haziran,Temmuz,Agustos,Eylul,Ekim,Kasim,Aralik",
abday: "Paz,Pzt,Sal,Car,Per,Cum,Cmt",
Expand Down
233 changes: 233 additions & 0 deletions apps/locale/sanitycheck.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
/**
* Maps the Espruino datetime format to min and max character lengths.
* Used when determining if a format can produce outputs that are too short or long.
*/
const datetime_length_map = {
// %A, %a, %B, %b vary depending on the locale, so they are calculated later
"%Y": [4, 4],
"%y": [2, 2],
"%m": [2, 2],
"%-m": [1, 2],
"%d": [2, 2],
"%-d": [1, 2],
"%HH": [2, 2],
"%MM": [2, 2],
"%SS": [2, 2],
};

/**
* Takes an Espruino datetime format string and returns the minumum and maximum possible length of characters that the format could use.
*
* @param {string} datetimeEspruino - The datetime Espruino format
* @returns first the minimum possible length, second the maximum possible length.
*/
function getLengthOfDatetimeFormat(name, datetimeEspruino, locale, errors) {

// Generate the length_map based on the actual names in the locale
const length_map = {...datetime_length_map};
for(const [symbol, values] of [
["%A", locale.day],
["%a", locale.abday],
["%B", locale.month],
["%b", locale.abmonth],
]){
const length = [Infinity, 0];
for(const value of values.split(",")){
if(length[0] > value.length) length[0] = value.length;
if(length[1] < value.length) length[1] = value.length;
}
length_map[symbol] = length;
}

// Find the length of the output
let formatLength = [0, 0];
let i = 0;
while (i < datetimeEspruino.length) {
if (datetimeEspruino[i] === "%") {
let match;
for(const symbolLength of [2, 3]){
const length = length_map[datetimeEspruino.substring(i, i+symbolLength)];
if(length){
match = {
length,
symbolLength,
}
}
}
if(match){
formatLength[0] += match.length[0];
formatLength[1] += match.length[1];
i += match.symbolLength;
}else{
errors.push({name, value: datetimeEspruino, lang: locale.lang, error: `uses an unsupported format symbol: ${datetimeEspruino.substring(i, i+3)}`});
formatLength[0]++;
formatLength[1]++;
i++;
}
} else {
formatLength[0]++;
formatLength[1]++;
i++;
}
}
return formatLength;
}

/**
* Checks that a locale conforms to some basic standards.
*
* @param {object} locale - The locale to test.
* @param {object} meta - Meta information that is needed to check if locales are supported.
* @param {object} meta.speedUnits - The table of speed units.
* @param {object} meta.distanceUnits - The table of distance units.
* @param {object} meta.codePages - Custom codepoint mappings.
* @param {object} meta.CODEPAGE_CONVERSIONS - The table of custom codepoint conversions.
* @returns an object with an array of errors and warnings.
*/
function checkLocale(locale, {speedUnits, distanceUnits, codePages, CODEPAGE_CONVERSIONS}){
const errors = [];
const warnings = [];

const speeds = Object.keys(speedUnits);
const distances = Object.keys(distanceUnits);

checkLength("lang", locale.lang, 5, undefined);
checkLength("decimal point", locale.decimal_point, 1, 1);
checkLength("thousands separator", locale.thousands_sep, 1, 1);
checkLength("speed", locale.speed, 2, 4);
checkIsIn("speed", locale.speed, "speedUnits", speeds);
checkLength("distance", locale.distance["0"], 1, 3);
checkLength("distance", locale.distance["1"], 1, 3);
checkIsIn("distance", locale.distance["0"], "distanceUnits", distances);
checkIsIn("distance", locale.distance["1"], "distanceUnits", distances);
checkLength("temperature", locale.temperature, 1, 2);
checkLength("meridian", locale.ampm["0"], 1, 3);
checkLength("meridian", locale.ampm["1"], 1, 3);
warnIfNot("long time format", locale.timePattern["0"], "%HH:%MM:%SS");
warnIfNot("short time format", locale.timePattern["1"], "%HH:%MM");
checkFormatLength("long time", locale.timePattern["0"], 8, 8);
checkFormatLength("short time", locale.timePattern["1"], 5, 5);
checkFormatLength("long date", locale.datePattern["0"], 6, 14);
checkFormatLength("short date", locale.datePattern["1"], 6, 11);
checkArrayLength("short months", locale.abmonth.split(","), 12, 12);
checkArrayLength("long months", locale.month.split(","), 12, 12);
checkArrayLength("short days", locale.abday.split(","), 7, 7);
checkArrayLength("long days", locale.day.split(","), 7, 7);
for (const abmonth of locale.abmonth.split(",")) {
checkLength("short month", abmonth, 2, 4);
}
for (const month of locale.month.split(",")) {
checkLength("month", month, 3, 11);
}
for (const abday of locale.abday.split(",")) {
checkLength("short day", abday, 2, 4);
}
for (const day of locale.day.split(",")) {
checkLength("day", day, 3, 13);
}
checkEncoding(locale);

function checkLength(name, value, min, max) {
if(typeof value !== "string"){
errors.push({name, value, lang: locale.lang, error: `must be defined and must be a string`});
return;
}
if (min && value.length < min) {
errors.push({name, value, lang: locale.lang, error: `must be longer than ${min-1} characters`});
}
if (max && value.length > max) {
errors.push({name, value, lang: locale.lang, error: `must be shorter than ${max+1} characters`});
}
}
function checkArrayLength(name, value, min, max){
if(!Array.isArray(value)){
errors.push({name, value, lang: locale.lang, error: `must be defined and must be an array`});
return;
}
if (min && value.length < min) {
errors.push({name, value, lang: locale.lang, error: `array must be longer than ${min-1} entries`});
}
if (max && value.length > max) {
errors.push({name, value, lang: locale.lang, error: `array must be shorter than ${max+1} entries`});
}
}
function checkFormatLength(name, value, min, max) {
const length = getLengthOfDatetimeFormat(name, value, locale, errors);
if (min && length[0] < min) {
errors.push({name, value, lang: locale.lang, error: `output must be longer than ${min-1} characters`});
}
if (max && length[1] > max) {
errors.push({name, value, lang: locale.lang, error: `output must be shorter than ${max+1} characters`});
}
}
function checkIsIn(name, value, listName, list) {
if (!list.includes(value)) {
errors.push({name, value, lang: locale.lang, error: `must be included in the ${listName} map`});
}
}
function warnIfNot(name, value, expected) {
if (value !== expected) {
warnings.push({name, value, lang: locale.lang, error: `might not work in some apps if it is not "${expected}"`});
}
}
function checkEncoding(object) {
if(!object){
return;
}else if(typeof object === "string"){
for(const char of object){
const charCode = char.charCodeAt();
if (charCode >= 32 && charCode < 128) {
// ASCII - fully supported
continue;
} else if (codePages["ISO8859-1"].map.indexOf(char) >= 0) {
// At upload time, the char can be converted to a custom codepage
continue;
} else if (CODEPAGE_CONVERSIONS[char]) {
// At upload time, the char can be converted to a similar supported char
continue;
}
errors.push({name: `character ${char}`, value: char, lang: locale.lang, error: `is not supported by BangleJS`});
}
}else{
for(const [key, value] of Object.entries(object)){
if(key === "icon") continue;
checkEncoding(value);
}
}
}

return {errors, warnings};
}

/**
* Checks that an array of locales conform to some basic standards.
*
* @param {object[]} locales - The locales to test.
* @param {object} meta.speedUnits - The table of speed units.
* @param {object} meta.distanceUnits - The table of distance units.
* @param {object} meta.codePages - Custom codepoint mappings.
* @param {object} meta.CODEPAGE_CONVERSIONS - The table of custom codepoint conversions.
* @returns an object with an array of errors and warnings.
*/
function checkLocales(locales, meta){
let errors = [];
let warnings = [];

for(const locale of Object.values(locales)){
const result = checkLocale(locale, meta);
errors = [...errors, ...result.errors];
warnings = [...warnings, ...result.warnings];
}

return {errors, warnings};
}

if(typeof module !== "undefined"){
module.exports = {
checkLocale,
checkLocales,
};
}else{
globalThis.checkLocale = checkLocale;
globalThis.checkLocales = checkLocales;
}
Loading
Loading