From 7ab9f268d93e9ad4a2039a9d2d5d3804e6bf5387 Mon Sep 17 00:00:00 2001 From: Quincy Morgan <2046746+quincylvania@users.noreply.github.com> Date: Tue, 15 Sep 2020 16:09:08 -0400 Subject: [PATCH] Don't fetch the English strings if the preferred locale has 100% coverage (close #7994) --- data/locales.json | 2 +- modules/core/localizer.js | 30 +++++++++++++------ scripts/update_locales.js | 61 ++++++++++++++++++++++++++++++++------- 3 files changed, 72 insertions(+), 21 deletions(-) diff --git a/data/locales.json b/data/locales.json index 53df5fad6..c27d055a1 100644 --- a/data/locales.json +++ b/data/locales.json @@ -1 +1 @@ -{"af": {"rtl": false}, "ar": {"rtl": true}, "ar-AA": {"rtl": true}, "as": {"rtl": false}, "ast": {"rtl": false}, "az": {"rtl": false}, "be": {"rtl": false}, "bg": {"rtl": false}, "bn": {"rtl": false}, "bs": {"rtl": false}, "ca": {"rtl": false}, "ckb": {"rtl": true}, "cs": {"rtl": false}, "cy": {"rtl": false}, "da": {"rtl": false}, "de": {"rtl": false}, "dv": {"rtl": true}, "el": {"rtl": false}, "en": {"rtl": false}, "en-AU": {"rtl": false}, "en-GB": {"rtl": false}, "en-NZ": {"rtl": false}, "en-US": {"rtl": false}, "eo": {"rtl": false}, "es": {"rtl": false}, "et": {"rtl": false}, "eu": {"rtl": false}, "fa": {"rtl": true}, "fi": {"rtl": false}, "fil": {"rtl": false}, "fr": {"rtl": false}, "gan": {"rtl": false}, "gl": {"rtl": false}, "gu": {"rtl": false}, "ha": {"rtl": false}, "he": {"rtl": true}, "hi": {"rtl": false}, "hr": {"rtl": false}, "hu": {"rtl": false}, "hy": {"rtl": false}, "ia": {"rtl": false}, "id": {"rtl": false}, "is": {"rtl": false}, "it": {"rtl": false}, "ja": {"rtl": false}, "jv": {"rtl": false}, "kk": {"rtl": false}, "km": {"rtl": false}, "kn": {"rtl": false}, "ko": {"rtl": false}, "ku": {"rtl": false}, "ky": {"rtl": false}, "lij": {"rtl": false}, "lt": {"rtl": false}, "lv": {"rtl": false}, "mg": {"rtl": false}, "mi": {"rtl": false}, "mk": {"rtl": false}, "ml": {"rtl": false}, "mn": {"rtl": false}, "mr": {"rtl": false}, "ms": {"rtl": false}, "my": {"rtl": false}, "ne": {"rtl": false}, "nl": {"rtl": false}, "nn": {"rtl": false}, "no": {"rtl": false}, "nv": {"rtl": false}, "oc": {"rtl": false}, "pa": {"rtl": false}, "pap": {"rtl": false}, "pl": {"rtl": false}, "pt": {"rtl": false}, "pt-BR": {"rtl": false}, "rm": {"rtl": false}, "ro": {"rtl": false}, "ru": {"rtl": false}, "sat": {"rtl": false}, "sc": {"rtl": false}, "si": {"rtl": false}, "sk": {"rtl": false}, "sl": {"rtl": false}, "so": {"rtl": false}, "sq": {"rtl": false}, "sr": {"rtl": false}, "sv": {"rtl": false}, "sw": {"rtl": false}, "ta": {"rtl": false}, "te": {"rtl": false}, "th": {"rtl": false}, "tl": {"rtl": false}, "tr": {"rtl": false}, "uk": {"rtl": false}, "ur": {"rtl": true}, "vi": {"rtl": false}, "yue": {"rtl": false}, "zh": {"rtl": false}, "zh-CN": {"rtl": false}, "zh-HK": {"rtl": false}, "zh-TW": {"rtl": false}} \ No newline at end of file +{"af": {"rtl": false, "pct": 0.03}, "ar": {"rtl": true, "pct": 0.41}, "ar-AA": {"rtl": true, "pct": 0}, "as": {"rtl": false, "pct": 0}, "ast": {"rtl": false, "pct": 0.3}, "az": {"rtl": false, "pct": 0}, "be": {"rtl": false, "pct": 0.23}, "bg": {"rtl": false, "pct": 0.19}, "bn": {"rtl": false, "pct": 0.07}, "bs": {"rtl": false, "pct": 0.06}, "ca": {"rtl": false, "pct": 0.39}, "ckb": {"rtl": true, "pct": 0.05}, "cs": {"rtl": false, "pct": 0.74}, "cy": {"rtl": false, "pct": 0.04}, "da": {"rtl": false, "pct": 0.67}, "de": {"rtl": false, "pct": 1}, "dv": {"rtl": true, "pct": 0.01}, "el": {"rtl": false, "pct": 0.37}, "en": {"rtl": false, "pct": 1}, "en-AU": {"rtl": false, "pct": 0}, "en-GB": {"rtl": false, "pct": 0.31}, "en-NZ": {"rtl": false, "pct": 0}, "en-US": {"rtl": false, "pct": 1}, "eo": {"rtl": false, "pct": 0.8}, "es": {"rtl": false, "pct": 0.99}, "et": {"rtl": false, "pct": 0.17}, "eu": {"rtl": false, "pct": 0.1}, "fa": {"rtl": true, "pct": 0.56}, "fi": {"rtl": false, "pct": 0.52}, "fil": {"rtl": false, "pct": 0}, "fr": {"rtl": false, "pct": 0.93}, "gan": {"rtl": false, "pct": 0}, "gl": {"rtl": false, "pct": 0.7}, "gu": {"rtl": false, "pct": 0.01}, "ha": {"rtl": false, "pct": 0}, "he": {"rtl": true, "pct": 0.93}, "hi": {"rtl": false, "pct": 0.01}, "hr": {"rtl": false, "pct": 0.27}, "hu": {"rtl": false, "pct": 0.68}, "hy": {"rtl": false, "pct": 0.03}, "ia": {"rtl": false, "pct": 0}, "id": {"rtl": false, "pct": 0.11}, "is": {"rtl": false, "pct": 0.51}, "it": {"rtl": false, "pct": 0.81}, "ja": {"rtl": false, "pct": 0.99}, "jv": {"rtl": false, "pct": 0}, "kk": {"rtl": false, "pct": 0}, "km": {"rtl": false, "pct": 0}, "kn": {"rtl": false, "pct": 0.07}, "ko": {"rtl": false, "pct": 0.51}, "ku": {"rtl": false, "pct": 0.01}, "ky": {"rtl": false, "pct": 0}, "lij": {"rtl": false, "pct": 0}, "lt": {"rtl": false, "pct": 0.24}, "lv": {"rtl": false, "pct": 0.42}, "mg": {"rtl": false, "pct": 0.05}, "mi": {"rtl": false, "pct": 0}, "mk": {"rtl": false, "pct": 0.72}, "ml": {"rtl": false, "pct": 0.01}, "mn": {"rtl": false, "pct": 0}, "mr": {"rtl": false, "pct": 0}, "ms": {"rtl": false, "pct": 0.19}, "my": {"rtl": false, "pct": 0}, "ne": {"rtl": false, "pct": 0.01}, "nl": {"rtl": false, "pct": 0.67}, "nn": {"rtl": false, "pct": 0.04}, "no": {"rtl": false, "pct": 0.76}, "nv": {"rtl": false, "pct": 0}, "oc": {"rtl": false, "pct": 0}, "pa": {"rtl": false, "pct": 0}, "pap": {"rtl": false, "pct": 0}, "pl": {"rtl": false, "pct": 0.97}, "pt": {"rtl": false, "pct": 0.86}, "pt-BR": {"rtl": false, "pct": 0.81}, "rm": {"rtl": false, "pct": 0}, "ro": {"rtl": false, "pct": 0.22}, "ru": {"rtl": false, "pct": 0.66}, "sat": {"rtl": false, "pct": 0}, "sc": {"rtl": false, "pct": 0}, "si": {"rtl": false, "pct": 0.02}, "sk": {"rtl": false, "pct": 0.4}, "sl": {"rtl": false, "pct": 0.28}, "so": {"rtl": false, "pct": 0.02}, "sq": {"rtl": false, "pct": 0.08}, "sr": {"rtl": false, "pct": 0.25}, "sv": {"rtl": false, "pct": 0.92}, "sw": {"rtl": false, "pct": 0}, "ta": {"rtl": false, "pct": 0.17}, "te": {"rtl": false, "pct": 0.03}, "th": {"rtl": false, "pct": 0.01}, "tl": {"rtl": false, "pct": 0.04}, "tr": {"rtl": false, "pct": 0.38}, "uk": {"rtl": false, "pct": 0.83}, "ur": {"rtl": true, "pct": 0}, "vi": {"rtl": false, "pct": 0.91}, "yue": {"rtl": false, "pct": 0.13}, "zh": {"rtl": false, "pct": 0.04}, "zh-CN": {"rtl": false, "pct": 0.79}, "zh-HK": {"rtl": false, "pct": 0.37}, "zh-TW": {"rtl": false, "pct": 0.91}} \ No newline at end of file diff --git a/modules/core/localizer.js b/modules/core/localizer.js index f8751d396..168dec9c1 100644 --- a/modules/core/localizer.js +++ b/modules/core/localizer.js @@ -20,10 +20,13 @@ export function coreLocalizer() { let _dataLanguages = {}; - // `localeData` is an object containing all _supported_ locale codes -> language info. + // `_dataLocales` is an object containing all _supported_ locale codes -> language info. + // * `rtl` - right-to-left or left-to-right text direction + // * `pct` - the percent of strings translated; 1 = 100%, full coverage + // // { - // en: { rtl: false, languageNames: {…}, scriptNames: {…} }, - // de: { rtl: false, languageNames: {…}, scriptNames: {…} }, + // en: { rtl: false, pct: {…} }, + // de: { rtl: false, pct: {…} }, // … // } let _dataLocales = {}; @@ -90,12 +93,21 @@ export function coreLocalizer() { .concat(utilDetect().browserLocales); _localeCode = bestSupportedLocale(requestedLocales); - return Promise.all([ - // always load the English locale strings as fallbacks - localizer.loadLocale('en'), - // load the preferred locale - localizer.loadLocale(_localeCode) - ]); + // always try to load the preferred locale + let loadStringsPromise = localizer.loadLocale(_localeCode); + + if (!_dataLocales[_localeCode] || + _dataLocales[_localeCode].pct !== 1) { + + loadStringsPromise = Promise.all([ + loadStringsPromise, + // Load the English locale as a fallback if the preferred locale + // isn't 100% complete + localizer.loadLocale('en') + ]); + } + + return loadStringsPromise; }) .then(() => { updateForCurrentLocale(); diff --git a/scripts/update_locales.js b/scripts/update_locales.js index 6df744a1d..b4c9e4937 100644 --- a/scripts/update_locales.js +++ b/scripts/update_locales.js @@ -6,7 +6,7 @@ const request = require('request').defaults({ maxSockets: 1 }); const YAML = require('js-yaml'); const colors = require('colors/safe'); -const resources = ['core', 'presets', 'imagery', 'community']; +const resourceIds = ['core', 'presets', 'imagery', 'community']; const outdir = 'dist/locales/'; const apiroot = 'https://www.transifex.com/api/2'; const projectURL = `${apiroot}/project/id-editor`; @@ -57,7 +57,36 @@ dataShortcuts.forEach(tab => { }); }); -asyncMap(resources, getResource, (err, results) => { +let coverageByLocaleCode = {}; + +// There's a race condition here, but it's highly unlikely that the info will +// return after the resources. There's an error check just in case. +asyncMap(resourceIds, getResourceInfo, gotResourceInfo); +asyncMap(resourceIds, getResource, gotResource); + +function getResourceInfo(resourceId, callback) { + let url = 'https://api.transifex.com/organizations/openstreetmap/projects/id-editor/resources/' + resourceId; + request.get(url, { auth : auth }, (err, resp, body) => { + if (err) return callback(err); + console.log(`${resp.statusCode}: ${url}`); + let content = JSON.parse(body); + callback(null, content); + }); +} +function gotResourceInfo(err, results) { + if (err) return console.log(err); + results.forEach(function(info) { + for (let code in info.stats) { + let coveragePart = info.stats[code].translated.percentage / results.length; + + code = code.replace(/_/g, '-'); + if (coverageByLocaleCode[code] === undefined) coverageByLocaleCode[code] = 0; + coverageByLocaleCode[code] += coveragePart; + } + }); +} + +function gotResource(err, results) { if (err) return console.log(err); // merge in strings fetched from transifex @@ -73,7 +102,7 @@ asyncMap(resources, getResource, (err, results) => { // write files and fetch language info for each locale let dataLocales = { - en: { rtl: false } + en: { rtl: false, pct: 1 } }; asyncMap(Object.keys(allStrings), (code, done) => { @@ -94,8 +123,18 @@ asyncMap(resources, getResource, (err, results) => { } else if (code === 'ku') { rtl = false; } + + let coverage = coverageByLocaleCode[code]; + if (coverage === undefined) { + console.log('Could not get language coverage'); + process.exit(1); + } + // we don't need high precision here, but we need to know if it's exactly 100% or not + coverage = Math.floor(coverage * 100) / 100; + dataLocales[code] = { - rtl: rtl + rtl: rtl, + pct: coverage }; done(); }); @@ -113,11 +152,11 @@ asyncMap(resources, getResource, (err, results) => { } } ); -}); +} -function getResource(resource, callback) { - let resourceURL = `${projectURL}/resource/${resource}`; +function getResource(resourceId, callback) { + let resourceURL = `${projectURL}/resource/${resourceId}`; getLanguages(resourceURL, (err, codes) => { if (err) return callback(err); @@ -126,11 +165,11 @@ function getResource(resource, callback) { let locale = {}; results.forEach((result, i) => { - if (resource === 'community' && Object.keys(result).length) { + if (resourceId === 'community' && Object.keys(result).length) { locale[codes[i]] = { community: result }; // add namespace } else { - if (resource === 'presets') { + if (resourceId === 'presets') { // remove terms that were not really translated let presets = (result.presets && result.presets.presets) || {}; for (const key of Object.keys(presets)) { @@ -144,7 +183,7 @@ function getResource(resource, callback) { } } } - } else if (resource === 'fields') { + } else if (resourceId === 'fields') { // remove terms that were not really translated let fields = (result.presets && result.presets.fields) || {}; for (const key of Object.keys(fields)) { @@ -158,7 +197,7 @@ function getResource(resource, callback) { } } } - } else if (resource === 'core') { + } else if (resourceId === 'core') { checkForDuplicateShortcuts(codes[i], result); }