diff --git a/data/update_locales.js b/data/update_locales.js deleted file mode 100644 index 9478a334f..000000000 --- a/data/update_locales.js +++ /dev/null @@ -1,384 +0,0 @@ -/* Downloads the latest translations from Transifex */ -const fs = require('fs'); -const prettyStringify = require('json-stringify-pretty-compact'); -const request = require('request').defaults({ maxSockets: 1 }); -const YAML = require('js-yaml'); -const colors = require('colors/safe'); - -const resources = ['core', 'presets', 'imagery', 'community']; -const outdir = './dist/locales/'; -const api = 'https://www.transifex.com/api/2/'; -const projectURL = api + 'project/id-editor/'; - - -/* - * Transifex oddly doesn't allow anonymous downloading - * - * auth is stored in transifex.auth in a json object: - * { - * "user": "username", - * "pass": "password" - * } - * */ - -const auth = JSON.parse(fs.readFileSync('./transifex.auth', 'utf8')); - -const sourceCore = YAML.load(fs.readFileSync('./data/core.yaml', 'utf8')); -const sourcePresets = YAML.load(fs.readFileSync('./data/presets.yaml', 'utf8')); -const sourceImagery = YAML.load(fs.readFileSync('./node_modules/editor-layer-index/i18n/en.yaml', 'utf8')); -const sourceCommunity = YAML.load(fs.readFileSync('./node_modules/osm-community-index/i18n/en.yaml', 'utf8')); - -const dataShortcuts = JSON.parse(fs.readFileSync('./data/shortcuts.json', 'utf8')).dataShortcuts; - -const cldrMainDir = './node_modules/cldr-localenames-full/main/'; - -var referencedScripts = []; - -const languageInfo = { - dataLanguages: getLangNamesInNativeLang() -}; -fs.writeFileSync('data/languages.json', JSON.stringify(languageInfo, null, 4)); - -var shortcuts = []; -dataShortcuts.forEach(function(tab) { - tab.columns.forEach(function(col) { - col.rows.forEach(function(row) { - if (!row.shortcuts) return; - row.shortcuts.forEach(function(shortcut) { - if (shortcut.includes('.')) { - var info = { - shortcut: shortcut - }; - if (row.modifiers) { - info.modifier = row.modifiers.join(''); - } - shortcuts.push(info); - } - }); - }); - }); -}); - -asyncMap(resources, getResource, function(err, results) { - if (err) return console.log(err); - - // merge in strings fetched from transifex - var allStrings = {}; - results.forEach(function(resourceStrings) { - Object.keys(resourceStrings).forEach(function(code) { - if (!allStrings[code]) { allStrings[code] = {}; } - var source = resourceStrings[code]; - var target = allStrings[code]; - Object.keys(source).forEach(function(k) { target[k] = source[k]; }); - }); - }); - - // write files and fetch language info for each locale - var dataLocales = { - en: { rtl: false, languageNames: languageNamesInLanguageOf('en'), scriptNames: scriptNamesInLanguageOf('en') } - }; - asyncMap(Object.keys(allStrings), - function(code, done) { - if (code === 'en' || !Object.keys(allStrings[code]).length) { - done(); - } else { - var obj = {}; - obj[code] = allStrings[code]; - fs.writeFileSync(outdir + code + '.json', JSON.stringify(obj, null, 4)); - - getLanguageInfo(code, function(err, info) { - var rtl = info && info.rtl; - // exceptions: see #4783 - if (code === 'ckb') { - rtl = true; - } else if (code === 'ku') { - rtl = false; - } - dataLocales[code] = { - rtl: rtl, - languageNames: languageNamesInLanguageOf(code) || {}, - scriptNames: scriptNamesInLanguageOf(code) || {} - }; - done(); - }); - } - }, function(err) { - if (!err) { - const keys = Object.keys(dataLocales).sort(); - var sorted = {}; - keys.forEach(function (k) { sorted[k] = dataLocales[k]; }); - fs.writeFileSync('data/locales.json', prettyStringify({ dataLocales: sorted }, { maxLength: 99999 })); - } - } - ); -}); - - -function getResource(resource, callback) { - var resourceURL = projectURL + 'resource/' + resource + '/'; - getLanguages(resourceURL, function(err, codes) { - if (err) return callback(err); - - asyncMap(codes, getLanguage(resourceURL), function(err, results) { - if (err) return callback(err); - - var locale = {}; - results.forEach(function(result, i) { - if (resource === 'community' && Object.keys(result).length) { - locale[codes[i]] = { community: result }; // add namespace - - } else { - if (resource === 'presets') { - // remove terms that were not really translated - var presets = (result.presets && result.presets.presets) || {}; - for (const key of Object.keys(presets)) { - var preset = presets[key]; - if (!preset.terms) continue; - preset.terms = preset.terms.replace(/<.*>/, '').trim(); - if (!preset.terms) { - delete preset.terms; - if (!Object.keys(preset).length) { - delete presets[key]; - } - } - } - } else if (resource === 'fields') { - // remove terms that were not really translated - var fields = (result.presets && result.presets.fields) || {}; - for (const key of Object.keys(fields)) { - var field = fields[key]; - if (!field.terms) continue; - field.terms = field.terms.replace(/\[.*\]/, '').trim(); - if (!field.terms) { - delete field.terms; - if (!Object.keys(preset).length) { - delete fields[key]; - } - } - } - } else if (resource === 'core') { - checkForDuplicateShortcuts(codes[i], result); - } - - locale[codes[i]] = result; - } - }); - - callback(null, locale); - }); - }); -} - - -function getLanguage(resourceURL) { - return function(code, callback) { - code = code.replace(/-/g, '_'); - var url = resourceURL + 'translation/' + code; - if (code === 'vi') { url += '?mode=reviewed'; } - - request.get(url, { auth : auth }, function(err, resp, body) { - if (err) return callback(err); - console.log(resp.statusCode + ': ' + url); - var content = JSON.parse(body).content; - callback(null, YAML.safeLoad(content)[code]); - }); - }; -} - - -function getLanguageInfo(code, callback) { - code = code.replace(/-/g, '_'); - var url = api + 'language/' + code; - request.get(url, { auth : auth }, function(err, resp, body) { - if (err) return callback(err); - console.log(resp.statusCode + ': ' + url); - callback(null, JSON.parse(body)); - }); -} - - -function getLanguages(resource, callback) { - var url = resource + '?details'; - request.get(url, { auth: auth }, - function(err, resp, body) { - if (err) return callback(err); - console.log(resp.statusCode + ': ' + url); - callback(null, JSON.parse(body).available_languages - .map(function(d) { return d.code.replace(/_/g, '-'); }) - .filter(function(d) { return d !== 'en'; }) - ); - }); -} - - -function asyncMap(inputs, func, callback) { - var index = 0; - var remaining = inputs.length; - var results = []; - var error; - - next(); - - function next() { - callFunc(index++); - if (index < inputs.length) { - setTimeout(next, 200); - } - } - - function callFunc(i) { - var d = inputs[i]; - func(d, function done(err, data) { - if (err) error = err; - results[i] = data; - remaining--; - if (!remaining) callback(error, results); - }); - } -} - -function checkForDuplicateShortcuts(code, coreStrings) { - var usedShortcuts = {}; - - shortcuts.forEach(function(shortcutInfo) { - var shortcutPathString = shortcutInfo.shortcut; - var modifier = shortcutInfo.modifier || ''; - - var path = shortcutPathString - .split('.') - .map(function (s) { return s.replace(//g, '.'); }) - .reverse(); - - var rep = coreStrings; - - while (rep !== undefined && path.length) { - rep = rep[path.pop()]; - } - - if (rep !== undefined) { - var shortcut = modifier + rep; - if (usedShortcuts[shortcut] && usedShortcuts[shortcut] !== shortcutPathString) { - var message = code + ': duplicate shortcut "' + shortcut + '" for "' + usedShortcuts[shortcut] + '" and "' + shortcutPathString + '"'; - console.warn(colors.yellow(message)); - } else { - usedShortcuts[shortcut] = shortcutPathString; - } - } - }); -} - -function getLangNamesInNativeLang() { - // manually add languages we want that aren't in CLDR - var unordered = { - 'oc': { - nativeName: 'Occitan' - }, - 'ja-Hira': { - base: 'ja', - script: 'Hira' - }, - 'ja-Latn': { - base: 'ja', - script: 'Latn' - }, - 'ko-Latn': { - base: 'ko', - script: 'Latn' - }, - 'zh_pinyin': { - base: 'zh', - script: 'Latn' - } - }; - var langDirectoryPaths = fs.readdirSync(cldrMainDir); - langDirectoryPaths.forEach(function(code) { - - var languagesPath = cldrMainDir + code + '/languages.json'; - - //if (!fs.existsSync(languagesPath)) return; - var languageObj = JSON.parse(fs.readFileSync(languagesPath, 'utf8')).main[code]; - - var identity = languageObj.identity; - - // skip locale-specific languages - if (identity.variant || identity.territory) return; - - var info = {}; - - var script = identity.script; - if (script) { - referencedScripts.push(script); - - info.base = identity.language; - info.script = script; - } - - var nativeName = languageObj.localeDisplayNames.languages[code]; - if (nativeName) { - info.nativeName = nativeName; - } - - unordered[code] = info; - }); - var ordered = {}; - Object.keys(unordered).sort().forEach(function(key) { - ordered[key] = unordered[key]; - }); - return ordered; -} - -var rematchCodes = { 'ar-AA': 'ar', 'zh-CN': 'zh', 'zh-HK': 'zh-Hant-HK', 'zh-TW': 'zh-Hant', 'pt-BR': 'pt', 'pt': 'pt-PT' }; - -function languageNamesInLanguageOf(code) { - - if (rematchCodes[code]) code = rematchCodes[code]; - - var languageFilePath = cldrMainDir + code + '/languages.json'; - if (!fs.existsSync(languageFilePath)) { - return null; - } - var translatedLangsByCode = JSON.parse(fs.readFileSync(languageFilePath, 'utf8')).main[code].localeDisplayNames.languages; - - // ignore codes for non-languages - for (var nonLangCode in { mis: true, mul: true, und: true, zxx: true }) { - delete translatedLangsByCode[nonLangCode]; - } - - for (var langCode in translatedLangsByCode) { - var altLongIndex = langCode.indexOf('-alt-long'); - if (altLongIndex !== -1) { - // prefer long names (e.g. Chinese -> Mandarin Chinese) - var base = langCode.substring(0, altLongIndex); - translatedLangsByCode[base] = translatedLangsByCode[langCode]; - } - - if (langCode.includes('-alt-')) { - // remove alternative names - delete translatedLangsByCode[langCode]; - } else if (langCode === translatedLangsByCode[langCode]) { - // no localized value available - delete translatedLangsByCode[langCode]; - } - } - - return translatedLangsByCode; -} - -function scriptNamesInLanguageOf(code) { - if (rematchCodes[code]) code = rematchCodes[code]; - - var languageFilePath = cldrMainDir + code + '/scripts.json'; - if (!fs.existsSync(languageFilePath)) { - return null; - } - var allTranslatedScriptsByCode = JSON.parse(fs.readFileSync(languageFilePath, 'utf8')).main[code].localeDisplayNames.scripts; - - var translatedScripts = {}; - referencedScripts.forEach(function(script) { - if (!allTranslatedScriptsByCode[script] || script === allTranslatedScriptsByCode[script]) return; - - translatedScripts[script] = allTranslatedScriptsByCode[script]; - }); - - return translatedScripts; -} diff --git a/package.json b/package.json index b12738459..5022c3424 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,10 @@ ], "license": "ISC", "scripts": { - "all": "npm-run-all -s clean build:css build:data build:dev build:legacy dist", - "build:css": "node build_css.js", - "build:data": "node build_data.js", + "all": "npm-run-all -s clean build build:legacy dist", + "build": "npm-run-all -s build:css build:data build:dev", + "build:css": "node scripts/build_css.js", + "build:data": "node scripts/build_data.js", "build:dev": "rollup --config config/rollup.config.dev.js", "build:legacy": "rollup --config config/rollup.config.legacy.js", "build:stats": "rollup --config config/rollup.config.stats.js", @@ -20,8 +21,8 @@ "dist": "npm-run-all -p dist:**", "dist:mapillary": "shx mkdir -p dist/mapillary-js && shx cp -R node_modules/mapillary-js/dist/* dist/mapillary-js/", "dist:pannellum": "shx mkdir -p dist/pannellum-streetside && shx cp -R node_modules/pannellum/build/* dist/pannellum-streetside/", - "dist:min": "uglifyjs dist/iD.legacy.js --compress --mangle --output dist/iD.min.js", - "dist:svg:id": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"iD-%s\" --symbol-sprite dist/img/iD-sprite.svg \"svg/iD-sprite/**/*.svg\"", + "dist:min:iD": "uglifyjs dist/iD.legacy.js --compress --mangle --output dist/iD.min.js", + "dist:svg:iD": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"iD-%s\" --symbol-sprite dist/img/iD-sprite.svg \"svg/iD-sprite/**/*.svg\"", "dist:svg:community": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"community-%s\" --symbol-sprite dist/img/community-sprite.svg node_modules/osm-community-index/dist/img/*.svg", "dist:svg:fa": "svg-sprite --symbol --symbol-dest . --symbol-sprite dist/img/fa-sprite.svg svg/fontawesome/*.svg", "dist:svg:tnp": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"tnp-%s\" --symbol-sprite dist/img/tnp-sprite.svg svg/the-noun-project/*.svg", @@ -29,14 +30,14 @@ "dist:svg:mapillary:signs": "svg-sprite --symbol --symbol-dest . --symbol-sprite dist/img/mapillary-sprite.svg node_modules/mapillary_sprite_source/package_signs/*.svg", "dist:svg:mapillary:objects": "svg-sprite --symbol --symbol-dest . --symbol-sprite dist/img/mapillary-object-sprite.svg node_modules/mapillary_sprite_source/package_objects/*.svg", "dist:svg:temaki": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"temaki-%s\" --symbol-sprite dist/img/temaki-sprite.svg node_modules/@ideditor/temaki/icons/*.svg", - "imagery": "node data/update_imagery", - "lint": "eslint *.js test/spec modules", - "start": "npm-run-all -s build:css build:data build:dev start:server", - "start:server": "node server.js", + "imagery": "node scripts/update_imagery.js", + "lint": "eslint scripts test/spec modules", + "start": "npm-run-all -s build start:server", + "start:server": "node scripts/server.js", "test": "npm-run-all -s lint build:css build:data build:legacy test:**", "test:phantom": "phantomjs --web-security=no node_modules/mocha-phantomjs-core/mocha-phantomjs-core.js test/phantom.html spec", "test:iD": "phantomjs --web-security=no node_modules/mocha-phantomjs-core/mocha-phantomjs-core.js test/index.html spec", - "translations": "node data/update_locales" + "translations": "node scripts/update_locales.js" }, "dependencies": { "@ideditor/country-coder": "^3.0.1", diff --git a/build_css.js b/scripts/build_css.js similarity index 93% rename from build_css.js rename to scripts/build_css.js index d2f1b679b..ddb2a6a28 100644 --- a/build_css.js +++ b/scripts/build_css.js @@ -26,13 +26,13 @@ function buildCSS() { return _currBuild = Promise.resolve() .then(() => doGlob('css/**/*.css')) - .then((files) => doConcat(files, 'dist/iD.css')) + .then(files => doConcat(files, 'dist/iD.css')) .then(() => { console.timeEnd(END); console.log(''); _currBuild = null; }) - .catch((err) => { + .catch(err => { console.error(err); console.log(''); _currBuild = null; diff --git a/build_data.js b/scripts/build_data.js similarity index 90% rename from build_data.js rename to scripts/build_data.js index 140c4fd0c..5d00fe148 100644 --- a/build_data.js +++ b/scripts/build_data.js @@ -3,15 +3,15 @@ const colors = require('colors/safe'); const fs = require('fs'); const glob = require('glob'); const jsonschema = require('jsonschema'); +const nsi = require('name-suggestion-index'); const path = require('path'); const prettyStringify = require('json-stringify-pretty-compact'); const shell = require('shelljs'); const YAML = require('js-yaml'); -const fieldSchema = require('./data/presets/schema/field.json'); -const presetSchema = require('./data/presets/schema/preset.json'); -const nsi = require('name-suggestion-index'); -const deprecated = require('./data/deprecated.json').dataDeprecated; +const fieldSchema = require('../data/presets/schema/field.json'); +const presetSchema = require('../data/presets/schema/preset.json'); +const deprecated = require('../data/deprecated.json').dataDeprecated; // fontawesome icons const fontawesome = require('@fortawesome/fontawesome-svg-core'); @@ -141,8 +141,8 @@ function validate(file, instance, schema) { let validationErrors = jsonschema.validate(instance, schema).errors; if (validationErrors.length) { - console.error(file + ': '); - validationErrors.forEach((error) => { + console.error(`${file}: `); + validationErrors.forEach(error => { if (error.property) { console.error(error.property + ' ' + error.message); } else { @@ -158,7 +158,7 @@ function validate(file, instance, schema) { function generateCategories(tstrings, faIcons, tnpIcons) { let categories = {}; - glob.sync(__dirname + '/data/presets/categories/*.json').forEach(file => { + glob.sync('data/presets/categories/*.json').forEach(file => { let category = read(file); let id = 'category-' + path.basename(file, '.json'); tstrings.categories[id] = { name: category.name }; @@ -181,7 +181,7 @@ function generateCategories(tstrings, faIcons, tnpIcons) { function generateFields(tstrings, faIcons, tnpIcons, searchableFieldIDs) { let fields = {}; - glob.sync(__dirname + '/data/presets/fields/**/*.json').forEach(file => { + glob.sync('data/presets/fields/**/*.json').forEach(file => { let field = read(file); let id = stripLeadingUnderscores(file.match(/presets\/fields\/([^.]*)\.json/)[1]); @@ -273,7 +273,7 @@ function suggestionsToPresets(presets) { // still no match? if (!preset) { - console.log('Warning: No preset "' + presetID + '" for name-suggestion "' + name + '"'); + console.log(`Warning: No preset "${presetID}" for name-suggestion "${name}"`); return; } @@ -329,7 +329,7 @@ function suggestionsToPresets(presets) { function stripLeadingUnderscores(str) { return str.split('/') - .map(function(s) { return s.replace(/^_/,''); }) + .map(s => s.replace(/^_/,'')) .join('/'); } @@ -337,7 +337,7 @@ function stripLeadingUnderscores(str) { function generatePresets(tstrings, faIcons, tnpIcons, searchableFieldIDs) { let presets = {}; - glob.sync(__dirname + '/data/presets/presets/**/*.json').forEach(file => { + glob.sync('data/presets/presets/**/*.json').forEach(file => { let preset = read(file); let id = stripLeadingUnderscores(file.match(/presets\/presets\/([^.]*)\.json/)[1]); @@ -379,23 +379,23 @@ function generateTranslations(fields, presets, tstrings, searchableFieldIDs) { let optkeys = Object.keys(options); if (f.keys) { - field['label#'] = f.keys.map(k => { return k + '=*'; }).join(', '); + field['label#'] = f.keys.map(k => `${k}=*`).join(', '); optkeys.forEach(k => { if (id === 'access') { - options[k]['title#'] = options[k]['description#'] = 'access=' + k; + options[k]['title#'] = options[k]['description#'] = `access=${k}`; } else { - options[k + '#'] = k + '=yes'; + options[k + '#'] = `${k}=yes`; } }); } else if (f.key) { - field['label#'] = f.key + '=*'; + field['label#'] = `${f.key}=*`; optkeys.forEach(k => { - options[k + '#'] = f.key + '=' + k; + options[k + '#'] = `${f.key}=${k}`; }); } if (f.placeholder) { - field['placeholder#'] = id + ' field placeholder'; + field['placeholder#'] = `${id} field placeholder`; } if (searchableFieldIDs[id]) { @@ -417,14 +417,14 @@ function generateTranslations(fields, presets, tstrings, searchableFieldIDs) { let keys = Object.keys(tags); if (keys.length) { - preset['name#'] = keys.map(k => { return k + '=' + tags[k]; }).join(', '); + preset['name#'] = keys.map(k => `${k}=${tags[k]}`).join(', '); } if (p.searchable !== false) { if (p.terms && p.terms.length) { preset['terms#'] = 'terms: ' + p.terms.join(); } - preset.terms = ''; + preset.terms = ``; } else { delete tstrings.presets[id].terms; delete p.terms; @@ -466,7 +466,7 @@ function generateTaginfo(presets, fields) { } if (preset.name) { let legacy = (preset.searchable === false) ? ' (unsearchable)' : ''; - tag.description = [ '🄿 ' + preset.name + legacy ]; + tag.description = [ `🄿 ${preset.name}${legacy}` ]; } if (preset.geometry) { setObjectType(tag, preset); @@ -505,14 +505,14 @@ function generateTaginfo(presets, fields) { if (value === 'undefined' || value === '*' || value === '') return; let tag = { key: key, value: value }; if (field.label) { - tag.description = [ '🄵 ' + field.label ]; + tag.description = [ `🄵 ${field.label}` ]; } coalesceTags(taginfo, tag); }); } else { let tag = { key: key }; if (field.label) { - tag.description = [ '🄵 ' + field.label ]; + tag.description = [ `🄵 ${field.label}` ]; } coalesceTags(taginfo, tag); } @@ -532,7 +532,7 @@ function generateTaginfo(presets, fields) { for (let replaceKey in elem.replace) { let replaceValue = elem.replace[replaceKey]; if (replaceValue === '$1') replaceValue = '*'; - replacementStrings.push(replaceKey + '=' + replaceValue); + replacementStrings.push(`${replaceKey}=${replaceValue}`); } let description = '🄳'; if (replacementStrings.length > 0) { @@ -553,9 +553,8 @@ function generateTaginfo(presets, fields) { function coalesceTags(taginfo, tag) { if (!tag.key) return; - let currentTaginfoEntries = taginfo.tags.filter(t => { - return (t.key === tag.key && t.value === tag.value); - }); + let currentTaginfoEntries = taginfo.tags + .filter(t => (t.key === tag.key && t.value === tag.value)); if (currentTaginfoEntries.length === 0) { taginfo.tags.push(tag); @@ -600,7 +599,7 @@ function generateTaginfo(presets, fields) { function generateTerritoryLanguages() { - let allRawInfo = read('./node_modules/cldr-core/supplemental/territoryInfo.json').supplemental.territoryInfo; + let allRawInfo = require('cldr-core/supplemental/territoryInfo.json').supplemental.territoryInfo; let territoryLanguages = {}; Object.keys(allRawInfo).forEach(territoryCode => { @@ -608,14 +607,14 @@ function generateTerritoryLanguages() { if (!territoryLangInfo) return; let langCodes = Object.keys(territoryLangInfo); - territoryLanguages[territoryCode.toLowerCase()] = langCodes.sort(function(langCode1, langCode2) { + territoryLanguages[territoryCode.toLowerCase()] = langCodes.sort((langCode1, langCode2) => { let popPercent1 = parseFloat(territoryLangInfo[langCode1]._populationPercent); let popPercent2 = parseFloat(territoryLangInfo[langCode2]._populationPercent); if (popPercent1 === popPercent2) { return langCode1.localeCompare(langCode2, 'en', { sensitivity: 'base' }); } return popPercent2 - popPercent1; - }).map(langCode => { return langCode.replace('_', '-'); }); + }).map(langCode => langCode.replace('_', '-')); }); return territoryLanguages; @@ -692,7 +691,7 @@ function validatePresetFields(presets, fields) { if (fieldCount > maxFieldsBeforeWarning) { // Fields with `prerequisiteTag` probably won't show up initially, // so don't count them against the limits. - let fieldsWithoutPrerequisites = preset.fields.filter(fieldID => { + const fieldsWithoutPrerequisites = preset.fields.filter(fieldID => { if (fields[fieldID] && fields[fieldID].prerequisiteTag) return false; return true; }); @@ -715,7 +714,7 @@ function validateDefaults(defaults, categories, presets) { let members = defaults.defaults[name]; members.forEach(id => { if (!presets[id] && !categories[id]) { - console.error('Unknown category or preset: ' + id + ' in default ' + name); + console.error(`Unknown category or preset: ${id} in default ${name}`); console.log(''); process.exit(1); } @@ -763,9 +762,9 @@ function writeFaIcons(faIcons) { const name = key.substring(4); const def = fontawesome.findIconDefinition({ prefix: prefix, iconName: name }); try { - writeFileProm('svg/fontawesome/' + key + '.svg', fontawesome.icon(def).html); + writeFileProm(`svg/fontawesome/${key}.svg`, fontawesome.icon(def).html); } catch (error) { - console.error('Error: No FontAwesome icon for ' + key); + console.error(`Error: No FontAwesome icon for ${key}`); throw (error); } } @@ -782,13 +781,13 @@ function writeTnpIcons(tnpIcons) { * } */ let nounAuth; - if (fs.existsSync('./the_noun_project.auth')) { - nounAuth = JSON.parse(fs.readFileSync('./the_noun_project.auth', 'utf8')); + if (fs.existsSync('../the_noun_project.auth')) { + nounAuth = JSON.parse(fs.readFileSync('../the_noun_project.auth', 'utf8')); } const baseURL = 'http://api.thenounproject.com/icon/'; let unusedSvgFiles = fs.readdirSync('svg/the-noun-project', 'utf8') - .reduce(function(obj, name) { + .reduce((obj, name) => { if (name.endsWith('.svg')) { obj[name] = true; } @@ -797,19 +796,19 @@ function writeTnpIcons(tnpIcons) { for (const key in tnpIcons) { const id = key.substring(4); - const fileName = id + '.svg'; + const fileName = `${id}.svg`; if (unusedSvgFiles[fileName]) { delete unusedSvgFiles[fileName]; } - const localPath = 'svg/the-noun-project/' + fileName; + const localPath = `svg/the-noun-project/${fileName}`; // don't redownload existing icons if (fs.existsSync(localPath)) continue; if (!nounAuth) { - console.error('No authentication file for The Noun Project. Cannot download icon: ' + key); + console.error(`No authentication file for The Noun Project. Cannot download icon: ${key}`); continue; } @@ -819,7 +818,7 @@ function writeTnpIcons(tnpIcons) { // remove icons that are not needed for (const unusedFileName in unusedSvgFiles) { - shell.rm('-f', ['svg/the-noun-project/' + unusedFileName]); + shell.rm('-f', [`svg/the-noun-project/${unusedFileName}`]); } } @@ -839,13 +838,13 @@ function handleTheNounProjectResponse(err, resp, body) { console.error('The Noun Project has not provided a URL to download the icon "' + icon.term + '" (tnp-' + icon.id + ').'); return; } - request.get(iconURL, function(err2, resp2, svg) { + request.get(iconURL, (err2, resp2, svg) => { if (err2) { console.error(err2); return; } try { - writeFileProm('svg/the-noun-project/' + icon.id + '.svg', svg); + writeFileProm(`svg/the-noun-project/${icon.id}.svg`, svg); } catch (error) { console.error(error); throw (error); diff --git a/server.js b/scripts/server.js similarity index 66% rename from server.js rename to scripts/server.js index 4835102ea..cddc45791 100644 --- a/server.js +++ b/scripts/server.js @@ -10,7 +10,7 @@ gaze(['css/**/*.css'], (err, watcher) => { watcher.on('all', () => buildCSS()); }); -const server = new StaticServer({ rootPath: __dirname, port: 8080, followSymlink: true }); +const server = new StaticServer({ rootPath: process.cwd(), port: 8080, followSymlink: true }); server.start(() => { - console.log(colors.yellow('Listening on ' + server.port)); + console.log(colors.yellow(`Listening on ${server.port}`)); }); diff --git a/data/update_imagery.js b/scripts/update_imagery.js similarity index 99% rename from data/update_imagery.js rename to scripts/update_imagery.js index 18f7190a4..328043aae 100644 --- a/data/update_imagery.js +++ b/scripts/update_imagery.js @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ const fs = require('fs'); const sources = require('editor-layer-index/imagery.json'); const prettyStringify = require('json-stringify-pretty-compact'); diff --git a/scripts/update_locales.js b/scripts/update_locales.js new file mode 100644 index 000000000..dc7712e74 --- /dev/null +++ b/scripts/update_locales.js @@ -0,0 +1,372 @@ +/* eslint-disable no-console */ +/* Downloads the latest translations from Transifex */ +const fs = require('fs'); +const prettyStringify = require('json-stringify-pretty-compact'); +const request = require('request').defaults({ maxSockets: 1 }); +const YAML = require('js-yaml'); +const colors = require('colors/safe'); + +const resources = ['core', 'presets', 'imagery', 'community']; +const outdir = 'dist/locales/'; +const apiroot = 'https://www.transifex.com/api/2'; +const projectURL = `${apiroot}/project/id-editor/`; + + +/* + * Transifex oddly doesn't allow anonymous downloading + * + * auth is stored in transifex.auth in a json object: + * { + * "user": "username", + * "pass": "password" + * } + * */ + +const auth = JSON.parse(fs.readFileSync('../transifex.auth', 'utf8')); + +// const sourceCore = YAML.load(fs.readFileSync('../data/core.yaml', 'utf8')); +// const sourcePresets = YAML.load(fs.readFileSync('../data/presets.yaml', 'utf8')); +// const sourceImagery = YAML.load(fs.readFileSync('../node_modules/editor-layer-index/i18n/en.yaml', 'utf8')); +// const sourceCommunity = YAML.load(fs.readFileSync('../node_modules/osm-community-index/i18n/en.yaml', 'utf8')); + +const dataShortcuts = JSON.parse(fs.readFileSync('../data/shortcuts.json', 'utf8')).dataShortcuts; + +const cldrMainDir = '../node_modules/cldr-localenames-full/main/'; + +let referencedScripts = []; + +const languageInfo = { dataLanguages: getLangNamesInNativeLang() }; +fs.writeFileSync('data/languages.json', JSON.stringify(languageInfo, null, 4)); + +let shortcuts = []; +dataShortcuts.forEach(tab => { + tab.columns.forEach(col => { + col.rows.forEach(row => { + if (!row.shortcuts) return; + row.shortcuts.forEach(shortcut => { + if (shortcut.includes('.')) { + let info = { shortcut: shortcut }; + if (row.modifiers) { + info.modifier = row.modifiers.join(''); + } + shortcuts.push(info); + } + }); + }); + }); +}); + +asyncMap(resources, getResource, (err, results) => { + if (err) return console.log(err); + + // merge in strings fetched from transifex + let allStrings = {}; + results.forEach(resourceStrings => { + Object.keys(resourceStrings).forEach(code => { + if (!allStrings[code]) { allStrings[code] = {}; } + let source = resourceStrings[code]; + let target = allStrings[code]; + Object.keys(source).forEach(k => target[k] = source[k]); + }); + }); + + // write files and fetch language info for each locale + let dataLocales = { + en: { rtl: false, languageNames: languageNamesInLanguageOf('en'), scriptNames: scriptNamesInLanguageOf('en') } + }; + asyncMap(Object.keys(allStrings), + (code, done) => { + if (code === 'en' || !Object.keys(allStrings[code]).length) { + done(); + } else { + let obj = {}; + obj[code] = allStrings[code]; + fs.writeFileSync(outdir + code + '.json', JSON.stringify(obj, null, 4)); + + getLanguageInfo(code, (err, info) => { + let rtl = info && info.rtl; + // exceptions: see #4783 + if (code === 'ckb') { + rtl = true; + } else if (code === 'ku') { + rtl = false; + } + dataLocales[code] = { + rtl: rtl, + languageNames: languageNamesInLanguageOf(code) || {}, + scriptNames: scriptNamesInLanguageOf(code) || {} + }; + done(); + }); + } + }, + (err) => { + if (!err) { + const keys = Object.keys(dataLocales).sort(); + let sorted = {}; + keys.forEach(k => sorted[k] = dataLocales[k]); + fs.writeFileSync('data/locales.json', prettyStringify({ dataLocales: sorted }, { maxLength: 99999 })); + } + } + ); +}); + + +function getResource(resource, callback) { + let resourceURL = `${projectURL}resource/${resource}/`; + getLanguages(resourceURL, (err, codes) => { + if (err) return callback(err); + + asyncMap(codes, getLanguage(resourceURL), (err, results) => { + if (err) return callback(err); + + let locale = {}; + results.forEach((result, i) => { + if (resource === 'community' && Object.keys(result).length) { + locale[codes[i]] = { community: result }; // add namespace + + } else { + if (resource === 'presets') { + // remove terms that were not really translated + let presets = (result.presets && result.presets.presets) || {}; + for (const key of Object.keys(presets)) { + let preset = presets[key]; + if (!preset.terms) continue; + preset.terms = preset.terms.replace(/<.*>/, '').trim(); + if (!preset.terms) { + delete preset.terms; + if (!Object.keys(preset).length) { + delete presets[key]; + } + } + } + } else if (resource === 'fields') { + // remove terms that were not really translated + let fields = (result.presets && result.presets.fields) || {}; + for (const key of Object.keys(fields)) { + let field = fields[key]; + if (!field.terms) continue; + field.terms = field.terms.replace(/\[.*\]/, '').trim(); + if (!field.terms) { + delete field.terms; + if (!Object.keys(field).length) { + delete fields[key]; + } + } + } + } else if (resource === 'core') { + checkForDuplicateShortcuts(codes[i], result); + } + + locale[codes[i]] = result; + } + }); + + callback(null, locale); + }); + }); +} + + +function getLanguage(resourceURL) { + return (code, callback) => { + code = code.replace(/-/g, '_'); + let url = `${resourceURL}/translation/${code}`; + if (code === 'vi') { url += '?mode=reviewed'; } + + request.get(url, { auth : auth }, (err, resp, body) => { + if (err) return callback(err); + console.log(`${resp.statusCode}: ${url}`); + let content = JSON.parse(body).content; + callback(null, YAML.safeLoad(content)[code]); + }); + }; +} + + +function getLanguageInfo(code, callback) { + code = code.replace(/-/g, '_'); + let url = `${apiroot}/language/${code}`; + request.get(url, { auth : auth }, (err, resp, body) => { + if (err) return callback(err); + console.log(`${resp.statusCode}: ${url}`); + callback(null, JSON.parse(body)); + }); +} + + +function getLanguages(resource, callback) { + let url = `${resource}?details`; + request.get(url, { auth: auth }, (err, resp, body) => { + if (err) return callback(err); + console.log(`${resp.statusCode}: ${url}`); + callback(null, JSON.parse(body).available_languages + .map(d => d.code.replace(/_/g, '-')) + .filter(d => d !== 'en') + ); + }); +} + + +function asyncMap(inputs, func, callback) { + let index = 0; + let remaining = inputs.length; + let results = []; + let error; + + next(); + + function next() { + callFunc(index++); + if (index < inputs.length) { + setTimeout(next, 200); + } + } + + function callFunc(i) { + let d = inputs[i]; + func(d, (err, data) => { + if (err) error = err; + results[i] = data; + remaining--; + if (!remaining) callback(error, results); + }); + } +} + + +function checkForDuplicateShortcuts(code, coreStrings) { + let usedShortcuts = {}; + + shortcuts.forEach(shortcutInfo => { + let shortcutPathString = shortcutInfo.shortcut; + let modifier = shortcutInfo.modifier || ''; + + let path = shortcutPathString + .split('.') + .map(s => s.replace(//g, '.')) + .reverse(); + + let rep = coreStrings; + + while (rep !== undefined && path.length) { + rep = rep[path.pop()]; + } + + if (rep !== undefined) { + let shortcut = modifier + rep; + if (usedShortcuts[shortcut] && usedShortcuts[shortcut] !== shortcutPathString) { + let message = code + ': duplicate shortcut "' + shortcut + '" for "' + usedShortcuts[shortcut] + '" and "' + shortcutPathString + '"'; + console.warn(colors.yellow(message)); + } else { + usedShortcuts[shortcut] = shortcutPathString; + } + } + }); +} + +function getLangNamesInNativeLang() { + // manually add languages we want that aren't in CLDR + let unordered = { + 'oc': { + nativeName: 'Occitan' + }, + 'ja-Hira': { + base: 'ja', + script: 'Hira' + }, + 'ja-Latn': { + base: 'ja', + script: 'Latn' + }, + 'ko-Latn': { + base: 'ko', + script: 'Latn' + }, + 'zh_pinyin': { + base: 'zh', + script: 'Latn' + } + }; + + let langDirectoryPaths = fs.readdirSync(cldrMainDir); + langDirectoryPaths.forEach(code => { + let languagesPath = `${cldrMainDir}${code}/languages.json`; + //if (!fs.existsSync(languagesPath)) return; + let languageObj = JSON.parse(fs.readFileSync(languagesPath, 'utf8')).main[code]; + let identity = languageObj.identity; + + // skip locale-specific languages + if (identity.letiant || identity.territory) return; + + let info = {}; + const script = identity.script; + if (script) { + referencedScripts.push(script); + info.base = identity.language; + info.script = script; + } + + const nativeName = languageObj.localeDisplayNames.languages[code]; + if (nativeName) { + info.nativeName = nativeName; + } + + unordered[code] = info; + }); + + let ordered = {}; + Object.keys(unordered).sort().forEach(key => ordered[key] = unordered[key]); + return ordered; +} + + +const rematchCodes = { 'ar-AA': 'ar', 'zh-CN': 'zh', 'zh-HK': 'zh-Hant-HK', 'zh-TW': 'zh-Hant', 'pt-BR': 'pt', 'pt': 'pt-PT' }; + +function languageNamesInLanguageOf(code) { + if (rematchCodes[code]) code = rematchCodes[code]; + + let languageFilePath = `${cldrMainDir}${code}/languages.json`; + if (!fs.existsSync(languageFilePath)) return null; + + let translatedLangsByCode = JSON.parse(fs.readFileSync(languageFilePath, 'utf8')).main[code].localeDisplayNames.languages; + + // ignore codes for non-languages + for (let nonLangCode in { mis: true, mul: true, und: true, zxx: true }) { + delete translatedLangsByCode[nonLangCode]; + } + + for (let langCode in translatedLangsByCode) { + let altLongIndex = langCode.indexOf('-alt-long'); + if (altLongIndex !== -1) { // prefer long names (e.g. Chinese -> Mandarin Chinese) + let base = langCode.substring(0, altLongIndex); + translatedLangsByCode[base] = translatedLangsByCode[langCode]; + } + + if (langCode.includes('-alt-')) { // remove alternative names + delete translatedLangsByCode[langCode]; + } else if (langCode === translatedLangsByCode[langCode]) { // no localized value available + delete translatedLangsByCode[langCode]; + } + } + + return translatedLangsByCode; +} + + +function scriptNamesInLanguageOf(code) { + if (rematchCodes[code]) code = rematchCodes[code]; + + let languageFilePath = `${cldrMainDir}${code}/scripts.json`; + if (!fs.existsSync(languageFilePath)) return null; + + let allTranslatedScriptsByCode = JSON.parse(fs.readFileSync(languageFilePath, 'utf8')).main[code].localeDisplayNames.scripts; + + let translatedScripts = {}; + referencedScripts.forEach(script => { + if (!allTranslatedScriptsByCode[script] || script === allTranslatedScriptsByCode[script]) return; + translatedScripts[script] = allTranslatedScriptsByCode[script]; + }); + + return translatedScripts; +} diff --git a/test/spec/core/data.js b/test/spec/core/data.js index 908b812c4..c727b2a64 100644 --- a/test/spec/core/data.js +++ b/test/spec/core/data.js @@ -1,8 +1,8 @@ describe('iD.coreData', function() { var _context; - var _oldData; before(function() { + iD.data = iD.data || {}; iD.data.test = { hello: 'world' }; });