CalvinBackup/iD
1
0
Fork 0
mirror of https://github.com/FoggedLens/iD.git synced 2026-02-12 16:52:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move all the build scripts into scripts/ folder, ES6ify more stuff

Browse Source
This commit is contained in:
Bryan Housel
2020-01-28 15:00:02 -05:00
parent aadbddba2f
commit ce7cac6738
8 changed files with 430 additions and 441 deletions
Download Patch File Download Diff File Expand all files Collapse all files

384
data/update_locales.js
View File

@@ -1,384 +0,0 @@
/* Downloads the latest translations from Transifex */
const fs = require('fs');
const prettyStringify = require('json-stringify-pretty-compact');
const request = require('request').defaults({ maxSockets: 1 });
const YAML = require('js-yaml');
const colors = require('colors/safe');
const resources = ['core', 'presets', 'imagery', 'community'];
const outdir = './dist/locales/';
const api = 'https://www.transifex.com/api/2/';
const projectURL = api + 'project/id-editor/';
/*
* Transifex oddly doesn't allow anonymous downloading
*
* auth is stored in transifex.auth in a json object:
* {
* "user": "username",
* "pass": "password"
* }
* */
const auth = JSON.parse(fs.readFileSync('./transifex.auth', 'utf8'));
const sourceCore = YAML.load(fs.readFileSync('./data/core.yaml', 'utf8'));
const sourcePresets = YAML.load(fs.readFileSync('./data/presets.yaml', 'utf8'));
const sourceImagery = YAML.load(fs.readFileSync('./node_modules/editor-layer-index/i18n/en.yaml', 'utf8'));
const sourceCommunity = YAML.load(fs.readFileSync('./node_modules/osm-community-index/i18n/en.yaml', 'utf8'));
const dataShortcuts = JSON.parse(fs.readFileSync('./data/shortcuts.json', 'utf8')).dataShortcuts;
const cldrMainDir = './node_modules/cldr-localenames-full/main/';
var referencedScripts = [];
const languageInfo = {
dataLanguages: getLangNamesInNativeLang()
};
fs.writeFileSync('data/languages.json', JSON.stringify(languageInfo, null, 4));
var shortcuts = [];
dataShortcuts.forEach(function(tab) {
tab.columns.forEach(function(col) {
col.rows.forEach(function(row) {
if (!row.shortcuts) return;
row.shortcuts.forEach(function(shortcut) {
if (shortcut.includes('.')) {
var info = {
shortcut: shortcut
};
if (row.modifiers) {
info.modifier = row.modifiers.join('');
}
shortcuts.push(info);
}
});
});
});
});
asyncMap(resources, getResource, function(err, results) {
if (err) return console.log(err);
// merge in strings fetched from transifex
var allStrings = {};
results.forEach(function(resourceStrings) {
Object.keys(resourceStrings).forEach(function(code) {
if (!allStrings[code]) { allStrings[code] = {}; }
var source = resourceStrings[code];
var target = allStrings[code];
Object.keys(source).forEach(function(k) { target[k] = source[k]; });
});
});
// write files and fetch language info for each locale
var dataLocales = {
en: { rtl: false, languageNames: languageNamesInLanguageOf('en'), scriptNames: scriptNamesInLanguageOf('en') }
};
asyncMap(Object.keys(allStrings),
function(code, done) {
if (code === 'en' || !Object.keys(allStrings[code]).length) {
done();
} else {
var obj = {};
obj[code] = allStrings[code];
fs.writeFileSync(outdir + code + '.json', JSON.stringify(obj, null, 4));
getLanguageInfo(code, function(err, info) {
var rtl = info && info.rtl;
// exceptions: see #4783
if (code === 'ckb') {
rtl = true;
} else if (code === 'ku') {
rtl = false;
}
dataLocales[code] = {
rtl: rtl,
languageNames: languageNamesInLanguageOf(code) || {},
scriptNames: scriptNamesInLanguageOf(code) || {}
};
done();
});
}
}, function(err) {
if (!err) {
const keys = Object.keys(dataLocales).sort();
var sorted = {};
keys.forEach(function (k) { sorted[k] = dataLocales[k]; });
fs.writeFileSync('data/locales.json', prettyStringify({ dataLocales: sorted }, { maxLength: 99999 }));
}
}
);
});
function getResource(resource, callback) {
var resourceURL = projectURL + 'resource/' + resource + '/';
getLanguages(resourceURL, function(err, codes) {
if (err) return callback(err);
asyncMap(codes, getLanguage(resourceURL), function(err, results) {
if (err) return callback(err);
var locale = {};
results.forEach(function(result, i) {
if (resource === 'community' && Object.keys(result).length) {
locale[codes[i]] = { community: result }; // add namespace
} else {
if (resource === 'presets') {
// remove terms that were not really translated
var presets = (result.presets && result.presets.presets) || {};
for (const key of Object.keys(presets)) {
var preset = presets[key];
if (!preset.terms) continue;
preset.terms = preset.terms.replace(/<.*>/, '').trim();
if (!preset.terms) {
delete preset.terms;
if (!Object.keys(preset).length) {
delete presets[key];
}
}
}
} else if (resource === 'fields') {
// remove terms that were not really translated
var fields = (result.presets && result.presets.fields) || {};
for (const key of Object.keys(fields)) {
var field = fields[key];
if (!field.terms) continue;
field.terms = field.terms.replace(/\[.*\]/, '').trim();
if (!field.terms) {
delete field.terms;
if (!Object.keys(preset).length) {
delete fields[key];
}
}
}
} else if (resource === 'core') {
checkForDuplicateShortcuts(codes[i], result);
}
locale[codes[i]] = result;
}
});
callback(null, locale);
});
});
}
function getLanguage(resourceURL) {
return function(code, callback) {
code = code.replace(/-/g, '_');
var url = resourceURL + 'translation/' + code;
if (code === 'vi') { url += '?mode=reviewed'; }
request.get(url, { auth : auth }, function(err, resp, body) {
if (err) return callback(err);
console.log(resp.statusCode + ': ' + url);
var content = JSON.parse(body).content;
callback(null, YAML.safeLoad(content)[code]);
});
};
}
function getLanguageInfo(code, callback) {
code = code.replace(/-/g, '_');
var url = api + 'language/' + code;
request.get(url, { auth : auth }, function(err, resp, body) {
if (err) return callback(err);
console.log(resp.statusCode + ': ' + url);
callback(null, JSON.parse(body));
});
}
function getLanguages(resource, callback) {
var url = resource + '?details';
request.get(url, { auth: auth },
function(err, resp, body) {
if (err) return callback(err);
console.log(resp.statusCode + ': ' + url);
callback(null, JSON.parse(body).available_languages
.map(function(d) { return d.code.replace(/_/g, '-'); })
.filter(function(d) { return d !== 'en'; })
);
});
}
function asyncMap(inputs, func, callback) {
var index = 0;
var remaining = inputs.length;
var results = [];
var error;
next();
function next() {
callFunc(index++);
if (index < inputs.length) {
setTimeout(next, 200);
}
}
function callFunc(i) {
var d = inputs[i];
func(d, function done(err, data) {
if (err) error = err;
results[i] = data;
remaining--;
if (!remaining) callback(error, results);
});
}
}
function checkForDuplicateShortcuts(code, coreStrings) {
var usedShortcuts = {};
shortcuts.forEach(function(shortcutInfo) {
var shortcutPathString = shortcutInfo.shortcut;
var modifier = shortcutInfo.modifier || '';
var path = shortcutPathString
.split('.')
.map(function (s) { return s.replace(/<TX_DOT>/g, '.'); })
.reverse();
var rep = coreStrings;
while (rep !== undefined && path.length) {
rep = rep[path.pop()];
}
if (rep !== undefined) {
var shortcut = modifier + rep;
if (usedShortcuts[shortcut] && usedShortcuts[shortcut] !== shortcutPathString) {
var message = code + ': duplicate shortcut "' + shortcut + '" for "' + usedShortcuts[shortcut] + '" and "' + shortcutPathString + '"';
console.warn(colors.yellow(message));
} else {
usedShortcuts[shortcut] = shortcutPathString;
}
}
});
}
function getLangNamesInNativeLang() {
// manually add languages we want that aren't in CLDR
var unordered = {
'oc': {
nativeName: 'Occitan'
},
'ja-Hira': {
base: 'ja',
script: 'Hira'
},
'ja-Latn': {
base: 'ja',
script: 'Latn'
},
'ko-Latn': {
base: 'ko',
script: 'Latn'
},
'zh_pinyin': {
base: 'zh',
script: 'Latn'
}
};
var langDirectoryPaths = fs.readdirSync(cldrMainDir);
langDirectoryPaths.forEach(function(code) {
var languagesPath = cldrMainDir + code + '/languages.json';
//if (!fs.existsSync(languagesPath)) return;
var languageObj = JSON.parse(fs.readFileSync(languagesPath, 'utf8')).main[code];
var identity = languageObj.identity;
// skip locale-specific languages
if (identity.variant || identity.territory) return;
var info = {};
var script = identity.script;
if (script) {
referencedScripts.push(script);
info.base = identity.language;
info.script = script;
}
var nativeName = languageObj.localeDisplayNames.languages[code];
if (nativeName) {
info.nativeName = nativeName;
}
unordered[code] = info;
});
var ordered = {};
Object.keys(unordered).sort().forEach(function(key) {
ordered[key] = unordered[key];
});
return ordered;
}
var rematchCodes = { 'ar-AA': 'ar', 'zh-CN': 'zh', 'zh-HK': 'zh-Hant-HK', 'zh-TW': 'zh-Hant', 'pt-BR': 'pt', 'pt': 'pt-PT' };
function languageNamesInLanguageOf(code) {
if (rematchCodes[code]) code = rematchCodes[code];
var languageFilePath = cldrMainDir + code + '/languages.json';
if (!fs.existsSync(languageFilePath)) {
return null;
}
var translatedLangsByCode = JSON.parse(fs.readFileSync(languageFilePath, 'utf8')).main[code].localeDisplayNames.languages;
// ignore codes for non-languages
for (var nonLangCode in { mis: true, mul: true, und: true, zxx: true }) {
delete translatedLangsByCode[nonLangCode];
}
for (var langCode in translatedLangsByCode) {
var altLongIndex = langCode.indexOf('-alt-long');
if (altLongIndex !== -1) {
// prefer long names (e.g. Chinese -> Mandarin Chinese)
var base = langCode.substring(0, altLongIndex);
translatedLangsByCode[base] = translatedLangsByCode[langCode];
}
if (langCode.includes('-alt-')) {
// remove alternative names
delete translatedLangsByCode[langCode];
} else if (langCode === translatedLangsByCode[langCode]) {
// no localized value available
delete translatedLangsByCode[langCode];
}
}
return translatedLangsByCode;
}
function scriptNamesInLanguageOf(code) {
if (rematchCodes[code]) code = rematchCodes[code];
var languageFilePath = cldrMainDir + code + '/scripts.json';
if (!fs.existsSync(languageFilePath)) {
return null;
}
var allTranslatedScriptsByCode = JSON.parse(fs.readFileSync(languageFilePath, 'utf8')).main[code].localeDisplayNames.scripts;
var translatedScripts = {};
referencedScripts.forEach(function(script) {
if (!allTranslatedScriptsByCode[script] || script === allTranslatedScriptsByCode[script]) return;
translatedScripts[script] = allTranslatedScriptsByCode[script];
});
return translatedScripts;
}

21
package.json
View File

@@ -10,9 +10,10 @@
],
"license": "ISC",
"scripts": {
"all": "npm-run-all -s clean build:css build:data build:dev build:legacy dist",
"build:css": "node build_css.js",
"build:data": "node build_data.js",
"all": "npm-run-all -s clean build build:legacy dist",
"build": "npm-run-all -s build:css build:data build:dev",
"build:css": "node scripts/build_css.js",
"build:data": "node scripts/build_data.js",
"build:dev": "rollup --config config/rollup.config.dev.js",
"build:legacy": "rollup --config config/rollup.config.legacy.js",
"build:stats": "rollup --config config/rollup.config.stats.js",
@@ -20,8 +21,8 @@
"dist": "npm-run-all -p dist:**",
"dist:mapillary": "shx mkdir -p dist/mapillary-js && shx cp -R node_modules/mapillary-js/dist/* dist/mapillary-js/",
"dist:pannellum": "shx mkdir -p dist/pannellum-streetside && shx cp -R node_modules/pannellum/build/* dist/pannellum-streetside/",
"dist:min": "uglifyjs dist/iD.legacy.js --compress --mangle --output dist/iD.min.js",
"dist:svg:id": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"iD-%s\" --symbol-sprite dist/img/iD-sprite.svg \"svg/iD-sprite/**/*.svg\"",
"dist:min:iD": "uglifyjs dist/iD.legacy.js --compress --mangle --output dist/iD.min.js",
"dist:svg:iD": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"iD-%s\" --symbol-sprite dist/img/iD-sprite.svg \"svg/iD-sprite/**/*.svg\"",
"dist:svg:community": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"community-%s\" --symbol-sprite dist/img/community-sprite.svg node_modules/osm-community-index/dist/img/*.svg",
"dist:svg:fa": "svg-sprite --symbol --symbol-dest . --symbol-sprite dist/img/fa-sprite.svg svg/fontawesome/*.svg",
"dist:svg:tnp": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"tnp-%s\" --symbol-sprite dist/img/tnp-sprite.svg svg/the-noun-project/*.svg",
@@ -29,14 +30,14 @@
"dist:svg:mapillary:signs": "svg-sprite --symbol --symbol-dest . --symbol-sprite dist/img/mapillary-sprite.svg node_modules/mapillary_sprite_source/package_signs/*.svg",
"dist:svg:mapillary:objects": "svg-sprite --symbol --symbol-dest . --symbol-sprite dist/img/mapillary-object-sprite.svg node_modules/mapillary_sprite_source/package_objects/*.svg",
"dist:svg:temaki": "svg-sprite --symbol --symbol-dest . --shape-id-generator \"temaki-%s\" --symbol-sprite dist/img/temaki-sprite.svg node_modules/@ideditor/temaki/icons/*.svg",
"imagery": "node data/update_imagery",
"lint": "eslint *.js test/spec modules",
"start": "npm-run-all -s build:css build:data build:dev start:server",
"start:server": "node server.js",
"imagery": "node scripts/update_imagery.js",
"lint": "eslint scripts test/spec modules",
"start": "npm-run-all -s build start:server",
"start:server": "node scripts/server.js",
"test": "npm-run-all -s lint build:css build:data build:legacy test:**",
"test:phantom": "phantomjs --web-security=no node_modules/mocha-phantomjs-core/mocha-phantomjs-core.js test/phantom.html spec",
"test:iD": "phantomjs --web-security=no node_modules/mocha-phantomjs-core/mocha-phantomjs-core.js test/index.html spec",
"translations": "node data/update_locales"
"translations": "node scripts/update_locales.js"
},
"dependencies": {
"@ideditor/country-coder": "^3.0.1",

4
build_css.js → scripts/build_css.js
View File

@@ -26,13 +26,13 @@ function buildCSS() {
return _currBuild =
Promise.resolve()
.then(() => doGlob('css/**/*.css'))
.then((files) => doConcat(files, 'dist/iD.css'))
.then(files => doConcat(files, 'dist/iD.css'))
.then(() => {
console.timeEnd(END);
console.log('');
_currBuild = null;
})
.catch((err) => {
.catch(err => {
console.error(err);
console.log('');
_currBuild = null;

83
build_data.js → scripts/build_data.js
View File

@@ -3,15 +3,15 @@ const colors = require('colors/safe');
const fs = require('fs');
const glob = require('glob');
const jsonschema = require('jsonschema');
const nsi = require('name-suggestion-index');
const path = require('path');
const prettyStringify = require('json-stringify-pretty-compact');
const shell = require('shelljs');
const YAML = require('js-yaml');
const fieldSchema = require('./data/presets/schema/field.json');
const presetSchema = require('./data/presets/schema/preset.json');
const nsi = require('name-suggestion-index');
const deprecated = require('./data/deprecated.json').dataDeprecated;
const fieldSchema = require('../data/presets/schema/field.json');
const presetSchema = require('../data/presets/schema/preset.json');
const deprecated = require('../data/deprecated.json').dataDeprecated;
// fontawesome icons
const fontawesome = require('@fortawesome/fontawesome-svg-core');
@@ -141,8 +141,8 @@ function validate(file, instance, schema) {
let validationErrors = jsonschema.validate(instance, schema).errors;
if (validationErrors.length) {
console.error(file + ': ');
validationErrors.forEach((error) => {
console.error(`${file}: `);
validationErrors.forEach(error => {
if (error.property) {
console.error(error.property + ' ' + error.message);
} else {
@@ -158,7 +158,7 @@ function validate(file, instance, schema) {
function generateCategories(tstrings, faIcons, tnpIcons) {
let categories = {};
glob.sync(__dirname + '/data/presets/categories/*.json').forEach(file => {
glob.sync('data/presets/categories/*.json').forEach(file => {
let category = read(file);
let id = 'category-' + path.basename(file, '.json');
tstrings.categories[id] = { name: category.name };
@@ -181,7 +181,7 @@ function generateCategories(tstrings, faIcons, tnpIcons) {
function generateFields(tstrings, faIcons, tnpIcons, searchableFieldIDs) {
let fields = {};
glob.sync(__dirname + '/data/presets/fields/**/*.json').forEach(file => {
glob.sync('data/presets/fields/**/*.json').forEach(file => {
let field = read(file);
let id = stripLeadingUnderscores(file.match(/presets\/fields\/([^.]*)\.json/)[1]);
@@ -273,7 +273,7 @@ function suggestionsToPresets(presets) {
// still no match?
if (!preset) {
console.log('Warning: No preset "' + presetID + '" for name-suggestion "' + name + '"');
console.log(`Warning: No preset "${presetID}" for name-suggestion "${name}"`);
return;
}
@@ -329,7 +329,7 @@ function suggestionsToPresets(presets) {
function stripLeadingUnderscores(str) {
return str.split('/')
.map(function(s) { return s.replace(/^_/,''); })
.map(s => s.replace(/^_/,''))
.join('/');
}
@@ -337,7 +337,7 @@ function stripLeadingUnderscores(str) {
function generatePresets(tstrings, faIcons, tnpIcons, searchableFieldIDs) {
let presets = {};
glob.sync(__dirname + '/data/presets/presets/**/*.json').forEach(file => {
glob.sync('data/presets/presets/**/*.json').forEach(file => {
let preset = read(file);
let id = stripLeadingUnderscores(file.match(/presets\/presets\/([^.]*)\.json/)[1]);
@@ -379,23 +379,23 @@ function generateTranslations(fields, presets, tstrings, searchableFieldIDs) {
let optkeys = Object.keys(options);
if (f.keys) {
field['label#'] = f.keys.map(k => { return k + '=*'; }).join(', ');
field['label#'] = f.keys.map(k => `${k}=*`).join(', ');
optkeys.forEach(k => {
if (id === 'access') {
options[k]['title#'] = options[k]['description#'] = 'access=' + k;
options[k]['title#'] = options[k]['description#'] = `access=${k}`;
} else {
options[k + '#'] = k + '=yes';
options[k + '#'] = `${k}=yes`;
}
});
} else if (f.key) {
field['label#'] = f.key + '=*';
field['label#'] = `${f.key}=*`;
optkeys.forEach(k => {
options[k + '#'] = f.key + '=' + k;
options[k + '#'] = `${f.key}=${k}`;
});
}
if (f.placeholder) {
field['placeholder#'] = id + ' field placeholder';
field['placeholder#'] = `${id} field placeholder`;
}
if (searchableFieldIDs[id]) {
@@ -417,14 +417,14 @@ function generateTranslations(fields, presets, tstrings, searchableFieldIDs) {
let keys = Object.keys(tags);
if (keys.length) {
preset['name#'] = keys.map(k => { return k + '=' + tags[k]; }).join(', ');
preset['name#'] = keys.map(k => `${k}=${tags[k]}`).join(', ');
}
if (p.searchable !== false) {
if (p.terms && p.terms.length) {
preset['terms#'] = 'terms: ' + p.terms.join();
}
preset.terms = '<translate with synonyms or related terms for \'' + preset.name + '\', separated by commas>';
preset.terms = `<translate with synonyms or related terms for '${preset.name}', separated by commas>`;
} else {
delete tstrings.presets[id].terms;
delete p.terms;
@@ -466,7 +466,7 @@ function generateTaginfo(presets, fields) {
}
if (preset.name) {
let legacy = (preset.searchable === false) ? ' (unsearchable)' : '';
tag.description = [ '🄿 ' + preset.name + legacy ];
tag.description = [ `🄿 ${preset.name}${legacy}` ];
}
if (preset.geometry) {
setObjectType(tag, preset);
@@ -505,14 +505,14 @@ function generateTaginfo(presets, fields) {
if (value === 'undefined' || value === '*' || value === '') return;
let tag = { key: key, value: value };
if (field.label) {
tag.description = [ '🄵 ' + field.label ];
tag.description = [ `🄵 ${field.label}` ];
}
coalesceTags(taginfo, tag);
});
} else {
let tag = { key: key };
if (field.label) {
tag.description = [ '🄵 ' + field.label ];
tag.description = [ `🄵 ${field.label}` ];
}
coalesceTags(taginfo, tag);
}
@@ -532,7 +532,7 @@ function generateTaginfo(presets, fields) {
for (let replaceKey in elem.replace) {
let replaceValue = elem.replace[replaceKey];
if (replaceValue === '$1') replaceValue = '*';
replacementStrings.push(replaceKey + '=' + replaceValue);
replacementStrings.push(`${replaceKey}=${replaceValue}`);
}
let description = '🄳';
if (replacementStrings.length > 0) {
@@ -553,9 +553,8 @@ function generateTaginfo(presets, fields) {
function coalesceTags(taginfo, tag) {
if (!tag.key) return;
let currentTaginfoEntries = taginfo.tags.filter(t => {
return (t.key === tag.key && t.value === tag.value);
});
let currentTaginfoEntries = taginfo.tags
.filter(t => (t.key === tag.key && t.value === tag.value));
if (currentTaginfoEntries.length === 0) {
taginfo.tags.push(tag);
@@ -600,7 +599,7 @@ function generateTaginfo(presets, fields) {
function generateTerritoryLanguages() {
let allRawInfo = read('./node_modules/cldr-core/supplemental/territoryInfo.json').supplemental.territoryInfo;
let allRawInfo = require('cldr-core/supplemental/territoryInfo.json').supplemental.territoryInfo;
let territoryLanguages = {};
Object.keys(allRawInfo).forEach(territoryCode => {
@@ -608,14 +607,14 @@ function generateTerritoryLanguages() {
if (!territoryLangInfo) return;
let langCodes = Object.keys(territoryLangInfo);
territoryLanguages[territoryCode.toLowerCase()] = langCodes.sort(function(langCode1, langCode2) {
territoryLanguages[territoryCode.toLowerCase()] = langCodes.sort((langCode1, langCode2) => {
let popPercent1 = parseFloat(territoryLangInfo[langCode1]._populationPercent);
let popPercent2 = parseFloat(territoryLangInfo[langCode2]._populationPercent);
if (popPercent1 === popPercent2) {
return langCode1.localeCompare(langCode2, 'en', { sensitivity: 'base' });
}
return popPercent2 - popPercent1;
}).map(langCode => { return langCode.replace('_', '-'); });
}).map(langCode => langCode.replace('_', '-'));
});
return territoryLanguages;
@@ -692,7 +691,7 @@ function validatePresetFields(presets, fields) {
if (fieldCount > maxFieldsBeforeWarning) {
// Fields with `prerequisiteTag` probably won't show up initially,
// so don't count them against the limits.
let fieldsWithoutPrerequisites = preset.fields.filter(fieldID => {
const fieldsWithoutPrerequisites = preset.fields.filter(fieldID => {
if (fields[fieldID] && fields[fieldID].prerequisiteTag) return false;
return true;
});
@@ -715,7 +714,7 @@ function validateDefaults(defaults, categories, presets) {
let members = defaults.defaults[name];
members.forEach(id => {
if (!presets[id] && !categories[id]) {
console.error('Unknown category or preset: ' + id + ' in default ' + name);
console.error(`Unknown category or preset: ${id} in default ${name}`);
console.log('');
process.exit(1);
}
@@ -763,9 +762,9 @@ function writeFaIcons(faIcons) {
const name = key.substring(4);
const def = fontawesome.findIconDefinition({ prefix: prefix, iconName: name });
try {
writeFileProm('svg/fontawesome/' + key + '.svg', fontawesome.icon(def).html);
writeFileProm(`svg/fontawesome/${key}.svg`, fontawesome.icon(def).html);
} catch (error) {
console.error('Error: No FontAwesome icon for ' + key);
console.error(`Error: No FontAwesome icon for ${key}`);
throw (error);
}
}
@@ -782,13 +781,13 @@ function writeTnpIcons(tnpIcons) {
* }
*/
let nounAuth;
if (fs.existsSync('./the_noun_project.auth')) {
nounAuth = JSON.parse(fs.readFileSync('./the_noun_project.auth', 'utf8'));
if (fs.existsSync('../the_noun_project.auth')) {
nounAuth = JSON.parse(fs.readFileSync('../the_noun_project.auth', 'utf8'));
}
const baseURL = 'http://api.thenounproject.com/icon/';
let unusedSvgFiles = fs.readdirSync('svg/the-noun-project', 'utf8')
.reduce(function(obj, name) {
.reduce((obj, name) => {
if (name.endsWith('.svg')) {
obj[name] = true;
}
@@ -797,19 +796,19 @@ function writeTnpIcons(tnpIcons) {
for (const key in tnpIcons) {
const id = key.substring(4);
const fileName = id + '.svg';
const fileName = `${id}.svg`;
if (unusedSvgFiles[fileName]) {
delete unusedSvgFiles[fileName];
}
const localPath = 'svg/the-noun-project/' + fileName;
const localPath = `svg/the-noun-project/${fileName}`;
// don't redownload existing icons
if (fs.existsSync(localPath)) continue;
if (!nounAuth) {
console.error('No authentication file for The Noun Project. Cannot download icon: ' + key);
console.error(`No authentication file for The Noun Project. Cannot download icon: ${key}`);
continue;
}
@@ -819,7 +818,7 @@ function writeTnpIcons(tnpIcons) {
// remove icons that are not needed
for (const unusedFileName in unusedSvgFiles) {
shell.rm('-f', ['svg/the-noun-project/' + unusedFileName]);
shell.rm('-f', [`svg/the-noun-project/${unusedFileName}`]);
}
}
@@ -839,13 +838,13 @@ function handleTheNounProjectResponse(err, resp, body) {
console.error('The Noun Project has not provided a URL to download the icon "' + icon.term + '" (tnp-' + icon.id + ').');
return;
}
request.get(iconURL, function(err2, resp2, svg) {
request.get(iconURL, (err2, resp2, svg) => {
if (err2) {
console.error(err2);
return;
}
try {
writeFileProm('svg/the-noun-project/' + icon.id + '.svg', svg);
writeFileProm(`svg/the-noun-project/${icon.id}.svg`, svg);
} catch (error) {
console.error(error);
throw (error);

4
server.js → scripts/server.js
View File

@@ -10,7 +10,7 @@ gaze(['css/**/*.css'], (err, watcher) => {
watcher.on('all', () => buildCSS());
});
const server = new StaticServer({ rootPath: __dirname, port: 8080, followSymlink: true });
const server = new StaticServer({ rootPath: process.cwd(), port: 8080, followSymlink: true });
server.start(() => {
console.log(colors.yellow('Listening on ' + server.port));
console.log(colors.yellow(`Listening on ${server.port}`));
});

1
data/update_imagery.js → scripts/update_imagery.js
View File

@@ -1,3 +1,4 @@
/* eslint-disable no-console */
const fs = require('fs');
const sources = require('editor-layer-index/imagery.json');
const prettyStringify = require('json-stringify-pretty-compact');

372
scripts/update_locales.js Normal file
View File

@@ -0,0 +1,372 @@
/* eslint-disable no-console */
/* Downloads the latest translations from Transifex */
const fs = require('fs');
const prettyStringify = require('json-stringify-pretty-compact');
const request = require('request').defaults({ maxSockets: 1 });
const YAML = require('js-yaml');
const colors = require('colors/safe');
const resources = ['core', 'presets', 'imagery', 'community'];
const outdir = 'dist/locales/';
const apiroot = 'https://www.transifex.com/api/2';
const projectURL = `${apiroot}/project/id-editor/`;
/*
* Transifex oddly doesn't allow anonymous downloading
*
* auth is stored in transifex.auth in a json object:
* {
* "user": "username",
* "pass": "password"
* }
* */
const auth = JSON.parse(fs.readFileSync('../transifex.auth', 'utf8'));
// const sourceCore = YAML.load(fs.readFileSync('../data/core.yaml', 'utf8'));
// const sourcePresets = YAML.load(fs.readFileSync('../data/presets.yaml', 'utf8'));
// const sourceImagery = YAML.load(fs.readFileSync('../node_modules/editor-layer-index/i18n/en.yaml', 'utf8'));
// const sourceCommunity = YAML.load(fs.readFileSync('../node_modules/osm-community-index/i18n/en.yaml', 'utf8'));
const dataShortcuts = JSON.parse(fs.readFileSync('../data/shortcuts.json', 'utf8')).dataShortcuts;
const cldrMainDir = '../node_modules/cldr-localenames-full/main/';
let referencedScripts = [];
const languageInfo = { dataLanguages: getLangNamesInNativeLang() };
fs.writeFileSync('data/languages.json', JSON.stringify(languageInfo, null, 4));
let shortcuts = [];
dataShortcuts.forEach(tab => {
tab.columns.forEach(col => {
col.rows.forEach(row => {
if (!row.shortcuts) return;
row.shortcuts.forEach(shortcut => {
if (shortcut.includes('.')) {
let info = { shortcut: shortcut };
if (row.modifiers) {
info.modifier = row.modifiers.join('');
}
shortcuts.push(info);
}
});
});
});
});
asyncMap(resources, getResource, (err, results) => {
if (err) return console.log(err);
// merge in strings fetched from transifex
let allStrings = {};
results.forEach(resourceStrings => {
Object.keys(resourceStrings).forEach(code => {
if (!allStrings[code]) { allStrings[code] = {}; }
let source = resourceStrings[code];
let target = allStrings[code];
Object.keys(source).forEach(k => target[k] = source[k]);
});
});
// write files and fetch language info for each locale
let dataLocales = {
en: { rtl: false, languageNames: languageNamesInLanguageOf('en'), scriptNames: scriptNamesInLanguageOf('en') }
};
asyncMap(Object.keys(allStrings),
(code, done) => {
if (code === 'en' || !Object.keys(allStrings[code]).length) {
done();
} else {
let obj = {};
obj[code] = allStrings[code];
fs.writeFileSync(outdir + code + '.json', JSON.stringify(obj, null, 4));
getLanguageInfo(code, (err, info) => {
let rtl = info && info.rtl;
// exceptions: see #4783
if (code === 'ckb') {
rtl = true;
} else if (code === 'ku') {
rtl = false;
}
dataLocales[code] = {
rtl: rtl,
languageNames: languageNamesInLanguageOf(code) || {},
scriptNames: scriptNamesInLanguageOf(code) || {}
};
done();
});
}
},
(err) => {
if (!err) {
const keys = Object.keys(dataLocales).sort();
let sorted = {};
keys.forEach(k => sorted[k] = dataLocales[k]);
fs.writeFileSync('data/locales.json', prettyStringify({ dataLocales: sorted }, { maxLength: 99999 }));
}
}
);
});
function getResource(resource, callback) {
let resourceURL = `${projectURL}resource/${resource}/`;
getLanguages(resourceURL, (err, codes) => {
if (err) return callback(err);
asyncMap(codes, getLanguage(resourceURL), (err, results) => {
if (err) return callback(err);
let locale = {};
results.forEach((result, i) => {
if (resource === 'community' && Object.keys(result).length) {
locale[codes[i]] = { community: result }; // add namespace
} else {
if (resource === 'presets') {
// remove terms that were not really translated
let presets = (result.presets && result.presets.presets) || {};
for (const key of Object.keys(presets)) {
let preset = presets[key];
if (!preset.terms) continue;
preset.terms = preset.terms.replace(/<.*>/, '').trim();
if (!preset.terms) {
delete preset.terms;
if (!Object.keys(preset).length) {
delete presets[key];
}
}
}
} else if (resource === 'fields') {
// remove terms that were not really translated
let fields = (result.presets && result.presets.fields) || {};
for (const key of Object.keys(fields)) {
let field = fields[key];
if (!field.terms) continue;
field.terms = field.terms.replace(/\[.*\]/, '').trim();
if (!field.terms) {
delete field.terms;
if (!Object.keys(field).length) {
delete fields[key];
}
}
}
} else if (resource === 'core') {
checkForDuplicateShortcuts(codes[i], result);
}
locale[codes[i]] = result;
}
});
callback(null, locale);
});
});
}
function getLanguage(resourceURL) {
return (code, callback) => {
code = code.replace(/-/g, '_');
let url = `${resourceURL}/translation/${code}`;
if (code === 'vi') { url += '?mode=reviewed'; }
request.get(url, { auth : auth }, (err, resp, body) => {
if (err) return callback(err);
console.log(`${resp.statusCode}: ${url}`);
let content = JSON.parse(body).content;
callback(null, YAML.safeLoad(content)[code]);
});
};
}
function getLanguageInfo(code, callback) {
code = code.replace(/-/g, '_');
let url = `${apiroot}/language/${code}`;
request.get(url, { auth : auth }, (err, resp, body) => {
if (err) return callback(err);
console.log(`${resp.statusCode}: ${url}`);
callback(null, JSON.parse(body));
});
}
function getLanguages(resource, callback) {
let url = `${resource}?details`;
request.get(url, { auth: auth }, (err, resp, body) => {
if (err) return callback(err);
console.log(`${resp.statusCode}: ${url}`);
callback(null, JSON.parse(body).available_languages
.map(d => d.code.replace(/_/g, '-'))
.filter(d => d !== 'en')
);
});
}
function asyncMap(inputs, func, callback) {
let index = 0;
let remaining = inputs.length;
let results = [];
let error;
next();
function next() {
callFunc(index++);
if (index < inputs.length) {
setTimeout(next, 200);
}
}
function callFunc(i) {
let d = inputs[i];
func(d, (err, data) => {
if (err) error = err;
results[i] = data;
remaining--;
if (!remaining) callback(error, results);
});
}
}
function checkForDuplicateShortcuts(code, coreStrings) {
let usedShortcuts = {};
shortcuts.forEach(shortcutInfo => {
let shortcutPathString = shortcutInfo.shortcut;
let modifier = shortcutInfo.modifier || '';
let path = shortcutPathString
.split('.')
.map(s => s.replace(/<TX_DOT>/g, '.'))
.reverse();
let rep = coreStrings;
while (rep !== undefined && path.length) {
rep = rep[path.pop()];
}
if (rep !== undefined) {
let shortcut = modifier + rep;
if (usedShortcuts[shortcut] && usedShortcuts[shortcut] !== shortcutPathString) {
let message = code + ': duplicate shortcut "' + shortcut + '" for "' + usedShortcuts[shortcut] + '" and "' + shortcutPathString + '"';
console.warn(colors.yellow(message));
} else {
usedShortcuts[shortcut] = shortcutPathString;
}
}
});
}
function getLangNamesInNativeLang() {
// manually add languages we want that aren't in CLDR
let unordered = {
'oc': {
nativeName: 'Occitan'
},
'ja-Hira': {
base: 'ja',
script: 'Hira'
},
'ja-Latn': {
base: 'ja',
script: 'Latn'
},
'ko-Latn': {
base: 'ko',
script: 'Latn'
},
'zh_pinyin': {
base: 'zh',
script: 'Latn'
}
};
let langDirectoryPaths = fs.readdirSync(cldrMainDir);
langDirectoryPaths.forEach(code => {
let languagesPath = `${cldrMainDir}${code}/languages.json`;
//if (!fs.existsSync(languagesPath)) return;
let languageObj = JSON.parse(fs.readFileSync(languagesPath, 'utf8')).main[code];
let identity = languageObj.identity;
// skip locale-specific languages
if (identity.letiant || identity.territory) return;
let info = {};
const script = identity.script;
if (script) {
referencedScripts.push(script);
info.base = identity.language;
info.script = script;
}
const nativeName = languageObj.localeDisplayNames.languages[code];
if (nativeName) {
info.nativeName = nativeName;
}
unordered[code] = info;
});
let ordered = {};
Object.keys(unordered).sort().forEach(key => ordered[key] = unordered[key]);
return ordered;
}
const rematchCodes = { 'ar-AA': 'ar', 'zh-CN': 'zh', 'zh-HK': 'zh-Hant-HK', 'zh-TW': 'zh-Hant', 'pt-BR': 'pt', 'pt': 'pt-PT' };
function languageNamesInLanguageOf(code) {
if (rematchCodes[code]) code = rematchCodes[code];
let languageFilePath = `${cldrMainDir}${code}/languages.json`;
if (!fs.existsSync(languageFilePath)) return null;
let translatedLangsByCode = JSON.parse(fs.readFileSync(languageFilePath, 'utf8')).main[code].localeDisplayNames.languages;
// ignore codes for non-languages
for (let nonLangCode in { mis: true, mul: true, und: true, zxx: true }) {
delete translatedLangsByCode[nonLangCode];
}
for (let langCode in translatedLangsByCode) {
let altLongIndex = langCode.indexOf('-alt-long');
if (altLongIndex !== -1) { // prefer long names (e.g. Chinese -> Mandarin Chinese)
let base = langCode.substring(0, altLongIndex);
translatedLangsByCode[base] = translatedLangsByCode[langCode];
}
if (langCode.includes('-alt-')) { // remove alternative names
delete translatedLangsByCode[langCode];
} else if (langCode === translatedLangsByCode[langCode]) { // no localized value available
delete translatedLangsByCode[langCode];
}
}
return translatedLangsByCode;
}
function scriptNamesInLanguageOf(code) {
if (rematchCodes[code]) code = rematchCodes[code];
let languageFilePath = `${cldrMainDir}${code}/scripts.json`;
if (!fs.existsSync(languageFilePath)) return null;
let allTranslatedScriptsByCode = JSON.parse(fs.readFileSync(languageFilePath, 'utf8')).main[code].localeDisplayNames.scripts;
let translatedScripts = {};
referencedScripts.forEach(script => {
if (!allTranslatedScriptsByCode[script] || script === allTranslatedScriptsByCode[script]) return;
translatedScripts[script] = allTranslatedScriptsByCode[script];
});
return translatedScripts;
}

2
test/spec/core/data.js
View File

@@ -1,8 +1,8 @@
describe('iD.coreData', function() {
var _context;
var _oldData;
before(function() {
iD.data = iD.data || {};
iD.data.test = { hello: 'world' };
});