/* eslint-disable no-console */ const requireESM = require('esm')(module); const _cloneDeep = requireESM('lodash-es/cloneDeep').default; const _forEach = requireESM('lodash-es/forEach').default; const _isEmpty = requireESM('lodash-es/isEmpty').default; const _merge = requireESM('lodash-es/merge').default; const _toPairs = requireESM('lodash-es/toPairs').default; const _filter = requireESM('lodash-es/filter').default; const colors = require('colors/safe'); const fs = require('fs'); const glob = require('glob'); const jsonschema = require('jsonschema'); const path = require('path'); const prettyStringify = require('json-stringify-pretty-compact'); const shell = require('shelljs'); const YAML = require('js-yaml'); const fieldSchema = require('./data/presets/schema/field.json'); const presetSchema = require('./data/presets/schema/preset.json'); const suggestions = require('name-suggestion-index').names; // fontawesome icons const fontawesome = require('@fortawesome/fontawesome-svg-core'); const fas = require('@fortawesome/free-solid-svg-icons').fas; const far = require('@fortawesome/free-regular-svg-icons').far; const fab = require('@fortawesome/free-brands-svg-icons').fab; fontawesome.library.add(fas, far, fab); module.exports = function buildData() { var building; return function() { // Note: even though this function is sync adding // the `building` variable for consistency and future proofing if (building) return; building = true; console.log('building data'); console.time(colors.green('data built')); // Create symlinks if necessary.. { 'target': 'source' } const symlinks = { 'land.html': 'dist/land.html', img: 'dist/img' }; for (var target of Object.keys(symlinks)) { if (!shell.test('-L', target)) { console.log(`Creating symlink: ${target} -> ${symlinks[target]}`); shell.ln('-sf', symlinks[target], target); } } // Translation strings var tstrings = { categories: {}, fields: {}, presets: {} }; // Font Awesome icons used var faIcons = { 'fas-long-arrow-alt-right': {} }; // Start clean shell.rm('-f', [ 'data/presets/categories.json', 'data/presets/fields.json', 'data/presets/presets.json', 'data/presets.yaml', 'data/taginfo.json', 'dist/locales/en.json', 'svg/fontawesome/*.svg', ]); var categories = generateCategories(tstrings, faIcons); var fields = generateFields(tstrings, faIcons); var presets = generatePresets(tstrings, faIcons); var defaults = read('data/presets/defaults.json'); var translations = generateTranslations(fields, presets, tstrings); var taginfo = generateTaginfo(presets, fields); // Additional consistency checks validateCategoryPresets(categories, presets); validatePresetFields(presets, fields); validateDefaults(defaults, categories, presets); // Save individual data files var tasks = [ writeFileProm( 'data/presets/categories.json', prettyStringify({ categories: categories }) ), writeFileProm( 'data/presets/fields.json', prettyStringify({ fields: fields }, { maxLength: 9999 }) ), writeFileProm( 'data/presets/presets.json', prettyStringify({ presets: presets }, { maxLength: 9999 }) ), writeFileProm( 'data/presets.yaml', translationsToYAML(translations) ), writeFileProm( 'data/taginfo.json', prettyStringify(taginfo, { maxLength: 9999 }) ), writeEnJson(tstrings), writeFaIcons(faIcons) ]; return Promise.all(tasks) .then(function () { console.timeEnd(colors.green('data built')); building = false; }) .catch(function (err) { console.error(err); process.exit(1); }); }; }; function read(f) { return JSON.parse(fs.readFileSync(f, 'utf8')); } function validate(file, instance, schema) { var validationErrors = jsonschema.validate(instance, schema).errors; if (validationErrors.length) { console.error(file + ': '); validationErrors.forEach(function(error) { if (error.property) { console.error(error.property + ' ' + error.message); } else { console.error(error); } }); process.exit(1); } } function generateCategories(tstrings, faIcons) { var categories = {}; glob.sync(__dirname + '/data/presets/categories/*.json').forEach(function(file) { var category = read(file); var id = 'category-' + path.basename(file, '.json'); tstrings.categories[id] = { name: category.name }; categories[id] = category; // fontawesome icon, remember for later if (/^fa[srb]-/.test(category.icon)) { faIcons[category.icon] = {}; } }); return categories; } function generateFields(tstrings, faIcons) { var fields = {}; glob.sync(__dirname + '/data/presets/fields/**/*.json').forEach(function(file) { var field = read(file); var id = stripLeadingUnderscores(file.match(/presets\/fields\/([^.]*)\.json/)[1]); validate(file, field, fieldSchema); var t = tstrings.fields[id] = { label: field.label }; if (field.placeholder) { t.placeholder = field.placeholder; } if (field.strings) { for (var i in field.strings) { t[i] = field.strings[i]; } } fields[id] = field; // fontawesome icon, remember for later if (/^fa[srb]-/.test(field.icon)) { faIcons[field.icon] = {}; } }); return fields; } function suggestionsToPresets(presets) { for (var key in suggestions) { for (var value in suggestions[key]) { for (var name in suggestions[key][value]) { addSuggestion(key, value, name); } } } function addSuggestion(key, value, name) { var suggestion = suggestions[key][value][name]; var presetID, preset; // sometimes we can find a more specific preset then key/value.. if (suggestion.tags.cuisine) { presetID = key + '/' + value + '/' + suggestion.tags.cuisine; preset = presets[presetID]; } else if (suggestion.tags.vending) { if (suggestion.tags.vending === 'parcel_pickup;parcel_mail_in') { presetID = key + '/' + value + '/parcel_pickup_dropoff'; } else { presetID = key + '/' + value + '/' + suggestion.tags.vending; } preset = presets[presetID]; } // fallback to key/value if (!preset) { presetID = key + '/' + value; preset = presets[presetID]; } // still no match? if (!preset) { console.log('Warning: No preset "' + presetID + '" for name-suggestion "' + name + '"'); return; } var wikidataTag = { 'brand:wikidata': suggestion.tags['brand:wikidata'] }; var suggestionID = presetID + '/' + name; presets[suggestionID] = { name: name, icon: preset.icon, geometry: preset.geometry, tags: _merge({}, preset.tags, wikidataTag), addTags: suggestion.tags, removeTags: suggestion.tags, reference: preset.reference, matchScore: 2, suggestion: true }; } return presets; } function stripLeadingUnderscores(str) { return str.split('/') .map(function(s) { return s.replace(/^_/,''); }) .join('/'); } function generatePresets(tstrings, faIcons) { var presets = {}; glob.sync(__dirname + '/data/presets/presets/**/*.json').forEach(function(file) { var preset = read(file); var id = stripLeadingUnderscores(file.match(/presets\/presets\/([^.]*)\.json/)[1]); validate(file, preset, presetSchema); tstrings.presets[id] = { name: preset.name, terms: (preset.terms || []).join(',') }; presets[id] = preset; // fontawesome icon, remember for later if (/^fa[srb]-/.test(preset.icon)) { faIcons[preset.icon] = {}; } }); presets = _merge(presets, suggestionsToPresets(presets)); return presets; } function generateTranslations(fields, presets, tstrings) { var translations = _cloneDeep(tstrings); _forEach(translations.fields, function(field, id) { var f = fields[id]; if (f.keys) { field['label#'] = _forEach(f.keys).map(function(key) { return key + '=*'; }).join(', '); if (!_isEmpty(field.options)) { _forEach(field.options, function(v,k) { if (id === 'access') { field.options[k]['title#'] = field.options[k]['description#'] = 'access=' + k; } else { field.options[k + '#'] = k + '=yes'; } }); } } else if (f.key) { field['label#'] = f.key + '=*'; if (!_isEmpty(field.options)) { _forEach(field.options, function(v,k) { field.options[k + '#'] = f.key + '=' + k; }); } } if (f.placeholder) { field['placeholder#'] = id + ' field placeholder'; } }); _forEach(translations.presets, function(preset, id) { var p = presets[id]; if (!_isEmpty(p.tags)) preset['name#'] = _toPairs(p.tags).map(function(pair) { return pair[0] + '=' + pair[1]; }).join(', '); if (p.searchable !== false) { if (p.terms && p.terms.length) preset['terms#'] = 'terms: ' + p.terms.join(); preset.terms = ''; } else { delete preset.terms; } }); return translations; } function generateTaginfo(presets, fields) { var taginfo = { 'data_format': 1, 'data_url': 'https://raw.githubusercontent.com/openstreetmap/iD/master/data/taginfo.json', 'project': { 'name': 'iD Editor', 'description': 'Online editor for OSM data.', 'project_url': 'https://github.com/openstreetmap/iD', 'doc_url': 'https://github.com/openstreetmap/iD/blob/master/data/presets/README.md', 'icon_url': 'https://raw.githubusercontent.com/openstreetmap/iD/master/dist/img/logo.png', 'keywords': [ 'editor' ] }, 'tags': [] }; _forEach(presets, function(preset) { if (preset.suggestion) return; var keys = Object.keys(preset.tags); var last = keys[keys.length - 1]; var tag = { key: last }; if (!last) return; if (preset.tags[last] !== '*') { tag.value = preset.tags[last]; } if (preset.name) { var legacy = (preset.searchable === false) ? ' (unsearchable)' : ''; tag.description = [ '🄿 ' + preset.name + legacy ]; } if (preset.geometry) { setObjectType(tag, preset); } // add icon if (/^maki-/.test(preset.icon)) { tag.icon_url = 'https://raw.githubusercontent.com/mapbox/maki/master/icons/' + preset.icon.replace(/^maki-/, '') + '-15.svg?sanitize=true'; } else if (/^temaki-/.test(preset.icon)) { tag.icon_url = 'https://raw.githubusercontent.com/bhousel/temaki/master/icons/' + preset.icon.replace(/^temaki-/, '') + '.svg?sanitize=true'; } else if (/^fa[srb]-/.test(preset.icon)) { tag.icon_url = 'https://raw.githubusercontent.com/openstreetmap/iD/master/svg/fontawesome/' + preset.icon + '.svg?sanitize=true'; } else if (/^iD-/.test(preset.icon)) { tag.icon_url = 'https://raw.githubusercontent.com/openstreetmap/iD/master/svg/iD-sprite/presets/' + preset.icon.replace(/^iD-/, '') + '.svg?sanitize=true'; } coalesceTags(taginfo, tag); }); _forEach(fields, function(field) { var keys = field.keys || [ field.key ] || []; var isRadio = (field.type === 'radio' || field.type === 'structureRadio'); keys.forEach(function(key) { if (field.strings && field.strings.options && !isRadio) { var values = Object.keys(field.strings.options); values.forEach(function(value) { if (value === 'undefined' || value === '*' || value === '') return; var tag = { key: key, value: value }; if (field.label) { tag.description = [ '🄵 ' + field.label ]; } coalesceTags(taginfo, tag); }); } else { var tag = { key: key }; if (field.label) { tag.description = [ '🄵 ' + field.label ]; } coalesceTags(taginfo, tag); } }); }); _forEach(taginfo.tags, function(elem) { if (elem.description) elem.description = elem.description.join(', '); }); function coalesceTags(taginfo, tag) { if (!tag.key) return; var currentTaginfoEntries = taginfo.tags.filter(function(t) { return (t.key === tag.key && t.value === tag.value); }); if (currentTaginfoEntries.length === 0) { taginfo.tags.push(tag); return; } if (!tag.description) return; if (!currentTaginfoEntries[0].description) { currentTaginfoEntries[0].description = tag.description; return; } var isNewDescription = currentTaginfoEntries[0].description .indexOf(tag.description[0]) === -1; if (isNewDescription) { currentTaginfoEntries[0].description.push(tag.description[0]); } } function setObjectType(tag, input) { tag.object_types = []; const mapping = { 'point' : 'node', 'vertex' : 'node', 'line' : 'way', 'relation' : 'relation', 'area' : 'area' }; input.geometry.forEach(function(geom) { if (tag.object_types.indexOf(mapping[geom]) === -1) { tag.object_types.push(mapping[geom]); } }); } return taginfo; } function validateCategoryPresets(categories, presets) { _forEach(categories, function(category) { if (category.members) { category.members.forEach(function(preset) { if (presets[preset] === undefined) { console.error('Unknown preset: ' + preset + ' in category ' + category.name); process.exit(1); } }); } }); } function validatePresetFields(presets, fields) { var betweenBracketsRegex = /([^{]*?)(?=\})/; var maxFieldsBeforeError = 12; var maxFieldsBeforeWarning = 8; for (var presetID in presets) { var preset = presets[presetID]; // the keys for properties that contain arrays of field ids var fieldKeys = ['fields', 'moreFields']; for (var fieldsKey in fieldKeys) { if (preset[fieldsKey]) { for (var field in preset[fieldsKey]) { if (fields[field] === undefined) { var regexResult = betweenBracketsRegex.exec(field); if (regexResult) { var foreignPresetID = regexResult[0]; if (presets[foreignPresetID] === undefined) { console.error('Unknown preset "' + foreignPresetID + '" referenced in "' + fieldsKey + '" array of preset ' + preset.name); process.exit(1); } } else { console.error('Unknown preset field "' + field + '" in "' + fieldsKey + '" array of preset ' + preset.name); process.exit(1); } } } } } if (preset.fields) { // since `moreFields` is available, check that `fields` doesn't get too cluttered var fieldCount = preset.fields.length; if (fieldCount > maxFieldsBeforeWarning) { // Fields with `prerequisiteTag` probably won't show up initially, // so don't count them against the limits. var fieldsWithoutPrerequisites = _filter(preset.fields, function(fieldID) { if (fields[fieldID] && fields[fieldID].prerequisiteTag) { return false; } return true; }); fieldCount = fieldsWithoutPrerequisites.length; } if (fieldCount > maxFieldsBeforeError) { console.error(fieldCount + ' values in "fields" of "' + preset.name + '" (' + presetID + '). Limit: ' + maxFieldsBeforeError + '. Please move lower-priority fields to "moreFields".'); process.exit(1); } else if (fieldCount > maxFieldsBeforeWarning) { console.log('Warning: ' + fieldCount + ' values in "fields" of "' + preset.name + '" (' + presetID + '). Recommended: ' + maxFieldsBeforeWarning + ' or fewer. Consider moving lower-priority fields to "moreFields".'); } } } } function validateDefaults (defaults, categories, presets) { _forEach(defaults.defaults, function (members, name) { members.forEach(function (id) { if (!presets[id] && !categories[id]) { console.error('Unknown category or preset: ' + id + ' in default ' + name); process.exit(1); } }); }); } function translationsToYAML(translations) { // comment keys end with '#' and should sort immediately before their related key. function commentFirst(a, b) { return (a === b + '#') ? -1 : (b === a + '#') ? 1 : (a > b ? 1 : a < b ? -1 : 0); } return YAML.safeDump({ en: { presets: translations }}, { sortKeys: commentFirst, lineWidth: -1 }) .replace(/\'.*#\':/g, '#'); } function writeEnJson(tstrings) { var readCoreYaml = readFileProm('data/core.yaml', 'utf8'); var readImagery = readFileProm('node_modules/editor-layer-index/i18n/en.yaml', 'utf8'); var readCommunity = readFileProm('node_modules/osm-community-index/i18n/en.yaml', 'utf8'); return Promise.all([readCoreYaml, readImagery, readCommunity]).then(function(data) { var core = YAML.load(data[0]); var imagery = YAML.load(data[1]); var community = YAML.load(data[2]); var en = _merge( core, { en: { presets: tstrings } }, imagery, { en: { community: community.en } } ); return writeFileProm('dist/locales/en.json', JSON.stringify(en, null, 4)); }); } function writeFaIcons(faIcons) { for (var key in faIcons) { var prefix = key.substring(0, 3); // `fas`, `far`, `fab` var name = key.substring(4); var def = fontawesome.findIconDefinition({ prefix: prefix, iconName: name }); try { writeFileProm('svg/fontawesome/' + key + '.svg', fontawesome.icon(def).html); } catch (error) { console.error('Error: No FontAwesome icon for ' + key); throw (error); } } } function writeFileProm(path, content) { return new Promise(function(res, rej) { fs.writeFile(path, content, function(err) { if (err) { return rej(err); } res(); }); }); } function readFileProm(path, options) { return new Promise(function(res, rej) { fs.readFile(path, options, function(err, data) { if (err) { return rej(err); } res(data); }); }); }