Make several data external instead of bundled:

Also dist/data/* contains minified files now

These are now external and will fetch as needed:
-export { dataAddressFormats } from './address-formats.json';
-export { dataDeprecated } from './deprecated.json';
-export { dataDiscarded } from './discarded.json';
-export { dataLanguages } from './languages.json';
-export { dataPhoneFormats } from './phone-formats.json';
-export { dataShortcuts } from './shortcuts.json';
-export { dataTerritoryLanguages } from './territory-languages.json';

scripts/build_data.js

@@ -11,7 +11,7 @@ const YAML = require('js-yaml');
 
 const fieldSchema = require('../data/presets/schema/field.json');
 const presetSchema = require('../data/presets/schema/preset.json');
-const deprecated = require('../data/deprecated.json').dataDeprecated;
+const deprecated = require('../data/deprecated.json');
 
 // fontawesome icons
 const fontawesome = require('@fortawesome/fontawesome-svg-core');
@@ -85,7 +85,7 @@ function buildData() {
     'data/presets/presets.json',
     'data/presets.yaml',
     'data/taginfo.json',
-    'data/territory-languages.json',
+    'data/territory_languages.json',
     'dist/locales/en.json',
     'svg/fontawesome/*.svg',
   ]);
@@ -110,10 +110,26 @@ function buildData() {
     writeFileProm('data/presets/presets.json', prettyStringify({ presets: presets }, { maxLength: 9999 }) ),
     writeFileProm('data/presets.yaml', translationsToYAML(translations) ),
     writeFileProm('data/taginfo.json', prettyStringify(taginfo, { maxLength: 9999 }) ),
-    writeFileProm('data/territory-languages.json', prettyStringify({ dataTerritoryLanguages: territoryLanguages }, { maxLength: 9999 }) ),
+    writeFileProm('data/territory_languages.json', prettyStringify(territoryLanguages, { maxLength: 9999 }) ),
     writeEnJson(tstrings),
     writeFaIcons(faIcons),
-    writeTnpIcons(tnpIcons)
+    writeTnpIcons(tnpIcons),
+    minifyJSON('data/presets/categories.json', 'dist/data/categories.min.json'),
+    minifyJSON('data/presets/fields.json', 'dist/data/fields.min.json'),
+    minifyJSON('data/presets/presets.json', 'dist/data/presets.min.json'),
+    minifyJSON('data/address_formats.json', 'dist/data/address_formats.min.json'),
+    minifyJSON('data/deprecated.json', 'dist/data/deprecated.min.json'),
+    minifyJSON('data/discarded.json', 'dist/data/discarded.min.json'),
+    minifyJSON('data/imagery.json', 'dist/data/imagery.min.json'),
+    minifyJSON('data/intro_graph.json', 'dist/data/intro_graph.min.json'),
+    minifyJSON('data/keepRight.json', 'dist/data/keepRight.min.json'),
+    minifyJSON('data/languages.json', 'dist/data/languages.min.json'),
+    minifyJSON('data/locales.json', 'dist/data/locales.min.json'),
+    minifyJSON('data/phone_formats.json', 'dist/data/phone_formats.min.json'),
+    minifyJSON('data/qa_errors.json', 'dist/data/qa_errors.min.json'),
+    minifyJSON('data/shortcuts.json', 'dist/data/shortcuts.min.json'),
+    minifyJSON('data/taginfo.json', 'dist/data/taginfo.min.json'),
+    minifyJSON('data/territory_languages.json', 'dist/data/territory_languages.min.json')
   ];
 
   return _currBuild =
@@ -873,4 +889,20 @@ function readFileProm(path, options) {
 }
 
 
+function minifyJSON(inPath, outPath) {
+  return new Promise((resolve, reject) => {
+    fs.readFile(inPath, 'utf8', (err, data) => {
+      if (err) return reject(err);
+
+      const minified = JSON.stringify(JSON.parse(data));
+      fs.writeFile(outPath, minified, (err) => {
+        if (err) return reject(err);
+        resolve();
+      });
+
+    });
+  });
+}
+
+
 module.exports = buildData;

scripts/update_locales.js

@@ -9,7 +9,7 @@ const colors = require('colors/safe');
 const resources = ['core', 'presets', 'imagery', 'community'];
 const outdir = 'dist/locales/';
 const apiroot = 'https://www.transifex.com/api/2';
-const projectURL = `${apiroot}/project/id-editor/`;
+const projectURL = `${apiroot}/project/id-editor`;
 
 
 /*
@@ -22,21 +22,14 @@ const projectURL = `${apiroot}/project/id-editor/`;
  *  }
  *  */
 
-const auth = JSON.parse(fs.readFileSync('../transifex.auth', 'utf8'));
-
-// const sourceCore = YAML.load(fs.readFileSync('../data/core.yaml', 'utf8'));
-// const sourcePresets = YAML.load(fs.readFileSync('../data/presets.yaml', 'utf8'));
-// const sourceImagery = YAML.load(fs.readFileSync('../node_modules/editor-layer-index/i18n/en.yaml', 'utf8'));
-// const sourceCommunity = YAML.load(fs.readFileSync('../node_modules/osm-community-index/i18n/en.yaml', 'utf8'));
-
-const dataShortcuts = JSON.parse(fs.readFileSync('../data/shortcuts.json', 'utf8')).dataShortcuts;
-
-const cldrMainDir = '../node_modules/cldr-localenames-full/main/';
+const auth = JSON.parse(fs.readFileSync('./transifex.auth', 'utf8'));
+const dataShortcuts = JSON.parse(fs.readFileSync('data/shortcuts.json', 'utf8'));
+const cldrMainDir = 'node_modules/cldr-localenames-full/main/';
 
 let referencedScripts = [];
 
-const languageInfo = { dataLanguages: getLangNamesInNativeLang() };
-fs.writeFileSync('data/languages.json', JSON.stringify(languageInfo, null, 4));
+const languageInfo = getLangNamesInNativeLang();
+fs.writeFileSync('data/languages.json', prettyStringify(languageInfo, { maxLength: 200 }));
 
 let shortcuts = [];
 dataShortcuts.forEach(tab => {
@@ -103,9 +96,9 @@ asyncMap(resources, getResource, (err, results) => {
     (err) => {
       if (!err) {
         const keys = Object.keys(dataLocales).sort();
-        let sorted = {};
-        keys.forEach(k => sorted[k] = dataLocales[k]);
-        fs.writeFileSync('data/locales.json', prettyStringify({ dataLocales: sorted }, { maxLength: 99999 }));
+        let sortedLocales = {};
+        keys.forEach(k => sortedLocales[k] = dataLocales[k]);
+        fs.writeFileSync('data/locales.json', prettyStringify({ dataLocales: sortedLocales }, { maxLength: 99999 }));
       }
     }
   );
@@ -113,7 +106,7 @@ asyncMap(resources, getResource, (err, results) => {
 
 
 function getResource(resource, callback) {
-  let resourceURL = `${projectURL}resource/${resource}/`;
+  let resourceURL = `${projectURL}/resource/${resource}`;
   getLanguages(resourceURL, (err, codes) => {
     if (err) return callback(err);
 
@@ -195,8 +188,8 @@ function getLanguageInfo(code, callback) {
 }
 
 
-function getLanguages(resource, callback) {
-  let url = `${resource}?details`;
+function getLanguages(resourceURL, callback) {
+  let url = `${resourceURL}?details`;
   request.get(url, { auth: auth }, (err, resp, body) => {
     if (err) return callback(err);
     console.log(`${resp.statusCode}: ${url}`);