Make several data external instead of bundled:

Also dist/data/* contains minified files now

These are now external and will fetch as needed:
-export { dataAddressFormats } from './address-formats.json';
-export { dataDeprecated } from './deprecated.json';
-export { dataDiscarded } from './discarded.json';
-export { dataLanguages } from './languages.json';
-export { dataPhoneFormats } from './phone-formats.json';
-export { dataShortcuts } from './shortcuts.json';
-export { dataTerritoryLanguages } from './territory-languages.json';

modules/actions/discard_tags.js

@@ -1,30 +1,29 @@
-import { dataDiscarded } from '../../data';
 
-export function actionDiscardTags(difference) {
+export function actionDiscardTags(difference, discardTags) {
+  discardTags = discardTags || {};
 
-    return function(graph) {
-        function discardTags(entity) {
-            var tags = {};
-            var keys = Object.keys(entity.tags);
-            var discarded = false;
+  return (graph) => {
+    difference.modified().forEach(checkTags);
+    difference.created().forEach(checkTags);
+    return graph;
 
-            for (var i = 0; i < keys.length; i++) {
-                var k = keys[i];
-                if (dataDiscarded[k] || !entity.tags[k]) {
-                    discarded = true;
-                } else {
-                    tags[k] = entity.tags[k];
-                }
-            }
+    function checkTags(entity) {
+      const keys = Object.keys(entity.tags);
+      let didDiscard = false;
+      let tags = {};
 
-            if (discarded) {
-                graph = graph.replace(entity.update({ tags: tags }));
-            }
+      for (let i = 0; i < keys.length; i++) {
+        const k = keys[i];
+        if (discardTags[k] || !entity.tags[k]) {
+          didDiscard = true;
+        } else {
+          tags[k] = entity.tags[k];
         }
+      }
+      if (didDiscard) {
+        graph = graph.replace(entity.update({ tags: tags }));
+      }
+    }
 
-        difference.modified().forEach(discardTags);
-        difference.created().forEach(discardTags);
-
-        return graph;
-    };
+  };
 }

modules/actions/merge_remote_changes.js

@@ -4,11 +4,11 @@ import { diff3Merge } from 'node-diff3';
 import { t } from '../util/locale';
 import { actionDeleteMultiple } from './delete_multiple';
 import { osmEntity } from '../osm';
-import { dataDiscarded } from '../../data';
 import { utilArrayUnion, utilArrayUniq } from '../util';
 
 
-export function actionMergeRemoteChanges(id, localGraph, remoteGraph, formatUser) {
+export function actionMergeRemoteChanges(id, localGraph, remoteGraph, discardTags, formatUser) {
+    discardTags = discardTags || {};
     var _option = 'safe';  // 'safe', 'force_local', 'force_remote'
     var _conflicts = [];
 
@@ -167,7 +167,7 @@ export function actionMergeRemoteChanges(id, localGraph, remoteGraph, formatUser
         var a = target.tags || {};
         var b = remote.tags || {};
         var keys = utilArrayUnion(utilArrayUnion(Object.keys(o), Object.keys(a)), Object.keys(b))
-            .filter(function(k) { return !dataDiscarded[k]; });
+            .filter(function(k) { return !discardTags[k]; });
         var tags = Object.assign({}, a);   // shallow copy
         var changed = false;
 

modules/core/data.js

@@ -9,7 +9,16 @@ export function coreData(context) {
   let _module = {};
   let _inflight = {};
   let _fileMap = {
-    'intro_graph': 'data/intro_graph.json'
+    'address_formats': 'data/address_formats.min.json',
+    'deprecated': 'data/deprecated.min.json',
+    'discarded': 'data/discarded.min.json',
+    'imagery': 'data/imagery.min.json',
+    'intro_graph': 'data/intro_graph.min.json',
+    'languages': 'data/languages.min.json',
+    'locales': 'data/locales.min.json',
+    'phone_formats': 'data/phone_formats.min.json',
+    'shortcuts': 'data/shortcuts.min.json',
+    'territory_languages': 'data/territory_languages.min.json'
   };
 
 

modules/modes/save.js

@@ -43,6 +43,11 @@ export function modeSave(context) {
     var _origChanges;
     var _location;
 
+    var _discardTags = {};
+    context.data().get('discarded')
+        .then(function(d) { _discardTags = d; })
+        .catch(function() { /* ignore */ });
+
 
     function cancel(selectedID) {
         if (selectedID) {
@@ -92,7 +97,7 @@ export function modeSave(context) {
         _errors = [];
 
         // Store original changes, in case user wants to download them as an .osc file
-        _origChanges = history.changes(actionDiscardTags(history.difference()));
+        _origChanges = history.changes(actionDiscardTags(history.difference(), _discardTags));
 
         // First time, `history.perform` a no-op action.
         // Any conflict resolutions will be done as `history.replace`
@@ -241,15 +246,15 @@ export function modeSave(context) {
                 if (sameVersions(local, remote)) return;
 
                 var action = actionMergeRemoteChanges;
-                var merge = action(id, localGraph, remoteGraph, formatUser);
+                var merge = action(id, localGraph, remoteGraph, _discardTags, formatUser);
 
                 history.replace(merge);
 
                 var mergeConflicts = merge.conflicts();
                 if (!mergeConflicts.length) return;  // merged safely
 
-                var forceLocal = action(id, localGraph, remoteGraph).withOption('force_local');
-                var forceRemote = action(id, localGraph, remoteGraph).withOption('force_remote');
+                var forceLocal = action(id, localGraph, remoteGraph, _discardTags).withOption('force_local');
+                var forceRemote = action(id, localGraph, remoteGraph, _discardTags).withOption('force_remote');
                 var keepMine = t('save.conflict.' + (remote.visible ? 'keep_local' : 'restore'));
                 var keepTheirs = t('save.conflict.' + (remote.visible ? 'keep_remote' : 'delete'));
 
@@ -285,7 +290,7 @@ export function modeSave(context) {
 
         } else {
             var history = context.history();
-            var changes = history.changes(actionDiscardTags(history.difference()));
+            var changes = history.changes(actionDiscardTags(history.difference(), _discardTags));
             if (changes.modified.length || changes.created.length || changes.deleted.length) {
                 loadLocation();  // so it is ready when we display the save screen
                 osm.putChangeset(changeset, changes, uploadCallback);

modules/osm/entity.js

@@ -1,6 +1,5 @@
 import { debug } from '../index';
 import { osmIsInterestingTag } from './tags';
-import { dataDeprecated } from '../../data/index';
 import { utilArrayUnion } from '../util';
 
 
@@ -52,7 +51,7 @@ osmEntity.key = function(entity) {
 
 var _deprecatedTagValuesByKey;
 
-osmEntity.deprecatedTagValuesByKey = function() {
+osmEntity.deprecatedTagValuesByKey = function(dataDeprecated) {
     if (!_deprecatedTagValuesByKey) {
         _deprecatedTagValuesByKey = {};
         dataDeprecated.forEach(function(d) {
@@ -187,7 +186,7 @@ osmEntity.prototype = {
         return true;
     },
 
-    deprecatedTags: function() {
+    deprecatedTags: function(dataDeprecated) {
         var tags = this.tags;
 
         // if there are no tags, none can be deprecated

modules/ui/commit_changes.js

@@ -18,6 +18,11 @@ export function uiCommitChanges(context) {
     var detected = utilDetect();
     var _entityID;
 
+    var _discardTags = {};
+    context.data().get('discarded')
+        .then(function(d) { _discardTags = d; })
+        .catch(function() { /* ignore */ });
+
 
     function commitChanges(selection) {
         var history = context.history();
@@ -96,7 +101,7 @@ export function uiCommitChanges(context) {
 
         // Download changeset link
         var changeset = new osmChangeset().update({ id: undefined });
-        var changes = history.changes(actionDiscardTags(history.difference()));
+        var changes = history.changes(actionDiscardTags(history.difference(), _discardTags));
 
         delete changeset.id;  // Export without chnageset_id
 
@@ -158,9 +163,9 @@ export function uiCommitChanges(context) {
     }
 
 
-    commitChanges.entityID = function(_) {
+    commitChanges.entityID = function(val) {
         if (!arguments.length) return _entityID;
-        _entityID = _;
+        _entityID = val;
         return commitChanges;
     };
 

modules/ui/fields/address.js

@@ -2,7 +2,6 @@ import { dispatch as d3_dispatch } from 'd3-dispatch';
 import { select as d3_select } from 'd3-selection';
 import * as countryCoder from '@ideditor/country-coder';
 
-import { dataAddressFormats } from '../../../data';
 import { geoExtent, geoChooseEdge, geoSphericalDistance } from '../../geo';
 import { uiCombobox } from '../combobox';
 import { utilArrayUniqBy, utilGetSetValue, utilNoAuto, utilRebind } from '../../util';
@@ -12,10 +11,21 @@ import { t } from '../../util/locale';
 export function uiFieldAddress(field, context) {
     var dispatch = d3_dispatch('init', 'change');
     var wrap = d3_select(null);
+    var addrField = context.presets().field('address');   // needed for placeholder strings
+
     var _isInitialized = false;
     var _entity;
-    // needed for placeholder strings
-    var addrField = context.presets().field('address');
+    var _addressFormats = [{
+        format: [
+          ['housenumber', 'street'],
+          ['city', 'postcode']
+        ]
+      }];
+
+    context.data().get('address_formats')
+        .then(function(d) { _addressFormats = d; })
+        .catch(function() { /* ignore */ });
+
 
     function getNearStreets() {
         var extent = _entity.extent(context.graph());
@@ -113,8 +123,8 @@ export function uiFieldAddress(field, context) {
         countryCode = countryCode.toLowerCase();
 
         var addressFormat;
-        for (var i = 0; i < dataAddressFormats.length; i++) {
-            var format = dataAddressFormats[i];
+        for (var i = 0; i < _addressFormats.length; i++) {
+            var format = _addressFormats[i];
             if (!format.countryCodes) {
                 addressFormat = format;   // choose the default format, keep going
             } else if (format.countryCodes.indexOf(countryCode) !== -1) {

modules/ui/fields/combo.js

@@ -38,6 +38,13 @@ export function uiFieldCombo(field, context) {
     var _entity;
     var _countryCode;
 
+    // initialize deprecated tags array
+    var _dataDeprecated = [];
+    context.data().get('deprecated')
+        .then(function(d) { _dataDeprecated = d; })
+        .catch(function() { /* ignore */ });
+
+
     // ensure multiCombo field.key ends with a ':'
     if (isMulti && /[^:]$/.test(field.key)) {
         field.key += ':';
@@ -193,7 +200,7 @@ export function uiFieldCombo(field, context) {
                 return !d.count || d.count > 10;
             });
 
-            var deprecatedValues = osmEntity.deprecatedTagValuesByKey()[field.key];
+            var deprecatedValues = osmEntity.deprecatedTagValuesByKey(_dataDeprecated)[field.key];
             if (deprecatedValues) {
                 // don't suggest deprecated tag values
                 data = data.filter(function(d) {

modules/ui/fields/input.js

@@ -3,7 +3,6 @@ import { select as d3_select, event as d3_event } from 'd3-selection';
 import * as countryCoder from '@ideditor/country-coder';
 
 import { t, textDirection } from '../../util/locale';
-import { dataPhoneFormats } from '../../../data';
 import { utilGetSetValue, utilNoAuto, utilRebind } from '../../util';
 import { svgIcon } from '../../svg/icon';
 
@@ -21,6 +20,13 @@ export function uiFieldText(field, context) {
     var input = d3_select(null);
     var outlinkButton = d3_select(null);
     var _entity;
+    var _phoneFormats = {};
+
+    if (field.type === 'tel') {
+        context.data().get('phone_formats')
+            .then(function(d) { _phoneFormats = d; })
+            .catch(function() { /* ignore */ });
+    }
 
     function i(selection) {
         var preset = _entity && context.presets().match(_entity, context.graph());
@@ -61,7 +67,7 @@ export function uiFieldText(field, context) {
         if (field.type === 'tel' && _entity) {
             var center = _entity.extent(context.graph()).center();
             var countryCode = countryCoder.iso1A2Code(center);
-            var format = countryCode && dataPhoneFormats[countryCode.toLowerCase()];
+            var format = countryCode && _phoneFormats[countryCode.toLowerCase()];
             if (format) {
                 wrap.selectAll('#' + fieldID)
                     .attr('placeholder', format);

modules/ui/fields/localized.js

@@ -3,8 +3,6 @@ import { select as d3_select, event as d3_event } from 'd3-selection';
 import * as countryCoder from '@ideditor/country-coder';
 
 import { currentLocale, t, languageName } from '../../util/locale';
-import { dataLanguages } from '../../../data';
-import { dataTerritoryLanguages } from '../../../data';
 import { services } from '../../services';
 import { svgIcon } from '../../svg';
 import { tooltip } from '../../util/tooltip';
@@ -12,29 +10,7 @@ import { uiCombobox } from '../combobox';
 import { utilDetect } from '../../util/detect';
 import { utilArrayUniq, utilEditDistance, utilGetSetValue, utilNoAuto, utilRebind } from '../../util';
 
-var languagesArray = [];
-function loadLanguagesArray() {
-    if (languagesArray.length !== 0) return;
-
-    // some conversion is needed to ensure correct OSM tags are used
-    var replacements = {
-        sr: 'sr-Cyrl',      // in OSM, `sr` implies Cyrillic
-        'sr-Cyrl': false    // `sr-Cyrl` isn't used in OSM
-    };
-
-    for (var code in dataLanguages) {
-        if (replacements[code] === false) continue;
-        var metaCode = code;
-        if (replacements[code]) metaCode = replacements[code];
-
-        languagesArray.push({
-            localName: languageName(metaCode, { localOnly: true }),
-            nativeName: dataLanguages[metaCode].nativeName,
-            code: code,
-            label: languageName(metaCode)
-        });
-    }
-}
+var _languagesArray = [];
 
 
 export function uiFieldLocalized(field, context) {
@@ -44,6 +20,16 @@ export function uiFieldLocalized(field, context) {
     var localizedInputs = d3_select(null);
     var _countryCode;
 
+    context.data().get('languages')
+        .then(loadLanguagesArray)
+        .catch(function() { /* ignore */ });
+
+    var _territoryLanguages = {};
+    context.data().get('territory_languages')
+        .then(function(d) { _territoryLanguages = d; })
+        .catch(function() { /* ignore */ });
+
+
     var allSuggestions = context.presets().collection.filter(function(p) {
         return p.suggestion === true;
     });
@@ -66,6 +52,30 @@ export function uiFieldLocalized(field, context) {
     var _entity;
 
 
+    function loadLanguagesArray(dataLanguages) {
+        if (_languagesArray.length !== 0) return;
+
+        // some conversion is needed to ensure correct OSM tags are used
+        var replacements = {
+            sr: 'sr-Cyrl',      // in OSM, `sr` implies Cyrillic
+            'sr-Cyrl': false    // `sr-Cyrl` isn't used in OSM
+        };
+
+        for (var code in dataLanguages) {
+            if (replacements[code] === false) continue;
+            var metaCode = code;
+            if (replacements[code]) metaCode = replacements[code];
+
+            _languagesArray.push({
+                localName: languageName(context, metaCode, { localOnly: true }),
+                nativeName: dataLanguages[metaCode].nativeName,
+                code: code,
+                label: languageName(context, metaCode)
+            });
+        }
+    }
+
+
     function calcLocked() {
         if (!_entity) {    // the original entity
             field.locked(false);
@@ -91,6 +101,7 @@ export function uiFieldLocalized(field, context) {
         field.locked(isLocked);
     }
 
+
     // update _multilingual, maintaining the existing order
     function calcMultilingual(tags) {
         var existingLangsOrdered = _multilingual.map(function(item) {
@@ -119,9 +130,6 @@ export function uiFieldLocalized(field, context) {
 
 
     function localized(selection) {
-        // load if needed
-        loadLanguagesArray();
-
         _selection = selection;
         calcLocked();
         var isLocked = field.locked();
@@ -376,7 +384,7 @@ export function uiFieldLocalized(field, context) {
     function changeLang(d) {
         var lang = utilGetSetValue(d3_select(this));
         var t = {};
-        var language = languagesArray.find(function(d) {
+        var language = _languagesArray.find(function(d) {
             return (d.localName && d.localName.toLowerCase() === lang.toLowerCase()) ||
                 d.label.toLowerCase() === lang.toLowerCase() ||
                 (d.nativeName && d.nativeName.toLowerCase() === lang.toLowerCase());
@@ -418,18 +426,18 @@ export function uiFieldLocalized(field, context) {
         // show the user's language first
         var langCodes = [currentLocale, currentLocale.split('-')[0]];
 
-        if (_countryCode && dataTerritoryLanguages[_countryCode]) {
-            langCodes = langCodes.concat(dataTerritoryLanguages[_countryCode]);
+        if (_countryCode && _territoryLanguages[_countryCode]) {
+            langCodes = langCodes.concat(_territoryLanguages[_countryCode]);
         }
 
         var langItems = [];
         langCodes.forEach(function(code) {
-            var langItem = languagesArray.find(function(item) {
+            var langItem = _languagesArray.find(function(item) {
                 return item.code === code;
             });
             if (langItem) langItems.push(langItem);
         });
-        langItems = utilArrayUniq(langItems.concat(languagesArray));
+        langItems = utilArrayUniq(langItems.concat(_languagesArray));
 
         cb(langItems.filter(function(d) {
             return d.label.toLowerCase().indexOf(v) >= 0 ||
@@ -537,7 +545,7 @@ export function uiFieldLocalized(field, context) {
         entries.order();
 
         utilGetSetValue(entries.select('.localized-lang'), function(d) {
-            return languageName(d.lang);
+            return languageName(context, d.lang);
         });
 
         utilGetSetValue(entries.select('.localized-value'),

modules/ui/intro/intro.js

@@ -56,7 +56,8 @@ export function uiIntro(context) {
           }
         }
         selection.call(startIntro);
-      });
+      })
+      .catch(function() { /* ignore */ });
   }
 
 

modules/ui/shortcuts.js

@@ -1,10 +1,6 @@
-import {
-    select as d3_select,
-    selectAll as d3_selectAll
-} from 'd3-selection';
+import { select as d3_select, selectAll as d3_selectAll } from 'd3-selection';
 
 import { t } from '../util/locale';
-import { dataShortcuts } from '../../data';
 import { svgIcon } from '../svg/icon';
 import { uiCmd } from './cmd';
 import { uiModal } from './modal';
@@ -28,7 +24,7 @@ export function uiShortcuts(context) {
                 }
             } else {
                 _modalSelection = uiModal(_selection);
-                shortcutsModal(_modalSelection);
+                _modalSelection.call(shortcutsModal);
             }
         });
 
@@ -37,20 +33,21 @@ export function uiShortcuts(context) {
         _modalSelection.select('.modal')
             .classed('modal-shortcuts', true);
 
-        var shortcutsModal = _modalSelection.select('.content');
+        var content = _modalSelection.select('.content');
 
-        shortcutsModal
+        content
             .append('div')
             .attr('class', 'modal-section')
             .append('h3')
             .text(t('shortcuts.title'));
 
-        shortcutsModal
-            .call(render);
+        context.data().get('shortcuts')
+            .then(function(data) { content.call(render, data); })
+            .catch(function() { /* ignore */ });
     }
 
 
-    function render(selection) {
+    function render(selection, dataShortcuts) {
         var wrapper = selection
             .selectAll('.wrapper')
             .data([0]);
@@ -80,7 +77,7 @@ export function uiShortcuts(context) {
             .attr('class', 'tab')
             .on('click', function (d, i) {
                 _activeTab = i;
-                render(selection);
+                render(selection, dataShortcuts);
             });
 
         tabsEnter
@@ -257,7 +254,7 @@ export function uiShortcuts(context) {
         _selection = selection;
         if (show) {
             _modalSelection = uiModal(selection);
-            shortcutsModal(_modalSelection);
+            _modalSelection.call(shortcutsModal);
         }
     };
 }

modules/ui/success.js

@@ -279,7 +279,7 @@ export function uiSuccess(context) {
 
       if (d.languageCodes && d.languageCodes.length) {
         const languageList = d.languageCodes
-          .map(code => languageName(code))
+          .map(code => languageName(context, code))
           .join(', ');
 
         moreEnter

modules/util/locale.js

@@ -1,6 +1,5 @@
-import { dataLanguages } from '../../../data';
-
 var translations = Object.create(null);
+var _dataLanguages = {};
 
 export var currentLocale = 'en';
 export var textDirection = 'ltr';
@@ -73,7 +72,6 @@ export function t(s, o, loc) {
  *
  * @param {string} dir ltr or rtl
  */
-
 export function setTextDirection(dir) {
     textDirection = dir;
 }
@@ -86,38 +84,42 @@ export function setScriptNames(obj) {
     scriptNames = obj;
 }
 
-export function languageName(code, options) {
-    if (languageNames[code]) { // name in locale langauge
+export function languageName(context, code, options) {
+    // Data access is async now, which makes this complicated.
+    // If _dataLanguages haven't been loaded yet, try to load them.
+    // Worst case, we fallback to the code until the file has been loaded.
+    if (!Object.keys(_dataLanguages).length) {
+        context.data().get('languages')
+            .then(function(d) { _dataLanguages = d; })
+            .catch(function() { /* ignore */ });
+    }
 
+    if (languageNames[code]) { // name in locale langauge
         // e.g. German
         return languageNames[code];
     }
+
     // sometimes we only want the local name
     if (options && options.localOnly) return null;
 
-    var langInfo = dataLanguages[code];
-
+    var langInfo = _dataLanguages[code];
     if (langInfo) {
         if (langInfo.nativeName) { // name in native language
-
             // e.g. Deutsch (de)
             return t('translate.language_and_code', { language: langInfo.nativeName, code: code });
 
         } else if (langInfo.base && langInfo.script) {
-
             var base = langInfo.base; // the code of the langauge this is based on
 
             if (languageNames[base]) { // base language name in locale langauge
                 var scriptCode = langInfo.script;
                 var script = scriptNames[scriptCode] || scriptCode;
-
                 // e.g. Serbian (Cyrillic)
                 return t('translate.language_and_code', { language: languageNames[base], code: script });
 
-            } else if (dataLanguages[base] && dataLanguages[base].nativeName) {
-
+            } else if (_dataLanguages[base] && _dataLanguages[base].nativeName) {
                 // e.g. српски (sr-Cyrl)
-                return t('translate.language_and_code', { language: dataLanguages[base].nativeName, code: code });
+                return t('translate.language_and_code', { language: _dataLanguages[base].nativeName, code: code });
             }
         }
     }

modules/validations/outdated_tags.js

@@ -13,6 +13,12 @@ import { validationIssue, validationIssueFix } from '../core/validation';
 export function validationOutdatedTags(context) {
     var type = 'outdated_tags';
 
+    // initialize deprecated tags array
+    var _dataDeprecated = [];
+    context.data().get('deprecated')
+        .then(function(d) { _dataDeprecated = d; })
+        .catch(function() { /* ignore */ });
+
     // initialize name-suggestion-index matcher
     var nsiMatcher = matcher();
     nsiMatcher.buildMatchIndex(brands.brands);
@@ -43,7 +49,7 @@ export function validationOutdatedTags(context) {
         }
 
         // upgrade tags..
-        var deprecatedTags = entity.deprecatedTags();
+        var deprecatedTags = entity.deprecatedTags(_dataDeprecated);
         if (deprecatedTags.length) {
             deprecatedTags.forEach(function(tag) {
                 graph = actionUpgradeTags(entity.id, tag.old, tag.replace)(graph);

modules/validations/suspicious_name.js

@@ -58,7 +58,7 @@ export function validationSuspiciousName() {
                 var entity = context.hasEntity(this.entityIds[0]);
                 if (!entity) return '';
                 var preset = utilPreset(entity, context);
-                var langName = langCode && languageName(langCode);
+                var langName = langCode && languageName(context, langCode);
                 return t('issues.generic_name.message' + (langName ? '_language' : ''),
                     { feature: preset.name(), name: genericName, language: langName }
                 );
@@ -105,7 +105,7 @@ export function validationSuspiciousName() {
                 var entity = context.hasEntity(this.entityIds[0]);
                 if (!entity) return '';
                 var preset = utilPreset(entity, context);
-                var langName = langCode && languageName(langCode);
+                var langName = langCode && languageName(context, langCode);
                 return t('issues.incorrect_name.message' + (langName ? '_language' : ''),
                     { feature: preset.name(), name: incorrectName, language: langName }
                 );