From 442f0831b16b496728114403d8fe41558b379f65 Mon Sep 17 00:00:00 2001 From: Kushan Joshi <0o3ko0@gmail.com> Date: Mon, 21 Aug 2017 15:56:50 +0530 Subject: [PATCH 1/8] add enitity cache --- modules/services/osm.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/services/osm.js b/modules/services/osm.js index a395f85cd..c9e93ce45 100644 --- a/modules/services/osm.js +++ b/modules/services/osm.js @@ -19,6 +19,7 @@ var dispatch = d3.dispatch('authLoading', 'authDone', 'change', 'loading', 'load blacklists = ['.*\.google(apis)?\..*/(vt|kh)[\?/].*([xyz]=.*){3}.*'], inflight = {}, loadedTiles = {}, + entityCache = {}, tileZoom = 16, oauth = osmAuth({ url: urlroot, @@ -158,7 +159,13 @@ function parse(xml) { var child = children[i], parser = parsers[child.nodeName]; if (parser) { + var uid = child.nodeName + child.attributes.id.value; + if (entityCache[uid]) { + console.log(uid, 'is cached'); + continue; + } entities.push(parser(child)); + entityCache[uid] = true; } } From 82686b10a57680b3e920e5907b3f30233e653646 Mon Sep 17 00:00:00 2001 From: Kushan Joshi <0o3ko0@gmail.com> Date: Tue, 22 Aug 2017 17:38:42 +0530 Subject: [PATCH 2/8] Add requestIdleCallback --- modules/services/osm.js | 43 +++++++++++++++++----------------- modules/util/idle_worker.js | 46 +++++++++++++++++++++++++++++++++++++ modules/util/index.js | 1 + 3 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 modules/util/idle_worker.js diff --git a/modules/services/osm.js b/modules/services/osm.js index c9e93ce45..977115363 100644 --- a/modules/services/osm.js +++ b/modules/services/osm.js @@ -11,7 +11,7 @@ import { osmWay } from '../osm'; -import { utilRebind } from '../util'; +import { utilRebind, utilIdleWorker } from '../util'; var dispatch = d3.dispatch('authLoading', 'authDone', 'change', 'loading', 'loaded'), @@ -101,10 +101,10 @@ function getVisible(attrs) { var parsers = { - node: function nodeData(obj) { + node: function nodeData(obj, uid) { var attrs = obj.attributes; return new osmNode({ - id: osmEntity.id.fromOSM('node', attrs.id.value), + id:uid, visible: getVisible(attrs), version: attrs.version.value, changeset: attrs.changeset && attrs.changeset.value, @@ -116,10 +116,10 @@ var parsers = { }); }, - way: function wayData(obj) { + way: function wayData(obj, uid) { var attrs = obj.attributes; return new osmWay({ - id: osmEntity.id.fromOSM('way', attrs.id.value), + id: uid, visible: getVisible(attrs), version: attrs.version.value, changeset: attrs.changeset && attrs.changeset.value, @@ -131,10 +131,10 @@ var parsers = { }); }, - relation: function relationData(obj) { + relation: function relationData(obj, uid) { var attrs = obj.attributes; return new osmRelation({ - id: osmEntity.id.fromOSM('relation', attrs.id.value), + id: uid, visible: getVisible(attrs), version: attrs.version.value, changeset: attrs.changeset && attrs.changeset.value, @@ -148,28 +148,23 @@ var parsers = { }; -function parse(xml) { +function parse(xml, callback) { if (!xml || !xml.childNodes) return; var root = xml.childNodes[0], - children = root.childNodes, - entities = []; + children = root.childNodes; - for (var i = 0, l = children.length; i < l; i++) { - var child = children[i], - parser = parsers[child.nodeName]; + function parseChild(child) { + var parser = parsers[child.nodeName]; if (parser) { - var uid = child.nodeName + child.attributes.id.value; + var uid = osmEntity.id.fromOSM(child.nodeName, child.attributes.id.value); if (entityCache[uid]) { - console.log(uid, 'is cached'); - continue; + return null; } - entities.push(parser(child)); - entityCache[uid] = true; + return parser(child, uid); } } - - return entities; + utilIdleWorker(children, parseChild, callback); } @@ -244,7 +239,13 @@ export default { } if (callback) { - callback(err, parse(xml)); + if (err) return callback(err, null); + parse(xml, function (entities) { + for (var i in entities) { + entityCache[entities[i].id] = true; + } + callback(null, entities); + }); } } } diff --git a/modules/util/idle_worker.js b/modules/util/idle_worker.js new file mode 100644 index 000000000..2d8ad1968 --- /dev/null +++ b/modules/util/idle_worker.js @@ -0,0 +1,46 @@ +export function utilIdleWorker(tasks, processor, callback) { + var processed = []; + var currentPos = 0; + var totalTasks = tasks.length; + + function worker(deadline) { + while (deadline.timeRemaining() > 0 && currentPos < totalTasks) { + var result = processor(tasks[currentPos]); + + // if falsy dont add to the processed list + if (result) processed.push(result); + currentPos++; + } + + // more tasks are left, we might need more idleCallbacks + if (currentPos < totalTasks) { + return window.requestIdleCallback(deadline => worker(deadline)); + } + + // tasks are completed + return callback(processed); + } + + window.requestIdleCallback(deadline => worker(deadline)); +} + +// shim +window.requestIdleCallback = + window.requestIdleCallback || + function(cb) { + var start = Date.now(); + return setTimeout(function() { + cb({ + didTimeout: false, + timeRemaining: function() { + return Math.max(0, 50 - (Date.now() - start)); + } + }); + }, 1); + }; + +window.cancelIdleCallback = + window.cancelIdleCallback || + function(id) { + clearTimeout(id); + }; diff --git a/modules/util/index.js b/modules/util/index.js index fc0c71673..1010cac21 100644 --- a/modules/util/index.js +++ b/modules/util/index.js @@ -22,3 +22,4 @@ export { utilSuggestNames } from './suggest_names'; export { utilTagText } from './util'; export { utilTriggerEvent } from './trigger_event'; export { utilWrap } from './util'; +export { utilIdleWorker} from './idle_worker'; \ No newline at end of file From 91a569985d16f64ad54727651e7d07f58276e9e6 Mon Sep 17 00:00:00 2001 From: Kushan Joshi <0o3ko0@gmail.com> Date: Tue, 22 Aug 2017 19:18:03 +0530 Subject: [PATCH 3/8] playing with map render --- modules/core/context.js | 10 ++-- modules/renderer/map.js | 102 ++++++++++++++++++++++++++++++++- modules/util/call_when_idle.js | 13 +++++ modules/util/index.js | 3 +- 4 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 modules/util/call_when_idle.js diff --git a/modules/core/context.js b/modules/core/context.js index 1acc4f96b..ed1d77f34 100644 --- a/modules/core/context.js +++ b/modules/core/context.js @@ -13,6 +13,8 @@ import { services } from '../services/index'; import { uiInit } from '../ui/init'; import { utilDetect } from '../util/detect'; import { utilRebind } from '../util/rebind'; +import { utilCallWhenIdle } from '../util/index'; + export var areaKeys = {}; @@ -83,9 +85,9 @@ export function coreContext() { /* Connection */ - function entitiesLoaded(err, result) { + var entitiesLoaded = utilCallWhenIdle(function entitiesLoaded(err, result) { if (!err) history.merge(result.data, result.extent); - } + }); context.preauth = function(options) { if (connection) { @@ -94,7 +96,7 @@ export function coreContext() { return context; }; - context.loadTiles = function(projection, dimensions, callback) { + context.loadTiles = utilCallWhenIdle(function(projection, dimensions, callback) { function done(err, result) { entitiesLoaded(err, result); if (callback) callback(err, result); @@ -102,7 +104,7 @@ export function coreContext() { if (connection) { connection.loadTiles(projection, dimensions, done); } - }; + }); context.loadEntity = function(id, callback) { function done(err, result) { diff --git a/modules/renderer/map.js b/modules/renderer/map.js index 8e82e12c8..6350405f2 100644 --- a/modules/renderer/map.js +++ b/modules/renderer/map.js @@ -32,6 +32,7 @@ import { utilGetDimensions } from '../util/dimensions'; + export function rendererMap(context) { var dimensions = [1, 1], @@ -57,6 +58,20 @@ export function rendererMap(context) { mouse, mousemove; + // var debouncedTransformStart = () => requestAnimationFrame(() => transformStart = projection.transform()); + // var setProjectionTransformStart = () => debSetTransformStart(projection.transform()); + + // var debSetTransformStart = _.debounce((val) => transformStart = val, 50); + + // const debouncedTransform + // var projectIdleCb; + var savedToDrawVertices; + var saveToDrawLines; + var saveToDrawArea; + var saveToDrawMidpoints; + var saveToDrawLabels; + var saveToDrawPoints; + var zoom = d3.zoom() .scaleExtent([ztok(2), ztok(24)]) .interpolate(d3.interpolate) @@ -64,8 +79,28 @@ export function rendererMap(context) { .on('zoom', zoomPan); var _selection = d3.select(null); + var isRenderScheduled = false; + var pendingRedrawCall; + function initiateRedraw() { + // Reset the boolean so future redraws can be set. + isRenderScheduled = false; + redraw.apply(this, arguments); + } + + function scheduleRedraw() { + // Only schedule the redraw if one has not already been set. + if (isRenderScheduled) return; + + isRenderScheduled = true; + + pendingRedrawCall = requestIdleCallback(() => initiateRedraw.apply(this, arguments), { timeout: 1400 }); + } - + function cancelPendingRedraw() { + isRenderScheduled = false; + window.cancelIdleCallback(pendingRedrawCall); + } + function map(selection) { _selection = selection; @@ -266,6 +301,67 @@ export function rendererMap(context) { .call(drawLabels, graph, data, filter, dimensions, !difference && !extent) .call(drawPoints, graph, data, filter); + // var toDrawVertices = (selection) => { + // // window.cancelIdleCallback(savedToDrawVertices); + // savedToDrawVertices = requestIdleCallback(() => { + // drawVertices(selection, graph, data, filter, map.extent(), map.zoom()); + // setProjectionTransformStart(); + // toDrawLines(selection); + // }, {timeout: 150}); + // }; + + // var toDrawLines = (selection) => { + // // window.cancelIdleCallback(saveToDrawLines); + // saveToDrawLines = requestIdleCallback(() => { + // drawLines(selection, graph, data, filter); + // // setProjectionTransformStart(); + // toDrawAreas(selection); + // }, {timeout: 150}); + // }; + + // var toDrawAreas = (selection) => { + // // window.cancelIdleCallback(saveToDrawArea); + // saveToDrawArea = requestIdleCallback(() => { + // drawAreas(selection, graph, data, filter); + // // setProjectionTransformStart(); + // toDrawMidpoints(selection); + // }, {timeout: 150}); + // }; + + // var toDrawMidpoints = (selection) => { + // // window.cancelIdleCallback(saveToDrawMidpoints); + // saveToDrawMidpoints = requestIdleCallback(() => { + // drawMidpoints(selection,graph, data, filter, map.trimmedExtent()); + // // setProjectionTransformStart(); + // toDrawLabels(selection); + // }, {timeout: 150}); + // }; + + // var toDrawLabels = (selection) => { + // // window.cancelIdleCallback(saveToDrawLabels); + // saveToDrawLabels = requestIdleCallback(() => { + // drawLabels(selection, graph, data, filter, dimensions, !difference && !extent); + // // setProjectionTransformStart(); + // toDrawPoints(selection); + // }, {timeout: 150}); + // }; + + // var toDrawPoints = (selection) => { + // // window.cancelIdleCallback(saveToDrawPoints); + // saveToDrawPoints = requestIdleCallback(() => { + // drawPoints(selection, graph, data, filter); + // setProjectionTransformStart(); + // }, {timeout: 150}); + // }; + + // surface.selectAll('.data-layer-osm') + // .call(toDrawVertices) + // .call(toDrawLines) + // .call(toDrawAreas) + // .call(toDrawMidpoints) + // .call(toDrawLabels) + // .call(toDrawPoints); + dispatch.call('drawn', this, {full: true}); } @@ -403,11 +499,11 @@ export function rendererMap(context) { } - var queueRedraw = _.throttle(redraw, 750); + var queueRedraw = scheduleRedraw; var immediateRedraw = function(difference, extent) { - if (!difference && !extent) queueRedraw.cancel(); + if (!difference && !extent) cancelPendingRedraw(); redraw(difference, extent); }; diff --git a/modules/util/call_when_idle.js b/modules/util/call_when_idle.js new file mode 100644 index 000000000..e86de4747 --- /dev/null +++ b/modules/util/call_when_idle.js @@ -0,0 +1,13 @@ +// note the function should be of low priority +// and should not be returning a value. +export function utilCallWhenIdle(func, timeout, name) { + return function() { + var args = arguments; + var that = this; + console.log('called ', name); + window.requestIdleCallback(function() { + console.log('idle succeed ', name); + func.apply(that, args); + }, {timeout: timeout}); + }; +} \ No newline at end of file diff --git a/modules/util/index.js b/modules/util/index.js index 1010cac21..b71c2fd4e 100644 --- a/modules/util/index.js +++ b/modules/util/index.js @@ -22,4 +22,5 @@ export { utilSuggestNames } from './suggest_names'; export { utilTagText } from './util'; export { utilTriggerEvent } from './trigger_event'; export { utilWrap } from './util'; -export { utilIdleWorker} from './idle_worker'; \ No newline at end of file +export { utilIdleWorker} from './idle_worker'; +export { utilCallWhenIdle } from './call_when_idle'; \ No newline at end of file From 909b72a39e884518062de7caa5fc34e59cd9c998 Mon Sep 17 00:00:00 2001 From: Kushan Joshi <0o3ko0@gmail.com> Date: Thu, 24 Aug 2017 16:53:40 +0530 Subject: [PATCH 4/8] cleaning up --- modules/core/context.js | 1 - modules/renderer/map.js | 115 ++++++--------------------------- modules/svg/labels.js | 5 +- modules/ui/background.js | 4 +- modules/util/call_when_idle.js | 6 +- modules/util/index.js | 2 +- 6 files changed, 26 insertions(+), 107 deletions(-) diff --git a/modules/core/context.js b/modules/core/context.js index ed1d77f34..0c4032e98 100644 --- a/modules/core/context.js +++ b/modules/core/context.js @@ -16,7 +16,6 @@ import { utilRebind } from '../util/rebind'; import { utilCallWhenIdle } from '../util/index'; - export var areaKeys = {}; export function setAreaKeys(value) { diff --git a/modules/renderer/map.js b/modules/renderer/map.js index 6350405f2..8fbad68e3 100644 --- a/modules/renderer/map.js +++ b/modules/renderer/map.js @@ -32,7 +32,6 @@ import { utilGetDimensions } from '../util/dimensions'; - export function rendererMap(context) { var dimensions = [1, 1], @@ -58,20 +57,6 @@ export function rendererMap(context) { mouse, mousemove; - // var debouncedTransformStart = () => requestAnimationFrame(() => transformStart = projection.transform()); - // var setProjectionTransformStart = () => debSetTransformStart(projection.transform()); - - // var debSetTransformStart = _.debounce((val) => transformStart = val, 50); - - // const debouncedTransform - // var projectIdleCb; - var savedToDrawVertices; - var saveToDrawLines; - var saveToDrawArea; - var saveToDrawMidpoints; - var saveToDrawLabels; - var saveToDrawPoints; - var zoom = d3.zoom() .scaleExtent([ztok(2), ztok(24)]) .interpolate(d3.interpolate) @@ -79,25 +64,24 @@ export function rendererMap(context) { .on('zoom', zoomPan); var _selection = d3.select(null); - var isRenderScheduled = false; + var isRedrawScheduled = false; var pendingRedrawCall; - function initiateRedraw() { - // Reset the boolean so future redraws can be set. - isRenderScheduled = false; - redraw.apply(this, arguments); - } - + function scheduleRedraw() { // Only schedule the redraw if one has not already been set. - if (isRenderScheduled) return; - - isRenderScheduled = true; - - pendingRedrawCall = requestIdleCallback(() => initiateRedraw.apply(this, arguments), { timeout: 1400 }); + if (isRedrawScheduled) return; + isRedrawScheduled = true; + var that = this; + var args = arguments; + pendingRedrawCall = requestIdleCallback(function () { + // Reset the boolean so future redraws can be set. + isRedrawScheduled = false; + redraw.apply(that, args); + }, { timeout: 1400 }); } function cancelPendingRedraw() { - isRenderScheduled = false; + isRedrawScheduled = false; window.cancelIdleCallback(pendingRedrawCall); } @@ -301,67 +285,6 @@ export function rendererMap(context) { .call(drawLabels, graph, data, filter, dimensions, !difference && !extent) .call(drawPoints, graph, data, filter); - // var toDrawVertices = (selection) => { - // // window.cancelIdleCallback(savedToDrawVertices); - // savedToDrawVertices = requestIdleCallback(() => { - // drawVertices(selection, graph, data, filter, map.extent(), map.zoom()); - // setProjectionTransformStart(); - // toDrawLines(selection); - // }, {timeout: 150}); - // }; - - // var toDrawLines = (selection) => { - // // window.cancelIdleCallback(saveToDrawLines); - // saveToDrawLines = requestIdleCallback(() => { - // drawLines(selection, graph, data, filter); - // // setProjectionTransformStart(); - // toDrawAreas(selection); - // }, {timeout: 150}); - // }; - - // var toDrawAreas = (selection) => { - // // window.cancelIdleCallback(saveToDrawArea); - // saveToDrawArea = requestIdleCallback(() => { - // drawAreas(selection, graph, data, filter); - // // setProjectionTransformStart(); - // toDrawMidpoints(selection); - // }, {timeout: 150}); - // }; - - // var toDrawMidpoints = (selection) => { - // // window.cancelIdleCallback(saveToDrawMidpoints); - // saveToDrawMidpoints = requestIdleCallback(() => { - // drawMidpoints(selection,graph, data, filter, map.trimmedExtent()); - // // setProjectionTransformStart(); - // toDrawLabels(selection); - // }, {timeout: 150}); - // }; - - // var toDrawLabels = (selection) => { - // // window.cancelIdleCallback(saveToDrawLabels); - // saveToDrawLabels = requestIdleCallback(() => { - // drawLabels(selection, graph, data, filter, dimensions, !difference && !extent); - // // setProjectionTransformStart(); - // toDrawPoints(selection); - // }, {timeout: 150}); - // }; - - // var toDrawPoints = (selection) => { - // // window.cancelIdleCallback(saveToDrawPoints); - // saveToDrawPoints = requestIdleCallback(() => { - // drawPoints(selection, graph, data, filter); - // setProjectionTransformStart(); - // }, {timeout: 150}); - // }; - - // surface.selectAll('.data-layer-osm') - // .call(toDrawVertices) - // .call(toDrawLines) - // .call(toDrawAreas) - // .call(toDrawMidpoints) - // .call(toDrawLabels) - // .call(toDrawPoints); - dispatch.call('drawn', this, {full: true}); } @@ -419,7 +342,7 @@ export function rendererMap(context) { surface.interrupt(); uiFlash().text(t('cannot_zoom')); setZoom(context.minEditableZoom(), true); - queueRedraw(); + scheduleRedraw(); dispatch.call('move', this, map); return; } @@ -442,7 +365,7 @@ export function rendererMap(context) { transformed = true; transformLast = eventTransform; utilSetTransform(supersurface, tX, tY, scale); - queueRedraw(); + scheduleRedraw(); dispatch.call('move', this, map); } @@ -499,8 +422,6 @@ export function rendererMap(context) { } - var queueRedraw = scheduleRedraw; - var immediateRedraw = function(difference, extent) { if (!difference && !extent) cancelPendingRedraw(); @@ -672,7 +593,7 @@ export function rendererMap(context) { mouse = utilFastMouse(supersurface.node()); setCenter(center); - queueRedraw(); + scheduleRedraw(); return map; }; @@ -701,7 +622,7 @@ export function rendererMap(context) { dispatch.call('move', this, map); } - queueRedraw(); + scheduleRedraw(); return map; }; @@ -721,7 +642,7 @@ export function rendererMap(context) { dispatch.call('move', this, map); } - queueRedraw(); + scheduleRedraw(); return map; }; @@ -744,7 +665,7 @@ export function rendererMap(context) { dispatch.call('move', this, map); } - queueRedraw(); + scheduleRedraw(); return map; }; diff --git a/modules/svg/labels.js b/modules/svg/labels.js index 90a3ea7f2..7b5fdb8e0 100644 --- a/modules/svg/labels.js +++ b/modules/svg/labels.js @@ -18,7 +18,8 @@ import { utilDetect } from '../util/detect'; import { utilDisplayName, utilDisplayNameForPath, - utilEntitySelector + utilEntitySelector, + utilCallWhenIdle } from '../util/index'; @@ -652,7 +653,7 @@ export function svgLabels(projection, context) { } - var throttleFilterLabels = _.throttle(filterLabels, 100); + var throttleFilterLabels = _.throttle(utilCallWhenIdle(filterLabels), 100); drawLabels.observe = function(selection) { diff --git a/modules/ui/background.js b/modules/ui/background.js index a9686fc16..419931fe0 100644 --- a/modules/ui/background.js +++ b/modules/ui/background.js @@ -4,7 +4,7 @@ import { d3keybinding } from '../lib/d3.keybinding.js'; import { t, textDirection } from '../util/locale'; import { geoMetersToOffset, geoOffsetToMeters } from '../geo/index'; import { utilDetect } from '../util/detect'; -import { utilSetTransform } from '../util/index'; +import { utilSetTransform, utilCallWhenIdle } from '../util/index'; import { svgIcon } from '../svg/index'; import { uiMapInMap } from './map_in_map'; import { uiCmd } from './cmd'; @@ -540,7 +540,7 @@ export function uiBackground(context) { ); context.map() - .on('move.background-update', _.debounce(update, 1000)); + .on('move.background-update', _.debounce(utilCallWhenIdle(update), 1000)); context.background() .on('change.background-update', update); diff --git a/modules/util/call_when_idle.js b/modules/util/call_when_idle.js index e86de4747..1b8ae482a 100644 --- a/modules/util/call_when_idle.js +++ b/modules/util/call_when_idle.js @@ -1,13 +1,11 @@ // note the function should be of low priority // and should not be returning a value. -export function utilCallWhenIdle(func, timeout, name) { +export function utilCallWhenIdle(func, timeout) { return function() { var args = arguments; var that = this; - console.log('called ', name); window.requestIdleCallback(function() { - console.log('idle succeed ', name); func.apply(that, args); }, {timeout: timeout}); }; -} \ No newline at end of file +} diff --git a/modules/util/index.js b/modules/util/index.js index b71c2fd4e..f3bf74842 100644 --- a/modules/util/index.js +++ b/modules/util/index.js @@ -23,4 +23,4 @@ export { utilTagText } from './util'; export { utilTriggerEvent } from './trigger_event'; export { utilWrap } from './util'; export { utilIdleWorker} from './idle_worker'; -export { utilCallWhenIdle } from './call_when_idle'; \ No newline at end of file +export { utilCallWhenIdle } from './call_when_idle'; From b356bb855fd269bc66806bacce8bd38c208ac055 Mon Sep 17 00:00:00 2001 From: Kushan Joshi <0o3ko0@gmail.com> Date: Thu, 24 Aug 2017 17:25:59 +0530 Subject: [PATCH 5/8] remove arrow functions :{ --- modules/util/idle_worker.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/util/idle_worker.js b/modules/util/idle_worker.js index 2d8ad1968..f1ff47451 100644 --- a/modules/util/idle_worker.js +++ b/modules/util/idle_worker.js @@ -14,14 +14,14 @@ export function utilIdleWorker(tasks, processor, callback) { // more tasks are left, we might need more idleCallbacks if (currentPos < totalTasks) { - return window.requestIdleCallback(deadline => worker(deadline)); + return window.requestIdleCallback(function(deadline) {worker(deadline);}); } // tasks are completed return callback(processed); } - window.requestIdleCallback(deadline => worker(deadline)); + window.requestIdleCallback(function(deadline) {worker(deadline);}); } // shim From cabc98649f16d99245a06808abcfff7f80013ca2 Mon Sep 17 00:00:00 2001 From: Bryan Housel Date: Thu, 24 Aug 2017 13:10:51 -0400 Subject: [PATCH 6/8] Clear entity id cache on reset() --- modules/services/osm.js | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/services/osm.js b/modules/services/osm.js index 977115363..77976531e 100644 --- a/modules/services/osm.js +++ b/modules/services/osm.js @@ -180,6 +180,7 @@ export default { userDetails = undefined; rateLimitError = undefined; _.forEach(inflight, abortRequest); + entityCache = {}; loadedTiles = {}; inflight = {}; return this; From cd05c514329d0693b5bbf97e71a0927083e101bd Mon Sep 17 00:00:00 2001 From: Bryan Housel Date: Thu, 24 Aug 2017 15:02:12 -0400 Subject: [PATCH 7/8] loadEntity can use entityCache, but loadEntityVersion should not --- modules/services/osm.js | 23 +++++++++++++++-------- modules/util/idle_worker.js | 4 ++-- test/spec/services/osm.js | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/modules/services/osm.js b/modules/services/osm.js index 77976531e..edcaa78db 100644 --- a/modules/services/osm.js +++ b/modules/services/osm.js @@ -148,7 +148,8 @@ var parsers = { }; -function parse(xml, callback) { +function parse(xml, callback, options) { + options = _.extend({ cache: true }, options); if (!xml || !xml.childNodes) return; var root = xml.childNodes[0], @@ -158,12 +159,13 @@ function parse(xml, callback) { var parser = parsers[child.nodeName]; if (parser) { var uid = osmEntity.id.fromOSM(child.nodeName, child.attributes.id.value); - if (entityCache[uid]) { + if (options.cache && entityCache[uid]) { return null; } return parser(child, uid); } } + utilIdleWorker(children, parseChild, callback); } @@ -216,7 +218,8 @@ export default { }, - loadFromAPI: function(path, callback) { + loadFromAPI: function(path, callback, options) { + options = _.extend({ cache: true }, options); var that = this; function done(err, xml) { @@ -242,11 +245,13 @@ export default { if (callback) { if (err) return callback(err, null); parse(xml, function (entities) { - for (var i in entities) { - entityCache[entities[i].id] = true; + if (options.cache) { + for (var i in entities) { + entityCache[entities[i].id] = true; + } } callback(null, entities); - }); + }, options); } } } @@ -275,13 +280,15 @@ export default { loadEntityVersion: function(id, version, callback) { var type = osmEntity.id.type(id), - osmID = osmEntity.id.toOSM(id); + osmID = osmEntity.id.toOSM(id), + options = { cache: false }; this.loadFromAPI( '/api/0.6/' + type + '/' + osmID + '/' + version, function(err, entities) { if (callback) callback(err, { data: entities }); - } + }, + options ); }, diff --git a/modules/util/idle_worker.js b/modules/util/idle_worker.js index f1ff47451..2a5cc5cfb 100644 --- a/modules/util/idle_worker.js +++ b/modules/util/idle_worker.js @@ -6,7 +6,7 @@ export function utilIdleWorker(tasks, processor, callback) { function worker(deadline) { while (deadline.timeRemaining() > 0 && currentPos < totalTasks) { var result = processor(tasks[currentPos]); - + // if falsy dont add to the processed list if (result) processed.push(result); currentPos++; @@ -15,7 +15,7 @@ export function utilIdleWorker(tasks, processor, callback) { // more tasks are left, we might need more idleCallbacks if (currentPos < totalTasks) { return window.requestIdleCallback(function(deadline) {worker(deadline);}); - } + } // tasks are completed return callback(processed); diff --git a/test/spec/services/osm.js b/test/spec/services/osm.js index f23c1d635..0ba3f5387 100644 --- a/test/spec/services/osm.js +++ b/test/spec/services/osm.js @@ -320,6 +320,23 @@ describe('iD.serviceOsm', function () { [200, { 'Content-Type': 'text/xml' }, wayXML]); server.respond(); }); + + it('ignores repeat requests using entityCache', function(done) { + var id = 'n1'; + connection.loadEntity(id, function(err, result) { + var entity = _.find(result.data, function(e) { return e.id === id; }); + expect(entity).to.be.an.instanceOf(iD.Node); + connection.loadEntity(id, function(err, result) { + expect(result.data).to.eql([]); + done(); + }); + server.respond(); + }); + + server.respondWith('GET', 'http://www.openstreetmap.org/api/0.6/node/1', + [200, { 'Content-Type': 'text/xml' }, nodeXML]); + server.respond(); + }); }); describe('#loadEntityVersion', function () { @@ -363,6 +380,24 @@ describe('iD.serviceOsm', function () { [200, { 'Content-Type': 'text/xml' }, wayXML]); server.respond(); }); + + it('does not ignore repeat requests', function(done) { + var id = 'n1'; + connection.loadEntityVersion(id, 1, function(err1, result1) { + var entity1 = _.find(result1.data, function(e1) { return e1.id === id; }); + expect(entity1).to.be.an.instanceOf(iD.Node); + connection.loadEntityVersion(id, 1, function(err2, result2) { + var entity2 = _.find(result2.data, function(e2) { return e2.id === id; }); + expect(entity2).to.be.an.instanceOf(iD.Node); + done(); + }); + server.respond(); + }); + + server.respondWith('GET', 'http://www.openstreetmap.org/api/0.6/node/1/1', + [200, { 'Content-Type': 'text/xml' }, nodeXML]); + server.respond(); + }); }); describe('#loadMultiple', function () { From b7d98b94fba4725a2e2c1e261cf3c24eebe7b6c3 Mon Sep 17 00:00:00 2001 From: Bryan Housel Date: Thu, 24 Aug 2017 21:00:39 -0400 Subject: [PATCH 8/8] loadEntity loadEntityVersion and loadMultiple all should ignore entityCache --- modules/services/osm.js | 13 +++++++++---- test/spec/services/osm.js | 14 ++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/modules/services/osm.js b/modules/services/osm.js index edcaa78db..498e5523f 100644 --- a/modules/services/osm.js +++ b/modules/services/osm.js @@ -267,13 +267,15 @@ export default { loadEntity: function(id, callback) { var type = osmEntity.id.type(id), - osmID = osmEntity.id.toOSM(id); + osmID = osmEntity.id.toOSM(id), + options = { cache: false }; this.loadFromAPI( '/api/0.6/' + type + '/' + osmID + (type !== 'node' ? '/full' : ''), function(err, entities) { if (callback) callback(err, { data: entities }); - } + }, + options ); }, @@ -295,16 +297,19 @@ export default { loadMultiple: function(ids, callback) { var that = this; + _.each(_.groupBy(_.uniq(ids), osmEntity.id.type), function(v, k) { var type = k + 's', - osmIDs = _.map(v, osmEntity.id.toOSM); + osmIDs = _.map(v, osmEntity.id.toOSM), + options = { cache: false }; _.each(_.chunk(osmIDs, 150), function(arr) { that.loadFromAPI( '/api/0.6/' + type + '?' + type + '=' + arr.join(), function(err, entities) { if (callback) callback(err, { data: entities }); - } + }, + options ); }); }); diff --git a/test/spec/services/osm.js b/test/spec/services/osm.js index 0ba3f5387..ec7bd3878 100644 --- a/test/spec/services/osm.js +++ b/test/spec/services/osm.js @@ -321,13 +321,14 @@ describe('iD.serviceOsm', function () { server.respond(); }); - it('ignores repeat requests using entityCache', function(done) { + it('does not ignore repeat requests', function(done) { var id = 'n1'; - connection.loadEntity(id, function(err, result) { - var entity = _.find(result.data, function(e) { return e.id === id; }); - expect(entity).to.be.an.instanceOf(iD.Node); - connection.loadEntity(id, function(err, result) { - expect(result.data).to.eql([]); + connection.loadEntity(id, function(err1, result1) { + var entity1 = _.find(result1.data, function(e1) { return e1.id === id; }); + expect(entity1).to.be.an.instanceOf(iD.Node); + connection.loadEntity(id, function(err2, result2) { + var entity2 = _.find(result2.data, function(e2) { return e2.id === id; }); + expect(entity2).to.be.an.instanceOf(iD.Node); done(); }); server.respond(); @@ -411,6 +412,7 @@ describe('iD.serviceOsm', function () { it('loads nodes'); it('loads ways'); + it('does not ignore repeat requests'); });