From 80b583a6f0e77dcd2033307b02aaa903af4ff88f Mon Sep 17 00:00:00 2001 From: Bryan Housel Date: Wed, 22 Aug 2018 23:16:31 -0400 Subject: [PATCH] WIP on feature deduplication across tile boundaries It seems like the ids stored in the features are not reliable, so I'm trying to generate ids --- modules/services/vector_tile.js | 59 +++++++++++--------- modules/svg/data.js | 97 ++++++++++++++++++++++----------- modules/svg/helpers.js | 12 +++- modules/util/index.js | 3 +- modules/util/util.js | 16 ++++++ 5 files changed, 128 insertions(+), 59 deletions(-) diff --git a/modules/services/vector_tile.js b/modules/services/vector_tile.js index 4c0556d5e..e6de67d87 100644 --- a/modules/services/vector_tile.js +++ b/modules/services/vector_tile.js @@ -7,7 +7,7 @@ import { request as d3_request } from 'd3-request'; import Protobuf from 'pbf'; import vt from '@mapbox/vector-tile'; -import { utilRebind, utilTiler } from '../util'; +import { utilHashcode, utilRebind, utilTiler } from '../util'; var tiler = utilTiler().tileSize(512); @@ -20,25 +20,28 @@ function abortRequest(i) { } -function vtToGeoJSON(bufferdata) { - var tile = new vt.VectorTile(new Protobuf(bufferdata.data.response)); - var layers = Object.keys(tile.layers); +function vtToGeoJSON(data, tile) { + var vectorTile = new vt.VectorTile(new Protobuf(data.response)); + var layers = Object.keys(vectorTile.layers); if (!Array.isArray(layers)) { layers = [layers]; } - var collection = { type: 'FeatureCollection', features: [] }; - - layers.forEach(function (layerID) { - var layer = tile.layers[layerID]; + var features = []; + layers.forEach(function(layerID) { + var layer = vectorTile.layers[layerID]; if (layer) { for (var i = 0; i < layer.length; i++) { - var feature = layer.feature(i).toGeoJSON(bufferdata.xyz[0], bufferdata.xyz[1], bufferdata.xyz[2]); - if (layers.length > 1) feature.properties.vt_layer = layerID; - collection.features.push(feature); + var feature = layer.feature(i).toGeoJSON(tile.xyz[0], tile.xyz[1], tile.xyz[2]); + if (layers.length > 1) { + feature.properties.vt_layer = layerID; + } + // force unique id generation + feature.__hashcode__ = utilHashcode(JSON.stringify(feature)); + features.push(feature); } } }); - return collection; + return features; } @@ -60,18 +63,13 @@ function loadTile(source, tile) { source.inflight[tile.id] = d3_request(url) .responseType('arraybuffer') .get(function(err, data) { - source.loaded[tile.id] = true; + source.loaded[tile.id] = {}; delete source.inflight[tile.id]; if (err || !data) return; - var bufferdata = { - data: data, - xyz: tile.xyz - }; - source.loaded[tile.id] = { - bufferdata: bufferdata, - geojson: vtToGeoJSON(bufferdata) + data: data, + features: vtToGeoJSON(data, tile) }; dispatch.call('loadedData'); @@ -112,12 +110,23 @@ export default { var source = _vtCache[sourceID]; if (!source) return []; - // for now, return the FeatureCollection for each tile var tiles = tiler.getTiles(projection); - return tiles.map(function(tile) { - var loaded = source.loaded[tile.id]; - return loaded && loaded.geojson; - }).filter(Boolean); + var seen = {}; + var results = []; + + for (var i = 0; i < tiles.length; i++) { + var loaded = source.loaded[tiles[i].id]; + if (!loaded || !loaded.features) continue; + + for (var j = 0; j < loaded.features.length; j++) { + var feature = loaded.features[j]; + if (seen[feature.__hashcode__]) continue; + seen[feature.__hashcode__] = true; + results.push(feature); + } + } + + return results; }, diff --git a/modules/svg/data.js b/modules/svg/data.js index 1cf4061fd..6d263233e 100644 --- a/modules/svg/data.js +++ b/modules/svg/data.js @@ -4,7 +4,11 @@ import _reduce from 'lodash-es/reduce'; import _union from 'lodash-es/union'; import _throttle from 'lodash-es/throttle'; -import { geoBounds as d3_geoBounds } from 'd3-geo'; +import { + geoBounds as d3_geoBounds, + geoPath as d3_geoPath +} from 'd3-geo'; + import { text as d3_text } from 'd3-request'; import { @@ -18,6 +22,7 @@ import { geoExtent, geoPolygonIntersectsPolygon } from '../geo'; import { services } from '../services'; import { svgPath } from './index'; import { utilDetect } from '../util/detect'; +import { utilHashcode } from '../util'; var _initialized = false; @@ -110,6 +115,41 @@ export function svgData(projection, context, dispatch) { } + // ensure that all geojson features in a collection have IDs + function ensureIDs(gj) { + if (!gj) return null; + + if (gj.type === 'FeatureCollection') { + for (var i = 0; i < gj.features.length; i++) { + ensureFeatureID(gj.features[i]); + } + } else { + ensureFeatureID(gj); + } + return gj; + } + + + // ensure that each single Feature object has a unique ID + function ensureFeatureID(feature) { + if (!feature) return; + feature.__hashcode__ = utilHashcode(JSON.stringify(feature)); + return feature; + } + + + // Prefer an array of Features instead of a FeatureCollection + function getFeatures(gj) { + if (!gj) return []; + + if (gj.type === 'FeatureCollection') { + return gj.features; + } else { + return [gj]; + } + } + + function drawData(selection) { var vtService = getService(); var getPath = svgPath(projection).geojson; @@ -133,45 +173,42 @@ export function svgData(projection, context, dispatch) { vtService.loadTiles(sourceID, _template, projection); geoData = vtService.data(sourceID, projection); } else { - geoData = _geojson ? [_geojson] : []; + geoData = getFeatures(_geojson); } + geoData = geoData.filter(getPath); + var paths = layer .selectAll('path') - .data(geoData); + .data(geoData, function(d) { return d.__hashcode__; }); + // exit paths.exit() .remove(); + // enter/update paths = paths.enter() .append('path') .attr('class', 'pathdata') - .merge(paths); - - paths + .merge(paths) .attr('d', getPath); - var labelData = []; if (_showLabels) { - geoData.forEach(function(f) { - if (f.type === 'FeatureCollection') { - labelData = labelData.concat(f.features); - } else { - labelData.push(f); - } - }); - labelData = labelData.filter(getPath); + layer + .call(drawLabels, 'label-halo', geoData) + .call(drawLabels, 'label', geoData); } - layer - .call(drawLabels, 'label-halo', labelData) - .call(drawLabels, 'label', labelData); - function drawLabels(selection, textClass, data) { + var labelPath = d3_geoPath(projection); + var labelData = data.filter(function(d) { + return d.properties && (d.properties.desc || d.properties.name); + }); + var labels = selection.selectAll('text.' + textClass) - .data(data); + .data(labelData, function(d) { return d.__hashcode__; }); // exit labels.exit() @@ -183,17 +220,14 @@ export function svgData(projection, context, dispatch) { .attr('class', textClass) .merge(labels) .text(function(d) { - if (d.properties) { - return d.properties.desc || d.properties.name; - } - return null; + return d.properties.desc || d.properties.name; }) .attr('x', function(d) { - var centroid = getPath.centroid(d); + var centroid = labelPath.centroid(d); return centroid[0] + 11; }) .attr('y', function(d) { - var centroid = getPath.centroid(d); + var centroid = labelPath.centroid(d); return centroid[1]; }); } @@ -236,7 +270,7 @@ export function svgData(projection, context, dispatch) { } if (!_isEmpty(gj)) { - _geojson = gj; + _geojson = ensureIDs(gj); _src = src || 'unknown.geojson'; return this.fitZoom(); } @@ -295,7 +329,7 @@ export function svgData(projection, context, dispatch) { _src = null; if (!_isEmpty(gj)) { - _geojson = gj; + _geojson = ensureIDs(gj); _src = src || 'unknown.geojson'; } @@ -343,7 +377,6 @@ export function svgData(projection, context, dispatch) { drawData.setFile(extension, data, url); } }); - } else { drawData.template(url); } @@ -358,12 +391,12 @@ export function svgData(projection, context, dispatch) { drawData.fitZoom = function() { - // note: only works on a FeatureCollection - if (_isEmpty(_geojson) || _isEmpty(_geojson.features)) return; + var features = getFeatures(_geojson); + if (!features.length) return; var map = context.map(); var viewport = map.trimmedExtent().polygon(); - var coords = _reduce(_geojson.features, function(coords, feature) { + var coords = _reduce(features, function(coords, feature) { var c = feature.geometry.coordinates; /* eslint-disable no-fallthrough */ diff --git a/modules/svg/helpers.js b/modules/svg/helpers.js index fe57f3926..c450e00f8 100644 --- a/modules/svg/helpers.js +++ b/modules/svg/helpers.js @@ -168,7 +168,17 @@ export function svgPath(projection, graph, isArea) { } }; - svgpath.geojson = path; + svgpath.geojson = function(d) { + if (d.id !== undefined) { + if (d.id in cache) { + return cache[d.id]; + } else { + return cache[d.id] = path(d); + } + } else { + return path(d); + } + }; return svgpath; } diff --git a/modules/util/index.js b/modules/util/index.js index bc06a0a41..16460cd14 100644 --- a/modules/util/index.js +++ b/modules/util/index.js @@ -12,6 +12,7 @@ export { utilFunctor } from './util'; export { utilGetAllNodes } from './util'; export { utilGetPrototypeOf } from './util'; export { utilGetSetValue } from './get_set_value'; +export { utilHashcode } from './util'; export { utilIdleWorker } from './idle_worker'; export { utilNoAuto } from './util'; export { utilPrefixCSSProperty } from './util'; @@ -25,4 +26,4 @@ export { utilSuggestNames } from './suggest_names'; export { utilTagText } from './util'; export { utilTiler } from './tiler'; export { utilTriggerEvent } from './trigger_event'; -export { utilWrap } from './util'; \ No newline at end of file +export { utilWrap } from './util'; diff --git a/modules/util/util.js b/modules/util/util.js index cf226f5ee..6b03c9f4c 100644 --- a/modules/util/util.js +++ b/modules/util/util.js @@ -266,3 +266,19 @@ export function utilNoAuto(selection) { .attr('autocapitalize', 'off') .attr('spellcheck', isText ? 'true' : 'false'); } + + +// https://stackoverflow.com/questions/194846/is-there-any-kind-of-hash-code-function-in-javascript +// https://werxltd.com/wp/2010/05/13/javascript-implementation-of-javas-string-hashcode-method/ +export function utilHashcode(str) { + var hash = 0; + if (str.length === 0) { + return hash; + } + for (var i = 0; i < str.length; i++) { + var char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; // Convert to 32bit integer + } + return hash; +}