Merge pull request #5647 from nyurik/osm_wikibase

OSM Multilingual data directly from wiki (wikibase data items)

data/core.yaml

@@ -405,7 +405,6 @@ en:
   inspector:
     no_documentation_combination: There is no documentation available for this tag combination
     no_documentation_key: There is no documentation available for this key
-    documentation_redirect: This documentation has been redirected to a new page
     show_more: Show More
     view_on_osm: View on openstreetmap.org
     all_fields: All fields
@@ -418,6 +417,7 @@ en:
     choose: Select feature type
     results: "{n} results for {search}"
     reference: View on OpenStreetMap Wiki
+    edit_reference: Edit or translate on OSM Wiki
     back_tooltip: Change feature
     remove: Remove
     search: Search

dist/locales/en.json

@@ -499,7 +499,6 @@
         "inspector": {
             "no_documentation_combination": "There is no documentation available for this tag combination",
             "no_documentation_key": "There is no documentation available for this key",
-            "documentation_redirect": "This documentation has been redirected to a new page",
             "show_more": "Show More",
             "view_on_osm": "View on openstreetmap.org",
             "all_fields": "All fields",
@@ -512,6 +511,7 @@
             "choose": "Select feature type",
             "results": "{n} results for {search}",
             "reference": "View on OpenStreetMap Wiki",
+            "edit_reference": "Edit or translate on OSM Wiki",
             "back_tooltip": "Change feature",
             "remove": "Remove",
             "search": "Search",

modules/services/index.js

@@ -3,6 +3,7 @@ import serviceMapRules from './maprules';
 import serviceNominatim from './nominatim';
 import serviceOpenstreetcam from './openstreetcam';
 import serviceOsm from './osm';
+import serviceOsmWikibase from './osm_wikibase';
 import serviceStreetside from './streetside';
 import serviceTaginfo from './taginfo';
 import serviceVectorTile from './vector_tile';
@@ -15,6 +16,7 @@ export var services = {
     mapillary: serviceMapillary,
     openstreetcam: serviceOpenstreetcam,
     osm: serviceOsm,
+    osmWikibase: serviceOsmWikibase,
     maprules: serviceMapRules,
     streetside: serviceStreetside,
     taginfo: serviceTaginfo,
@@ -29,6 +31,7 @@ export {
     serviceNominatim,
     serviceOpenstreetcam,
     serviceOsm,
+    serviceOsmWikibase,
     serviceStreetside,
     serviceTaginfo,
     serviceVectorTile,

modules/services/osm_wikibase.js

@@ -0,0 +1,201 @@
+import _debounce from 'lodash-es/debounce';
+import _forEach from 'lodash-es/forEach';
+
+import { json as d3_json } from 'd3-request';
+
+import { utilQsString } from '../util';
+
+
+var apibase = 'https://wiki.openstreetmap.org/w/api.php';
+var _inflight = {};
+var _wikibaseCache = {};
+var _localeIds = { en: false };
+
+
+var debouncedRequest = _debounce(request, 500, { leading: false });
+
+function request(url, callback) {
+    if (_inflight[url]) return;
+
+    _inflight[url] = d3_json(url, function (err, data) {
+        delete _inflight[url];
+        callback(err, data);
+    });
+}
+
+
+/**
+ * Get the best string value from the descriptions/labels result
+ * Note that if mediawiki doesn't recognize language code, it will return all values.
+ * In that case, fallback to use English.
+ * @param values object - either descriptions or labels
+ * @param langCode String
+ * @returns localized string
+ */
+function localizedToString(values, langCode) {
+    if (values) {
+        values = values[langCode] || values.en;
+    }
+    return values ? values.value : '';
+}
+
+
+export default {
+
+    init: function() {
+        _inflight = {};
+        _wikibaseCache = {};
+        _localeIds = {};
+    },
+
+
+    reset: function() {
+        _forEach(_inflight, function(req) { req.abort(); });
+        _inflight = {};
+    },
+
+
+    /**
+     * Get the best value for the property, or undefined if not found
+     * @param entity object from wikibase
+     * @param property string e.g. 'P4' for image
+     * @param langCode string e.g. 'fr' for French
+     */
+    claimToValue: function(entity, property, langCode) {
+        if (!entity.claims[property]) return undefined;
+        var locale = _localeIds[langCode];
+        var preferredPick, localePick;
+        _forEach(entity.claims[property], function(stmt) {
+            // If exists, use value limited to the needed language (has a qualifier P26 = locale)
+            // Or if not found, use the first value with the "preferred" rank
+            if (!preferredPick && stmt.rank === 'preferred') {
+                preferredPick = stmt;
+            }
+            if (locale && stmt.qualifiers && stmt.qualifiers.P26 &&
+                stmt.qualifiers.P26[0].datavalue.value.id === locale
+            ) {
+                localePick = stmt;
+            }
+        });
+        var result = localePick || preferredPick;
+
+        if (result) {
+            var datavalue = result.mainsnak.datavalue;
+            return datavalue.type === 'wikibase-entityid' ? datavalue.value.id : datavalue.value;
+        } else {
+            return undefined;
+        }
+    },
+
+
+    toSitelink: function(key, value) {
+        var result = value ? 'Tag:' + key + '=' + value : 'Key:' + key;
+        return result.replace(/_/g, ' ').trim();
+    },
+
+
+    getEntity: function(params, callback) {
+        var doRequest = params.debounce ? debouncedRequest : request;
+        var self = this;
+        var titles = [];
+        var result = {};
+        var keySitelink = this.toSitelink(params.key);
+        var tagSitelink = params.value ? this.toSitelink(params.key, params.value) : false;
+        var localeSitelink;
+
+        if (params.langCode && _localeIds[params.langCode] === undefined) {
+            // If this is the first time we are asking about this locale,
+            // fetch corresponding entity (if it exists), and cache it.
+            // If there is no such entry, cache `false` value to avoid re-requesting it.
+            localeSitelink = ('Locale:' + params.langCode).replace(/_/g, ' ').trim();
+            titles.push(localeSitelink);
+        }
+
+        if (_wikibaseCache[keySitelink]) {
+            result.key = _wikibaseCache[keySitelink];
+        } else {
+            titles.push(keySitelink);
+        }
+
+        if (tagSitelink) {
+            if (_wikibaseCache[tagSitelink]) {
+                result.tag = _wikibaseCache[tagSitelink];
+            } else {
+                titles.push(tagSitelink);
+            }
+        }
+
+        if (!titles.length) {
+            // Nothing to do, we already had everything in the cache
+            return callback(null, result);
+        }
+
+        // Requesting just the user language code
+        // If backend recognizes the code, it will perform proper fallbacks,
+        // and the result will contain the requested code. If not, all values are returned:
+        // {"zh-tw":{"value":"...","language":"zh-tw","source-language":"zh-hant"}
+        // {"pt-br":{"value":"...","language":"pt","for-language":"pt-br"}}
+        var obj = {
+            action: 'wbgetentities',
+            sites: 'wiki',
+            titles: titles.join('|'),
+            languages: params.langCode,
+            languagefallback: 1,
+            origin: '*',
+            format: 'json',
+            // There is an MW Wikibase API bug https://phabricator.wikimedia.org/T212069
+            // We shouldn't use v1 until it gets fixed, but should switch to it afterwards
+            // formatversion: 2,
+        };
+
+        var url = apibase + '?' + utilQsString(obj);
+        doRequest(url, function(err, d) {
+            if (err) {
+                callback(err);
+            } else if (!d.success || d.error) {
+                callback(d.error.messages.map(function(v) { return v.html['*']; }).join('<br>'));
+            } else {
+                var localeId = false;
+                _forEach(d.entities, function(res) {
+                    if (res.missing !== '') {
+                        var title = res.sitelinks.wiki.title;
+                        // Simplify access to the localized values
+                        res.description = localizedToString(res.descriptions, params.langCode);
+                        res.label = localizedToString(res.labels, params.langCode);
+                        if (title === keySitelink) {
+                            _wikibaseCache[keySitelink] = res;
+                            result.key = res;
+                        } else if (title === tagSitelink) {
+                            _wikibaseCache[tagSitelink] = res;
+                            result.tag = res;
+                        } else if (title === localeSitelink) {
+                            localeId = res.id;
+                        } else {
+                            console.log('Unexpected title ' + title);
+                        }
+                    }
+                });
+
+                if (localeSitelink) {
+                    // If locale ID is not found, store false to prevent repeated queries
+                    self.addLocale(params.langCode, localeId);
+                }
+
+                callback(null, result);
+            }
+        });
+    },
+
+
+    addLocale: function(langCode, qid) {
+        // Makes it easier to unit test
+        _localeIds[langCode] = qid;
+    },
+
+    apibase: function(_) {
+        if (!arguments.length) return apibase;
+        apibase = _;
+        return this;
+    }
+
+};

modules/ui/tag_reference.js

@@ -1,6 +1,3 @@
-import _find from 'lodash-es/find';
-import _omit from 'lodash-es/omit';
-
 import {
     event as d3_event,
     select as d3_select
@@ -10,10 +7,11 @@ import { t } from '../util/locale';
 import { utilDetect } from '../util/detect';
 import { services } from '../services';
 import { svgIcon } from '../svg';
+import { utilQsString } from '../util';
 
 
 export function uiTagReference(tag) {
-    var taginfo = services.taginfo;
+    var wikibase = services.osmWikibase;
     var tagReference = {};
 
     var _button = d3_select(null);
@@ -21,42 +19,49 @@ export function uiTagReference(tag) {
     var _loaded;
     var _showing;
 
-
+    /**
+     * @returns {{itemTitle: String, description: String, image: String|null}|null}
+     **/
     function findLocal(data) {
-        var locale = utilDetect().locale.toLowerCase();
-        var localized;
+        var entity = data.tag || data.key;
+        if (!entity) return null;
 
-        if (locale !== 'pt-br') {  // see #3776, prefer 'pt' over 'pt-br'
-            localized = _find(data, function(d) {
-                return d.lang.toLowerCase() === locale;
-            });
-            if (localized) return localized;
+        var result = {
+            title: entity.title,
+            description: entity.description,
+        };
+
+        if (entity.claims) {
+            var langCode = utilDetect().locale.toLowerCase();
+            var url;
+            var image = wikibase.claimToValue(entity, 'P4', langCode);
+            if (image) {
+                url = 'https://commons.wikimedia.org/w/index.php';
+            } else {
+                image = wikibase.claimToValue(entity, 'P28', langCode);
+                if (image) {
+                    url = 'https://wiki.openstreetmap.org/w/index.php';
+                }
+            }
+            if (image) {
+                result.image = {
+                    url: url,
+                    title: 'Special:Redirect/file/' + image
+                };
+            }
         }
 
-        // try the non-regional version of a language, like
-        // 'en' if the language is 'en-US'
-        if (locale.indexOf('-') !== -1) {
-            var first = locale.split('-')[0];
-            localized = _find(data, function(d) {
-                return d.lang.toLowerCase() === first;
-            });
-            if (localized) return localized;
-        }
-
-        // finally fall back to english
-        return _find(data, function(d) {
-            return d.lang.toLowerCase() === 'en';
-        });
+        return result;
     }
 
 
     function load(param) {
-        if (!taginfo) return;
+        if (!wikibase) return;
 
         _button
             .classed('tag-reference-loading', true);
 
-        taginfo.docs(param, function show(err, data) {
+        wikibase.getEntity(param, function show(err, data) {
             var docs;
             if (!err && data) {
                 docs = findLocal(data);
@@ -65,23 +70,25 @@ export function uiTagReference(tag) {
             _body.html('');
 
             if (!docs || !docs.title) {
-                if (param.hasOwnProperty('value')) {
-                    load(_omit(param, 'value'));   // retry with key only
-                } else {
-                    _body
-                        .append('p')
-                        .attr('class', 'tag-reference-description')
-                        .text(t('inspector.no_documentation_key'));
-                    done();
-                }
+                _body
+                    .append('p')
+                    .attr('class', 'tag-reference-description')
+                    .text(t('inspector.no_documentation_key'));
+                done();
                 return;
             }
 
-            if (docs.image && docs.image.thumb_url_prefix) {
+            if (docs.image) {
+                var imageUrl = docs.image.url + '?' + utilQsString({
+                    title: docs.image.title,
+                    width: 100,
+                    height: 100,
+                });
+
                 _body
                     .append('img')
                     .attr('class', 'tag-reference-wiki-image')
-                    .attr('src', docs.image.thumb_url_prefix + '100' + docs.image.thumb_url_suffix)
+                    .attr('src', imageUrl)
                     .on('load', function() { done(); })
                     .on('error', function() { d3_select(this).remove(); done(); });
             } else {
@@ -91,7 +98,7 @@ export function uiTagReference(tag) {
             _body
                 .append('p')
                 .attr('class', 'tag-reference-description')
-                .text(docs.description || t('inspector.documentation_redirect'));
+                .text(docs.description || t('inspector.no_documentation_key'));
 
             _body
                 .append('a')
@@ -101,7 +108,7 @@ export function uiTagReference(tag) {
                 .attr('href', 'https://wiki.openstreetmap.org/wiki/' + docs.title)
                 .call(svgIcon('#iD-icon-out-link', 'inline'))
                 .append('span')
-                .text(t('inspector.reference'));
+                .text(t('inspector.edit_reference'));
 
             // Add link to info about "good changeset comments" - #2923
             if (param.key === 'comment') {
@@ -171,6 +178,7 @@ export function uiTagReference(tag) {
                 } else if (_loaded) {
                     done();
                 } else {
+                    tag.langCode = utilDetect().locale.toLowerCase();
                     load(tag);
                 }
             });

test/index.html

@@ -110,6 +110,7 @@
     <script src='spec/services/nominatim.js'></script>
     <script src='spec/services/openstreetcam.js'></script>
     <script src='spec/services/osm.js'></script>
+    <script src='spec/services/osm_wikibase.js'></script>
     <script src='spec/services/streetside.js'></script>
     <script src='spec/services/taginfo.js'></script>
 

test/spec/services/osm_wikibase.js

@@ -0,0 +1,322 @@
+describe('iD.serviceOsmWikibase', function () {
+  var server, wikibase;
+
+  before(function () {
+    iD.services.osmWikibase = iD.serviceOsmWikibase;
+  });
+
+  after(function () {
+    delete iD.services.osmWikibase;
+  });
+
+  beforeEach(function () {
+    wikibase = iD.services.osmWikibase;
+    wikibase.init();
+    server = sinon.fakeServer.create();
+  });
+
+  afterEach(function () {
+    server.restore();
+  });
+
+
+  function query(url) {
+    return iD.utilStringQs(url.substring(url.indexOf('?') + 1));
+  }
+
+  function adjust(params, data) {
+    if (params) {
+      if (params.norm) {
+        data.description = data.descriptions.fr.value;
+        data.label = data.labels.fr.value;
+      }
+    }
+    return data;
+  }
+
+  function keyData(params) {
+    return adjust(params, {
+      pageid: 205725,
+      ns: 120,
+      title: 'Item:Q42',
+      lastrevid: 1721242,
+      modified: '2018-12-18T07:00:43Z',
+      type: 'item',
+      id: 'Q42',
+      labels: {
+        fr: {language: 'en', value: 'amenity', 'for-language': 'fr'}
+      },
+      descriptions: {
+        fr: {language: 'en', value: 'English description', 'for-language': 'fr'}
+      },
+      aliases: {},
+      claims: {
+        P2: [ // instance of
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q7'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            rank: 'normal'
+          }
+        ],
+        P16: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'string',
+              datavalue: {value: 'amenity', type: 'string'}
+            },
+            type: 'statement',
+            rank: 'normal'
+          }
+        ],
+        P25: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q4679'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            rank: 'normal'
+          }
+        ],
+        P9: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q8'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            rank: 'normal'
+          }
+        ],
+        P6: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q15'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            rank: 'preferred'
+          },
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q14'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            qualifiers: {
+              P26: [
+                {
+                  snaktype: 'value',
+                  datatype: 'wikibase-item',
+                  datavalue: {value: {'entity-type': 'item', id: 'Q6994'}, type: 'wikibase-entityid'}
+                }
+              ]
+            },
+            rank: 'normal'
+          }
+        ],
+        P28: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'string',
+              datavalue: {value: 'Mapping-Features-Parking-Lot.png', type: 'string'}
+            },
+            type: 'statement',
+            rank: 'normal'
+          }
+        ]
+      },
+      sitelinks: {
+        wiki: {
+          site: 'wiki',
+          title: 'Key:amenity',
+          badges: []
+        }
+      }
+    });
+  }
+
+  function tagData(params) {
+    return adjust(params, {
+      pageid: 210934,
+      ns: 120,
+      title: 'Item:Q13',
+      lastrevid: 1718041,
+      modified: '2018-12-18T03:51:05Z',
+      type: 'item',
+      id: 'Q13',
+      labels: {
+        fr: {language: 'en', value: 'amenity=parking', 'for-language': 'fr'}
+      },
+      descriptions: {
+        fr: {language: 'fr', value: 'French description'}
+      },
+      aliases: {},
+      claims: {
+        P2: [ // instance of = Q2 (tag)
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q2'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            rank: 'normal'
+          }
+        ],
+        P19: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'string',
+              datavalue: {value: 'amenity=parking', type: 'string'}
+            },
+            type: 'statement',
+            rank: 'normal'
+          }
+        ],
+        P10: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q42'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            rank: 'normal'
+          }
+        ],
+        P4: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'commonsMedia',
+              datavalue: {value: 'Primary image.jpg', type: 'string'}
+            },
+            type: 'statement',
+            rank: 'preferred'
+          }
+        ],
+        P6: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q14'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            rank: 'preferred'
+          },
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q13'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            qualifiers: {
+              P26: [
+                {
+                  snaktype: 'value',
+                  datatype: 'wikibase-item',
+                  datavalue: {value: {'entity-type': 'item', id: 'Q6994'}, type: 'wikibase-entityid'}
+                }
+              ]
+            },
+            rank: 'normal'
+          }
+        ],
+        P25: [
+          {
+            mainsnak: {
+              snaktype: 'value',
+              datatype: 'wikibase-item',
+              datavalue: {value: {'entity-type': 'item', id: 'Q4679'}, type: 'wikibase-entityid'}
+            },
+            type: 'statement',
+            rank: 'normal'
+          }
+        ]
+      },
+      sitelinks: {
+        wiki: {
+          site: 'wiki',
+          title: 'Tag:amenity=parking',
+          badges: []
+        }
+      }
+    });
+  }
+
+
+  var localeData = {
+    id: 'Q7792',
+    sitelinks: {wiki: {site: 'wiki', title: 'Locale:fr'}}
+  };
+
+  describe('#getEntity', function () {
+    it('calls the given callback with the results of the getEntity data item query', function () {
+      var callback = sinon.spy();
+      wikibase.getEntity({key: 'amenity', value: 'parking', langCode: 'fr'}, callback);
+
+      server.respondWith('GET', /action=wbgetentities/,
+        [200, {'Content-Type': 'application/json'}, JSON.stringify({
+          entities: {
+            Q42: keyData(),
+            Q13: tagData(),
+            Q7792: localeData,
+          },
+          success: 1
+        })]
+      );
+      server.respond();
+
+      expect(query(server.requests[0].url)).to.eql(
+        {
+          action: 'wbgetentities',
+          sites: 'wiki',
+          titles: 'Locale:fr|Key:amenity|Tag:amenity=parking',
+          languages: 'fr',
+          languagefallback: '1',
+          origin: '*',
+          format: 'json',
+        }
+      );
+      expect(callback).to.have.been.calledWith(null, {
+        key: keyData({norm: true}),
+        tag: tagData({norm: true})
+      });
+    });
+  });
+
+
+  it('creates correct sitelinks', function () {
+    expect(wikibase.toSitelink('amenity')).to.eql('Key:amenity');
+    expect(wikibase.toSitelink('amenity_')).to.eql('Key:amenity');
+    expect(wikibase.toSitelink('_amenity_')).to.eql('Key: amenity');
+    expect(wikibase.toSitelink('amenity or_not_')).to.eql('Key:amenity or not');
+    expect(wikibase.toSitelink('amenity', 'parking')).to.eql('Tag:amenity=parking');
+    expect(wikibase.toSitelink(' amenity_', '_parking_')).to.eql('Tag: amenity = parking');
+    expect(wikibase.toSitelink('amenity or_not', '_park ing_')).to.eql('Tag:amenity or not= park ing');
+  });
+
+  it('gets correct value from entity', function () {
+    wikibase.addLocale('de', 'Q6994');
+    wikibase.addLocale('fr', 'Q7792');
+    expect(wikibase.claimToValue(tagData(), 'P4', 'en')).to.eql('Primary image.jpg');
+    expect(wikibase.claimToValue(keyData(), 'P6', 'en')).to.eql('Q15');
+    expect(wikibase.claimToValue(keyData(), 'P6', 'fr')).to.eql('Q15');
+    expect(wikibase.claimToValue(keyData(), 'P6', 'de')).to.eql('Q14');
+  });
+
+});