Implement support for multilingual descriptions from wiki data items

* Takes data directly from the Wikibase data items (OSM Wiki)
     https://wiki.openstreetmap.org/wiki/OpenStreetMap:Data_Items
* Understands the difference in regions - e.g. will show different
  images depending on the local settings
* Perf: Single request will get both the tag and key description

data/core.yaml

@@ -405,7 +405,6 @@ en:
   inspector:
     no_documentation_combination: There is no documentation available for this tag combination
     no_documentation_key: There is no documentation available for this key
-    documentation_redirect: This documentation has been redirected to a new page
     show_more: Show More
     view_on_osm: View on openstreetmap.org
     all_fields: All fields
@@ -418,6 +417,7 @@ en:
     choose: Select feature type
     results: "{n} results for {search}"
     reference: View on OpenStreetMap Wiki
+    edit_reference: Edit or translate on OSM Wiki
     back_tooltip: Change feature
     remove: Remove
     search: Search

dist/locales/en.json

@@ -499,7 +499,6 @@
         "inspector": {
             "no_documentation_combination": "There is no documentation available for this tag combination",
             "no_documentation_key": "There is no documentation available for this key",
-            "documentation_redirect": "This documentation has been redirected to a new page",
             "show_more": "Show More",
             "view_on_osm": "View on openstreetmap.org",
             "all_fields": "All fields",
@@ -512,6 +511,7 @@
             "choose": "Select feature type",
             "results": "{n} results for {search}",
             "reference": "View on OpenStreetMap Wiki",
+            "edit_reference": "Edit or translate on OSM Wiki",
             "back_tooltip": "Change feature",
             "remove": "Remove",
             "search": "Search",

modules/services/osm_wikibase.js

@@ -4,7 +4,6 @@ import _forEach from 'lodash-es/forEach';
 import { json as d3_json } from 'd3-request';
 
 import { utilQsString } from '../util';
-import { currentLocale } from '../util/locale';
 
 
 var apibase = 'https://wiki.openstreetmap.org/w/api.php';
@@ -38,28 +37,140 @@ export default {
     },
 
 
-    docs: function(params, callback) {
+    /** List of data items representing language regions.
+     *  To regenerate, use Sophox query:  http://tinyurl.com/y6v9ne2c (every instance of Q6999)
+     *  A less accurate list can be seen here (everything that links to Q6999):
+     *  https://wiki.openstreetmap.org/w/index.php?title=Special%3AWhatLinksHere&target=Item%3AQ6999&namespace=120
+     */
+    regionCodes: {
+        ar: 'Q7780', az: 'Q7781', bg: 'Q7782', bn: 'Q7783', ca: 'Q7784', cs: 'Q7785', da: 'Q7786',
+        de: 'Q6994', el: 'Q7787', es: 'Q7788', et: 'Q7789', fa: 'Q7790', fi: 'Q7791', fr: 'Q7792',
+        gl: 'Q7793', hr: 'Q7794', ht: 'Q7795', hu: 'Q7796', id: 'Q7797', it: 'Q7798', ja: 'Q7799',
+        ko: 'Q7800', lt: 'Q7801', lv: 'Q7802', ms: 'Q7803', nl: 'Q7804', no: 'Q7805', pl: 'Q7806',
+        pt: 'Q7807', ro: 'Q7808', ru: 'Q7809', sk: 'Q7810', sq: 'Q7811', sv: 'Q7812', tr: 'Q7813',
+        uk: 'Q7814', vi: 'Q7815', yue: 'Q7816', 'zh-hans': 'Q7817', 'zh-hant': 'Q7818',
+    },
+
+
+    /**
+     * Get the best value for the property, or undefined if not found
+     * @param entity object from wikibase
+     * @param property string e.g. 'P4' for image
+     * @param langCode string e.g. 'fr' for French
+     */
+    claimToValue: function(entity, property, langCode) {
+        if (!entity.claims[property]) return undefined;
+        var region = this.regionCodes[langCode];
+        var preferredPick, regionPick;
+        _forEach(entity.claims[property], function(stmt) {
+            // If exists, use value limited to the needed language (has a qualifier P26 = region)
+            // Or if not found, use the first value with the "preferred" rank
+            if (!preferredPick && stmt.rank === 'preferred') {
+                preferredPick = stmt;
+            }
+            if (stmt.qualifiers && stmt.qualifiers.P26 && stmt.qualifiers.P26[0].datavalue.value.id === region) {
+                regionPick = stmt;
+            }
+        });
+        var result = regionPick || preferredPick;
+
+        if (result) {
+            var datavalue = result.mainsnak.datavalue;
+            return datavalue.type === 'wikibase-entityid' ? datavalue.value.id : datavalue.value;
+        } else {
+            return undefined;
+        }
+    },
+
+
+    getDescription: function(entity) {
+        if (entity.descriptions) {
+            // Assume that there will be at most two languages because of
+            // how we request it: English + possibly another one.
+            // Pick non-English description if available (if we have more than one)
+            var langs = Object.keys(entity.descriptions);
+            if (langs.length) {
+                var lng = langs.length > 1 && langs[0] === 'en' ? langs[1] : langs[0];
+                return entity.descriptions[lng].value;
+            }
+        }
+        return undefined;
+    },
+
+
+    toSitelink: function(key, value) {
+        var result = value ? 'Tag:' + key + '=' + value : 'Key:' + key;
+        return result.replace(/_/g, ' ').trim();
+    },
+
+
+    getEntity: function(params, callback) {
         var doRequest = params.debounce ? debouncedRequest : request;
 
-        // if (params.value) path = 'tag/wiki_pages?';
-        // else if (params.rtype) path = 'relation/wiki_pages?';
+        var titles = [];
+        var languages = ['en'];
+        var result = {};
+        var keySitelink = this.toSitelink(params.key);
+        var tagSitelink = params.value ? this.toSitelink(params.key, params.value) : false;
+
+        if (params.langCode && params.langCode !== 'en') {
+            languages.push(params.langCode);
+        }
+
+        if (_wikibaseCache[keySitelink]) {
+            result.key = _wikibaseCache[keySitelink];
+        } else {
+            titles.push(keySitelink);
+        }
+
+        if (tagSitelink) {
+            if (_wikibaseCache[tagSitelink]) {
+                result.key = _wikibaseCache[tagSitelink];
+            } else {
+                titles.push(tagSitelink);
+            }
+        }
+
+        if (!titles.length) {
+            // Nothing to do, we already had everything in the cache
+            return callback(null, result);
+        }
 
         var obj = {
             action: 'wbgetentities',
             sites: 'wiki',
-            titles: 'Tag:amenity=parking',
-            languages: 'en',
+            titles: titles.join('|'),
+            languages: languages.join('|'),
             origin: '*',
-            formatversion: 2,
-            format: 'json'
-        }
+            format: 'json',
+            // There is an MW Wikibase API bug https://phabricator.wikimedia.org/T212069
+            // We shouldn't use v1 until it gets fixed, but should switch to it afterwards
+            // formatversion: 2,
+        };
+
         var url = apibase + '?' + utilQsString(obj);
         doRequest(url, function(err, d) {
             if (err) {
                 callback(err);
+            } else if (!d.success || d.error) {
+                callback(d.error.messages.map(function(v) { return v.html['*']; }).join('<br>'));
             } else {
-                _wikibaseCache[url] = d.data;
-                callback(null, d.data);
+                _forEach(d.entities, function(res) {
+                    if (res.missing !== '') {
+                        var title = res.sitelinks.wiki.title;
+                        if (title === keySitelink) {
+                            _wikibaseCache[keySitelink] = res;
+                            result.key = res;
+                        } else if (title === tagSitelink) {
+                            _wikibaseCache[tagSitelink] = res;
+                            result.tag = res;
+                        } else {
+                            console.log('Unexpected title ' + title);
+                        }
+                    }
+                });
+
+                callback(null, result);
             }
         });
     },

modules/ui/tag_reference.js

@@ -1,6 +1,3 @@
-import _find from 'lodash-es/find';
-import _omit from 'lodash-es/omit';
-
 import {
     event as d3_event,
     select as d3_select
@@ -10,10 +7,10 @@ import { t } from '../util/locale';
 import { utilDetect } from '../util/detect';
 import { services } from '../services';
 import { svgIcon } from '../svg';
+import { utilQsString } from '../util';
 
 
 export function uiTagReference(tag) {
-    // var taginfo = services.taginfo;
     var wikibase = services.osmWikibase;
     var tagReference = {};
 
@@ -22,33 +19,40 @@ export function uiTagReference(tag) {
     var _loaded;
     var _showing;
 
+    /**
+     * @returns {{itemTitle: String, description: String, image: String|null}|null}
+     **/
+    function findLocal(data) {
+        var entity = data.tag || data.key;
+        if (!entity) return null;
 
-    // function findLocal(data) {
-    //     var locale = utilDetect().locale.toLowerCase();
-    //     var localized;
+        var result = {
+            title: entity.title,
+            description: wikibase.getDescription(entity),
+        };
 
-    //     if (locale !== 'pt-br') {  // see #3776, prefer 'pt' over 'pt-br'
-    //         localized = _find(data, function(d) {
-    //             return d.lang.toLowerCase() === locale;
-    //         });
-    //         if (localized) return localized;
-    //     }
+        if (entity.claims) {
+            var langCode = utilDetect().locale.toLowerCase();
+            var url;
+            var image = wikibase.claimToValue(entity, 'P4', langCode);
+            if (image) {
+                url = 'https://commons.wikimedia.org/w/index.php';
+            } else {
+                image = wikibase.claimToValue(entity, 'P28', langCode);
+                if (image) {
+                    url = 'https://wiki.openstreetmap.org/w/index.php';
+                }
+            }
+            if (image) {
+                result.image = {
+                    url: url,
+                    title: 'Special:Redirect/file/' + image
+                };
+            }
+        }
 
-    //     // try the non-regional version of a language, like
-    //     // 'en' if the language is 'en-US'
-    //     if (locale.indexOf('-') !== -1) {
-    //         var first = locale.split('-')[0];
-    //         localized = _find(data, function(d) {
-    //             return d.lang.toLowerCase() === first;
-    //         });
-    //         if (localized) return localized;
-    //     }
-
-    //     // finally fall back to english
-    //     return _find(data, function(d) {
-    //         return d.lang.toLowerCase() === 'en';
-    //     });
-    // }
+        return result;
+    }
 
 
     function load(param) {
@@ -57,33 +61,34 @@ export function uiTagReference(tag) {
         _button
             .classed('tag-reference-loading', true);
 
-        wikibase.docs(param, function show(err, data) {
+        wikibase.getEntity(param, function show(err, data) {
             var docs;
             if (!err && data) {
-                // docs = findLocal(data);
-                debugger;
+                docs = findLocal(data);
             }
 
             _body.html('');
 
             if (!docs || !docs.title) {
-                if (param.hasOwnProperty('value')) {
-                    load(_omit(param, 'value'));   // retry with key only
-                } else {
-                    _body
-                        .append('p')
-                        .attr('class', 'tag-reference-description')
-                        .text(t('inspector.no_documentation_key'));
-                    done();
-                }
+                _body
+                    .append('p')
+                    .attr('class', 'tag-reference-description')
+                    .text(t('inspector.no_documentation_key'));
+                done();
                 return;
             }
 
-            if (docs.image && docs.image.thumb_url_prefix) {
+            if (docs.image) {
+                var imageUrl = docs.image.url + '?' + utilQsString({
+                    title: docs.image.title,
+                    width: 100,
+                    height: 100,
+                });
+
                 _body
                     .append('img')
                     .attr('class', 'tag-reference-wiki-image')
-                    .attr('src', docs.image.thumb_url_prefix + '100' + docs.image.thumb_url_suffix)
+                    .attr('src', imageUrl)
                     .on('load', function() { done(); })
                     .on('error', function() { d3_select(this).remove(); done(); });
             } else {
@@ -93,7 +98,7 @@ export function uiTagReference(tag) {
             _body
                 .append('p')
                 .attr('class', 'tag-reference-description')
-                .text(docs.description || t('inspector.documentation_redirect'));
+                .text(docs.description || t('inspector.no_documentation_key'));
 
             _body
                 .append('a')
@@ -173,6 +178,7 @@ export function uiTagReference(tag) {
                 } else if (_loaded) {
                     done();
                 } else {
+                    tag.langCode = utilDetect().locale.toLowerCase();
                     load(tag);
                 }
             });

test/index.html

@@ -110,6 +110,7 @@
     <script src='spec/services/nominatim.js'></script>
     <script src='spec/services/openstreetcam.js'></script>
     <script src='spec/services/osm.js'></script>
+    <script src='spec/services/osm_wikibase.js'></script>
     <script src='spec/services/streetside.js'></script>
     <script src='spec/services/taginfo.js'></script>
 

test/spec/services/osm_wikibase.js

@@ -1,49 +1,301 @@
-describe('iD.serviceOsmWikibase', function() {
-    var server, wikibase;
+describe('iD.serviceOsmWikibase', function () {
+  var server, wikibase;
+
+  before(function () {
+    iD.services.osmWikibase = iD.serviceOsmWikibase;
+  });
+
+  after(function () {
+    delete iD.services.osmWikibase;
+  });
+
+  beforeEach(function () {
+    wikibase = iD.services.osmWikibase;
+    wikibase.init();
+    server = sinon.fakeServer.create();
+  });
+
+  afterEach(function () {
+    server.restore();
+  });
 
 
-    before(function() {
-        iD.services.osmWikibase = iD.serviceOsmWikibase;
-    });
+  function query(url) {
+    return iD.utilStringQs(url.substring(url.indexOf('?') + 1));
+  }
 
-    after(function() {
-        delete iD.services.osmWikibase;
-    });
-
-    beforeEach(function() {
-        wikibase = iD.services.osmWikibase;
-        wikibase.init();
-        server = sinon.fakeServer.create();
-    });
-
-    afterEach(function() {
-        server.restore();
-    });
-
-
-    function query(url) {
-        return iD.utilStringQs(url.substring(url.indexOf('?') + 1));
+  var keyData = {
+    pageid: 205725,
+    ns: 120,
+    title: 'Item:Q61',
+    lastrevid: 1721242,
+    modified: '2018-12-18T07:00:43Z',
+    type: 'item',
+    id: 'Q61',
+    labels: {
+      en: {language: 'en', value: 'amenity'}
+    },
+    descriptions: {
+      en: {language: 'en', value: 'For describing useful and important facilities for visitors and residents.'}
+    },
+    aliases: {},
+    claims: {
+      P2: [ // instance of
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q7'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          rank: 'normal'
+        }
+      ],
+      P16: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'string',
+            datavalue: {value: 'amenity', type: 'string'}
+          },
+          type: 'statement',
+          rank: 'normal'
+        }
+      ],
+      P25: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q4679'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          rank: 'normal'
+        }
+      ],
+      P9: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q8'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          rank: 'normal'
+        }
+      ],
+      P6: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q15'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          rank: 'preferred'
+        },
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q14'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          qualifiers: {
+            P26: [
+              {
+                snaktype: 'value',
+                datatype: 'wikibase-item',
+                datavalue: {value: {'entity-type': 'item', id: 'Q6994'}, type: 'wikibase-entityid'}
+              }
+            ]
+          },
+          rank: 'normal'
+        }
+      ],
+      P28: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'string',
+            datavalue: {value: 'Mapping-Features-Parking-Lot.png', type: 'string'}
+          },
+          type: 'statement',
+          rank: 'normal'
+        }
+      ]
+    },
+    sitelinks: {
+      wiki: {
+        site: 'wiki',
+        title: 'Key:amenity',
+        badges: []
+      }
     }
+  };
 
+  var tagData = {
+    pageid: 210934,
+    ns: 120,
+    title: 'Item:Q4904',
+    lastrevid: 1718041,
+    modified: '2018-12-18T03:51:05Z',
+    type: 'item',
+    id: 'Q4904',
+    labels: {
+      en: {language: 'en', value: 'amenity=parking'}
+    },
+    descriptions: {
+      en: {language: 'en', value: 'A place for parking cars'},
+      fr: {language: 'fr', value: 'Un lieu pour garer des voitures'}
+    },
+    aliases: {},
+    claims: {
+      P2: [ // instance of = Q2 (tag)
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q2'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          rank: 'normal'
+        }
+      ],
+      P19: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'string',
+            datavalue: {value: 'amenity=parking', type: 'string'}
+          },
+          type: 'statement',
+          rank: 'normal'
+        }
+      ],
+      P10: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q61'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          rank: 'normal'
+        }
+      ],
+      P4: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'commonsMedia',
+            datavalue: {value: 'Primary image.jpg', type: 'string'}
+          },
+          type: 'statement',
+          rank: 'preferred'
+        }
+      ],
+      P6: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q14'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          rank: 'preferred'
+        },
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q13'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          qualifiers: {
+            P26: [
+              {
+                snaktype: 'value',
+                datatype: 'wikibase-item',
+                datavalue: {value: {'entity-type': 'item', id: 'Q6994'}, type: 'wikibase-entityid'}
+              }
+            ]
+          },
+          rank: 'normal'
+        }
+      ],
+      P25: [
+        {
+          mainsnak: {
+            snaktype: 'value',
+            datatype: 'wikibase-item',
+            datavalue: {value: {'entity-type': 'item', id: 'Q4679'}, type: 'wikibase-entityid'}
+          },
+          type: 'statement',
+          rank: 'normal'
+        }
+      ]
+    },
+    sitelinks: {
+      wiki: {
+        site: 'wiki',
+        title: 'Tag:amenity=parking',
+        badges: []
+      }
+    }
+  };
 
-    describe('#docs', function() {
-        it('calls the given callback with the results of the docs query', function() {
-            var callback = sinon.spy();
-            wikibase.docs({key: 'amenity', value: 'parking'}, callback);
+  describe('#getEntity', function () {
+    it('calls the given callback with the results of the getEntity data item query', function () {
+      var callback = sinon.spy();
+      wikibase.getEntity({key: 'amenity', value: 'parking', langCode: 'fr'}, callback);
 
-            server.respondWith('GET', /\/tag\/wiki_page/,
-                [200, { 'Content-Type': 'application/json' },
-                    '{"data":[{"on_way":false,"lang":"en","on_area":true,"image":"File:Car park2.jpg"}]}']
-            );
-            server.respond();
+      server.respondWith('GET', /action=wbgetentities/,
+        [200, {'Content-Type': 'application/json'}, JSON.stringify({
+          entities: {
+            Q61: keyData,
+            Q4904: tagData
+          },
+          success: 1
+        })]
+      );
+      server.respond();
 
-            expect(query(server.requests[0].url)).to.eql(
-                {key: 'amenity', value: 'parking'}
-            );
-            expect(callback).to.have.been.calledWith(
-                null, [{'on_way':false,'lang':'en','on_area':true,'image':'File:Car park2.jpg'}]
-            );
-        });
+      expect(query(server.requests[0].url)).to.eql(
+        {
+          action: 'wbgetentities',
+          format: 'json',
+          languages: 'en|fr',
+          origin: '*',
+          sites: 'wiki',
+          titles: 'Key:amenity|Tag:amenity=parking',
+        }
+      );
+      expect(callback).to.have.been.calledWith(null, {key: keyData, tag: tagData});
     });
+  });
+
+
+  it('creates correct sitelinks', function () {
+    expect(wikibase.toSitelink('amenity')).to.eql('Key:amenity');
+    expect(wikibase.toSitelink('amenity_')).to.eql('Key:amenity');
+    expect(wikibase.toSitelink('_amenity_')).to.eql('Key: amenity');
+    expect(wikibase.toSitelink('amenity or_not_')).to.eql('Key:amenity or not');
+    expect(wikibase.toSitelink('amenity', 'parking')).to.eql('Tag:amenity=parking');
+    expect(wikibase.toSitelink(' amenity_', '_parking_')).to.eql('Tag: amenity = parking');
+    expect(wikibase.toSitelink('amenity or_not', '_park ing_')).to.eql('Tag:amenity or not= park ing');
+  });
+
+  it('gets correct value from entity', function () {
+    expect(wikibase.claimToValue(tagData, 'P4', 'en')).to.eql('Primary image.jpg');
+    expect(wikibase.claimToValue(keyData, 'P6', 'en')).to.eql('Q15');
+    expect(wikibase.claimToValue(keyData, 'P6', 'fr')).to.eql('Q15');
+    expect(wikibase.claimToValue(keyData, 'P6', 'de')).to.eql('Q14');
+  });
+
+  it('gets correct description from entity', function () {
+    expect(wikibase.getDescription(tagData)).to.eql('Un lieu pour garer des voitures');
+    expect(wikibase.getDescription(keyData)).to.eql('For describing useful and important facilities for visitors and residents.');
+  });
 
 });