From b2810105a5cd6d5b13df6a720c8fc8c23edf3eff Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 22 Dec 2018 00:19:10 -0500 Subject: [PATCH] Implement support for multilingual descriptions from wiki data items * Takes data directly from the Wikibase data items (OSM Wiki) https://wiki.openstreetmap.org/wiki/OpenStreetMap:Data_Items * Understands the difference in regions - e.g. will show different images depending on the local settings * Perf: Single request will get both the tag and key description --- data/core.yaml | 2 +- dist/locales/en.json | 2 +- modules/services/osm_wikibase.js | 133 +++++++++++- modules/ui/tag_reference.js | 92 ++++---- test/index.html | 1 + test/spec/services/osm_wikibase.js | 328 +++++++++++++++++++++++++---- 6 files changed, 464 insertions(+), 94 deletions(-) diff --git a/data/core.yaml b/data/core.yaml index c74f1a977..a06894368 100644 --- a/data/core.yaml +++ b/data/core.yaml @@ -405,7 +405,6 @@ en: inspector: no_documentation_combination: There is no documentation available for this tag combination no_documentation_key: There is no documentation available for this key - documentation_redirect: This documentation has been redirected to a new page show_more: Show More view_on_osm: View on openstreetmap.org all_fields: All fields @@ -418,6 +417,7 @@ en: choose: Select feature type results: "{n} results for {search}" reference: View on OpenStreetMap Wiki + edit_reference: Edit or translate on OSM Wiki back_tooltip: Change feature remove: Remove search: Search diff --git a/dist/locales/en.json b/dist/locales/en.json index da3671d75..f093e6e7d 100644 --- a/dist/locales/en.json +++ b/dist/locales/en.json @@ -499,7 +499,6 @@ "inspector": { "no_documentation_combination": "There is no documentation available for this tag combination", "no_documentation_key": "There is no documentation available for this key", - "documentation_redirect": "This documentation has been redirected to a new page", "show_more": "Show More", "view_on_osm": "View on openstreetmap.org", "all_fields": "All fields", @@ -512,6 +511,7 @@ "choose": "Select feature type", "results": "{n} results for {search}", "reference": "View on OpenStreetMap Wiki", + "edit_reference": "Edit or translate on OSM Wiki", "back_tooltip": "Change feature", "remove": "Remove", "search": "Search", diff --git a/modules/services/osm_wikibase.js b/modules/services/osm_wikibase.js index ef1d2eafa..22db05bd9 100644 --- a/modules/services/osm_wikibase.js +++ b/modules/services/osm_wikibase.js @@ -4,7 +4,6 @@ import _forEach from 'lodash-es/forEach'; import { json as d3_json } from 'd3-request'; import { utilQsString } from '../util'; -import { currentLocale } from '../util/locale'; var apibase = 'https://wiki.openstreetmap.org/w/api.php'; @@ -38,28 +37,140 @@ export default { }, - docs: function(params, callback) { + /** List of data items representing language regions. + * To regenerate, use Sophox query: http://tinyurl.com/y6v9ne2c (every instance of Q6999) + * A less accurate list can be seen here (everything that links to Q6999): + * https://wiki.openstreetmap.org/w/index.php?title=Special%3AWhatLinksHere&target=Item%3AQ6999&namespace=120 + */ + regionCodes: { + ar: 'Q7780', az: 'Q7781', bg: 'Q7782', bn: 'Q7783', ca: 'Q7784', cs: 'Q7785', da: 'Q7786', + de: 'Q6994', el: 'Q7787', es: 'Q7788', et: 'Q7789', fa: 'Q7790', fi: 'Q7791', fr: 'Q7792', + gl: 'Q7793', hr: 'Q7794', ht: 'Q7795', hu: 'Q7796', id: 'Q7797', it: 'Q7798', ja: 'Q7799', + ko: 'Q7800', lt: 'Q7801', lv: 'Q7802', ms: 'Q7803', nl: 'Q7804', no: 'Q7805', pl: 'Q7806', + pt: 'Q7807', ro: 'Q7808', ru: 'Q7809', sk: 'Q7810', sq: 'Q7811', sv: 'Q7812', tr: 'Q7813', + uk: 'Q7814', vi: 'Q7815', yue: 'Q7816', 'zh-hans': 'Q7817', 'zh-hant': 'Q7818', + }, + + + /** + * Get the best value for the property, or undefined if not found + * @param entity object from wikibase + * @param property string e.g. 'P4' for image + * @param langCode string e.g. 'fr' for French + */ + claimToValue: function(entity, property, langCode) { + if (!entity.claims[property]) return undefined; + var region = this.regionCodes[langCode]; + var preferredPick, regionPick; + _forEach(entity.claims[property], function(stmt) { + // If exists, use value limited to the needed language (has a qualifier P26 = region) + // Or if not found, use the first value with the "preferred" rank + if (!preferredPick && stmt.rank === 'preferred') { + preferredPick = stmt; + } + if (stmt.qualifiers && stmt.qualifiers.P26 && stmt.qualifiers.P26[0].datavalue.value.id === region) { + regionPick = stmt; + } + }); + var result = regionPick || preferredPick; + + if (result) { + var datavalue = result.mainsnak.datavalue; + return datavalue.type === 'wikibase-entityid' ? datavalue.value.id : datavalue.value; + } else { + return undefined; + } + }, + + + getDescription: function(entity) { + if (entity.descriptions) { + // Assume that there will be at most two languages because of + // how we request it: English + possibly another one. + // Pick non-English description if available (if we have more than one) + var langs = Object.keys(entity.descriptions); + if (langs.length) { + var lng = langs.length > 1 && langs[0] === 'en' ? langs[1] : langs[0]; + return entity.descriptions[lng].value; + } + } + return undefined; + }, + + + toSitelink: function(key, value) { + var result = value ? 'Tag:' + key + '=' + value : 'Key:' + key; + return result.replace(/_/g, ' ').trim(); + }, + + + getEntity: function(params, callback) { var doRequest = params.debounce ? debouncedRequest : request; - // if (params.value) path = 'tag/wiki_pages?'; - // else if (params.rtype) path = 'relation/wiki_pages?'; + var titles = []; + var languages = ['en']; + var result = {}; + var keySitelink = this.toSitelink(params.key); + var tagSitelink = params.value ? this.toSitelink(params.key, params.value) : false; + + if (params.langCode && params.langCode !== 'en') { + languages.push(params.langCode); + } + + if (_wikibaseCache[keySitelink]) { + result.key = _wikibaseCache[keySitelink]; + } else { + titles.push(keySitelink); + } + + if (tagSitelink) { + if (_wikibaseCache[tagSitelink]) { + result.key = _wikibaseCache[tagSitelink]; + } else { + titles.push(tagSitelink); + } + } + + if (!titles.length) { + // Nothing to do, we already had everything in the cache + return callback(null, result); + } var obj = { action: 'wbgetentities', sites: 'wiki', - titles: 'Tag:amenity=parking', - languages: 'en', + titles: titles.join('|'), + languages: languages.join('|'), origin: '*', - formatversion: 2, - format: 'json' - } + format: 'json', + // There is an MW Wikibase API bug https://phabricator.wikimedia.org/T212069 + // We shouldn't use v1 until it gets fixed, but should switch to it afterwards + // formatversion: 2, + }; + var url = apibase + '?' + utilQsString(obj); doRequest(url, function(err, d) { if (err) { callback(err); + } else if (!d.success || d.error) { + callback(d.error.messages.map(function(v) { return v.html['*']; }).join('
')); } else { - _wikibaseCache[url] = d.data; - callback(null, d.data); + _forEach(d.entities, function(res) { + if (res.missing !== '') { + var title = res.sitelinks.wiki.title; + if (title === keySitelink) { + _wikibaseCache[keySitelink] = res; + result.key = res; + } else if (title === tagSitelink) { + _wikibaseCache[tagSitelink] = res; + result.tag = res; + } else { + console.log('Unexpected title ' + title); + } + } + }); + + callback(null, result); } }); }, diff --git a/modules/ui/tag_reference.js b/modules/ui/tag_reference.js index a1dcf15d9..d4ce97f5c 100644 --- a/modules/ui/tag_reference.js +++ b/modules/ui/tag_reference.js @@ -1,6 +1,3 @@ -import _find from 'lodash-es/find'; -import _omit from 'lodash-es/omit'; - import { event as d3_event, select as d3_select @@ -10,10 +7,10 @@ import { t } from '../util/locale'; import { utilDetect } from '../util/detect'; import { services } from '../services'; import { svgIcon } from '../svg'; +import { utilQsString } from '../util'; export function uiTagReference(tag) { - // var taginfo = services.taginfo; var wikibase = services.osmWikibase; var tagReference = {}; @@ -22,33 +19,40 @@ export function uiTagReference(tag) { var _loaded; var _showing; + /** + * @returns {{itemTitle: String, description: String, image: String|null}|null} + **/ + function findLocal(data) { + var entity = data.tag || data.key; + if (!entity) return null; - // function findLocal(data) { - // var locale = utilDetect().locale.toLowerCase(); - // var localized; + var result = { + title: entity.title, + description: wikibase.getDescription(entity), + }; - // if (locale !== 'pt-br') { // see #3776, prefer 'pt' over 'pt-br' - // localized = _find(data, function(d) { - // return d.lang.toLowerCase() === locale; - // }); - // if (localized) return localized; - // } + if (entity.claims) { + var langCode = utilDetect().locale.toLowerCase(); + var url; + var image = wikibase.claimToValue(entity, 'P4', langCode); + if (image) { + url = 'https://commons.wikimedia.org/w/index.php'; + } else { + image = wikibase.claimToValue(entity, 'P28', langCode); + if (image) { + url = 'https://wiki.openstreetmap.org/w/index.php'; + } + } + if (image) { + result.image = { + url: url, + title: 'Special:Redirect/file/' + image + }; + } + } - // // try the non-regional version of a language, like - // // 'en' if the language is 'en-US' - // if (locale.indexOf('-') !== -1) { - // var first = locale.split('-')[0]; - // localized = _find(data, function(d) { - // return d.lang.toLowerCase() === first; - // }); - // if (localized) return localized; - // } - - // // finally fall back to english - // return _find(data, function(d) { - // return d.lang.toLowerCase() === 'en'; - // }); - // } + return result; + } function load(param) { @@ -57,33 +61,34 @@ export function uiTagReference(tag) { _button .classed('tag-reference-loading', true); - wikibase.docs(param, function show(err, data) { + wikibase.getEntity(param, function show(err, data) { var docs; if (!err && data) { - // docs = findLocal(data); - debugger; + docs = findLocal(data); } _body.html(''); if (!docs || !docs.title) { - if (param.hasOwnProperty('value')) { - load(_omit(param, 'value')); // retry with key only - } else { - _body - .append('p') - .attr('class', 'tag-reference-description') - .text(t('inspector.no_documentation_key')); - done(); - } + _body + .append('p') + .attr('class', 'tag-reference-description') + .text(t('inspector.no_documentation_key')); + done(); return; } - if (docs.image && docs.image.thumb_url_prefix) { + if (docs.image) { + var imageUrl = docs.image.url + '?' + utilQsString({ + title: docs.image.title, + width: 100, + height: 100, + }); + _body .append('img') .attr('class', 'tag-reference-wiki-image') - .attr('src', docs.image.thumb_url_prefix + '100' + docs.image.thumb_url_suffix) + .attr('src', imageUrl) .on('load', function() { done(); }) .on('error', function() { d3_select(this).remove(); done(); }); } else { @@ -93,7 +98,7 @@ export function uiTagReference(tag) { _body .append('p') .attr('class', 'tag-reference-description') - .text(docs.description || t('inspector.documentation_redirect')); + .text(docs.description || t('inspector.no_documentation_key')); _body .append('a') @@ -173,6 +178,7 @@ export function uiTagReference(tag) { } else if (_loaded) { done(); } else { + tag.langCode = utilDetect().locale.toLowerCase(); load(tag); } }); diff --git a/test/index.html b/test/index.html index b455624b3..8e78ab96d 100644 --- a/test/index.html +++ b/test/index.html @@ -110,6 +110,7 @@ + diff --git a/test/spec/services/osm_wikibase.js b/test/spec/services/osm_wikibase.js index 1c87a6f8e..2536fd246 100644 --- a/test/spec/services/osm_wikibase.js +++ b/test/spec/services/osm_wikibase.js @@ -1,49 +1,301 @@ -describe('iD.serviceOsmWikibase', function() { - var server, wikibase; +describe('iD.serviceOsmWikibase', function () { + var server, wikibase; + + before(function () { + iD.services.osmWikibase = iD.serviceOsmWikibase; + }); + + after(function () { + delete iD.services.osmWikibase; + }); + + beforeEach(function () { + wikibase = iD.services.osmWikibase; + wikibase.init(); + server = sinon.fakeServer.create(); + }); + + afterEach(function () { + server.restore(); + }); - before(function() { - iD.services.osmWikibase = iD.serviceOsmWikibase; - }); + function query(url) { + return iD.utilStringQs(url.substring(url.indexOf('?') + 1)); + } - after(function() { - delete iD.services.osmWikibase; - }); - - beforeEach(function() { - wikibase = iD.services.osmWikibase; - wikibase.init(); - server = sinon.fakeServer.create(); - }); - - afterEach(function() { - server.restore(); - }); - - - function query(url) { - return iD.utilStringQs(url.substring(url.indexOf('?') + 1)); + var keyData = { + pageid: 205725, + ns: 120, + title: 'Item:Q61', + lastrevid: 1721242, + modified: '2018-12-18T07:00:43Z', + type: 'item', + id: 'Q61', + labels: { + en: {language: 'en', value: 'amenity'} + }, + descriptions: { + en: {language: 'en', value: 'For describing useful and important facilities for visitors and residents.'} + }, + aliases: {}, + claims: { + P2: [ // instance of + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q7'}, type: 'wikibase-entityid'} + }, + type: 'statement', + rank: 'normal' + } + ], + P16: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'string', + datavalue: {value: 'amenity', type: 'string'} + }, + type: 'statement', + rank: 'normal' + } + ], + P25: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q4679'}, type: 'wikibase-entityid'} + }, + type: 'statement', + rank: 'normal' + } + ], + P9: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q8'}, type: 'wikibase-entityid'} + }, + type: 'statement', + rank: 'normal' + } + ], + P6: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q15'}, type: 'wikibase-entityid'} + }, + type: 'statement', + rank: 'preferred' + }, + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q14'}, type: 'wikibase-entityid'} + }, + type: 'statement', + qualifiers: { + P26: [ + { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q6994'}, type: 'wikibase-entityid'} + } + ] + }, + rank: 'normal' + } + ], + P28: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'string', + datavalue: {value: 'Mapping-Features-Parking-Lot.png', type: 'string'} + }, + type: 'statement', + rank: 'normal' + } + ] + }, + sitelinks: { + wiki: { + site: 'wiki', + title: 'Key:amenity', + badges: [] + } } + }; + var tagData = { + pageid: 210934, + ns: 120, + title: 'Item:Q4904', + lastrevid: 1718041, + modified: '2018-12-18T03:51:05Z', + type: 'item', + id: 'Q4904', + labels: { + en: {language: 'en', value: 'amenity=parking'} + }, + descriptions: { + en: {language: 'en', value: 'A place for parking cars'}, + fr: {language: 'fr', value: 'Un lieu pour garer des voitures'} + }, + aliases: {}, + claims: { + P2: [ // instance of = Q2 (tag) + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q2'}, type: 'wikibase-entityid'} + }, + type: 'statement', + rank: 'normal' + } + ], + P19: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'string', + datavalue: {value: 'amenity=parking', type: 'string'} + }, + type: 'statement', + rank: 'normal' + } + ], + P10: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q61'}, type: 'wikibase-entityid'} + }, + type: 'statement', + rank: 'normal' + } + ], + P4: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'commonsMedia', + datavalue: {value: 'Primary image.jpg', type: 'string'} + }, + type: 'statement', + rank: 'preferred' + } + ], + P6: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q14'}, type: 'wikibase-entityid'} + }, + type: 'statement', + rank: 'preferred' + }, + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q13'}, type: 'wikibase-entityid'} + }, + type: 'statement', + qualifiers: { + P26: [ + { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q6994'}, type: 'wikibase-entityid'} + } + ] + }, + rank: 'normal' + } + ], + P25: [ + { + mainsnak: { + snaktype: 'value', + datatype: 'wikibase-item', + datavalue: {value: {'entity-type': 'item', id: 'Q4679'}, type: 'wikibase-entityid'} + }, + type: 'statement', + rank: 'normal' + } + ] + }, + sitelinks: { + wiki: { + site: 'wiki', + title: 'Tag:amenity=parking', + badges: [] + } + } + }; - describe('#docs', function() { - it('calls the given callback with the results of the docs query', function() { - var callback = sinon.spy(); - wikibase.docs({key: 'amenity', value: 'parking'}, callback); + describe('#getEntity', function () { + it('calls the given callback with the results of the getEntity data item query', function () { + var callback = sinon.spy(); + wikibase.getEntity({key: 'amenity', value: 'parking', langCode: 'fr'}, callback); - server.respondWith('GET', /\/tag\/wiki_page/, - [200, { 'Content-Type': 'application/json' }, - '{"data":[{"on_way":false,"lang":"en","on_area":true,"image":"File:Car park2.jpg"}]}'] - ); - server.respond(); + server.respondWith('GET', /action=wbgetentities/, + [200, {'Content-Type': 'application/json'}, JSON.stringify({ + entities: { + Q61: keyData, + Q4904: tagData + }, + success: 1 + })] + ); + server.respond(); - expect(query(server.requests[0].url)).to.eql( - {key: 'amenity', value: 'parking'} - ); - expect(callback).to.have.been.calledWith( - null, [{'on_way':false,'lang':'en','on_area':true,'image':'File:Car park2.jpg'}] - ); - }); + expect(query(server.requests[0].url)).to.eql( + { + action: 'wbgetentities', + format: 'json', + languages: 'en|fr', + origin: '*', + sites: 'wiki', + titles: 'Key:amenity|Tag:amenity=parking', + } + ); + expect(callback).to.have.been.calledWith(null, {key: keyData, tag: tagData}); }); + }); + + + it('creates correct sitelinks', function () { + expect(wikibase.toSitelink('amenity')).to.eql('Key:amenity'); + expect(wikibase.toSitelink('amenity_')).to.eql('Key:amenity'); + expect(wikibase.toSitelink('_amenity_')).to.eql('Key: amenity'); + expect(wikibase.toSitelink('amenity or_not_')).to.eql('Key:amenity or not'); + expect(wikibase.toSitelink('amenity', 'parking')).to.eql('Tag:amenity=parking'); + expect(wikibase.toSitelink(' amenity_', '_parking_')).to.eql('Tag: amenity = parking'); + expect(wikibase.toSitelink('amenity or_not', '_park ing_')).to.eql('Tag:amenity or not= park ing'); + }); + + it('gets correct value from entity', function () { + expect(wikibase.claimToValue(tagData, 'P4', 'en')).to.eql('Primary image.jpg'); + expect(wikibase.claimToValue(keyData, 'P6', 'en')).to.eql('Q15'); + expect(wikibase.claimToValue(keyData, 'P6', 'fr')).to.eql('Q15'); + expect(wikibase.claimToValue(keyData, 'P6', 'de')).to.eql('Q14'); + }); + + it('gets correct description from entity', function () { + expect(wikibase.getDescription(tagData)).to.eql('Un lieu pour garer des voitures'); + expect(wikibase.getDescription(keyData)).to.eql('For describing useful and important facilities for visitors and residents.'); + }); });