Use CLDR for a translated language list in the localized name field instead of untranslated wmf site matrix (close #2457)

Translate language names in the community index list (close #4990)

data/core.yaml

@@ -391,6 +391,7 @@ en:
     localized_translation_label: Multilingual Name
     localized_translation_language: Choose language
     localized_translation_name: Name
+    language_and_code: "{language} ({code})"
   zoom_in_edit: Zoom in to edit
   login: Log In
   logout: Log Out

data/index.js

@@ -3,6 +3,7 @@ export { wikipedia as dataWikipedia } from 'wmf-sitematrix';
 export { dataAddressFormats } from './address-formats.json';
 export { dataDeprecated } from './deprecated.json';
 export { dataDiscarded } from './discarded.json';
+export { dataLanguages } from './languages.json';
 export { dataLocales } from './locales.json';
 export { dataPhoneFormats } from './phone-formats.json';
 export { dataShortcuts } from './shortcuts.json';

data/languages.json

@@ -0,0 +1,726 @@
+{
+    "dataLanguages": {
+        "af": {
+            "nativeName": "Afrikaans"
+        },
+        "agq": {
+            "nativeName": "Aghem"
+        },
+        "ak": {
+            "nativeName": "Akan"
+        },
+        "am": {
+            "nativeName": "አማርኛ"
+        },
+        "ar": {
+            "nativeName": "العربية"
+        },
+        "as": {
+            "nativeName": "অসমীয়া"
+        },
+        "asa": {
+            "nativeName": "Kipare"
+        },
+        "ast": {
+            "nativeName": "asturianu"
+        },
+        "az": {
+            "nativeName": "azərbaycan"
+        },
+        "az-Cyrl": {
+            "base": "az",
+            "script": "Cyrl"
+        },
+        "az-Latn": {
+            "base": "az",
+            "script": "Latn"
+        },
+        "bas": {
+            "nativeName": "Ɓàsàa"
+        },
+        "be": {
+            "nativeName": "беларуская"
+        },
+        "bem": {
+            "nativeName": "Ichibemba"
+        },
+        "bez": {
+            "nativeName": "Hibena"
+        },
+        "bg": {
+            "nativeName": "български"
+        },
+        "bm": {
+            "nativeName": "bamanakan"
+        },
+        "bn": {
+            "nativeName": "বাংলা"
+        },
+        "bo": {
+            "nativeName": "བོད་སྐད་"
+        },
+        "br": {
+            "nativeName": "brezhoneg"
+        },
+        "brx": {
+            "nativeName": "बड़ो"
+        },
+        "bs": {
+            "nativeName": "bosanski"
+        },
+        "bs-Cyrl": {
+            "base": "bs",
+            "script": "Cyrl"
+        },
+        "bs-Latn": {
+            "base": "bs",
+            "script": "Latn"
+        },
+        "ca": {
+            "nativeName": "català"
+        },
+        "ccp": {
+            "nativeName": "𑄌𑄋𑄴𑄟𑄳𑄦"
+        },
+        "ce": {
+            "nativeName": "нохчийн"
+        },
+        "ceb": {
+            "nativeName": "Cebuano"
+        },
+        "cgg": {
+            "nativeName": "Rukiga"
+        },
+        "chr": {
+            "nativeName": "ᏣᎳᎩ"
+        },
+        "ckb": {
+            "nativeName": "کوردیی ناوەندی"
+        },
+        "cs": {
+            "nativeName": "čeština"
+        },
+        "cu": {
+            "nativeName": "cu"
+        },
+        "cy": {
+            "nativeName": "Cymraeg"
+        },
+        "da": {
+            "nativeName": "dansk"
+        },
+        "dav": {
+            "nativeName": "Kitaita"
+        },
+        "de": {
+            "nativeName": "Deutsch"
+        },
+        "dje": {
+            "nativeName": "Zarmaciine"
+        },
+        "dsb": {
+            "nativeName": "dolnoserbšćina"
+        },
+        "dua": {
+            "nativeName": "duálá"
+        },
+        "dyo": {
+            "nativeName": "joola"
+        },
+        "dz": {
+            "nativeName": "རྫོང་ཁ"
+        },
+        "ebu": {
+            "nativeName": "Kĩembu"
+        },
+        "ee": {
+            "nativeName": "Eʋegbe"
+        },
+        "el": {
+            "nativeName": "Ελληνικά"
+        },
+        "en": {
+            "nativeName": "English"
+        },
+        "eo": {
+            "nativeName": "esperanto"
+        },
+        "es": {
+            "nativeName": "español"
+        },
+        "et": {
+            "nativeName": "eesti"
+        },
+        "eu": {
+            "nativeName": "euskara"
+        },
+        "ewo": {
+            "nativeName": "ewondo"
+        },
+        "fa": {
+            "nativeName": "فارسی"
+        },
+        "ff": {
+            "nativeName": "Pulaar"
+        },
+        "ff-Latn": {
+            "base": "ff",
+            "script": "Latn"
+        },
+        "fi": {
+            "nativeName": "suomi"
+        },
+        "fil": {
+            "nativeName": "Filipino"
+        },
+        "fo": {
+            "nativeName": "føroyskt"
+        },
+        "fr": {
+            "nativeName": "français"
+        },
+        "fur": {
+            "nativeName": "furlan"
+        },
+        "fy": {
+            "nativeName": "Frysk"
+        },
+        "ga": {
+            "nativeName": "Gaeilge"
+        },
+        "gd": {
+            "nativeName": "Gàidhlig"
+        },
+        "gl": {
+            "nativeName": "galego"
+        },
+        "gsw": {
+            "nativeName": "Schwiizertüütsch"
+        },
+        "gu": {
+            "nativeName": "ગુજરાતી"
+        },
+        "guz": {
+            "nativeName": "Ekegusii"
+        },
+        "gv": {
+            "nativeName": "Gaelg"
+        },
+        "ha": {
+            "nativeName": "Hausa"
+        },
+        "haw": {
+            "nativeName": "ʻŌlelo Hawaiʻi"
+        },
+        "he": {
+            "nativeName": "עברית"
+        },
+        "hi": {
+            "nativeName": "हिन्दी"
+        },
+        "hr": {
+            "nativeName": "hrvatski"
+        },
+        "hsb": {
+            "nativeName": "hornjoserbšćina"
+        },
+        "hu": {
+            "nativeName": "magyar"
+        },
+        "hy": {
+            "nativeName": "հայերեն"
+        },
+        "ia": {
+            "nativeName": "interlingua"
+        },
+        "id": {
+            "nativeName": "Indonesia"
+        },
+        "ig": {
+            "nativeName": "Asụsụ Igbo"
+        },
+        "ii": {
+            "nativeName": "ꆈꌠꉙ"
+        },
+        "is": {
+            "nativeName": "íslenska"
+        },
+        "it": {
+            "nativeName": "italiano"
+        },
+        "ja": {
+            "nativeName": "日本語"
+        },
+        "ja-Hira": {
+            "base": "ja",
+            "script": "Hira"
+        },
+        "ja-Latn": {
+            "base": "ja",
+            "script": "Latn"
+        },
+        "jgo": {
+            "nativeName": "Ndaꞌa"
+        },
+        "jmc": {
+            "nativeName": "Kimachame"
+        },
+        "jv": {
+            "nativeName": "Jawa"
+        },
+        "ka": {
+            "nativeName": "ქართული"
+        },
+        "kab": {
+            "nativeName": "Taqbaylit"
+        },
+        "kam": {
+            "nativeName": "Kikamba"
+        },
+        "kde": {
+            "nativeName": "Chimakonde"
+        },
+        "kea": {
+            "nativeName": "kabuverdianu"
+        },
+        "khq": {
+            "nativeName": "Koyra ciini"
+        },
+        "ki": {
+            "nativeName": "Gikuyu"
+        },
+        "kk": {
+            "nativeName": "қазақ тілі"
+        },
+        "kkj": {
+            "nativeName": "kakɔ"
+        },
+        "kl": {
+            "nativeName": "kalaallisut"
+        },
+        "kln": {
+            "nativeName": "Kalenjin"
+        },
+        "km": {
+            "nativeName": "ខ្មែរ"
+        },
+        "kn": {
+            "nativeName": "ಕನ್ನಡ"
+        },
+        "ko": {
+            "nativeName": "한국어"
+        },
+        "ko-Latn": {
+            "base": "ko",
+            "script": "Latn"
+        },
+        "kok": {
+            "nativeName": "कोंकणी"
+        },
+        "ks": {
+            "nativeName": "کٲشُر"
+        },
+        "ksb": {
+            "nativeName": "Kishambaa"
+        },
+        "ksf": {
+            "nativeName": "rikpa"
+        },
+        "ksh": {
+            "nativeName": "Kölsch"
+        },
+        "ku": {
+            "nativeName": "kurdî"
+        },
+        "kw": {
+            "nativeName": "kernewek"
+        },
+        "ky": {
+            "nativeName": "кыргызча"
+        },
+        "lag": {
+            "nativeName": "Kɨlaangi"
+        },
+        "lb": {
+            "nativeName": "Lëtzebuergesch"
+        },
+        "lg": {
+            "nativeName": "Luganda"
+        },
+        "lkt": {
+            "nativeName": "Lakȟólʼiyapi"
+        },
+        "ln": {
+            "nativeName": "lingála"
+        },
+        "lo": {
+            "nativeName": "ລາວ"
+        },
+        "lrc": {
+            "nativeName": "لۊری شومالی"
+        },
+        "lt": {
+            "nativeName": "lietuvių"
+        },
+        "lu": {
+            "nativeName": "Tshiluba"
+        },
+        "luo": {
+            "nativeName": "Dholuo"
+        },
+        "luy": {
+            "nativeName": "Luluhia"
+        },
+        "lv": {
+            "nativeName": "latviešu"
+        },
+        "mas": {
+            "nativeName": "Maa"
+        },
+        "mer": {
+            "nativeName": "Kĩmĩrũ"
+        },
+        "mfe": {
+            "nativeName": "kreol morisien"
+        },
+        "mg": {
+            "nativeName": "Malagasy"
+        },
+        "mgh": {
+            "nativeName": "Makua"
+        },
+        "mgo": {
+            "nativeName": "metaʼ"
+        },
+        "mi": {
+            "nativeName": "Māori"
+        },
+        "mk": {
+            "nativeName": "македонски"
+        },
+        "ml": {
+            "nativeName": "മലയാളം"
+        },
+        "mn": {
+            "nativeName": "монгол"
+        },
+        "mr": {
+            "nativeName": "मराठी"
+        },
+        "ms": {
+            "nativeName": "Melayu"
+        },
+        "mt": {
+            "nativeName": "Malti"
+        },
+        "mua": {
+            "nativeName": "MUNDAŊ"
+        },
+        "my": {
+            "nativeName": "မြန်မာ"
+        },
+        "mzn": {
+            "nativeName": "مازرونی"
+        },
+        "naq": {
+            "nativeName": "Khoekhoegowab"
+        },
+        "nb": {
+            "nativeName": "norsk bokmål"
+        },
+        "nd": {
+            "nativeName": "isiNdebele"
+        },
+        "nds": {
+            "nativeName": "nds"
+        },
+        "ne": {
+            "nativeName": "नेपाली"
+        },
+        "nl": {
+            "nativeName": "Nederlands"
+        },
+        "nmg": {
+            "nativeName": "nmg"
+        },
+        "nn": {
+            "nativeName": "nynorsk"
+        },
+        "nnh": {
+            "nativeName": "Shwóŋò ngiembɔɔn"
+        },
+        "nus": {
+            "nativeName": "Thok Nath"
+        },
+        "nyn": {
+            "nativeName": "Runyankore"
+        },
+        "om": {
+            "nativeName": "Oromoo"
+        },
+        "or": {
+            "nativeName": "ଓଡ଼ିଆ"
+        },
+        "os": {
+            "nativeName": "ирон"
+        },
+        "pa": {
+            "nativeName": "ਪੰਜਾਬੀ"
+        },
+        "pa-Arab": {
+            "base": "pa",
+            "script": "Arab"
+        },
+        "pa-Guru": {
+            "base": "pa",
+            "script": "Guru"
+        },
+        "pl": {
+            "nativeName": "polski"
+        },
+        "prg": {
+            "nativeName": "prūsiskan"
+        },
+        "ps": {
+            "nativeName": "پښتو"
+        },
+        "pt": {
+            "nativeName": "português"
+        },
+        "qu": {
+            "nativeName": "Runasimi"
+        },
+        "rm": {
+            "nativeName": "rumantsch"
+        },
+        "rn": {
+            "nativeName": "Ikirundi"
+        },
+        "ro": {
+            "nativeName": "română"
+        },
+        "rof": {
+            "nativeName": "Kihorombo"
+        },
+        "root": {
+            "nativeName": "root"
+        },
+        "ru": {
+            "nativeName": "русский"
+        },
+        "rw": {
+            "nativeName": "Kinyarwanda"
+        },
+        "rwk": {
+            "nativeName": "Kiruwa"
+        },
+        "sah": {
+            "nativeName": "саха тыла"
+        },
+        "saq": {
+            "nativeName": "Kisampur"
+        },
+        "sbp": {
+            "nativeName": "Ishisangu"
+        },
+        "sd": {
+            "nativeName": "سنڌي"
+        },
+        "se": {
+            "nativeName": "davvisámegiella"
+        },
+        "seh": {
+            "nativeName": "sena"
+        },
+        "ses": {
+            "nativeName": "Koyraboro senni"
+        },
+        "sg": {
+            "nativeName": "Sängö"
+        },
+        "shi": {
+            "nativeName": "ⵜⴰⵛⵍⵃⵉⵜ"
+        },
+        "shi-Latn": {
+            "base": "shi",
+            "script": "Latn"
+        },
+        "shi-Tfng": {
+            "base": "shi",
+            "script": "Tfng"
+        },
+        "si": {
+            "nativeName": "සිංහල"
+        },
+        "sk": {
+            "nativeName": "slovenčina"
+        },
+        "sl": {
+            "nativeName": "slovenščina"
+        },
+        "smn": {
+            "nativeName": "anarâškielâ"
+        },
+        "sn": {
+            "nativeName": "chiShona"
+        },
+        "so": {
+            "nativeName": "Soomaali"
+        },
+        "sq": {
+            "nativeName": "shqip"
+        },
+        "sr": {
+            "nativeName": "српски"
+        },
+        "sr-Cyrl": {
+            "base": "sr",
+            "script": "Cyrl"
+        },
+        "sr-Latn": {
+            "base": "sr",
+            "script": "Latn"
+        },
+        "sv": {
+            "nativeName": "svenska"
+        },
+        "sw": {
+            "nativeName": "Kiswahili"
+        },
+        "ta": {
+            "nativeName": "தமிழ்"
+        },
+        "te": {
+            "nativeName": "తెలుగు"
+        },
+        "teo": {
+            "nativeName": "Kiteso"
+        },
+        "tg": {
+            "nativeName": "тоҷикӣ"
+        },
+        "th": {
+            "nativeName": "ไทย"
+        },
+        "ti": {
+            "nativeName": "ትግርኛ"
+        },
+        "tk": {
+            "nativeName": "türkmen dili"
+        },
+        "to": {
+            "nativeName": "lea fakatonga"
+        },
+        "tr": {
+            "nativeName": "Türkçe"
+        },
+        "tt": {
+            "nativeName": "татар"
+        },
+        "twq": {
+            "nativeName": "Tasawaq senni"
+        },
+        "tzm": {
+            "nativeName": "Tamaziɣt n laṭlaṣ"
+        },
+        "ug": {
+            "nativeName": "ئۇيغۇرچە"
+        },
+        "uk": {
+            "nativeName": "українська"
+        },
+        "ur": {
+            "nativeName": "اردو"
+        },
+        "uz": {
+            "nativeName": "o‘zbek"
+        },
+        "uz-Arab": {
+            "base": "uz",
+            "script": "Arab"
+        },
+        "uz-Cyrl": {
+            "base": "uz",
+            "script": "Cyrl"
+        },
+        "uz-Latn": {
+            "base": "uz",
+            "script": "Latn"
+        },
+        "vai": {
+            "nativeName": "ꕙꔤ"
+        },
+        "vai-Latn": {
+            "base": "vai",
+            "script": "Latn"
+        },
+        "vai-Vaii": {
+            "base": "vai",
+            "script": "Vaii"
+        },
+        "vi": {
+            "nativeName": "Tiếng Việt"
+        },
+        "vo": {
+            "nativeName": "vo"
+        },
+        "vun": {
+            "nativeName": "Kyivunjo"
+        },
+        "wae": {
+            "nativeName": "Walser"
+        },
+        "wo": {
+            "nativeName": "Wolof"
+        },
+        "xh": {
+            "nativeName": "isiXhosa"
+        },
+        "xog": {
+            "nativeName": "Olusoga"
+        },
+        "yav": {
+            "nativeName": "nuasue"
+        },
+        "yi": {
+            "nativeName": "ייִדיש"
+        },
+        "yo": {
+            "nativeName": "Èdè Yorùbá"
+        },
+        "yue": {
+            "nativeName": "粵語"
+        },
+        "yue-Hans": {
+            "base": "yue",
+            "script": "Hans"
+        },
+        "yue-Hant": {
+            "base": "yue",
+            "script": "Hant"
+        },
+        "zgh": {
+            "nativeName": "ⵜⴰⵎⴰⵣⵉⵖⵜ"
+        },
+        "zh": {
+            "nativeName": "中文"
+        },
+        "zh-Hans": {
+            "base": "zh",
+            "script": "Hans",
+            "nativeName": "简体中文"
+        },
+        "zh-Hant": {
+            "base": "zh",
+            "script": "Hant",
+            "nativeName": "繁體中文"
+        },
+        "zh_pinyin": {
+            "base": "zh",
+            "script": "Latn"
+        },
+        "zu": {
+            "nativeName": "isiZulu"
+        }
+    }
+}

data/update_locales.js

@@ -27,6 +27,12 @@ const sourcePresets = YAML.load(fs.readFileSync('./data/presets.yaml', 'utf8'));
 const sourceImagery = YAML.load(fs.readFileSync('./node_modules/editor-layer-index/i18n/en.yaml', 'utf8'));
 const sourceCommunity = YAML.load(fs.readFileSync('./node_modules/osm-community-index/i18n/en.yaml', 'utf8'));
 
+const cldrMainDir = './node_modules/cldr-localenames-full/main/';
+
+const languageInfo = {
+    dataLanguages: getLangNamesInNativeLang()
+};
+fs.writeFileSync('data/languages.json', JSON.stringify(languageInfo, null, 4));
 
 asyncMap(resources, getResource, function(err, results) {
     if (err) return console.log(err);
@@ -43,7 +49,9 @@ asyncMap(resources, getResource, function(err, results) {
     });
 
     // write files and fetch language info for each locale
-    var dataLocales = {};
+    var dataLocales = {
+        en: { rtl: false, languageNames: foreignLanguageNamesInLanguageOf('en') }
+    };
     asyncMap(Object.keys(allStrings),
         function(code, done) {
             if (code === 'en' || !Object.keys(allStrings[code]).length) {
@@ -61,7 +69,8 @@ asyncMap(resources, getResource, function(err, results) {
                     } else if (code === 'ku') {
                         rtl = false;
                     }
-                    dataLocales[code] = { rtl: rtl };
+                    var langTranslations = foreignLanguageNamesInLanguageOf(code);
+                    dataLocales[code] = { rtl: rtl, languageNames: langTranslations || {} };
                     done();
                 });
             }
@@ -70,7 +79,7 @@ asyncMap(resources, getResource, function(err, results) {
                 const keys = Object.keys(dataLocales).sort();
                 var sorted = {};
                 keys.forEach(function (k) { sorted[k] = dataLocales[k]; });
-                fs.writeFileSync('data/locales.json', prettyStringify({ dataLocales: sorted }));
+                fs.writeFileSync('data/locales.json', prettyStringify({ dataLocales: sorted }, { maxLength: 99999 }));
             }
         }
     );
@@ -183,3 +192,95 @@ function asyncMap(inputs, func, callback) {
         });
     }
 }
+
+function getLangNamesInNativeLang() {
+    // manually add languages we want that aren't in CLDR
+    var unordered = {
+        'ja-Hira': {
+            base: 'ja',
+            script: 'Hira'
+        },
+        'ja-Latn': {
+            base: 'ja',
+            script: 'Latn'
+        },
+        'ko-Latn': {
+            base: 'ko',
+            script: 'Latn'
+        },
+        'zh_pinyin': {
+            base: 'zh',
+            script: 'Latn'
+        }
+    };
+    var langDirectoryPaths = fs.readdirSync(cldrMainDir);
+    langDirectoryPaths.forEach(function(code) {
+
+        var languagesPath = cldrMainDir + code + '/languages.json';
+
+        //if (!fs.existsSync(languagesPath)) return;
+        var languageObj = JSON.parse(fs.readFileSync(languagesPath, 'utf8')).main[code];
+
+        var identity = languageObj.identity;
+
+        // skip locale-specific languages
+        if (identity.variant || identity.territory) return;
+
+        var info = {};
+
+        var script = identity.script;
+        if (script) {
+            info.base = identity.language;
+            info.script = script;
+        }
+
+        var nativeName = languageObj.localeDisplayNames.languages[code];
+        if (nativeName) {
+            info.nativeName = nativeName;
+        }
+
+        unordered[code] = info;
+    });
+    var ordered = {};
+    Object.keys(unordered).sort().forEach(function(key) {
+        ordered[key] = unordered[key];
+    });
+    return ordered;
+}
+
+var rematchCodes = { 'ar-AA': 'ar', 'zh-CN': 'zh', 'zh-HK': 'zh-Hant-HK', 'zh-TW': 'zh', 'pt-BR': 'pt', 'pt': 'pt-PT' };
+
+function foreignLanguageNamesInLanguageOf(code) {
+
+    if (rematchCodes[code]) code = rematchCodes[code];
+
+    var languageFilePath = cldrMainDir + code + '/languages.json';
+    if (!fs.existsSync(languageFilePath)) {
+        return null;
+    }
+    var translatedLangsByCode = JSON.parse(fs.readFileSync(languageFilePath, 'utf8')).main[code].localeDisplayNames.languages;
+
+    // ignore codes for non-languages
+    for (var nonLangCode in { mis: true, mul: true, und: true, zxx: true }) {
+        delete translatedLangsByCode[nonLangCode];
+    }
+
+    for (var langCode in translatedLangsByCode) {
+        var altLongIndex = langCode.indexOf('-alt-long');
+        if (altLongIndex !== -1) {
+            // prefer long names (e.g. Chinese -> Mandarin Chinese)
+            var base = langCode.substring(0, altLongIndex);
+            translatedLangsByCode[base] = translatedLangsByCode[langCode];
+        }
+
+        if (langCode.includes('-alt-')) {
+            // remove alternative names
+            delete translatedLangsByCode[langCode];
+        } else if (langCode === translatedLangsByCode[langCode]) {
+            // no localized value available
+            delete translatedLangsByCode[langCode];
+        }
+    }
+
+    return translatedLangsByCode;
+}

modules/ui/fields/localized.js

@@ -1,8 +1,8 @@
 import { dispatch as d3_dispatch } from 'd3-dispatch';
 import { select as d3_select, event as d3_event } from 'd3-selection';
 
-import { t } from '../../util/locale';
-import { dataWikipedia } from '../../../data';
+import { t, languageName } from '../../util/locale';
+import { dataLanguages } from '../../../data';
 import { services } from '../../services';
 import { svgIcon } from '../../svg';
 import { tooltip } from '../../util/tooltip';
@@ -10,6 +10,20 @@ import { uiCombobox } from '../combobox';
 import { utilDetect } from '../../util/detect';
 import { utilEditDistance, utilGetSetValue, utilNoAuto, utilRebind } from '../../util';
 
+var languagesArray = [];
+function loadLanguagesArray() {
+    if (languagesArray.length !== 0) return;
+
+    for (var code in dataLanguages) {
+        languagesArray.push({
+            localName: languageName(code, { localOnly: true }),
+            nativeName: dataLanguages[code].nativeName,
+            code: code,
+            label: languageName(code)
+        });
+    }
+}
+
 
 export function uiFieldLocalized(field, context) {
     var dispatch = d3_dispatch('change', 'input');
@@ -92,6 +106,9 @@ export function uiFieldLocalized(field, context) {
 
 
     function localized(selection) {
+        // load if needed
+        loadLanguagesArray();
+
         _selection = selection;
         calcLocked();
         var isLocked = field.locked();
@@ -340,12 +357,13 @@ export function uiFieldLocalized(field, context) {
     function changeLang(d) {
         var lang = utilGetSetValue(d3_select(this));
         var t = {};
-        var language = dataWikipedia.find(function(d) {
-            return d[0].toLowerCase() === lang.toLowerCase() ||
-                d[1].toLowerCase() === lang.toLowerCase();
+        var language = languagesArray.find(function(d) {
+            return (d.localName && d.localName.toLowerCase() === lang.toLowerCase()) ||
+                d.label.toLowerCase() === lang.toLowerCase() ||
+                (d.nativeName && d.nativeName.toLowerCase() === lang.toLowerCase());
         });
 
-        if (language) lang = language[2];
+        if (language) lang = language.code;
 
         if (d.lang && d.lang !== lang) {
             t[key(d.lang)] = undefined;
@@ -378,12 +396,13 @@ export function uiFieldLocalized(field, context) {
     function fetchLanguages(value, cb) {
         var v = value.toLowerCase();
 
-        cb(dataWikipedia.filter(function(d) {
-            return d[0].toLowerCase().indexOf(v) >= 0 ||
-            d[1].toLowerCase().indexOf(v) >= 0 ||
-            d[2].toLowerCase().indexOf(v) >= 0;
+        cb(languagesArray.filter(function(d) {
+            return d.label.toLowerCase().indexOf(v) >= 0 ||
+                (d.localName && d.localName.toLowerCase().indexOf(v) >= 0) ||
+                (d.nativeName && d.nativeName.toLowerCase().indexOf(v) >= 0) ||
+                d.code.toLowerCase().indexOf(v) >= 0;
         }).map(function(d) {
-            return { value: d[1] };
+            return { value: d.label };
         }));
     }
 
@@ -482,8 +501,7 @@ export function uiFieldLocalized(field, context) {
         entries.order();
 
         utilGetSetValue(entries.select('.localized-lang'), function(d) {
-            var lang = dataWikipedia.find(function(lang) { return lang[2] === d.lang; });
-            return lang ? lang[1] : d.lang;
+            return languageName(d.lang);
         });
 
         utilGetSetValue(entries.select('.localized-value'),

modules/ui/success.js

@@ -1,7 +1,7 @@
 import { dispatch as d3_dispatch } from 'd3-dispatch';
 import { select as d3_select } from 'd3-selection';
 
-import { t } from '../util/locale';
+import { t, languageName } from '../util/locale';
 import { data } from '../../data';
 import { svgIcon } from '../svg/icon';
 import { uiDisclosure } from '../ui/disclosure';
@@ -296,10 +296,14 @@ export function uiSuccess(context) {
             }
 
             if (d.languageCodes && d.languageCodes.length) {
+                var languageList = d.languageCodes.map(function(code) {
+                    return languageName(code);
+                }).join(', ');
+
                 moreEnter
                     .append('div')
                     .attr('class', 'community-languages')
-                    .text(t('success.languages', { languages: d.languageCodes.join(', ') }));
+                    .text(t('success.languages', { languages: languageList }));
             }
         }
 

modules/util/detect.js

@@ -1,4 +1,4 @@
-import { currentLocale, setTextDirection } from './locale';
+import { currentLocale, setTextDirection, setLanguageNames } from './locale';
 import { dataLocales } from '../../data/index';
 import { utilStringQs } from './util';
 
@@ -97,13 +97,14 @@ export function utilDetect(force) {
     }
 
     // detect text direction
-    var lang = dataLocales[detected.locale];
+    var lang = dataLocales[detected.locale] || dataLocales[detected.language];
     if ((lang && lang.rtl) || (q.rtl === 'true')) {
         detected.textDirection = 'rtl';
     } else {
         detected.textDirection = 'ltr';
     }
     setTextDirection(detected.textDirection);
+    setLanguageNames((lang && lang.languageNames) || {});
 
     // detect host
     var loc = window.top.location;

modules/util/locale.js

@@ -1,7 +1,10 @@
+import { dataLanguages } from '../../../data';
+
 var translations = Object.create(null);
 
 export var currentLocale = 'en';
 export var textDirection = 'ltr';
+export var languageNames = {};
 
 export function setLocale(val) {
     if (translations[val] !== undefined) {
@@ -73,3 +76,29 @@ export function t(s, o, loc) {
 export function setTextDirection(dir) {
     textDirection = dir;
 }
+
+export function setLanguageNames(obj) {
+    languageNames = obj;
+}
+
+export function languageName(code, options = {}) {
+    if (languageNames[code]) { // name in locale langauge
+        return languageNames[code];
+    }
+    // sometimes we only want the local name
+    if (options.localOnly) return null;
+
+    if (dataLanguages[code]) { // language info
+        if (dataLanguages[code].nativeName) { // name in native language
+            return t('translate.language_and_code', { language: dataLanguages[code].nativeName, code: code });
+        } else if (dataLanguages[code].base && dataLanguages[code].script) {
+            var base = dataLanguages[code].base;
+            if (languageNames[base]) { // base name in locale langauge
+                return t('translate.language_and_code', { language: languageNames[base], code: code });
+            } else if (dataLanguages[code] && dataLanguages[code].nativeName) {
+                return t('translate.language_and_code', { language: dataLanguages[base].nativeName, code: code });
+            }
+        }
+    }
+    return code; // if not found, use the code
+}

package.json

@@ -61,6 +61,7 @@
     "@fortawesome/free-solid-svg-icons": "^5.8.2",
     "@mapbox/maki": "^6.0.0",
     "chai": "^4.1.0",
+    "cldr-localenames-full": "35.1.0",
     "colors": "^1.1.2",
     "concat-files": "^0.1.1",
     "d3": "~5.9.2",