upgrade translation fetcher to transifex API v3

see https://www.transifex.com/blog/2020/transifex-api-version-3/

package.json

@@ -76,8 +76,8 @@
     "@ideditor/temaki": "~5.2.0",
     "@mapbox/maki": "^8.0.0",
     "@openstreetmap/id-tagging-schema": "^5.0.1",
+    "@transifex/api": "^4.2.5",
     "autoprefixer": "^10.0.1",
-    "btoa": "^1.2.1",
     "chai": "^4.3.4",
     "chalk": "^4.1.2",
     "cldr-core": "^41.0.0",

scripts/update_locales.js

@@ -3,44 +3,32 @@
 const chalk = require('chalk');
 const fs = require('fs');
 const fetch = require('node-fetch');
-const btoa = require('btoa');
 const YAML = require('js-yaml');
+const transifexApi = require('@transifex/api').transifexApi;
 
 const resourceIds = ['core', 'imagery', 'community'];
 const reviewedOnlyLangs = ['vi'];
 const outdir = 'dist/locales/';
-const apiroot = 'https://www.transifex.com/api/2';
-const projectURL = `${apiroot}/project/id-editor`;
 
 const languageNames = require('./language_names.js');
 
+const transifexOrganization = 'openstreetmap';
+const transifexProject = 'id-editor';
 
 // Transifex doesn't allow anonymous downloading
-let auth;
-/* eslint-disable no-process-env */
 if (process.env.transifex_password) {
   // Deployment scripts may prefer environment variables
-  auth = {
-    user: process.env.transifex_user || 'api',
-    password: process.env.transifex_password
-  };
+  /* eslint-disable no-process-env */
+  transifexApi.setup({ auth: process.env.transifex_password });
+  /* eslint-enable no-process-env */
 } else {
   // Credentials can be stored in transifex.auth as a json object. This file is gitignored.
-  // You can use an API key instead of your password: https://docs.transifex.com/api/introduction#authentication
-  // in which case for user parameter value should be: "api"
+  // You must use an API token for authentication: You can generate one at https://www.transifex.com/user/settings/api/.
   // {
-  //   "user": "username",
-  //   "password": "password"
+  //   "password": "<api_key>"
   // }
-  auth = JSON.parse(fs.readFileSync('./transifex.auth', 'utf8'));
+  transifexApi.setup({ auth: JSON.parse(fs.readFileSync('./transifex.auth', 'utf8')).password });
 }
-/* eslint-enable no-process-env */
-
-const fetchOpts = {
-  headers: {
-    'Authorization': 'Basic ' + btoa(auth.user + ':' + auth.password),
-  }
-};
 
 const dataShortcuts = JSON.parse(fs.readFileSync('data/shortcuts.json', 'utf8'));
 
@@ -69,33 +57,36 @@ let coverageByLocaleCode = {};
 asyncMap(resourceIds, getResourceInfo, gotResourceInfo);
 asyncMap(resourceIds, getResource, gotResource);
 
-function getResourceInfo(resourceId, callback) {
-  let url = 'https://api.transifex.com/organizations/openstreetmap/projects/id-editor/resources/' + resourceId;
-  fetch(url, fetchOpts)
-    .then(res => {
-      console.log(`${res.status}: ${url}`);
-      return res.json();
-    })
-    .then(json => {
-      callback(null, json);
-    })
-    .catch(err => callback(err));
+async function getResourceInfo(resourceId, callback) {
+  try {
+    const result = [];
+    for await (const stat of transifexApi.ResourceLanguageStats.filter({
+      project: `o:${transifexOrganization}:p:${transifexProject}`,
+      resource: `o:${transifexOrganization}:p:${transifexProject}:r:${resourceId}`
+    }).all()) {
+      result.push(stat);
+    }
+    console.log(`got resource language stats collection for ${resourceId}`);
+    callback(null, result);
+  } catch(err) {
+    console.error(`error while getting resource language stats collection for ${resourceId}`);
+    callback(err);
+  }
 }
 function gotResourceInfo(err, results) {
   if (err) return console.log(err);
-  results.forEach(function(info) {
-    for (let code in info.stats) {
-      let type = 'translated';
+  results.forEach(info => {
+    info.forEach(stat => {
+      let code = stat.relationships.language.data.id.substr(2).replace(/_/g, '-');
+      let type = 'translated_strings';
       if (reviewedOnlyLangs.indexOf(code) !== -1) {
-        // reviewed_1 = reviewed, reviewed_2 = proofread
-        type = 'reviewed_1';
+        type = 'reviewed_strings';
       }
-      let coveragePart = info.stats[code][type].percentage / results.length;
+      let coveragePart = (stat.attributes[type] /  stat.attributes.total_strings) / results.length;
 
-      code = code.replace(/_/g, '-');
       if (coverageByLocaleCode[code] === undefined) coverageByLocaleCode[code] = 0;
       coverageByLocaleCode[code] += coveragePart;
-    }
+    });
   });
 }
 
@@ -135,7 +126,9 @@ function gotResource(err, results) {
         fs.writeFileSync(`${outdir}${code}.min.json`, JSON.stringify(obj));
 
         getLanguageInfo(code, (err, info) => {
-          let rtl = info && info.rtl;
+          if (err) return console.log(err);
+
+          let rtl = info && info.attributes && info.attributes.rtl;
           // exceptions: see #4783
           if (code === 'ckb') {
             rtl = true;
@@ -145,7 +138,7 @@ function gotResource(err, results) {
 
           let coverage = coverageByLocaleCode[code];
           if (coverage === undefined) {
-            console.log('Could not get language coverage');
+            console.log(`Could not get language coverage for ${code}`, coverageByLocaleCode);
             process.exit(1);
           }
           // we don't need high precision here, but we need to know if it's exactly 100% or not
@@ -173,12 +166,11 @@ function gotResource(err, results) {
 }
 
 
-function getResource(resourceId, callback) {
-  let resourceURL = `${projectURL}/resource/${resourceId}`;
-  getLanguages(resourceURL, (err, codes) => {
+async function getResource(resourceId, callback) {
+  getLanguages((err, codes) => {
     if (err) return callback(err);
 
-    asyncMap(codes, getLanguage(resourceURL), (err, results) => {
+    asyncMap(codes, getLanguage(resourceId), (err, results) => {
       if (err) return callback(err);
 
       let locale = {};
@@ -229,56 +221,60 @@ function getResource(resourceId, callback) {
 }
 
 
-function getLanguage(resourceURL) {
-  return (code, callback) => {
-    code = code.replace(/-/g, '_');
-    let url = `${resourceURL}/translation/${code}`;
-    // fetch only reviewed strings for some languages
-    if (reviewedOnlyLangs.indexOf(code) !== -1) {
-      url += '?mode=reviewed';
+function getLanguage(resourceId) {
+  return async (code, callback) => {
+    try {
+      code = code.replace(/-/g, '_');
+      const url = await transifexApi.ResourceTranslationsAsyncDownload.download({
+        resource: {data:{type:'resources', id:`o:${transifexOrganization}:p:${transifexProject}:r:${resourceId}`}},
+        language: {data:{type:'languages', id:`l:${code}`}},
+        // fetch only reviewed strings for some languages
+        mode: reviewedOnlyLangs.indexOf(code) !== -1 ? 'reviewed' : 'default'
+      });
+      const data = await fetch(url).then(d => d.text());
+      console.log(`got translations for ${resourceId}, language ${code}`);
+      callback(null, YAML.load(data)[code]);
+    } catch(err) {
+      console.error(`error while getting translations for ${resourceId}, language ${code}`, err);
+      callback(err);
     }
-    fetch(url, fetchOpts)
-      .then(res => {
-        console.log(`${res.status}: ${url}`);
-        return res.json();
-      })
-      .then(json => {
-        callback(null, YAML.load(json.content)[code]);
-      })
-      .catch(err => callback(err));
   };
 }
 
 
-function getLanguageInfo(code, callback) {
+async function getLanguageInfo(code, callback) {
   code = code.replace(/-/g, '_');
-  let url = `${apiroot}/language/${code}`;
-  fetch(url, fetchOpts)
-    .then(res => {
-      console.log(`${res.status}: ${url}`);
-      return res.json();
-    })
-    .then(json => {
-      callback(null, json);
-    })
-    .catch(err => callback(err));
+  try {
+    const lng = await transifexApi.Language.get({
+      code: code
+    });
+    console.log(`got language details for ${code}`);
+    callback(null, lng);
+  } catch(err) {
+    console.error(`error while getting language details for ${code}`);
+    callback(err);
+  }
 }
 
 
-function getLanguages(resourceURL, callback) {
-  let url = `${resourceURL}?details`;
-  fetch(url, fetchOpts)
-    .then(res => {
-      console.log(`${res.status}: ${url}`);
-      return res.json();
-    })
-    .then(json => {
-      callback(null, json.available_languages
-        .map(d => d.code.replace(/_/g, '-'))
-        .filter(d => d !== 'en')
-      );
-    })
-    .catch(err => callback(err));
+async function getLanguages(callback) {
+  try {
+    const result = [];
+    const project = await transifexApi.Project.get({
+      organization: `o:${transifexOrganization}`,
+      slug: transifexProject
+    });
+    const lngs = await project.fetch('languages');
+    for await (const lng of lngs.all()) {
+      if (lng.attributes.code === 'en') continue;
+      result.push(lng.attributes.code.replace(/_/g, '-'));
+    }
+    console.log('got project languages');
+    callback(null, result);
+  } catch(err) {
+    console.error('error while getting project languages');
+    callback(err);
+  }
 }
 
 
@@ -293,7 +289,7 @@ function asyncMap(inputs, func, callback) {
   function next() {
     callFunc(index++);
     if (index < inputs.length) {
-      setTimeout(next, 200);
+      setTimeout(next, 50);
     }
   }