manage_translations.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360
  1. #!/usr/bin/env python
  2. #
  3. # This Python file contains utility scripts to manage Django translations.
  4. # It has to be run inside the django git root directory.
  5. #
  6. # The following commands are available:
  7. #
  8. # * update_catalogs: check for new strings in core and contrib catalogs, and
  9. # output how much strings are new/changed.
  10. #
  11. # * lang_stats: output statistics for each catalog/language combination
  12. #
  13. # * fetch: fetch translations from transifex.com
  14. #
  15. # Each command support the --languages and --resources options to limit their
  16. # operation to the specified language or resource. For example, to get stats
  17. # for Spanish in contrib.admin, run:
  18. #
  19. # $ python scripts/manage_translations.py lang_stats --language=es --resources=admin
  20. import os
  21. from argparse import ArgumentParser
  22. from collections import defaultdict
  23. from configparser import ConfigParser
  24. from datetime import datetime
  25. from subprocess import run
  26. import requests
  27. import django
  28. from django.conf import settings
  29. from django.core.management import call_command
  30. HAVE_JS = ["admin"]
  31. LANG_OVERRIDES = {
  32. "zh_CN": "zh_Hans",
  33. "zh_TW": "zh_Hant",
  34. }
  35. def list_resources_with_updates(date_since, date_skip=None, verbose=False):
  36. resource_lang_changed = defaultdict(list)
  37. resource_lang_unchanged = defaultdict(list)
  38. # Read token from ENV, otherwise read from the ~/.transifexrc file.
  39. api_token = os.getenv("TRANSIFEX_API_TOKEN")
  40. if not api_token:
  41. parser = ConfigParser()
  42. parser.read(os.path.expanduser("~/.transifexrc"))
  43. api_token = parser.get("https://www.transifex.com", "token")
  44. assert api_token, "Please define the TRANSIFEX_API_TOKEN env var."
  45. headers = {"Authorization": f"Bearer {api_token}"}
  46. base_url = "https://rest.api.transifex.com"
  47. base_params = {"filter[project]": "o:django:p:django"}
  48. resources_url = base_url + "/resources"
  49. resource_stats_url = base_url + "/resource_language_stats"
  50. response = requests.get(resources_url, headers=headers, params=base_params)
  51. assert response.ok, response.content
  52. data = response.json()["data"]
  53. for item in data:
  54. if item["type"] != "resources":
  55. continue
  56. resource_id = item["id"]
  57. resource_name = item["attributes"]["name"]
  58. params = base_params.copy()
  59. params.update({"filter[resource]": resource_id})
  60. stats = requests.get(resource_stats_url, headers=headers, params=params)
  61. stats_data = stats.json()["data"]
  62. for lang_data in stats_data:
  63. lang_id = lang_data["id"].split(":")[-1]
  64. lang_attributes = lang_data["attributes"]
  65. last_update = lang_attributes["last_translation_update"]
  66. if verbose:
  67. print(
  68. f"CHECKING {resource_name} for {lang_id=} updated on {last_update}"
  69. )
  70. if last_update is None:
  71. resource_lang_unchanged[resource_name].append(lang_id)
  72. continue
  73. last_update = datetime.strptime(last_update, "%Y-%m-%dT%H:%M:%SZ")
  74. if last_update > date_since and (
  75. date_skip is None or last_update.date() != date_skip.date()
  76. ):
  77. if verbose:
  78. print(f"=> CHANGED {lang_attributes=} {date_skip=}")
  79. resource_lang_changed[resource_name].append(lang_id)
  80. else:
  81. resource_lang_unchanged[resource_name].append(lang_id)
  82. if verbose:
  83. unchanged = "\n".join(
  84. f"\n * resource {res} languages {' '.join(sorted(langs))}"
  85. for res, langs in resource_lang_unchanged.items()
  86. )
  87. print(f"== SUMMARY for unchanged resources ==\n{unchanged}")
  88. return resource_lang_changed
  89. def _get_locale_dirs(resources, include_core=True):
  90. """
  91. Return a tuple (contrib name, absolute path) for all locale directories,
  92. optionally including the django core catalog.
  93. If resources list is not None, filter directories matching resources content.
  94. """
  95. contrib_dir = os.path.join(os.getcwd(), "django", "contrib")
  96. dirs = []
  97. # Collect all locale directories
  98. for contrib_name in os.listdir(contrib_dir):
  99. path = os.path.join(contrib_dir, contrib_name, "locale")
  100. if os.path.isdir(path):
  101. dirs.append((contrib_name, path))
  102. if contrib_name in HAVE_JS:
  103. dirs.append(("%s-js" % contrib_name, path))
  104. if include_core:
  105. dirs.insert(0, ("core", os.path.join(os.getcwd(), "django", "conf", "locale")))
  106. # Filter by resources, if any
  107. if resources is not None:
  108. res_names = [d[0] for d in dirs]
  109. dirs = [ld for ld in dirs if ld[0] in resources]
  110. if len(resources) > len(dirs):
  111. print(
  112. "You have specified some unknown resources. "
  113. "Available resource names are: %s" % (", ".join(res_names),)
  114. )
  115. exit(1)
  116. return dirs
  117. def _tx_resource_for_name(name):
  118. """Return the Transifex resource name"""
  119. if name == "core":
  120. return "django.core"
  121. else:
  122. return "django.contrib-%s" % name
  123. def _check_diff(cat_name, base_path):
  124. """
  125. Output the approximate number of changed/added strings in the en catalog.
  126. """
  127. po_path = "%(path)s/en/LC_MESSAGES/django%(ext)s.po" % {
  128. "path": base_path,
  129. "ext": "js" if cat_name.endswith("-js") else "",
  130. }
  131. p = run(
  132. "git diff -U0 %s | egrep '^[-+]msgid' | wc -l" % po_path,
  133. capture_output=True,
  134. shell=True,
  135. )
  136. num_changes = int(p.stdout.strip())
  137. print("%d changed/added messages in '%s' catalog." % (num_changes, cat_name))
  138. def update_catalogs(resources=None, languages=None):
  139. """
  140. Update the en/LC_MESSAGES/django.po (main and contrib) files with
  141. new/updated translatable strings.
  142. """
  143. settings.configure()
  144. django.setup()
  145. if resources is not None:
  146. print("`update_catalogs` will always process all resources.")
  147. contrib_dirs = _get_locale_dirs(None, include_core=False)
  148. os.chdir(os.path.join(os.getcwd(), "django"))
  149. print("Updating en catalogs for Django and contrib apps...")
  150. call_command("makemessages", locale=["en"])
  151. print("Updating en JS catalogs for Django and contrib apps...")
  152. call_command("makemessages", locale=["en"], domain="djangojs")
  153. # Output changed stats
  154. _check_diff("core", os.path.join(os.getcwd(), "conf", "locale"))
  155. for name, dir_ in contrib_dirs:
  156. _check_diff(name, dir_)
  157. def lang_stats(resources=None, languages=None):
  158. """
  159. Output language statistics of committed translation files for each
  160. Django catalog.
  161. If resources is provided, it should be a list of translation resource to
  162. limit the output (e.g. ['core', 'gis']).
  163. """
  164. locale_dirs = _get_locale_dirs(resources)
  165. for name, dir_ in locale_dirs:
  166. print("\nShowing translations stats for '%s':" % name)
  167. langs = sorted(d for d in os.listdir(dir_) if not d.startswith("_"))
  168. for lang in langs:
  169. if languages and lang not in languages:
  170. continue
  171. # TODO: merge first with the latest en catalog
  172. po_path = "{path}/{lang}/LC_MESSAGES/django{ext}.po".format(
  173. path=dir_, lang=lang, ext="js" if name.endswith("-js") else ""
  174. )
  175. p = run(
  176. ["msgfmt", "-vc", "-o", "/dev/null", po_path],
  177. capture_output=True,
  178. env={"LANG": "C"},
  179. encoding="utf-8",
  180. )
  181. if p.returncode == 0:
  182. # msgfmt output stats on stderr
  183. print("%s: %s" % (lang, p.stderr.strip()))
  184. else:
  185. print(
  186. "Errors happened when checking %s translation for %s:\n%s"
  187. % (lang, name, p.stderr)
  188. )
  189. def fetch(resources=None, languages=None):
  190. """
  191. Fetch translations from Transifex, wrap long lines, generate mo files.
  192. """
  193. locale_dirs = _get_locale_dirs(resources)
  194. errors = []
  195. for name, dir_ in locale_dirs:
  196. cmd = [
  197. "tx",
  198. "pull",
  199. "-r",
  200. _tx_resource_for_name(name),
  201. "-f",
  202. "--minimum-perc=5",
  203. ]
  204. # Transifex pull
  205. if languages is None:
  206. run(cmd + ["--all"])
  207. target_langs = sorted(
  208. d for d in os.listdir(dir_) if not d.startswith("_") and d != "en"
  209. )
  210. else:
  211. for lang in languages:
  212. run(cmd + ["-l", lang])
  213. target_langs = languages
  214. target_langs = [LANG_OVERRIDES.get(d, d) for d in target_langs]
  215. # msgcat to wrap lines and msgfmt for compilation of .mo file
  216. for lang in target_langs:
  217. po_path = "%(path)s/%(lang)s/LC_MESSAGES/django%(ext)s.po" % {
  218. "path": dir_,
  219. "lang": lang,
  220. "ext": "js" if name.endswith("-js") else "",
  221. }
  222. if not os.path.exists(po_path):
  223. print(
  224. "No %(lang)s translation for resource %(name)s"
  225. % {"lang": lang, "name": name}
  226. )
  227. continue
  228. run(["msgcat", "--no-location", "-o", po_path, po_path])
  229. msgfmt = run(["msgfmt", "-c", "-o", "%s.mo" % po_path[:-3], po_path])
  230. if msgfmt.returncode != 0:
  231. errors.append((name, lang))
  232. if errors:
  233. print("\nWARNING: Errors have occurred in following cases:")
  234. for resource, lang in errors:
  235. print("\tResource %s for language %s" % (resource, lang))
  236. exit(1)
  237. def fetch_since(date_since, date_skip=None, verbose=False, dry_run=False):
  238. """
  239. Fetch translations from Transifex that were modified since the given date.
  240. """
  241. changed = list_resources_with_updates(
  242. date_since=date_since, date_skip=date_skip, verbose=verbose
  243. )
  244. if verbose:
  245. print(f"== SUMMARY for changed resources {dry_run=} ==\n")
  246. for res, langs in changed.items():
  247. if verbose:
  248. print(f"\n * resource {res} languages {' '.join(sorted(langs))}")
  249. if not dry_run:
  250. fetch(resources=[res], languages=sorted(langs))
  251. if not changed and verbose:
  252. print(f"\n No resource changed since {date_since}")
  253. def add_common_arguments(parser):
  254. parser.add_argument(
  255. "-r",
  256. "--resources",
  257. action="append",
  258. help="limit operation to the specified resources",
  259. )
  260. parser.add_argument(
  261. "-l",
  262. "--languages",
  263. action="append",
  264. help="limit operation to the specified languages",
  265. )
  266. if __name__ == "__main__":
  267. parser = ArgumentParser()
  268. subparsers = parser.add_subparsers(
  269. dest="cmd", help="choose the operation to perform"
  270. )
  271. parser_update = subparsers.add_parser(
  272. "update_catalogs",
  273. help="update English django.po files with new/updated translatable strings",
  274. )
  275. add_common_arguments(parser_update)
  276. parser_stats = subparsers.add_parser(
  277. "lang_stats",
  278. help="print the approximate number of changed/added strings in the en catalog",
  279. )
  280. add_common_arguments(parser_stats)
  281. parser_fetch = subparsers.add_parser(
  282. "fetch",
  283. help="fetch translations from Transifex, wrap long lines, generate mo files",
  284. )
  285. add_common_arguments(parser_fetch)
  286. parser_fetch = subparsers.add_parser(
  287. "fetch_since",
  288. help=(
  289. "fetch translations from Transifex modified since a given date "
  290. "(for all languages and all resources)"
  291. ),
  292. )
  293. parser_fetch.add_argument("-v", "--verbose", action="store_true")
  294. parser_fetch.add_argument(
  295. "-s",
  296. "--since",
  297. required=True,
  298. dest="date_since",
  299. metavar="YYYY-MM-DD",
  300. type=datetime.fromisoformat,
  301. help="fetch new translations since this date (ISO format YYYY-MM-DD).",
  302. )
  303. parser_fetch.add_argument(
  304. "--skip",
  305. dest="date_skip",
  306. metavar="YYYY-MM-DD",
  307. type=datetime.fromisoformat,
  308. help="skip changes from this date (ISO format YYYY-MM-DD).",
  309. )
  310. parser_fetch.add_argument("--dry-run", dest="dry_run", action="store_true")
  311. options = parser.parse_args()
  312. kwargs = options.__dict__
  313. cmd = kwargs.pop("cmd")
  314. eval(cmd)(**kwargs)