From 303a33c3bfd97009c96b160df0449996b4feda2c Mon Sep 17 00:00:00 2001 From: Dalai Felinto Date: Sat, 9 Sep 2017 00:02:24 +0200 Subject: [PATCH] Internationalization: Backend support to localization based on user browser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User experience =============== For users it means we can provide localized web-sites to enrich their overall experiences. Although for the Blender Cloud this doesn't make much sense (since the content is in English), Flamenco and Attract can really benefit from this. New configuration settings ========================== There are two new parameters in config.py: * DEFAULT_LOCALE='en_US' * SUPPORT_ENGLISH=True They are both properly documented in the `config.py` file. Technicall details ================== We are using the 'Accept-Languages' header to match the available translations with the user supported languages. If an extension has a `translations` folder, it's used for translations. However the main application (e.g., Blender Cloud) is the one that determines the supported languages based on its `languages` folder. How to mark strings for translation =================================== See the documentation in README.md. But as an example, 404.pug and pillar/__init__.py::handle_sdk_resource_invalid have marked up strings that will be extracted once you install pillar, or run any of the translations commangs. Remember to **gulp** after you update the template files. How to setup translations ========================= You will need to create translation for the main project, and for each extension that you want to see translated. I added a new entry-point to the installation of Pillar. So all you need is to use the `translations` script to initialize, update and compile your translations. Pending tasks ============= Aside from marking more strings for extraction and start the translation effort it would be interesting to replace the pretty_date routine with momentjs. Acknowledgement =============== Many thanks for Sybren Stüvel for the suggestions and throughout code review. Thanks also to Francesco Siddi for the original documentation and suggesting me to tackle this. And Kudos for Pablo Vazquez for the motivational support and for the upcoming "strings mark up" task force! The core of the implementation is based on Miguel Grinberg i18n chapter of his great 'The Mega Flask Tutorial'. Reviewers: sybren Differential Revision: https://developer.blender.org/D2826 --- .gitignore | 2 + README.md | 12 ++++ pillar/__init__.py | 89 +++++++++++++++++++++++++++++- pillar/cli/__init__.py | 2 +- pillar/cli/translations.py | 104 +++++++++++++++++++++++++++++++++++ pillar/config.py | 20 +++++++ pillar/extension.py | 19 +++++++ requirements.txt | 1 + setup.py | 31 +++++++++++ src/templates/errors/404.pug | 19 +++++-- translations.cfg | 14 +++++ 11 files changed, 305 insertions(+), 8 deletions(-) create mode 100644 pillar/cli/translations.py create mode 100644 translations.cfg diff --git a/.gitignore b/.gitignore index 0cf72cd4..cf0a6565 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,8 @@ profile.stats *.css.map *.js.map +/translations/*/LC_MESSAGES/*.mo + pillar/web/static/assets/css/*.css pillar/web/static/assets/js/*.min.js pillar/web/static/storage/ diff --git a/README.md b/README.md index 8af7d93c..5f4a74de 100644 --- a/README.md +++ b/README.md @@ -64,3 +64,15 @@ RabbitMQ. You can run the Celery Worker using `manage.py celery worker`. Find other Celery operations with the `manage.py celery` command. + +## Translations + +If the language you want to support doesn't exist, you need to run: `translations init es_AR`. + +Every time a new string is marked for translation you need to update the entire catalog: `translations update` + +And once more strings are translated, you need to compile the translations: `translations compile` + +*To mark strings strings for translations in Python scripts you need to +wrap them with the `flask_babel.gettext` function. +For .pug templates wrap them with `_()`.* diff --git a/pillar/__init__.py b/pillar/__init__.py index e4cd3db2..13175f1a 100644 --- a/pillar/__init__.py +++ b/pillar/__init__.py @@ -11,11 +11,13 @@ import tempfile import typing import os import os.path +import pathlib import jinja2 from eve import Eve import flask -from flask import render_template, request +from flask import g, render_template, request +from flask_babel import Babel, gettext as _ from flask.templating import TemplateNotFound import pymongo.collection import pymongo.database @@ -118,6 +120,8 @@ class PillarServer(Eve): self._config_caching() + self._config_translations() + # Celery itself is configured after all extensions have loaded. self.celery: Celery = None @@ -234,6 +238,74 @@ class PillarServer(Eve): from flask_cache import Cache self.cache = Cache(self) + def set_languages(self, translations_folder: pathlib.Path): + """Set the supported languages based on translations folders + + English is an optional language included by default, since we will + never have a translations folder for it. + """ + self.default_locale = self.config['DEFAULT_LOCALE'] + self.config['BABEL_DEFAULT_LOCALE'] = self.default_locale + + # Determine available languages. + languages = list() + + # The available languages will be determined based on available + # translations in the //translations/ folder. The exception is (American) English + # since all the text is originally in English already. + # That said, if rare occasions we may want to never show + # the site in English. + + if self.config['SUPPORT_ENGLISH']: + languages.append('en_US') + + base_path = pathlib.Path(self.app_root) / 'translations' + + if not base_path.is_dir(): + self.log.debug('Project has no translations folder: %s', base_path) + else: + languages.extend(i.name for i in base_path.iterdir() if i.is_dir()) + + # Use set for quicker lookup + self.languages = set(languages) + + self.log.info('Available languages: %s' % ', '.join(self.languages)) + + def _config_translations(self): + """ + Initialize translations variable. + + The BABEL_TRANSLATION_DIRECTORIES has the folder for the compiled + translations files. It uses ; separation for the extension folders. + """ + self.log.info('Configure translations') + translations_path = pathlib.Path(__file__).parents[1].joinpath('translations') + + self.config['BABEL_TRANSLATION_DIRECTORIES'] = str(translations_path) + babel = Babel(self) + + self.set_languages(translations_path) + + # get_locale() is registered as a callback for locale selection. + # That prevents the function from being garbage collected. + @babel.localeselector + def get_locale() -> str: + """ + Callback runs before each request to give us a chance to choose the + language to use when producing its response. + + We set g.locale to be able to access it from the template pages. + We still need to return it explicitly, since this function is + called as part of the babel translation framework. + + We are using the 'Accept-Languages' header to match the available + translations with the user supported languages. + """ + locale = request.accept_languages.best_match( + self.languages, self.default_locale) + g.locale = locale + return locale + def load_extension(self, pillar_extension, url_prefix): from .extension import PillarExtension @@ -293,6 +365,19 @@ class PillarServer(Eve): self.config['DOMAIN'].update(eve_settings['DOMAIN']) + # Configure the extension translations + trpath = pillar_extension.translations_path + if not trpath: + self.log.debug('Extension %s does not have a translations folder', + pillar_extension.name) + return + + self.log.info('Extension %s: adding translations path %s', + pillar_extension.name, trpath) + + # Babel requires semi-colon string separation + self.config['BABEL_TRANSLATION_DIRECTORIES'] += ';' + str(trpath) + def _config_jinja_env(self): # Start with the extensions... paths_list = [ @@ -512,7 +597,7 @@ class PillarServer(Eve): self.log.info('Forwarding ResourceInvalid exception to client: %s', error, exc_info=True) # Raising a Werkzeug 422 exception doens't work, as Flask turns it into a 500. - return 'The submitted data could not be validated.', 422 + return _('The submitted data could not be validated.'), 422 def handle_sdk_method_not_allowed(self, error): """Forwards 405 Method Not Allowed to the client. diff --git a/pillar/cli/__init__.py b/pillar/cli/__init__.py index dcfc07a7..3cb59ef5 100644 --- a/pillar/cli/__init__.py +++ b/pillar/cli/__init__.py @@ -19,4 +19,4 @@ manager = Manager(current_app) manager.add_command('celery', manager_celery) manager.add_command("maintenance", manager_maintenance) manager.add_command("setup", manager_setup) -manager.add_command("operations", manager_operations) +manager.add_command("operations", manager_operations) \ No newline at end of file diff --git a/pillar/cli/translations.py b/pillar/cli/translations.py new file mode 100644 index 00000000..67debbbf --- /dev/null +++ b/pillar/cli/translations.py @@ -0,0 +1,104 @@ +import argparse +import contextlib +import pathlib +import subprocess +import sys + +BABEL_CONFIG = pathlib.Path('translations.cfg') + + +@contextlib.contextmanager +def create_messages_pot() -> pathlib.Path: + """Extract the translatable strings from the source code + + This creates a temporary messages.pot file, to be used to init or + update the translation .mo files. + + It works as a generator, yielding the temporarily created pot file. + The messages.pot file will be deleted at the end of it if all went well. + + :return The path of the messages.pot file created. + """ + if not BABEL_CONFIG.is_file(): + print("No translations config file found: %s" % (BABEL_CONFIG)) + sys.exit(-1) + return + + messages_pot = pathlib.Path('messages.pot') + subprocess.run(('pybabel', 'extract', '-F', BABEL_CONFIG, '-k', 'lazy_gettext', '-o', messages_pot, '.')) + yield messages_pot + messages_pot.unlink() + + +def init(locale): + """ + Initialize the translations for a new language. + """ + with create_messages_pot() as messages_pot: + subprocess.run(('pybabel', 'init', '-i', messages_pot, '-d', 'translations', '-l', locale)) + + +def update(): + """ + Update the strings to be translated. + """ + with create_messages_pot() as messages_pot: + subprocess.run(('pybabel', 'update', '-i', messages_pot, '-d', 'translations')) + + +def compile(): + """ + Compile the translation to be used. + """ + if pathlib.Path('translations').is_dir(): + subprocess.run(('pybabel', 'compile','-d', 'translations')) + else: + print("No translations folder available") + + +def parse_arguments() -> argparse.Namespace: + """ + Parse command-line arguments. + """ + parser = argparse.ArgumentParser(description='Translate Pillar') + + parser.add_argument( + 'mode', + type=str, + help='Init once, update often, compile before deploying.', + choices=['init', 'update', 'compile']) + + parser.add_argument( + 'languages', + nargs='*', + type=str, + help='Languages to initialize: pt it es ...') + + args = parser.parse_args() + if args.mode == 'init' and not args.languages: + parser.error("init requires languages") + + return args + + +def main(): + """ + When calling from the setup.py entry-point we need to parse the arguments + and init/update/compile the translations strings + """ + args = parse_arguments() + + if args.mode == 'init': + for language in args.languages: + init(language) + + elif args.mode == 'update': + update() + + else: # mode == 'compile' + compile() + + +if __name__ == '__main__': + main() + diff --git a/pillar/config.py b/pillar/config.py index e67a4bc5..057a5b5c 100644 --- a/pillar/config.py +++ b/pillar/config.py @@ -187,3 +187,23 @@ USER_CAPABILITIES = defaultdict(**{ 'view-pending-nodes', 'edit-project-node-types', 'create-organization'}, 'org-subscriber': {'subscriber', 'home-project'}, }, default_factory=frozenset) + + +# Internationalization and localization + +# The default locale is US English. +# A locale can include a territory, a codeset and a modifier. +# We only support locale strings with or without territories though. +# For example, nl_NL and pt_BR are not the same language as nl_BE, and pt_PT. +# However we can have a nl, or a pt translation, to be used as a common +# translation when no territorial specific locale is available. +# All translations should be in UTF-8. +# This setting is used as a fallback when there is no good match between the +# browser language and the available translations. +DEFAULT_LOCALE = 'en_US' +# All the available languages will be determined based on available translations +# in the //translations/ folder. The exception is English, since all the text is +# originally in English already. That said, if rare occasions we may want to +# never show the site in English. +SUPPORT_ENGLISH = True + diff --git a/pillar/extension.py b/pillar/extension.py index adb51302..e98b14ae 100644 --- a/pillar/extension.py +++ b/pillar/extension.py @@ -16,6 +16,8 @@ can then be registered to the application at app creation time: """ import abc +import inspect +import pathlib import typing import flask @@ -112,6 +114,23 @@ class PillarExtension(object, metaclass=abc.ABCMeta): """ return None + @property + def translations_path(self) -> typing.Union[pathlib.Path, None]: + """Returns the path where the translations for this extension are stored. + + This is top folder that contains a "translations" sub-folder + + May return None, in which case English will always be used for this extension. + """ + class_filename = pathlib.Path(inspect.getfile(self.__class__)) + + # Pillar extensions instantiate the PillarExtension from a sub-folder in + # the main project (e.g. //blender_cloud/blender_cloud/__init__.py), but + # the translations folders is in the main project folder. + translations_path = class_filename.parents[1] / 'translations' + + return translations_path if translations_path.is_dir() else None + def setup_app(self, app): """Called during app startup, after all extensions have loaded.""" diff --git a/requirements.txt b/requirements.txt index 6e78994f..93c9d774 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ celery[redis]==4.0.2 CommonMark==0.7.2 Eve==0.7.3 Flask==0.12 +Flask-Babel==0.11.2 Flask-Cache==0.13.1 Flask-Script==2.0.5 Flask-Login==0.3.2 diff --git a/setup.py b/setup.py index 6c6c1151..33cce541 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,30 @@ """Setup file for testing, not for packaging/distribution.""" import setuptools +from setuptools.command.develop import develop +from setuptools.command.install import install + + +def translations_compile(): + """Compile any existent translation. + """ + from pillar import cli + cli.translations.compile() + + +class PostDevelopCommand(develop): + """Post-installation for develop mode.""" + def run(self): + super().run() + translations_compile() + + +class PostInstallCommand(install): + """Post-installation for installation mode.""" + def run(self): + super().run() + translations_compile() + setuptools.setup( name='pillar', @@ -36,5 +60,12 @@ setuptools.setup( 'pytest-cov>=2.2.1', 'mock>=2.0.0', ], + entry_points = {'console_scripts': [ + 'translations = pillar.cli.translations:main', + ]}, + cmdclass={ + 'install': PostInstallCommand, + 'develop': PostDevelopCommand, + }, zip_safe=False, ) diff --git a/src/templates/errors/404.pug b/src/templates/errors/404.pug index e1216177..8c75c96f 100644 --- a/src/templates/errors/404.pug +++ b/src/templates/errors/404.pug @@ -3,15 +3,24 @@ #error-container(class="error-404") #error-box .error-top-container - .error-title Not found. + .error-title {{ _("Not found.") }} .error-lead - p Whatever you're looking for, it's not here. + p {{ _("Whatever you're looking for, it's not here.") }} .error-lead.extra p. - Looking for the Open Movies? Check out Blender Foundation's YouTube channel.
Were you looking for tutorials instead? blender.org has a good selection. + {% autoescape false %} + {{ gettext("Looking for the Open Movies? Check out %(link_start)s Blender Foundation's Youtube %(link_end)s channel.", + link_start='', link_end='') }}
+ {{ gettext("Were you looking for tutorials instead? %(blender_org)s has a good selection.", + blender_org='blender.org') }} + {% endautoescape %} p. - Is this content missing? Let us know on Twitter - or e-mail. + {% autoescape false %} + {{ gettext("Is this content missing? Let us know on %(twitter)s or %(email_start)s e-mail %(email_end)s", + twitter='Twitter', + email_start='', + email_end='') }} + {% endautoescape %} | {% endblock %} diff --git a/translations.cfg b/translations.cfg new file mode 100644 index 00000000..c8052d0b --- /dev/null +++ b/translations.cfg @@ -0,0 +1,14 @@ +# This file is used to crawl over the source code looking for +# strings to be extracted for translation. +# +# This file is structured to be parsed by the ConfigParser Python module +# +# For more information please visit: +# http://babel.pocoo.org/en/latest/messages.html + +# Extraction from Python source files +[python: pillar/**.py] + +# Extraction from the template files +[jinja2: src/templates/**.pug] +extensions=jinja2.ext.autoescape,jinja2.ext.with_