Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/i18n.py: 21%
230 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-03 13:41 +0000
1"""
2This module provides translation, also known as internationalization -- short: i18n.
4Project translations must be stored in the datastore. There are only some
5static translation tables in the viur-core to have some basic ones.
7The viur-core's own "translation" module (routed as _translation) provides
8an API to manage these translations, for example in the vi-admin.
10How to use translations?
11First, make sure that the languages are configured:
12.. code-block:: python
13 from viur.core.config import conf
14 # These are the main languages (for which translated values exist)
15 # that should be available for the project.
16 conf.i18n.available_languages = = ["en", "de", "fr"]
18 # These are some aliases for languages that should use the translated
19 # values of a particular main language, but don't have their own values.
20 conf.i18n.language_alias_map = {
21 "at": "de", # Austria uses German
22 "ch": "de", # Switzerland uses German
23 "be": "fr", # Belgian uses France
24 "us": "en", # US uses English
25 }
27Now translations can be used
291. In python
30.. code-block:: python
31 from viur.core.i18n import translate
32 # Just the translation key, the minimal case
33 print(translate("translation-key"))
34 # also provide a default value to use if there's no value in the datastore
35 # set and a hint to provide some context.
36 print(translate("translation-key", "the default value", "a hint"))
37 # Use string interpolation with variables
38 print(translate("hello", "Hello {{name}}!", "greeting a user")(name=current.user.get()["firstname"]))
402. In jinja
41.. code-block:: jinja
42 {# Use the ViUR translation extension, it can be compiled with the template,
43 caches the translation values and is therefore efficient #}
44 {% do translate "hello", "Hello {{name}}!", "greet a user", name="ViUR" %}
46 {# But in some cases the key or interpolation variables are dynamic and
47 not available during template compilation.
48 For this you can use the translate function: #}
49 {{ translate("hello", "Hello {{name}}!", "greet a user", name=skel["firstname"]) }}
52How to add translations
53There are two ways to add translations:
541. Manually
55With the vi-admin. Entries can be added manually by creating a new skeleton
56and filling in of the key and values.
582. Automatically
59The add_missing_translations option must be enabled for this.
60.. code-block:: python
62 from viur.core.config import conf
63 conf.i18n.add_missing_translations = True
66If a translation is now printed and the key is unknown (because someone has
67just added the related print code), an entry is added in the datastore kind.
68In addition, the default text and the hint are filled in and the filename
69and the line from the call from the code are set in the skeleton.
70This is the recommended way, as ViUR collects all the information you need
71and you only have to enter the translated values.
72(3. own way
73Of course you can create skeletons / entries in the datastore in your project
74on your own. Just use the TranslateSkel).
75""" # FIXME: grammar, rst syntax
76import datetime
77import jinja2.ext as jinja2
78import logging
79import traceback
80import typing as t
81from pathlib import Path
83from viur.core import current, db, languages, tasks
84from viur.core.config import conf
86systemTranslations = {}
87"""Memory storage for translation methods"""
89KINDNAME = "viur-translations"
90"""Kindname for the translations"""
93class LanguageWrapper(dict):
94 """
95 Wrapper-class for a multi-language value.
97 It's a dictionary, allowing accessing each stored language,
98 but can also be used as a string, in which case it tries to
99 guess the correct language.
100 Used by the HTML renderer to provide multi-lang bones values for templates.
101 """
103 def __init__(self, languages: list[str] | tuple[str]):
104 """
105 :param languages: Languages which are set in the bone.
106 """
107 super(LanguageWrapper, self).__init__()
108 self.languages = languages
110 def __str__(self) -> str:
111 return str(self.resolve())
113 def __bool__(self) -> bool:
114 # Overridden to support if skel["bone"] tests in html render
115 # (otherwise that test is always true as this dict contains keys)
116 return bool(str(self))
118 def resolve(self) -> str:
119 """
120 Causes this wrapper to evaluate to the best language available for the current request.
122 :returns: An item stored inside this instance or the empty string.
123 """
124 lang = current.language.get()
125 if lang:
126 lang = conf.i18n.language_alias_map.get(lang, lang)
127 else:
128 logging.warning(f"No lang set to current! {lang = }")
129 lang = self.languages[0]
130 if (value := self.get(lang)) and str(value).strip():
131 # The site language is available and not empty
132 return value
133 else: # Choose the first not-empty value as alternative
134 for lang in self.languages:
135 if (value := self.get(lang)) and str(value).strip():
136 return value
137 return "" # TODO: maybe we should better use sth like None or N/A
140class translate:
141 """
142 Translate class which chooses the correct translation according to the request language
144 This class is the replacement for the old translate() function provided by ViUR2. This classes __init__
145 takes the unique translation key (a string usually something like "user.auth_user_password.loginfailed" which
146 uniquely defines this text fragment), a default text that will be used if no translation for this key has been
147 added yet (in the projects default language) and a hint (an optional text that can convey context information
148 for the persons translating these texts - they are not shown to the end-user). This class will resolve its
149 translations upfront, so the actual resolving (by casting this class to string) is fast. This resolves most
150 translation issues with bones, which can now take an instance of this class as it's description/hints.
151 """
153 __slots__ = ["key", "defaultText", "hint", "translationCache", "force_lang"]
155 def __init__(self, key: str, defaultText: str = None, hint: str = None, force_lang: str = None):
156 """
157 :param key: The unique key defining this text fragment.
158 Usually it's a path/filename and a unique descriptor in that file
159 :param defaultText: The text to use if no translation has been added yet.
160 While optional, it's recommended to set this, as the key is used
161 instead if neither are available.
162 :param hint: A text only shown to the person translating this text,
163 as the key/defaultText may have different meanings in the
164 target language.
165 :param force_lang: Use this language instead the one of the request.
166 """
167 super().__init__()
168 key = str(key) # ensure key is a str
169 self.key = key.lower()
170 self.defaultText = defaultText or key
171 self.hint = hint
172 self.translationCache = None
173 if force_lang is not None and force_lang not in conf.i18n.available_dialects: 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true
174 raise ValueError(f"The language {force_lang=} is not available")
175 self.force_lang = force_lang
177 def __repr__(self) -> str:
178 return f"<translate object for {self.key} with force_lang={self.force_lang}>"
181 def __str__(self) -> str:
182 if self.translationCache is None:
183 global systemTranslations
185 from viur.core.render.html.env.viur import translate as jinja_translate
187 if self.key not in systemTranslations and conf.i18n.add_missing_translations:
188 # This translation seems to be new and should be added
189 filename = lineno = None
190 is_jinja = False
191 for frame, line in traceback.walk_stack(None):
192 if filename is None:
193 # Use the first frame as fallback.
194 # In case of calling this class directly,
195 # this is anyway the caller we're looking for.
196 filename = frame.f_code.co_filename
197 lineno = frame.f_lineno
198 if frame.f_code == jinja_translate.__code__:
199 # The call was caused by our jinja method
200 is_jinja = True
201 if is_jinja and not frame.f_code.co_filename.endswith(".py"):
202 # Look for the latest html, macro (not py) where the
203 # translate method has been used, that's our caller
204 filename = frame.f_code.co_filename
205 lineno = line
206 break
208 add_missing_translation(
209 key=self.key,
210 hint=self.hint,
211 default_text=self.defaultText,
212 filename=filename,
213 lineno=lineno,
214 )
216 self.translationCache = self.merge_alias(systemTranslations.get(self.key, {}))
218 if (lang := self.force_lang) is None:
219 # The default case: use the request language
220 lang = current.language.get()
221 if value := self.translationCache.get(lang):
222 return value
223 # Use the default text from datastore or from the caller arguments
224 return self.translationCache.get("_default_text_") or self.defaultText
226 def translate(self, **kwargs) -> str:
227 """Substitute the given kwargs in the translated or default text."""
228 return self.substitute_vars(str(self), **kwargs)
230 def __call__(self, **kwargs):
231 """Just an alias for translate"""
232 return self.translate(**kwargs)
234 @staticmethod
235 def substitute_vars(value: str, **kwargs):
236 """Substitute vars in a translation
238 Variables has to start with two braces (`{{`), followed by the variable
239 name and end with two braces (`}}`).
240 Values can be anything, they are cast to string anyway.
241 "Hello {{name}}!" becomes with name="Bob": "Hello Bob!"
242 """
243 res = str(value)
244 for k, v in kwargs.items():
245 # 2 braces * (escape + real brace) + 1 for variable = 5
246 res = res.replace(f"{ { {k}} } ", str(v))
247 return res
249 @staticmethod
250 def merge_alias(translations: dict[str, str]):
251 """Make sure each aliased language has a value
253 If an aliased language does not have a value in the translation dict,
254 the value of the main language is copied.
255 """
256 for alias, main in conf.i18n.language_alias_map.items():
257 if not (value := translations.get(alias)) or not value.strip():
258 if main_value := translations.get(main):
259 # Use only not empty value
260 translations[alias] = main_value
261 return translations
264class TranslationExtension(jinja2.Extension):
265 """
266 Default translation extension for jinja2 render.
267 Use like {% translate "translationKey", "defaultText", "translationHint", replaceValue1="replacedText1" %}
268 All except translationKey is optional. translationKey is the same Key supplied to _() before.
269 defaultText will be printed if no translation is available.
270 translationHint is an optional hint for anyone adding a now translation how/where that translation is used.
271 `force_lang` can be used as a keyword argument (the only allowed way) to
272 force the use of a specific language, not the language of the request.
273 """
275 tags = {"translate"}
277 def parse(self, parser):
278 # Parse the translate tag
279 global systemTranslations
280 args = [] # positional args for the `_translate()` method
281 kwargs = {} # keyword args (force_lang + substitute vars) for the `_translate()` method
282 lineno = parser.stream.current.lineno
283 filename = parser.stream.filename
284 # Parse arguments (args and kwargs) until the current block ends
285 lastToken = None
286 while parser.stream.current.type != 'block_end':
287 lastToken = parser.parse_expression()
288 if parser.stream.current.type == "comma": # It's a positional arg
289 args.append(lastToken.value)
290 next(parser.stream) # Advance pointer
291 lastToken = None
292 elif parser.stream.current.type == "assign":
293 next(parser.stream) # Advance beyond =
294 expr = parser.parse_expression()
295 kwargs[lastToken.name] = expr.value
296 if parser.stream.current.type == "comma":
297 next(parser.stream)
298 elif parser.stream.current.type == "block_end":
299 lastToken = None
300 break
301 else:
302 raise SyntaxError()
303 lastToken = None
304 if lastToken: # TODO: what's this? what it is doing?
305 logging.debug(f"final append {lastToken = }")
306 args.append(lastToken.value)
307 if not 0 < len(args) <= 3:
308 raise SyntaxError("Translation-Key missing or excess parameters!")
309 args += [""] * (3 - len(args))
310 args += [kwargs]
311 tr_key = args[0].lower()
312 if tr_key not in systemTranslations:
313 add_missing_translation(
314 key=tr_key,
315 hint=args[1],
316 default_text=args[2],
317 filename=filename,
318 lineno=lineno,
319 variables=list(kwargs.keys()),
320 )
322 translations = translate.merge_alias(systemTranslations.get(tr_key, {}))
323 args[1] = translations.get("_default_text_") or args[1]
324 args = [jinja2.nodes.Const(x) for x in args]
325 args.append(jinja2.nodes.Const(translations))
326 return jinja2.nodes.CallBlock(self.call_method("_translate", args), [], [], []).set_lineno(lineno)
328 def _translate(
329 self, key: str, default_text: str, hint: str, kwargs: dict[str, t.Any],
330 translations: dict[str, str], caller
331 ) -> str:
332 """Perform the actual translation during render"""
333 lang = kwargs.pop("force_lang", current.language.get())
334 res = str(translations.get(lang, default_text))
335 return translate.substitute_vars(res, **kwargs)
338def initializeTranslations() -> None:
339 """
340 Fetches all translations from the datastore and populates the *systemTranslations* dictionary of this module.
341 Currently, the translate-class will resolve using that dictionary; but as we expect projects to grow and
342 accumulate translations that are no longer/not yet used, we plan to made the translation-class fetch it's
343 translations directly from the datastore, so we don't have to allocate memory for unused translations.
344 """
345 # Load translations from static languages module into systemTranslations
346 # If they're in the datastore, they will be overwritten below.
347 for lang in dir(languages):
348 if lang.startswith("__"):
349 continue
350 for tr_key, tr_value in getattr(languages, lang).items():
351 systemTranslations.setdefault(tr_key, {})[lang] = tr_value
353 # Load translations from datastore into systemTranslations
354 # TODO: iter() would be more memory efficient, but unfortunately takes much longer than run()
355 # for entity in db.Query(KINDNAME).iter():
356 for entity in db.Query(KINDNAME).run(10_000):
357 if "tr_key" not in entity:
358 logging.warning(f"translations entity {entity.key} has no tr_key set --> Call migration")
359 migrate_translation(entity.key)
360 # Before the migration has run do a quick modification to get it loaded as is
361 entity["tr_key"] = entity["key"] or entity.key.name
362 if not entity.get("tr_key"):
363 logging.error(f'translations entity {entity.key} has an empty {entity["tr_key"]=} set. Skipping.')
364 continue
365 if entity and not isinstance(entity["translations"], dict):
366 logging.error(f'translations entity {entity.key} has invalid '
367 f'translations set: {entity["translations"]}. Skipping.')
368 continue
370 translations = {
371 "_default_text_": entity.get("default_text") or None,
372 }
373 for lang, translation in entity["translations"].items():
374 if lang not in conf.i18n.available_dialects:
375 # Don't store unknown languages in the memory
376 continue
377 if not translation or not str(translation).strip():
378 # Skip empty values
379 continue
380 translations[lang] = translation
381 systemTranslations[entity["tr_key"]] = translations
384@tasks.CallDeferred
385@tasks.retry_n_times(20)
386def add_missing_translation(
387 key: str,
388 hint: str | None = None,
389 default_text: str | None = None,
390 filename: str | None = None,
391 lineno: int | None = None,
392 variables: list[str] = None,
393) -> None:
394 """Add missing translations to datastore"""
395 try:
396 from viur.core.modules.translation import TranslationSkel, Creator
397 except ImportError as exc:
398 # We use translate inside the TranslationSkel, this causes circular dependencies which can be ignored
399 logging.warning(f"ImportError (probably during warmup), "
400 f"cannot add translation {key}: {exc}", exc_info=True)
401 return
403 # Ensure lowercase key
404 key = key.lower()
405 entity = db.Query(KINDNAME).filter("tr_key =", key).getEntry()
406 if entity is not None:
407 # Ensure it doesn't exist to avoid datastore conflicts
408 logging.warning(f"Found an entity with tr_key={key}. "
409 f"Probably an other instance was faster.")
410 return
412 if isinstance(filename, str):
413 try:
414 filename = str(Path(filename)
415 .relative_to(conf.instance.project_base_path,
416 conf.instance.core_base_path))
417 except ValueError:
418 pass # not a subpath
420 logging.info(f"Add missing translation {key}")
421 skel = TranslationSkel()
422 skel["tr_key"] = key
423 skel["default_text"] = default_text or None
424 skel["hint"] = hint or None
425 skel["usage_filename"] = filename
426 skel["usage_lineno"] = lineno
427 skel["usage_variables"] = variables or []
428 skel["creator"] = Creator.VIUR
429 skel.toDB()
431 # Add to system translation to avoid triggering this method again
432 systemTranslations[key] = {
433 "_default_text_": default_text or None,
434 }
437@tasks.CallDeferred
438@tasks.retry_n_times(20)
439def migrate_translation(
440 key: db.Key,
441) -> None:
442 """Migrate entities, if required.
444 With viur-core 3.6 translations are now managed as Skeletons and require
445 some changes, which are performed in this method.
446 """
447 from viur.core.modules.translation import TranslationSkel
448 logging.info(f"Migrate translation {key}")
449 entity: db.Entity = db.Get(key)
450 if "tr_key" not in entity:
451 entity["tr_key"] = entity["key"] or key.name
452 if "translation" in entity:
453 if not isinstance(dict, entity["translation"]):
454 logging.error("translation is not a dict?")
455 entity["translation"]["_viurLanguageWrapper_"] = True
456 skel = TranslationSkel()
457 skel.setEntity(entity)
458 skel["key"] = key
459 try:
460 skel.toDB()
461 except ValueError as exc:
462 logging.exception(exc)
463 if "unique value" in exc.args[0] and "recently claimed" in exc.args[0]:
464 logging.info(f"Delete duplicate entry {key}: {entity}")
465 db.Delete(key)
466 else:
467 raise exc
470localizedDateTime = translate("const_datetimeformat", "%a %b %d %H:%M:%S %Y", "Localized Time and Date format string")
471localizedDate = translate("const_dateformat", "%m/%d/%Y", "Localized Date only format string")
472localizedTime = translate("const_timeformat", "%H:%M:%S", "Localized Time only format string")
473localizedAbbrevDayNames = {
474 0: translate("const_day_0_short", "Sun", "Abbreviation for Sunday"),
475 1: translate("const_day_1_short", "Mon", "Abbreviation for Monday"),
476 2: translate("const_day_2_short", "Tue", "Abbreviation for Tuesday"),
477 3: translate("const_day_3_short", "Wed", "Abbreviation for Wednesday"),
478 4: translate("const_day_4_short", "Thu", "Abbreviation for Thursday"),
479 5: translate("const_day_5_short", "Fri", "Abbreviation for Friday"),
480 6: translate("const_day_6_short", "Sat", "Abbreviation for Saturday"),
481}
482localizedDayNames = {
483 0: translate("const_day_0_long", "Sunday", "Sunday"),
484 1: translate("const_day_1_long", "Monday", "Monday"),
485 2: translate("const_day_2_long", "Tuesday", "Tuesday"),
486 3: translate("const_day_3_long", "Wednesday", "Wednesday"),
487 4: translate("const_day_4_long", "Thursday", "Thursday"),
488 5: translate("const_day_5_long", "Friday", "Friday"),
489 6: translate("const_day_6_long", "Saturday", "Saturday"),
490}
491localizedAbbrevMonthNames = {
492 1: translate("const_month_1_short", "Jan", "Abbreviation for January"),
493 2: translate("const_month_2_short", "Feb", "Abbreviation for February"),
494 3: translate("const_month_3_short", "Mar", "Abbreviation for March"),
495 4: translate("const_month_4_short", "Apr", "Abbreviation for April"),
496 5: translate("const_month_5_short", "May", "Abbreviation for May"),
497 6: translate("const_month_6_short", "Jun", "Abbreviation for June"),
498 7: translate("const_month_7_short", "Jul", "Abbreviation for July"),
499 8: translate("const_month_8_short", "Aug", "Abbreviation for August"),
500 9: translate("const_month_9_short", "Sep", "Abbreviation for September"),
501 10: translate("const_month_10_short", "Oct", "Abbreviation for October"),
502 11: translate("const_month_11_short", "Nov", "Abbreviation for November"),
503 12: translate("const_month_12_short", "Dec", "Abbreviation for December"),
504}
505localizedMonthNames = {
506 1: translate("const_month_1_long", "January", "January"),
507 2: translate("const_month_2_long", "February", "February"),
508 3: translate("const_month_3_long", "March", "March"),
509 4: translate("const_month_4_long", "April", "April"),
510 5: translate("const_month_5_long", "May", "May"),
511 6: translate("const_month_6_long", "June", "June"),
512 7: translate("const_month_7_long", "July", "July"),
513 8: translate("const_month_8_long", "August", "August"),
514 9: translate("const_month_9_long", "September", "September"),
515 10: translate("const_month_10_long", "October", "October"),
516 11: translate("const_month_11_long", "November", "November"),
517 12: translate("const_month_12_long", "December", "December"),
518}
521def localizedStrfTime(datetimeObj: datetime.datetime, format: str) -> str:
522 """
523 Provides correct localized names for directives like %a which don't get translated on GAE properly as we can't
524 set the locale (for each request).
525 This currently replaces %a, %A, %b, %B, %c, %x and %X.
527 :param datetimeObj: Datetime-instance to call strftime on
528 :param format: String containing the Format to apply.
529 :returns: Date and time formatted according to format with correct localization
530 """
531 if "%c" in format:
532 format = format.replace("%c", str(localizedDateTime))
533 if "%x" in format:
534 format = format.replace("%x", str(localizedDate))
535 if "%X" in format:
536 format = format.replace("%X", str(localizedTime))
537 if "%a" in format:
538 format = format.replace("%a", str(localizedAbbrevDayNames[int(datetimeObj.strftime("%w"))]))
539 if "%A" in format:
540 format = format.replace("%A", str(localizedDayNames[int(datetimeObj.strftime("%w"))]))
541 if "%b" in format:
542 format = format.replace("%b", str(localizedAbbrevMonthNames[int(datetimeObj.strftime("%m"))]))
543 if "%B" in format:
544 format = format.replace("%B", str(localizedMonthNames[int(datetimeObj.strftime("%m"))]))
545 return datetimeObj.strftime(format)