Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/i18n.py: 17%

1"""

2This module provides translation, also known as internationalization -- short: i18n.

4Project translations must be stored in the datastore. There are only some

5static translation tables in the viur-core to have some basic ones.

7The viur-core's own "translation" module (routed as _translation) provides

8an API to manage these translations, for example in the vi-admin.

10How to use translations?

11First, make sure that the languages are configured:

12.. code-block:: python

13 from viur.core.config import conf

14 # These are the main languages (for which translated values exist)

15 # that should be available for the project.

16 conf.i18n.available_languages = = ["en", "de", "fr"]

18 # These are some aliases for languages that should use the translated

19 # values of a particular main language, but don't have their own values.

20 conf.i18n.language_alias_map = {

21 "at": "de", # Austria uses German

22 "ch": "de", # Switzerland uses German

23 "be": "fr", # Belgian uses France

24 "us": "en", # US uses English

25 }

27Now translations can be used

291. In python

30.. code-block:: python

31 from viur.core.i18n import translate

32 # Just the translation key, the minimal case

33 print(translate("translation-key"))

34 # also provide a default value to use if there's no value in the datastore

35 # set and a hint to provide some context.

36 print(translate("translation-key", "the default value", "a hint"))

37 # Use string interpolation with variables

38 print(translate("hello", "Hello {{name}}!", "greeting a user")(name=current.user.get()["firstname"]))

402. In jinja

41.. code-block:: jinja

42 {# Use the ViUR translation extension, it can be compiled with the template,

43 caches the translation values and is therefore efficient #}

44 {% do translate "hello", "Hello {{name}}!", "greet a user", name="ViUR" %}

46 {# But in some cases the key or interpolation variables are dynamic and

47 not available during template compilation.

48 For this you can use the translate function: #}

49 {{ translate("hello", "Hello {{name}}!", "greet a user", name=skel["firstname"]) }}

52How to add translations

53There are two ways to add translations:

541. Manually

55With the vi-admin. Entries can be added manually by creating a new skeleton

56and filling in of the key and values.

582. Automatically

59The add_missing_translations option must be enabled for this.

60.. code-block:: python

62 from viur.core.config import conf

63 conf.i18n.add_missing_translations = True

66If a translation is now printed and the key is unknown (because someone has

67just added the related print code), an entry is added in the datastore kind.

68In addition, the default text and the hint are filled in and the filename

69and the line from the call from the code are set in the skeleton.

70This is the recommended way, as ViUR collects all the information you need

71and you only have to enter the translated values.

72(3. own way

73Of course you can create skeletons / entries in the datastore in your project

74on your own. Just use the TranslateSkel).

75""" # FIXME: grammar, rst syntax

76import datetime

77import jinja2.ext as jinja2

78import logging

79import traceback

80import typing as t

81from pathlib import Path

83from viur.core import current, db, languages, tasks

84from viur.core.config import conf

86systemTranslations = {}

87"""Memory storage for translation methods"""

89KINDNAME = "viur-translations"

90"""Kindname for the translations"""

93class LanguageWrapper(dict):

94 """

95 Wrapper-class for a multi-language value.

97 It's a dictionary, allowing accessing each stored language,

98 but can also be used as a string, in which case it tries to

99 guess the correct language.

100 Used by the HTML renderer to provide multi-lang bones values for templates.

101 """

102

103 def __init__(self, languages: list[str] | tuple[str]):

104 """

105 :param languages: Languages which are set in the bone.

106 """

107 super(LanguageWrapper, self).__init__()

108 self.languages = languages

109

110 def __str__(self) -> str:

111 return str(self.resolve())

112

113 def __bool__(self) -> bool:

114 # Overridden to support if skel["bone"] tests in html render

115 # (otherwise that test is always true as this dict contains keys)

116 return bool(str(self))

117

118 def resolve(self) -> str:

119 """

120 Causes this wrapper to evaluate to the best language available for the current request.

121

122 :returns: An item stored inside this instance or the empty string.

123 """

124 lang = current.language.get()

125 if lang:

126 lang = conf.i18n.language_alias_map.get(lang, lang)

127 else:

128 logging.warning(f"No lang set to current! {lang = }")

129 lang = self.languages[0]

130 if (value := self.get(lang)) and str(value).strip():

131 # The site language is available and not empty

132 return value

133 else: # Choose the first not-empty value as alternative

134 for lang in self.languages:

135 if (value := self.get(lang)) and str(value).strip():

136 return value

137 return "" # TODO: maybe we should better use sth like None or N/A

138

139

140class translate:

141 """

142 Translate class which chooses the correct translation according to the request language

143

144 This class is the replacement for the old translate() function provided by ViUR2. This classes __init__

145 takes the unique translation key (a string usually something like "user.auth_user_password.loginfailed" which

146 uniquely defines this text fragment), a default text that will be used if no translation for this key has been

147 added yet (in the projects default language) and a hint (an optional text that can convey context information

148 for the persons translating these texts - they are not shown to the end-user). This class will resolve its

149 translations upfront, so the actual resolving (by casting this class to string) is fast. This resolves most

150 translation issues with bones, which can now take an instance of this class as it's description/hints.

151 """

152

153 __slots__ = ["key", "defaultText", "hint", "translationCache", "force_lang"]

154

155 def __init__(self, key: str, defaultText: str = None, hint: str = None, force_lang: str = None):

156 """

157 :param key: The unique key defining this text fragment.

158 Usually it's a path/filename and a unique descriptor in that file

159 :param defaultText: The text to use if no translation has been added yet.

160 While optional, it's recommended to set this, as the key is used

161 instead if neither are available.

162 :param hint: A text only shown to the person translating this text,

163 as the key/defaultText may have different meanings in the

164 target language.

165 :param force_lang: Use this language instead the one of the request.

166 """

167 super().__init__()

168 key = str(key) # ensure key is a str

169 self.key = key.lower()

170 self.defaultText = defaultText or key

171 self.hint = hint

172 self.translationCache = None

173 if force_lang is not None and force_lang not in conf.i18n.available_dialects: 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true

174 raise ValueError(f"The language {force_lang=} is not available")

175 self.force_lang = force_lang

176

177 def __repr__(self) -> str:

178 return f"<translate object for {self.key} with force_lang={self.force_lang}>"

179

180

181 def __str__(self) -> str:

182 if self.translationCache is None:

183 global systemTranslations

184

185 from viur.core.render.html.env.viur import translate as jinja_translate

186

187 if self.key not in systemTranslations and conf.i18n.add_missing_translations:

188 # This translation seems to be new and should be added

189 filename = lineno = None

190 is_jinja = False

191 for frame, line in traceback.walk_stack(None):

192 if filename is None:

193 # Use the first frame as fallback.

194 # In case of calling this class directly,

195 # this is anyway the caller we're looking for.

196 filename = frame.f_code.co_filename

197 lineno = frame.f_lineno

198 if frame.f_code == jinja_translate.__code__:

199 # The call was caused by our jinja method

200 is_jinja = True

201 if is_jinja and not frame.f_code.co_filename.endswith(".py"):

202 # Look for the latest html, macro (not py) where the

203 # translate method has been used, that's our caller

204 filename = frame.f_code.co_filename

205 lineno = line

206 break

207

208 add_missing_translation(

209 key=self.key,

210 hint=self.hint,

211 default_text=self.defaultText,

212 filename=filename,

213 lineno=lineno,

214 )

215

216 self.translationCache = self.merge_alias(systemTranslations.get(self.key, {}))

217

218 if (lang := self.force_lang) is None:

219 # The default case: use the request language

220 lang = current.language.get()

221 if value := self.translationCache.get(lang):

222 return value

223 # Use the default text from datastore or from the caller arguments

224 return self.translationCache.get("_default_text_") or self.defaultText

225

226 def translate(self, **kwargs) -> str:

227 """Substitute the given kwargs in the translated or default text."""

228 return self.substitute_vars(str(self), **kwargs)

229

230 def __call__(self, **kwargs):

231 """Just an alias for translate"""

232 return self.translate(**kwargs)

233

234 @staticmethod

235 def substitute_vars(value: str, **kwargs):

236 """Substitute vars in a translation

237

238 Variables has to start with two braces (`{{`), followed by the variable

239 name and end with two braces (`}}`).

240 Values can be anything, they are cast to string anyway.

241 "Hello {{name}}!" becomes with name="Bob": "Hello Bob!"

242 """

243 res = str(value)

244 for k, v in kwargs.items():

245 # 2 braces * (escape + real brace) + 1 for variable = 5

246 res = res.replace(f"{ { {k}} } ", str(v))

247 return res

248

249 @staticmethod

250 def merge_alias(translations: dict[str, str]):

251 """Make sure each aliased language has a value

252

253 If an aliased language does not have a value in the translation dict,

254 the value of the main language is copied.

255 """

256 for alias, main in conf.i18n.language_alias_map.items():

257 if not (value := translations.get(alias)) or not value.strip():

258 if main_value := translations.get(main):

259 # Use only not empty value

260 translations[alias] = main_value

261 return translations

262

263

264class TranslationExtension(jinja2.Extension):

265 """

266 Default translation extension for jinja2 render.

267 Use like {% translate "translationKey", "defaultText", "translationHint", replaceValue1="replacedText1" %}

268 All except translationKey is optional. translationKey is the same Key supplied to _() before.

269 defaultText will be printed if no translation is available.

270 translationHint is an optional hint for anyone adding a now translation how/where that translation is used.

271 `force_lang` can be used as a keyword argument (the only allowed way) to

272 force the use of a specific language, not the language of the request.

273 """

274

275 tags = {"translate"}

276

277 def parse(self, parser):

278 # Parse the translate tag

279 global systemTranslations

280 args = [] # positional args for the `_translate()` method

281 kwargs = {} # keyword args (force_lang + substitute vars) for the `_translate()` method

282 lineno = parser.stream.current.lineno

283 filename = parser.stream.filename

284 # Parse arguments (args and kwargs) until the current block ends

285 lastToken = None

286 while parser.stream.current.type != 'block_end':

287 lastToken = parser.parse_expression()

288 if parser.stream.current.type == "comma": # It's a positional arg

289 args.append(lastToken.value)

290 next(parser.stream) # Advance pointer

291 lastToken = None

292 elif parser.stream.current.type == "assign":

293 next(parser.stream) # Advance beyond =

294 expr = parser.parse_expression()

295 kwargs[lastToken.name] = expr.value

296 if parser.stream.current.type == "comma":

297 next(parser.stream)

298 elif parser.stream.current.type == "block_end":

299 lastToken = None

300 break

301 else:

302 raise SyntaxError()

303 lastToken = None

304 if lastToken: # TODO: what's this? what it is doing?

305 logging.debug(f"final append {lastToken = }")

306 args.append(lastToken.value)

307 if not 0 < len(args) <= 3:

308 raise SyntaxError("Translation-Key missing or excess parameters!")

309 args += [""] * (3 - len(args))

310 args += [kwargs]

311 tr_key = args[0].lower()

312 if tr_key not in systemTranslations:

313 add_missing_translation(

314 key=tr_key,

315 hint=args[1],

316 default_text=args[2],

317 filename=filename,

318 lineno=lineno,

319 variables=list(kwargs.keys()),

320 )

321

322 translations = translate.merge_alias(systemTranslations.get(tr_key, {}))

323 args[1] = translations.get("_default_text_") or args[1]

324 args = [jinja2.nodes.Const(x) for x in args]

325 args.append(jinja2.nodes.Const(translations))

326 return jinja2.nodes.CallBlock(self.call_method("_translate", args), [], [], []).set_lineno(lineno)

327

328 def _translate(

329 self, key: str, default_text: str, hint: str, kwargs: dict[str, t.Any],

330 translations: dict[str, str], caller

331 ) -> str:

332 """Perform the actual translation during render"""

333 lang = kwargs.pop("force_lang", current.language.get())

334 res = str(translations.get(lang, default_text))

335 return translate.substitute_vars(res, **kwargs)

336

337

338def initializeTranslations() -> None:

339 """

340 Fetches all translations from the datastore and populates the *systemTranslations* dictionary of this module.

341 Currently, the translate-class will resolve using that dictionary; but as we expect projects to grow and

342 accumulate translations that are no longer/not yet used, we plan to made the translation-class fetch it's

343 translations directly from the datastore, so we don't have to allocate memory for unused translations.

344 """

345 # Load translations from static languages module into systemTranslations

346 # If they're in the datastore, they will be overwritten below.

347 for lang in dir(languages):

348 if lang.startswith("__"):

349 continue

350 for tr_key, tr_value in getattr(languages, lang).items():

351 systemTranslations.setdefault(tr_key, {})[lang] = tr_value

352

353 # Load translations from datastore into systemTranslations

354 # TODO: iter() would be more memory efficient, but unfortunately takes much longer than run()

355 # for entity in db.Query(KINDNAME).iter():

356 for entity in db.Query(KINDNAME).run(10_000):

357 if "tr_key" not in entity:

358 logging.warning(f"translations entity {entity.key} has no tr_key set --> Call migration")

359 migrate_translation(entity.key)

360 # Before the migration has run do a quick modification to get it loaded as is

361 entity["tr_key"] = entity["key"] or entity.key.name

362 if not entity.get("tr_key"):

363 logging.error(f'translations entity {entity.key} has an empty {entity["tr_key"]=} set. Skipping.')

364 continue

365 if entity and not isinstance(entity["translations"], dict):

366 logging.error(f'translations entity {entity.key} has invalid '

367 f'translations set: {entity["translations"]}. Skipping.')

368 continue

369

370 translations = {

371 "_default_text_": entity.get("default_text") or None,

372 }

373 for lang, translation in entity["translations"].items():

374 if lang not in conf.i18n.available_dialects:

375 # Don't store unknown languages in the memory

376 continue

377 if not translation or not str(translation).strip():

378 # Skip empty values

379 continue

380 translations[lang] = translation

381 systemTranslations[entity["tr_key"]] = translations

382

383

384@tasks.CallDeferred

385@tasks.retry_n_times(20)

386def add_missing_translation(

387 key: str,

388 hint: str | None = None,

389 default_text: str | None = None,

390 filename: str | None = None,

391 lineno: int | None = None,

392 variables: list[str] = None,

393) -> None:

394 """Add missing translations to datastore"""

395 try:

396 from viur.core.modules.translation import TranslationSkel, Creator

397 except ImportError as exc:

398 # We use translate inside the TranslationSkel, this causes circular dependencies which can be ignored

399 logging.warning(f"ImportError (probably during warmup), "

400 f"cannot add translation {key}: {exc}", exc_info=True)

401 return

402

403 # Ensure lowercase key

404 key = key.lower()

405 entity = db.Query(KINDNAME).filter("tr_key =", key).getEntry()

406 if entity is not None:

407 # Ensure it doesn't exist to avoid datastore conflicts

408 logging.warning(f"Found an entity with tr_key={key}. "

409 f"Probably an other instance was faster.")

410 return

411

412 if isinstance(filename, str):

413 try:

414 filename = str(Path(filename)

415 .relative_to(conf.instance.project_base_path,

416 conf.instance.core_base_path))

417 except ValueError:

418 pass # not a subpath

419

420 logging.info(f"Add missing translation {key}")

421 skel = TranslationSkel()

422 skel["tr_key"] = key

423 skel["default_text"] = default_text or None

424 skel["hint"] = hint or None

425 skel["usage_filename"] = filename

426 skel["usage_lineno"] = lineno

427 skel["usage_variables"] = variables or []

428 skel["creator"] = Creator.VIUR

429 skel.toDB()

430

431 # Add to system translation to avoid triggering this method again

432 systemTranslations[key] = {

433 "_default_text_": default_text or None,

434 }

435

436

437@tasks.CallDeferred

438@tasks.retry_n_times(20)

439def migrate_translation(

440 key: db.Key,

441) -> None:

442 """Migrate entities, if required.

443

444 With viur-core 3.6 translations are now managed as Skeletons and require

445 some changes, which are performed in this method.

446 """

447 from viur.core.modules.translation import TranslationSkel

448 logging.info(f"Migrate translation {key}")

449 entity: db.Entity = db.Get(key)

450 if "tr_key" not in entity:

451 entity["tr_key"] = entity["key"] or key.name

452 if "translation" in entity:

453 if not isinstance(dict, entity["translation"]):

454 logging.error("translation is not a dict?")

455 entity["translation"]["_viurLanguageWrapper_"] = True

456 skel = TranslationSkel()

457 skel.setEntity(entity)

458 skel["key"] = key

459 try:

460 skel.toDB()

461 except ValueError as exc:

462 logging.exception(exc)

463 if "unique value" in exc.args[0] and "recently claimed" in exc.args[0]:

464 logging.info(f"Delete duplicate entry {key}: {entity}")

465 db.Delete(key)

466 else:

467 raise exc

468

469

470localizedDateTime = translate("const_datetimeformat", "%a %b %d %H:%M:%S %Y", "Localized Time and Date format string")

471localizedDate = translate("const_dateformat", "%m/%d/%Y", "Localized Date only format string")

472localizedTime = translate("const_timeformat", "%H:%M:%S", "Localized Time only format string")

473localizedAbbrevDayNames = {

474 0: translate("const_day_0_short", "Sun", "Abbreviation for Sunday"),

475 1: translate("const_day_1_short", "Mon", "Abbreviation for Monday"),

476 2: translate("const_day_2_short", "Tue", "Abbreviation for Tuesday"),

477 3: translate("const_day_3_short", "Wed", "Abbreviation for Wednesday"),

478 4: translate("const_day_4_short", "Thu", "Abbreviation for Thursday"),

479 5: translate("const_day_5_short", "Fri", "Abbreviation for Friday"),

480 6: translate("const_day_6_short", "Sat", "Abbreviation for Saturday"),

481}

482localizedDayNames = {

483 0: translate("const_day_0_long", "Sunday", "Sunday"),

484 1: translate("const_day_1_long", "Monday", "Monday"),

485 2: translate("const_day_2_long", "Tuesday", "Tuesday"),

486 3: translate("const_day_3_long", "Wednesday", "Wednesday"),

487 4: translate("const_day_4_long", "Thursday", "Thursday"),

488 5: translate("const_day_5_long", "Friday", "Friday"),

489 6: translate("const_day_6_long", "Saturday", "Saturday"),

490}

491localizedAbbrevMonthNames = {

492 1: translate("const_month_1_short", "Jan", "Abbreviation for January"),

493 2: translate("const_month_2_short", "Feb", "Abbreviation for February"),

494 3: translate("const_month_3_short", "Mar", "Abbreviation for March"),

495 4: translate("const_month_4_short", "Apr", "Abbreviation for April"),

496 5: translate("const_month_5_short", "May", "Abbreviation for May"),

497 6: translate("const_month_6_short", "Jun", "Abbreviation for June"),

498 7: translate("const_month_7_short", "Jul", "Abbreviation for July"),

499 8: translate("const_month_8_short", "Aug", "Abbreviation for August"),

500 9: translate("const_month_9_short", "Sep", "Abbreviation for September"),

501 10: translate("const_month_10_short", "Oct", "Abbreviation for October"),

502 11: translate("const_month_11_short", "Nov", "Abbreviation for November"),

503 12: translate("const_month_12_short", "Dec", "Abbreviation for December"),

504}

505localizedMonthNames = {

506 1: translate("const_month_1_long", "January", "January"),

507 2: translate("const_month_2_long", "February", "February"),

508 3: translate("const_month_3_long", "March", "March"),

509 4: translate("const_month_4_long", "April", "April"),

510 5: translate("const_month_5_long", "May", "May"),

511 6: translate("const_month_6_long", "June", "June"),

512 7: translate("const_month_7_long", "July", "July"),

513 8: translate("const_month_8_long", "August", "August"),

514 9: translate("const_month_9_long", "September", "September"),

515 10: translate("const_month_10_long", "October", "October"),

516 11: translate("const_month_11_long", "November", "November"),

517 12: translate("const_month_12_long", "December", "December"),

518}

519

520

521def localizedStrfTime(datetimeObj: datetime.datetime, format: str) -> str:

522 """

523 Provides correct localized names for directives like %a which don't get translated on GAE properly as we can't

524 set the locale (for each request).

525 This currently replaces %a, %A, %b, %B, %c, %x and %X.

526

527 :param datetimeObj: Datetime-instance to call strftime on

528 :param format: String containing the Format to apply.

529 :returns: Date and time formatted according to format with correct localization

530 """

531 if "%c" in format:

532 format = format.replace("%c", str(localizedDateTime))

533 if "%x" in format:

534 format = format.replace("%x", str(localizedDate))

535 if "%X" in format:

536 format = format.replace("%X", str(localizedTime))

537 if "%a" in format:

538 format = format.replace("%a", str(localizedAbbrevDayNames[int(datetimeObj.strftime("%w"))]))

539 if "%A" in format:

540 format = format.replace("%A", str(localizedDayNames[int(datetimeObj.strftime("%w"))]))

541 if "%b" in format:

542 format = format.replace("%b", str(localizedAbbrevMonthNames[int(datetimeObj.strftime("%m"))]))

543 if "%B" in format:

544 format = format.replace("%B", str(localizedMonthNames[int(datetimeObj.strftime("%m"))]))

545 return datetimeObj.strftime(format)