Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/skeleton.py: 0%
958 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-02-07 19:28 +0000
1from __future__ import annotations # noqa: required for pre-defined annotations
3import copy
4import fnmatch
5import inspect
6import logging
7import os
8import string
9import sys
10import time
11import typing as t
12import warnings
13from deprecated.sphinx import deprecated
14from functools import partial
15from itertools import chain
16from viur.core import conf, current, db, email, errors, translate, utils
17from viur.core.bones import (
18 BaseBone,
19 DateBone,
20 KeyBone,
21 ReadFromClientException,
22 RelationalBone,
23 RelationalConsistency,
24 RelationalUpdateLevel,
25 SelectBone,
26 StringBone,
27)
28from viur.core.bones.base import (
29 Compute,
30 ComputeInterval,
31 ComputeMethod,
32 ReadFromClientError,
33 ReadFromClientErrorSeverity,
34 getSystemInitialized,
35)
36from viur.core.tasks import CallDeferred, CallableTask, CallableTaskBase, QueryIter
38_UNDEFINED = object()
39ABSTRACT_SKEL_CLS_SUFFIX = "AbstractSkel"
40KeyType: t.TypeAlias = db.Key | str | int
43class MetaBaseSkel(type):
44 """
45 This is the metaclass for Skeletons.
46 It is used to enforce several restrictions on bone names, etc.
47 """
48 _skelCache = {} # Mapping kindName -> SkelCls
49 _allSkelClasses = set() # list of all known skeleton classes (including Ref and Mail-Skels)
51 # List of reserved keywords and function names
52 __reserved_keywords = {
53 "all",
54 "bounce",
55 "clone",
56 "cursor",
57 "delete",
58 "errors",
59 "fromClient",
60 "fromDB",
61 "get",
62 "getCurrentSEOKeys",
63 "items",
64 "keys",
65 "limit",
66 "orderby",
67 "orderdir",
68 "patch",
69 "postDeletedHandler",
70 "postSavedHandler",
71 "preProcessBlobLocks",
72 "preProcessSerializedData",
73 "read",
74 "refresh",
75 "self",
76 "serialize",
77 "setBoneValue",
78 "structure",
79 "style",
80 "toDB",
81 "unserialize",
82 "values",
83 "write",
84 }
86 __allowed_chars = string.ascii_letters + string.digits + "_"
88 def __init__(cls, name, bases, dct, **kwargs):
89 cls.__boneMap__ = MetaBaseSkel.generate_bonemap(cls)
91 if not getSystemInitialized() and not cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
92 MetaBaseSkel._allSkelClasses.add(cls)
94 super().__init__(name, bases, dct)
96 @staticmethod
97 def generate_bonemap(cls):
98 """
99 Recursively constructs a dict of bones from
100 """
101 map = {}
103 for base in cls.__bases__:
104 if "__viurBaseSkeletonMarker__" in dir(base):
105 map |= MetaBaseSkel.generate_bonemap(base)
107 for key in cls.__dict__:
108 prop = getattr(cls, key)
110 if isinstance(prop, BaseBone):
111 if not all([c in MetaBaseSkel.__allowed_chars for c in key]):
112 raise AttributeError(f"Invalid bone name: {key!r} contains invalid characters")
113 elif key in MetaBaseSkel.__reserved_keywords:
114 raise AttributeError(f"Invalid bone name: {key!r} is reserved and cannot be used")
116 map[key] = prop
118 elif prop is None and key in map: # Allow removing a bone in a subclass by setting it to None
119 del map[key]
121 return map
123 def __setattr__(self, key, value):
124 super().__setattr__(key, value)
125 if isinstance(value, BaseBone):
126 # Call BaseBone.__set_name__ manually for bones that are assigned at runtime
127 value.__set_name__(self, key)
130class SkeletonInstance:
131 """
132 The actual wrapper around a Skeleton-Class. An object of this class is what's actually returned when you
133 call a Skeleton-Class. With ViUR3, you don't get an instance of a Skeleton-Class any more - it's always this
134 class. This is much faster as this is a small class.
135 """
136 __slots__ = {
137 "accessedValues",
138 "boneMap",
139 "dbEntity",
140 "errors",
141 "is_cloned",
142 "renderAccessedValues",
143 "renderPreparation",
144 "skeletonCls",
145 }
147 def __init__(
148 self,
149 skel_cls: t.Type[Skeleton],
150 *,
151 bones: t.Iterable[str] = (),
152 bone_map: t.Optional[t.Dict[str, BaseBone]] = None,
153 clone: bool = False,
154 # FIXME: BELOW IS DEPRECATED!
155 clonedBoneMap: t.Optional[t.Dict[str, BaseBone]] = None,
156 ):
157 """
158 Creates a new SkeletonInstance based on `skel_cls`.
160 :param skel_cls: Is the base skeleton class to inherit from and reference to.
161 :param bones: If given, defines an iterable of bones that are take into the SkeletonInstance.
162 The order of the bones defines the order in the SkeletonInstance.
163 :param bone_map: A pre-defined bone map to use, or extend.
164 :param clone: If set True, performs a cloning of the used bone map, to be entirely stand-alone.
165 """
167 # TODO: Remove with ViUR-core 3.8; required by viur-datastore :'-(
168 if clonedBoneMap:
169 msg = "'clonedBoneMap' was renamed into 'bone_map'"
170 warnings.warn(msg, DeprecationWarning, stacklevel=2)
171 # logging.warning(msg, stacklevel=2)
173 if bone_map:
174 raise ValueError("Can't provide both 'bone_map' and 'clonedBoneMap'")
176 bone_map = clonedBoneMap
178 bone_map = bone_map or {}
180 if bones:
181 names = ("key", ) + tuple(bones)
183 # generate full keys sequence based on definition; keeps order of patterns!
184 keys = []
185 for name in names:
186 if name in skel_cls.__boneMap__:
187 keys.append(name)
188 else:
189 keys.extend(fnmatch.filter(skel_cls.__boneMap__.keys(), name))
191 if clone:
192 bone_map |= {k: copy.deepcopy(skel_cls.__boneMap__[k]) for k in keys if skel_cls.__boneMap__[k]}
193 else:
194 bone_map |= {k: skel_cls.__boneMap__[k] for k in keys if skel_cls.__boneMap__[k]}
196 elif clone:
197 if bone_map:
198 bone_map = copy.deepcopy(bone_map)
199 else:
200 bone_map = copy.deepcopy(skel_cls.__boneMap__)
202 # generated or use provided bone_map
203 if bone_map:
204 self.boneMap = bone_map
206 else: # No Subskel, no Clone
207 self.boneMap = skel_cls.__boneMap__.copy()
209 if clone:
210 for v in self.boneMap.values():
211 v.isClonedInstance = True
213 self.accessedValues = {}
214 self.dbEntity = None
215 self.errors = []
216 self.is_cloned = clone
217 self.renderAccessedValues = {}
218 self.renderPreparation = None
219 self.skeletonCls = skel_cls
221 def items(self, yieldBoneValues: bool = False) -> t.Iterable[tuple[str, BaseBone]]:
222 if yieldBoneValues:
223 for key in self.boneMap.keys():
224 yield key, self[key]
225 else:
226 yield from self.boneMap.items()
228 def keys(self) -> t.Iterable[str]:
229 yield from self.boneMap.keys()
231 def values(self) -> t.Iterable[t.Any]:
232 yield from self.boneMap.values()
234 def __iter__(self) -> t.Iterable[str]:
235 yield from self.keys()
237 def __contains__(self, item):
238 return item in self.boneMap
240 def get(self, item, default=None):
241 if item not in self:
242 return default
244 return self[item]
246 def update(self, *args, **kwargs) -> None:
247 self.__ior__(dict(*args, **kwargs))
249 def __setitem__(self, key, value):
250 assert self.renderPreparation is None, "Cannot modify values while rendering"
251 if isinstance(value, BaseBone):
252 raise AttributeError(f"Don't assign this bone object as skel[\"{key}\"] = ... anymore to the skeleton. "
253 f"Use skel.{key} = ... for bone to skeleton assignment!")
254 self.accessedValues[key] = value
256 def __getitem__(self, key):
257 if self.renderPreparation:
258 if key in self.renderAccessedValues:
259 return self.renderAccessedValues[key]
260 if key not in self.accessedValues:
261 boneInstance = self.boneMap.get(key, None)
262 if boneInstance:
263 if self.dbEntity is not None:
264 boneInstance.unserialize(self, key)
265 else:
266 self.accessedValues[key] = boneInstance.getDefaultValue(self)
267 if not self.renderPreparation:
268 return self.accessedValues.get(key)
269 value = self.renderPreparation(getattr(self, key), self, key, self.accessedValues.get(key))
270 self.renderAccessedValues[key] = value
271 return value
273 def __getattr__(self, item: str):
274 """
275 Get a special attribute from the SkeletonInstance
277 __getattr__ is called when an attribute access fails with an
278 AttributeError. So we know that this is not a real attribute of
279 the SkeletonInstance. But there are still a few special cases in which
280 attributes are loaded from the skeleton class.
281 """
282 if item == "boneMap":
283 return {} # There are __setAttr__ calls before __init__ has run
285 # Load attribute value from the Skeleton class
286 elif item in {
287 "database_adapters",
288 "interBoneValidations",
289 "kindName",
290 }:
291 return getattr(self.skeletonCls, item)
293 # FIXME: viur-datastore backward compatiblity REMOVE WITH VIUR4
294 elif item == "customDatabaseAdapter":
295 if prop := getattr(self.skeletonCls, "database_adapters"):
296 return prop[0] # viur-datastore assumes there is only ONE!
298 return None
300 # Load a @classmethod from the Skeleton class and bound this SkeletonInstance
301 elif item in {
302 "all",
303 "delete",
304 "patch",
305 "fromClient",
306 "fromDB",
307 "getCurrentSEOKeys",
308 "postDeletedHandler",
309 "postSavedHandler",
310 "preProcessBlobLocks",
311 "preProcessSerializedData",
312 "read",
313 "refresh",
314 "serialize",
315 "setBoneValue",
316 "toDB",
317 "unserialize",
318 "write",
319 }:
320 return partial(getattr(self.skeletonCls, item), self)
322 # Load a @property from the Skeleton class
323 try:
324 # Use try/except to save an if check
325 class_value = getattr(self.skeletonCls, item)
327 except AttributeError:
328 # Not inside the Skeleton class, okay at this point.
329 pass
331 else:
332 if isinstance(class_value, property):
333 # The attribute is a @property and can be called
334 # Note: `self` is this SkeletonInstance, not the Skeleton class.
335 # Therefore, you can access values inside the property method
336 # with item-access like `self["key"]`.
337 try:
338 return class_value.fget(self)
339 except AttributeError as exc:
340 # The AttributeError cannot be re-raised any further at this point.
341 # Since this would then be evaluated as an access error
342 # to the property attribute.
343 # Otherwise, it would be lost that it is an incorrect attribute access
344 # within this property (during the method call).
345 msg, *args = exc.args
346 msg = f"AttributeError: {msg}"
347 raise ValueError(msg, *args) from exc
348 # Load the bone instance from the bone map of this SkeletonInstance
349 try:
350 return self.boneMap[item]
351 except KeyError as exc:
352 raise AttributeError(f"{self.__class__.__name__!r} object has no attribute '{item}'") from exc
354 def __delattr__(self, item):
355 del self.boneMap[item]
356 if item in self.accessedValues:
357 del self.accessedValues[item]
358 if item in self.renderAccessedValues:
359 del self.renderAccessedValues[item]
361 def __setattr__(self, key, value):
362 if key in self.boneMap or isinstance(value, BaseBone):
363 if value is None:
364 del self.boneMap[key]
365 else:
366 value.__set_name__(self.skeletonCls, key)
367 self.boneMap[key] = value
368 elif key == "renderPreparation":
369 super().__setattr__(key, value)
370 self.renderAccessedValues.clear()
371 else:
372 super().__setattr__(key, value)
374 def __repr__(self) -> str:
375 return f"<SkeletonInstance of {self.skeletonCls.__name__} with {dict(self)}>"
377 def __str__(self) -> str:
378 return str(dict(self))
380 def __len__(self) -> int:
381 return len(self.boneMap)
383 def __ior__(self, other: dict | SkeletonInstance | db.Entity) -> SkeletonInstance:
384 if isinstance(other, dict):
385 for key, value in other.items():
386 self.setBoneValue(key, value)
387 elif isinstance(other, db.Entity):
388 new_entity = self.dbEntity or db.Entity()
389 # We're not overriding the key
390 for key, value in other.items():
391 new_entity[key] = value
392 self.setEntity(new_entity)
393 elif isinstance(other, SkeletonInstance):
394 for key, value in other.accessedValues.items():
395 self.accessedValues[key] = value
396 for key, value in other.dbEntity.items():
397 self.dbEntity[key] = value
398 else:
399 raise ValueError("Unsupported Type")
400 return self
402 def clone(self):
403 """
404 Clones a SkeletonInstance into a modificable, stand-alone instance.
405 This will also allow to modify the underlying data model.
406 """
407 res = SkeletonInstance(self.skeletonCls, bone_map=self.boneMap, clone=True)
408 res.accessedValues = copy.deepcopy(self.accessedValues)
409 res.dbEntity = copy.deepcopy(self.dbEntity)
410 res.is_cloned = True
411 res.renderAccessedValues = copy.deepcopy(self.renderAccessedValues)
412 return res
414 def ensure_is_cloned(self):
415 """
416 Ensured this SkeletonInstance is a stand-alone clone, which can be modified.
417 Does nothing in case it was already cloned before.
418 """
419 if not self.is_cloned:
420 return self.clone()
422 return self
424 def setEntity(self, entity: db.Entity):
425 self.dbEntity = entity
426 self.accessedValues = {}
427 self.renderAccessedValues = {}
429 def structure(self) -> dict:
430 return {
431 key: bone.structure() | {"sortindex": i}
432 for i, (key, bone) in enumerate(self.items())
433 }
435 def __deepcopy__(self, memodict):
436 res = self.clone()
437 memodict[id(self)] = res
438 return res
441class BaseSkeleton(object, metaclass=MetaBaseSkel):
442 """
443 This is a container-object holding information about one database entity.
445 It has to be sub-classed with individual information about the kindName of the entities
446 and its specific data attributes, the so called bones.
447 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
448 contained bones remains constant.
450 :ivar key: This bone stores the current database key of this entity. \
451 Assigning to this bones value is dangerous and does *not* affect the actual key its stored in.
453 :vartype key: server.bones.BaseBone
455 :ivar creationdate: The date and time where this entity has been created.
456 :vartype creationdate: server.bones.DateBone
458 :ivar changedate: The date and time of the last change to this entity.
459 :vartype changedate: server.bones.DateBone
460 """
461 __viurBaseSkeletonMarker__ = True
462 boneMap = None
464 @classmethod
465 @deprecated(
466 version="3.7.0",
467 reason="Function renamed. Use subskel function as alternative implementation.",
468 )
469 def subSkel(cls, *subskel_names, fullClone: bool = False, **kwargs) -> SkeletonInstance:
470 return cls.subskel(*subskel_names, clone=fullClone) # FIXME: REMOVE WITH VIUR4
472 @classmethod
473 def subskel(
474 cls,
475 *names: str,
476 bones: t.Iterable[str] = (),
477 clone: bool = False,
478 ) -> SkeletonInstance:
479 """
480 Creates a new sub-skeleton from the current skeleton.
482 A sub-skeleton is a copy of the original skeleton, containing only a subset of its bones.
484 Sub-skeletons can either be defined using the the subSkels property of the Skeleton object,
485 or freely by giving patterns for bone names which shall be part of the sub-skeleton.
487 1. Giving names as parameter merges the bones of all Skeleton.subSkels-configurations together.
488 This is the usual behavior. By passing multiple sub-skeleton names to this function, a sub-skeleton
489 with the union of all bones of the specified sub-skeletons is returned. If an entry called "*"
490 exists in the subSkels-dictionary, the bones listed in this entry will always be part of the
491 generated sub-skeleton.
492 2. Given the *bones* parameter allows to freely specify a sub-skeleton; One specialty here is,
493 that the order of the bones can also be changed in this mode. This mode is the new way of defining
494 sub-skeletons, and might become the primary way to define sub-skeletons in future.
495 3. Both modes (1 + 2) can be combined, but then the original order of the bones is kept.
496 4. The "key" bone is automatically available in each sub-skeleton.
497 5. An fnmatch-compatible wildcard pattern is allowed both in the subSkels-bone-list and the
498 free bone list.
500 Example (TodoSkel is the example skeleton from viur-base):
501 ```py
502 # legacy mode (see 1)
503 subskel = TodoSkel.subskel("add")
504 # creates subskel: key, firstname, lastname, subject
506 # free mode (see 2) allows to specify a different order!
507 subskel = TodoSkel.subskel(bones=("subject", "message", "*stname"))
508 # creates subskel: key, subject, message, firstname, lastname
510 # mixed mode (see 3)
511 subskel = TodoSkel.subskel("add", bones=("message", ))
512 # creates subskel: key, firstname, lastname, subject, message
513 ```
515 :param bones: Allows to specify an iterator of bone names (more precisely, fnmatch-wildards) which allow
516 to freely define a subskel. If *only* this parameter is given, the order of the specification also
517 defines, the order of the list. Otherwise, the original order as defined in the skeleton is kept.
518 :param clone: If set True, performs a cloning of the used bone map, to be entirely stand-alone.
520 :return: The sub-skeleton of the specified type.
521 """
522 from_subskel = False
523 bones = list(bones)
525 for name in names:
526 # a str refers to a subskel name from the cls.subSkel dict
527 if isinstance(name, str):
528 # add bones from "*" subskel once
529 if not from_subskel:
530 bones.extend(cls.subSkels.get("*") or ())
531 from_subskel = True
533 bones.extend(cls.subSkels.get(name) or ())
535 else:
536 raise ValueError(f"Invalid subskel definition: {name!r}")
538 if from_subskel:
539 # when from_subskel is True, create bone names based on the order of the bones in the original skeleton
540 bones = tuple(k for k in cls.__boneMap__.keys() if any(fnmatch.fnmatch(k, n) for n in bones))
542 if not bones:
543 raise ValueError("The given subskel definition doesn't contain any bones!")
545 return cls(bones=bones, clone=clone)
547 @classmethod
548 def setSystemInitialized(cls):
549 for attrName in dir(cls):
550 bone = getattr(cls, attrName)
551 if isinstance(bone, BaseBone):
552 bone.setSystemInitialized()
554 @classmethod
555 def setBoneValue(
556 cls,
557 skel: SkeletonInstance,
558 boneName: str,
559 value: t.Any,
560 append: bool = False,
561 language: t.Optional[str] = None
562 ) -> bool:
563 """
564 Allows for setting a bones value without calling fromClient or assigning a value directly.
565 Sanity-Checks are performed; if the value is invalid, that bone flips back to its original
566 (default) value and false is returned.
568 :param boneName: The name of the bone to be modified
569 :param value: The value that should be assigned. It's type depends on the type of that bone
570 :param append: If True, the given value is appended to the values of that bone instead of
571 replacing it. Only supported on bones with multiple=True
572 :param language: Language to set
574 :return: Wherever that operation succeeded or not.
575 """
576 bone = getattr(skel, boneName, None)
578 if not isinstance(bone, BaseBone):
579 raise ValueError(f"{boneName!r} is no valid bone on this skeleton ({skel!r})")
581 if language:
582 if not bone.languages:
583 raise ValueError("The bone {boneName!r} has no language setting")
584 elif language not in bone.languages:
585 raise ValueError("The language {language!r} is not available for bone {boneName!r}")
587 if value is None:
588 if append:
589 raise ValueError("Cannot append None-value to bone {boneName!r}")
591 if language:
592 skel[boneName][language] = [] if bone.multiple else None
593 else:
594 skel[boneName] = [] if bone.multiple else None
596 return True
598 _ = skel[boneName] # ensure the bone is being unserialized first
599 return bone.setBoneValue(skel, boneName, value, append, language)
601 @classmethod
602 def fromClient(
603 cls,
604 skel: SkeletonInstance,
605 data: dict[str, list[str] | str],
606 *,
607 amend: bool = False,
608 ignore: t.Optional[t.Iterable[str]] = None,
609 ) -> bool:
610 """
611 Load supplied *data* into Skeleton.
613 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
614 the values retrieved from *data* are checked against the bones and their validity checks.
616 Even if this function returns False, all bones are guaranteed to be in a valid state.
617 The ones which have been read correctly are set to their valid values;
618 Bones with invalid values are set back to a safe default (None in most cases).
619 So its possible to call :func:`~viur.core.skeleton.Skeleton.write` afterwards even if reading
620 data with this function failed (through this might violates the assumed consistency-model).
622 :param skel: The skeleton instance to be filled.
623 :param data: Dictionary from which the data is read.
624 :param amend: Defines whether content of data may be incomplete to amend the skel,
625 which is useful for edit-actions.
626 :param ignore: optional list of bones to be ignored; Defaults to all readonly-bones when set to None.
628 :returns: True if all data was successfully read and complete. \
629 False otherwise (e.g. some required fields where missing or where invalid).
630 """
631 complete = True
632 skel.errors = []
634 for key, bone in skel.items():
635 if (ignore is None and bone.readOnly) or key in (ignore or ()):
636 continue
638 if errors := bone.fromClient(skel, key, data):
639 for error in errors:
640 # insert current bone name into error's fieldPath
641 error.fieldPath.insert(0, str(key))
643 # logging.debug(f"BaseSkel.fromClient {key=} {error=}")
645 incomplete = (
646 # always when something is invalid
647 error.severity == ReadFromClientErrorSeverity.Invalid
648 or (
649 # only when path is top-level
650 len(error.fieldPath) == 1
651 and (
652 # bone is generally required
653 bool(bone.required)
654 and (
655 # and value is either empty
656 error.severity == ReadFromClientErrorSeverity.Empty
657 # or when not amending, not set
658 or (not amend and error.severity == ReadFromClientErrorSeverity.NotSet)
659 )
660 )
661 )
662 )
664 # in case there are language requirements, test additionally
665 if bone.languages and isinstance(bone.required, (list, tuple)):
666 incomplete &= any([key, lang] == error.fieldPath for lang in bone.required)
668 # logging.debug(f"BaseSkel.fromClient {incomplete=} {error.severity=} {bone.required=}")
670 if incomplete:
671 complete = False
673 if conf.debug.skeleton_from_client:
674 logging.error(
675 f"""{getattr(cls, "kindName", cls.__name__)}: {".".join(error.fieldPath)}: """
676 f"""({error.severity}) {error.errorMessage}"""
677 )
679 skel.errors += errors
681 return complete
683 @classmethod
684 def refresh(cls, skel: SkeletonInstance):
685 """
686 Refresh the bones current content.
688 This function causes a refresh of all relational bones and their associated
689 information.
690 """
691 logging.debug(f"""Refreshing {skel["key"]!r} ({skel.get("name")!r})""")
693 for key, bone in skel.items():
694 if not isinstance(bone, BaseBone):
695 continue
697 _ = skel[key] # Ensure value gets loaded
698 bone.refresh(skel, key)
700 def __new__(cls, *args, **kwargs) -> SkeletonInstance:
701 return SkeletonInstance(cls, *args, **kwargs)
704class MetaSkel(MetaBaseSkel):
706 def __init__(cls, name, bases, dct, **kwargs):
707 super().__init__(name, bases, dct, **kwargs)
709 relNewFileName = inspect.getfile(cls) \
710 .replace(str(conf.instance.project_base_path), "") \
711 .replace(str(conf.instance.core_base_path), "")
713 # Check if we have an abstract skeleton
714 if cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
715 # Ensure that it doesn't have a kindName
716 assert cls.kindName is _UNDEFINED or cls.kindName is None, "Abstract Skeletons can't have a kindName"
717 # Prevent any further processing by this class; it has to be sub-classed before it can be used
718 return
720 # Automatic determination of the kindName, if the class is not part of viur.core.
721 if (cls.kindName is _UNDEFINED
722 and not relNewFileName.strip(os.path.sep).startswith("viur")
723 and not "viur_doc_build" in dir(sys)):
724 if cls.__name__.endswith("Skel"):
725 cls.kindName = cls.__name__.lower()[:-4]
726 else:
727 cls.kindName = cls.__name__.lower()
729 # Try to determine which skeleton definition takes precedence
730 if cls.kindName and cls.kindName is not _UNDEFINED and cls.kindName in MetaBaseSkel._skelCache:
731 relOldFileName = inspect.getfile(MetaBaseSkel._skelCache[cls.kindName]) \
732 .replace(str(conf.instance.project_base_path), "") \
733 .replace(str(conf.instance.core_base_path), "")
734 idxOld = min(
735 [x for (x, y) in enumerate(conf.skeleton_search_path) if relOldFileName.startswith(y)] + [999])
736 idxNew = min(
737 [x for (x, y) in enumerate(conf.skeleton_search_path) if relNewFileName.startswith(y)] + [999])
738 if idxNew == 999:
739 # We could not determine a priority for this class as its from a path not listed in the config
740 raise NotImplementedError(
741 "Skeletons must be defined in a folder listed in conf.skeleton_search_path")
742 elif idxOld < idxNew: # Lower index takes precedence
743 # The currently processed skeleton has a lower priority than the one we already saw - just ignore it
744 return
745 elif idxOld > idxNew:
746 # The currently processed skeleton has a higher priority, use that from now
747 MetaBaseSkel._skelCache[cls.kindName] = cls
748 else: # They seem to be from the same Package - raise as something is messed up
749 raise ValueError(f"Duplicate definition for {cls.kindName} in {relNewFileName} and {relOldFileName}")
751 # Ensure that all skeletons are defined in folders listed in conf.skeleton_search_path
752 if (not any([relNewFileName.startswith(x) for x in conf.skeleton_search_path])
753 and not "viur_doc_build" in dir(sys)): # Do not check while documentation build
754 raise NotImplementedError(
755 f"""{relNewFileName} must be defined in a folder listed in {conf.skeleton_search_path}""")
757 if cls.kindName and cls.kindName is not _UNDEFINED:
758 MetaBaseSkel._skelCache[cls.kindName] = cls
760 # Auto-Add ViUR Search Tags Adapter if the skeleton has no adapter attached
761 if cls.database_adapters is _UNDEFINED:
762 cls.database_adapters = ViurTagsSearchAdapter()
764 # Always ensure that skel.database_adapters is an iterable
765 cls.database_adapters = utils.ensure_iterable(cls.database_adapters)
768class DatabaseAdapter:
769 """
770 Adapter class used to bind or use other databases and hook operations when working with a Skeleton.
771 """
773 providesFulltextSearch: bool = False
774 """Set to True if we can run a fulltext search using this database."""
776 fulltextSearchGuaranteesQueryConstrains = False
777 """Are results returned by `meth:fulltextSearch` guaranteed to also match the databaseQuery"""
779 providesCustomQueries: bool = False
780 """Indicate that we can run more types of queries than originally supported by datastore"""
782 def prewrite(self, skel: SkeletonInstance, is_add: bool, change_list: t.Iterable[str] = ()):
783 """
784 Hook being called on a add, edit or delete operation before the skeleton-specific action is performed.
786 The hook can be used to modifiy the skeleton before writing.
787 The raw entity can be obainted using `skel.dbEntity`.
789 :param action: Either contains "add", "edit" or "delete", depending on the operation.
790 :param skel: is the skeleton that is being read before written.
791 :param change_list: is a list of bone names which are being changed within the write.
792 """
793 pass
795 def write(self, skel: SkeletonInstance, is_add: bool, change_list: t.Iterable[str] = ()):
796 """
797 Hook being called on a write operations after the skeleton is written.
799 The raw entity can be obainted using `skel.dbEntity`.
801 :param action: Either contains "add" or "edit", depending on the operation.
802 :param skel: is the skeleton that is being read before written.
803 :param change_list: is a list of bone names which are being changed within the write.
804 """
805 pass
807 def delete(self, skel: SkeletonInstance):
808 """
809 Hook being called on a delete operation after the skeleton is deleted.
810 """
811 pass
813 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
814 """
815 If this database supports fulltext searches, this method has to implement them.
816 If it's a plain fulltext search engine, leave 'prop:fulltextSearchGuaranteesQueryConstrains' set to False,
817 then the server will post-process the list of entries returned from this function and drop any entry that
818 cannot be returned due to other constrains set in 'param:databaseQuery'. If you can obey *every* constrain
819 set in that Query, we can skip this post-processing and save some CPU-cycles.
820 :param queryString: the string as received from the user (no quotation or other safety checks applied!)
821 :param databaseQuery: The query containing any constrains that returned entries must also match
822 :return:
823 """
824 raise NotImplementedError
827class ViurTagsSearchAdapter(DatabaseAdapter):
828 """
829 This Adapter implements a simple fulltext search on top of the datastore.
831 On skel.write(), all words from String-/TextBones are collected with all *min_length* postfixes and dumped
832 into the property `viurTags`. When queried, we'll run a prefix-match against this property - thus returning
833 entities with either an exact match or a match within a word.
835 Example:
836 For the word "hello" we'll write "hello", "ello" and "llo" into viurTags.
837 When queried with "hello" we'll have an exact match.
838 When queried with "hel" we'll match the prefix for "hello"
839 When queried with "ell" we'll prefix-match "ello" - this is only enabled when substring_matching is True.
841 We'll automatically add this adapter if a skeleton has no other database adapter defined.
842 """
843 providesFulltextSearch = True
844 fulltextSearchGuaranteesQueryConstrains = True
846 def __init__(self, min_length: int = 2, max_length: int = 50, substring_matching: bool = False):
847 super().__init__()
848 self.min_length = min_length
849 self.max_length = max_length
850 self.substring_matching = substring_matching
852 def _tags_from_str(self, value: str) -> set[str]:
853 """
854 Extract all words including all min_length postfixes from given string
855 """
856 res = set()
858 for tag in value.split(" "):
859 tag = "".join([x for x in tag.lower() if x in conf.search_valid_chars])
861 if len(tag) >= self.min_length:
862 res.add(tag)
864 if self.substring_matching:
865 for i in range(1, 1 + len(tag) - self.min_length):
866 res.add(tag[i:])
868 return res
870 def prewrite(self, skel: SkeletonInstance, *args, **kwargs):
871 """
872 Collect searchTags from skeleton and build viurTags
873 """
874 tags = set()
876 for name, bone in skel.items():
877 if bone.searchable:
878 tags = tags.union(bone.getSearchTags(skel, name))
880 skel.dbEntity["viurTags"] = list(
881 chain(*[self._tags_from_str(tag) for tag in tags if len(tag) <= self.max_length])
882 )
884 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
885 """
886 Run a fulltext search
887 """
888 keywords = list(self._tags_from_str(queryString))[:10]
889 resultScoreMap = {}
890 resultEntryMap = {}
892 for keyword in keywords:
893 qryBase = databaseQuery.clone()
894 for entry in qryBase.filter("viurTags >=", keyword).filter("viurTags <", keyword + "\ufffd").run():
895 if not entry.key in resultScoreMap:
896 resultScoreMap[entry.key] = 1
897 else:
898 resultScoreMap[entry.key] += 1
899 if not entry.key in resultEntryMap:
900 resultEntryMap[entry.key] = entry
902 resultList = [(k, v) for k, v in resultScoreMap.items()]
903 resultList.sort(key=lambda x: x[1], reverse=True)
905 return [resultEntryMap[x[0]] for x in resultList[:databaseQuery.queries.limit]]
908class SeoKeyBone(StringBone):
909 """
910 Special kind of StringBone saving its contents as `viurCurrentSeoKeys` into the entity's `viur` dict.
911 """
913 def unserialize(self, skel: SkeletonInstance, name: str) -> bool:
914 try:
915 skel.accessedValues[name] = skel.dbEntity["viur"]["viurCurrentSeoKeys"]
916 except KeyError:
917 skel.accessedValues[name] = self.getDefaultValue(skel)
919 def serialize(self, skel: SkeletonInstance, name: str, parentIndexed: bool) -> bool:
920 # Serialize also to skel["viur"]["viurCurrentSeoKeys"], so we can use this bone in relations
921 if name in skel.accessedValues:
922 newVal = skel.accessedValues[name]
923 if not skel.dbEntity.get("viur"):
924 skel.dbEntity["viur"] = db.Entity()
925 res = db.Entity()
926 res["_viurLanguageWrapper_"] = True
927 for language in (self.languages or []):
928 if not self.indexed:
929 res.exclude_from_indexes.add(language)
930 res[language] = None
931 if language in newVal:
932 res[language] = self.singleValueSerialize(newVal[language], skel, name, parentIndexed)
933 skel.dbEntity["viur"]["viurCurrentSeoKeys"] = res
934 return True
937class Skeleton(BaseSkeleton, metaclass=MetaSkel):
938 kindName: str = _UNDEFINED
939 """
940 Specifies the entity kind name this Skeleton is associated with.
941 Will be determined automatically when not explicitly set.
942 """
944 database_adapters: DatabaseAdapter | t.Iterable[DatabaseAdapter] | None = _UNDEFINED
945 """
946 Custom database adapters.
947 Allows to hook special functionalities that during skeleton modifications.
948 """
950 subSkels = {} # List of pre-defined sub-skeletons of this type
952 interBoneValidations: list[
953 t.Callable[[Skeleton], list[ReadFromClientError]]] = [] # List of functions checking inter-bone dependencies
955 __seo_key_trans = str.maketrans(
956 {"<": "",
957 ">": "",
958 "\"": "",
959 "'": "",
960 "\n": "",
961 "\0": "",
962 "/": "",
963 "\\": "",
964 "?": "",
965 "&": "",
966 "#": ""
967 })
969 # The "key" bone stores the current database key of this skeleton.
970 # Warning: Assigning to this bones value now *will* set the key
971 # it gets stored in. Must be kept readOnly to avoid security-issues with add/edit.
972 key = KeyBone(
973 descr="Key"
974 )
976 name = StringBone(
977 descr="Name",
978 visible=False,
979 compute=Compute(
980 fn=lambda skel: str(skel["key"]),
981 interval=ComputeInterval(ComputeMethod.OnWrite)
982 )
983 )
985 # The date (including time) when this entry has been created
986 creationdate = DateBone(
987 descr="created at",
988 readOnly=True,
989 visible=False,
990 indexed=True,
991 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.Once)),
992 )
994 # The last date (including time) when this entry has been updated
996 changedate = DateBone(
997 descr="updated at",
998 readOnly=True,
999 visible=False,
1000 indexed=True,
1001 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.OnWrite)),
1002 )
1004 viurCurrentSeoKeys = SeoKeyBone(
1005 descr="SEO-Keys",
1006 readOnly=True,
1007 visible=False,
1008 languages=conf.i18n.available_languages
1009 )
1011 def __repr__(self):
1012 return "<skeleton %s with data=%r>" % (self.kindName, {k: self[k] for k in self.keys()})
1014 def __str__(self):
1015 return str({k: self[k] for k in self.keys()})
1017 def __init__(self, *args, **kwargs):
1018 super(Skeleton, self).__init__(*args, **kwargs)
1019 assert self.kindName and self.kindName is not _UNDEFINED, "You must set kindName on this skeleton!"
1021 @classmethod
1022 def all(cls, skel, **kwargs) -> db.Query:
1023 """
1024 Create a query with the current Skeletons kindName.
1026 :returns: A db.Query object which allows for entity filtering and sorting.
1027 """
1028 return db.Query(skel.kindName, srcSkelClass=skel, **kwargs)
1030 @classmethod
1031 def fromClient(
1032 cls,
1033 skel: SkeletonInstance,
1034 data: dict[str, list[str] | str],
1035 *,
1036 amend: bool = False,
1037 ignore: t.Optional[t.Iterable[str]] = None,
1038 ) -> bool:
1039 """
1040 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
1041 the values retrieved from *data* are checked against the bones and their validity checks.
1043 Even if this function returns False, all bones are guaranteed to be in a valid state.
1044 The ones which have been read correctly are set to their valid values;
1045 Bones with invalid values are set back to a safe default (None in most cases).
1046 So its possible to call :func:`~viur.core.skeleton.Skeleton.write` afterwards even if reading
1047 data with this function failed (through this might violates the assumed consistency-model).
1049 :param skel: The skeleton instance to be filled.
1050 :param data: Dictionary from which the data is read.
1051 :param amend: Defines whether content of data may be incomplete to amend the skel,
1052 which is useful for edit-actions.
1053 :param ignore: optional list of bones to be ignored; Defaults to all readonly-bones when set to None.
1055 :returns: True if all data was successfully read and complete. \
1056 False otherwise (e.g. some required fields where missing or where invalid).
1057 """
1058 assert skel.renderPreparation is None, "Cannot modify values while rendering"
1060 # Load data into this skeleton
1061 complete = bool(data) and super().fromClient(skel, data, amend=amend, ignore=ignore)
1063 if (
1064 not data # in case data is empty
1065 or (len(data) == 1 and "key" in data)
1066 or (utils.parse.bool(data.get("nomissing")))
1067 ):
1068 skel.errors = []
1070 # Check if all unique values are available
1071 for boneName, boneInstance in skel.items():
1072 if boneInstance.unique:
1073 lockValues = boneInstance.getUniquePropertyIndexValues(skel, boneName)
1074 for lockValue in lockValues:
1075 dbObj = db.Get(db.Key(f"{skel.kindName}_{boneName}_uniquePropertyIndex", lockValue))
1076 if dbObj and (not skel["key"] or dbObj["references"] != skel["key"].id_or_name):
1077 # This value is taken (sadly, not by us)
1078 complete = False
1079 errorMsg = boneInstance.unique.message
1080 skel.errors.append(
1081 ReadFromClientError(ReadFromClientErrorSeverity.Invalid, errorMsg, [boneName]))
1083 # Check inter-Bone dependencies
1084 for checkFunc in skel.interBoneValidations:
1085 errors = checkFunc(skel)
1086 if errors:
1087 for error in errors:
1088 if error.severity.value > 1:
1089 complete = False
1090 if conf.debug.skeleton_from_client:
1091 logging.debug(f"{cls.kindName}: {error.fieldPath}: {error.errorMessage!r}")
1093 skel.errors.extend(errors)
1095 return complete
1097 @classmethod
1098 @deprecated(
1099 version="3.7.0",
1100 reason="Use skel.read() instead of skel.fromDB()",
1101 )
1102 def fromDB(cls, skel: SkeletonInstance, key: KeyType) -> bool:
1103 """
1104 Deprecated function, replaced by Skeleton.read().
1105 """
1106 return bool(cls.read(skel, key, _check_legacy=False))
1108 @classmethod
1109 def read(
1110 cls,
1111 skel: SkeletonInstance,
1112 key: t.Optional[KeyType] = None,
1113 *,
1114 create: bool | dict | t.Callable[[SkeletonInstance], None] = False,
1115 _check_legacy: bool = True
1116 ) -> t.Optional[SkeletonInstance]:
1117 """
1118 Read Skeleton with *key* from the datastore into the Skeleton.
1119 If not key is given, skel["key"] will be used.
1121 Reads all available data of entity kind *kindName* and the key *key*
1122 from the Datastore into the Skeleton structure's bones. Any previous
1123 data of the bones will discard.
1125 To store a Skeleton object to the Datastore, see :func:`~viur.core.skeleton.Skeleton.write`.
1127 :param key: A :class:`viur.core.db.Key`, string, or int; from which the data shall be fetched.
1128 If not provided, skel["key"] will be used.
1129 :param create: Allows to specify a dict or initial callable that is executed in case the Skeleton with the
1130 given key does not exist, it will be created.
1132 :returns: None on error, or the given SkeletonInstance on success.
1134 """
1135 # FIXME VIUR4: Stay backward compatible, call sub-classed fromDB if available first!
1136 if _check_legacy and "fromDB" in cls.__dict__:
1137 with warnings.catch_warnings():
1138 warnings.simplefilter("ignore", DeprecationWarning)
1139 return cls.fromDB(skel, key=key)
1141 assert skel.renderPreparation is None, "Cannot modify values while rendering"
1143 try:
1144 db_key = db.keyHelper(key or skel["key"], skel.kindName)
1145 except (ValueError, NotImplementedError): # This key did not parse
1146 return None
1148 if db_res := db.Get(db_key):
1149 skel.setEntity(db_res)
1150 return skel
1151 elif create in (False, None):
1152 return None
1153 elif isinstance(create, dict):
1154 if create and not skel.fromClient(create, amend=True):
1155 raise ReadFromClientException(skel.errors)
1156 elif callable(create):
1157 create(skel)
1158 elif create is not True:
1159 raise ValueError("'create' must either be dict, a callable or True.")
1161 return skel.write()
1163 @classmethod
1164 @deprecated(
1165 version="3.7.0",
1166 reason="Use skel.write() instead of skel.toDB()",
1167 )
1168 def toDB(cls, skel: SkeletonInstance, update_relations: bool = True, **kwargs) -> db.Key:
1169 """
1170 Deprecated function, replaced by Skeleton.write().
1171 """
1173 # TODO: Remove with ViUR4
1174 if "clearUpdateTag" in kwargs:
1175 msg = "clearUpdateTag was replaced by update_relations"
1176 warnings.warn(msg, DeprecationWarning, stacklevel=3)
1177 logging.warning(msg, stacklevel=3)
1178 update_relations = not kwargs["clearUpdateTag"]
1180 skel = cls.write(skel, update_relations=update_relations, _check_legacy=False)
1181 return skel["key"]
1183 @classmethod
1184 def write(
1185 cls,
1186 skel: SkeletonInstance,
1187 key: t.Optional[KeyType] = None,
1188 *,
1189 update_relations: bool = True,
1190 _check_legacy: bool = True,
1191 ) -> SkeletonInstance:
1192 """
1193 Write current Skeleton to the datastore.
1195 Stores the current data of this instance into the database.
1196 If an *key* value is set to the object, this entity will ne updated;
1197 Otherwise a new entity will be created.
1199 To read a Skeleton object from the data store, see :func:`~viur.core.skeleton.Skeleton.read`.
1201 :param key: Allows to specify a key that is set to the skeleton and used for writing.
1202 :param update_relations: If False, this entity won't be marked dirty;
1203 This avoids from being fetched by the background task updating relations.
1205 :returns: The Skeleton.
1206 """
1207 # FIXME VIUR4: Stay backward compatible, call sub-classed toDB if available first!
1208 if _check_legacy and "toDB" in cls.__dict__:
1209 with warnings.catch_warnings():
1210 warnings.simplefilter("ignore", DeprecationWarning)
1211 return cls.toDB(skel, update_relations=update_relations)
1213 assert skel.renderPreparation is None, "Cannot modify values while rendering"
1215 def __txn_write(write_skel):
1216 db_key = write_skel["key"]
1217 skel = write_skel.skeletonCls()
1219 blob_list = set()
1220 change_list = []
1221 old_copy = {}
1222 # Load the current values from Datastore or create a new, empty db.Entity
1223 if not db_key:
1224 # We'll generate the key we'll be stored under early so we can use it for locks etc
1225 db_key = db.AllocateIDs(db.Key(skel.kindName))
1226 skel.dbEntity = db.Entity(db_key)
1227 is_add = True
1228 else:
1229 db_key = db.keyHelper(db_key, skel.kindName)
1230 if db_obj := db.Get(db_key):
1231 skel.dbEntity = db_obj
1232 old_copy = {k: v for k, v in skel.dbEntity.items()}
1233 is_add = False
1234 else:
1235 skel.dbEntity = db.Entity(db_key)
1236 is_add = True
1238 skel.dbEntity.setdefault("viur", {})
1240 # Merge values and assemble unique properties
1241 # Move accessed Values from srcSkel over to skel
1242 skel.accessedValues = write_skel.accessedValues
1243 skel["key"] = db_key # Ensure key stays set
1245 for bone_name, bone in skel.items():
1246 if bone_name == "key": # Explicitly skip key on top-level - this had been set above
1247 continue
1249 # Allow bones to perform outstanding "magic" operations before saving to db
1250 bone.performMagic(skel, bone_name, isAdd=is_add) # FIXME VIUR4: ANY MAGIC IN OUR CODE IS DEPRECATED!!!
1252 if not (bone_name in skel.accessedValues or bone.compute) and bone_name not in skel.dbEntity:
1253 _ = skel[bone_name] # Ensure the datastore is filled with the default value
1255 if (
1256 bone_name in skel.accessedValues or bone.compute # We can have a computed value on store
1257 or bone_name not in skel.dbEntity # It has not been written and is not in the database
1258 ):
1259 # Serialize bone into entity
1260 try:
1261 bone.serialize(skel, bone_name, True)
1262 except Exception as e:
1263 logging.error(
1264 f"Failed to serialize {bone_name=} ({bone=}): {skel.accessedValues[bone_name]=}"
1265 )
1266 raise e
1268 # Obtain referenced blobs
1269 blob_list.update(bone.getReferencedBlobs(skel, bone_name))
1271 # Check if the value has actually changed
1272 if skel.dbEntity.get(bone_name) != old_copy.get(bone_name):
1273 change_list.append(bone_name)
1275 # Lock hashes from bones that must have unique values
1276 if bone.unique:
1277 # Remember old hashes for bones that must have an unique value
1278 old_unique_values = []
1280 if f"{bone_name}_uniqueIndexValue" in skel.dbEntity["viur"]:
1281 old_unique_values = skel.dbEntity["viur"][f"{bone_name}_uniqueIndexValue"]
1282 # Check if the property is unique
1283 new_unique_values = bone.getUniquePropertyIndexValues(skel, bone_name)
1284 new_lock_kind = f"{skel.kindName}_{bone_name}_uniquePropertyIndex"
1285 for new_lock_value in new_unique_values:
1286 new_lock_key = db.Key(new_lock_kind, new_lock_value)
1287 if lock_db_obj := db.Get(new_lock_key):
1289 # There's already a lock for that value, check if we hold it
1290 if lock_db_obj["references"] != skel.dbEntity.key.id_or_name:
1291 # This value has already been claimed, and not by us
1292 # TODO: Use a custom exception class which is catchable with an try/except
1293 raise ValueError(
1294 f"The unique value {skel[bone_name]!r} of bone {bone_name!r} "
1295 f"has been recently claimed!")
1296 else:
1297 # This value is locked for the first time, create a new lock-object
1298 lock_obj = db.Entity(new_lock_key)
1299 lock_obj["references"] = skel.dbEntity.key.id_or_name
1300 db.Put(lock_obj)
1301 if new_lock_value in old_unique_values:
1302 old_unique_values.remove(new_lock_value)
1303 skel.dbEntity["viur"][f"{bone_name}_uniqueIndexValue"] = new_unique_values
1305 # Remove any lock-object we're holding for values that we don't have anymore
1306 for old_unique_value in old_unique_values:
1307 # Try to delete the old lock
1309 old_lock_key = db.Key(f"{skel.kindName}_{bone_name}_uniquePropertyIndex", old_unique_value)
1310 if old_lock_obj := db.Get(old_lock_key):
1311 if old_lock_obj["references"] != skel.dbEntity.key.id_or_name:
1313 # We've been supposed to have that lock - but we don't.
1314 # Don't remove that lock as it now belongs to a different entry
1315 logging.critical("Detected Database corruption! A Value-Lock had been reassigned!")
1316 else:
1317 # It's our lock which we don't need anymore
1318 db.Delete(old_lock_key)
1319 else:
1320 logging.critical("Detected Database corruption! Could not delete stale lock-object!")
1322 # Delete legacy property (PR #1244) #TODO: Remove in ViUR4
1323 skel.dbEntity.pop("viur_incomming_relational_locks", None)
1325 # Ensure the SEO-Keys are up-to-date
1326 last_requested_seo_keys = skel.dbEntity["viur"].get("viurLastRequestedSeoKeys") or {}
1327 last_set_seo_keys = skel.dbEntity["viur"].get("viurCurrentSeoKeys") or {}
1328 # Filter garbage serialized into this field by the SeoKeyBone
1329 last_set_seo_keys = {k: v for k, v in last_set_seo_keys.items() if not k.startswith("_") and v}
1331 if not isinstance(skel.dbEntity["viur"].get("viurCurrentSeoKeys"), dict):
1332 skel.dbEntity["viur"]["viurCurrentSeoKeys"] = {}
1334 if current_seo_keys := skel.getCurrentSEOKeys():
1335 # Convert to lower-case and remove certain characters
1336 for lang, value in current_seo_keys.items():
1337 current_seo_keys[lang] = value.lower().translate(Skeleton.__seo_key_trans).strip()
1339 for language in (conf.i18n.available_languages or [conf.i18n.default_language]):
1340 if current_seo_keys and language in current_seo_keys:
1341 current_seo_key = current_seo_keys[language]
1343 if current_seo_key != last_requested_seo_keys.get(language): # This one is new or has changed
1344 new_seo_key = current_seo_keys[language]
1346 for _ in range(0, 3):
1347 entry_using_key = db.Query(skel.kindName).filter(
1348 "viur.viurActiveSeoKeys =", new_seo_key).getEntry()
1350 if entry_using_key and entry_using_key.key != skel.dbEntity.key:
1351 # It's not unique; append a random string and try again
1352 new_seo_key = f"{current_seo_keys[language]}-{utils.string.random(5).lower()}"
1354 else:
1355 # We found a new SeoKey
1356 break
1357 else:
1358 raise ValueError("Could not generate an unique seo key in 3 attempts")
1359 else:
1360 new_seo_key = current_seo_key
1361 last_set_seo_keys[language] = new_seo_key
1363 else:
1364 # We'll use the database-key instead
1365 last_set_seo_keys[language] = str(skel.dbEntity.key.id_or_name)
1367 # Store the current, active key for that language
1368 skel.dbEntity["viur"]["viurCurrentSeoKeys"][language] = last_set_seo_keys[language]
1370 skel.dbEntity["viur"].setdefault("viurActiveSeoKeys", [])
1371 for language, seo_key in last_set_seo_keys.items():
1372 if skel.dbEntity["viur"]["viurCurrentSeoKeys"][language] not in \
1373 skel.dbEntity["viur"]["viurActiveSeoKeys"]:
1374 # Ensure the current, active seo key is in the list of all seo keys
1375 skel.dbEntity["viur"]["viurActiveSeoKeys"].insert(0, seo_key)
1376 if str(skel.dbEntity.key.id_or_name) not in skel.dbEntity["viur"]["viurActiveSeoKeys"]:
1377 # Ensure that key is also in there
1378 skel.dbEntity["viur"]["viurActiveSeoKeys"].insert(0, str(skel.dbEntity.key.id_or_name))
1379 # Trim to the last 200 used entries
1380 skel.dbEntity["viur"]["viurActiveSeoKeys"] = skel.dbEntity["viur"]["viurActiveSeoKeys"][:200]
1381 # Store lastRequestedKeys so further updates can run more efficient
1382 skel.dbEntity["viur"]["viurLastRequestedSeoKeys"] = current_seo_keys
1384 # mark entity as "dirty" when update_relations is set, to zero otherwise.
1385 skel.dbEntity["viur"]["delayedUpdateTag"] = time.time() if update_relations else 0
1387 skel.dbEntity = skel.preProcessSerializedData(skel.dbEntity)
1389 # Allow the database adapter to apply last minute changes to the object
1390 for adapter in skel.database_adapters:
1391 adapter.prewrite(skel, is_add, change_list)
1393 # ViUR2 import compatibility - remove properties containing. if we have a dict with the same name
1394 def fixDotNames(entity):
1395 for k, v in list(entity.items()):
1396 if isinstance(v, dict):
1397 for k2, v2 in list(entity.items()):
1398 if k2.startswith(f"{k}."):
1399 del entity[k2]
1400 backupKey = k2.replace(".", "__")
1401 entity[backupKey] = v2
1402 entity.exclude_from_indexes = set(entity.exclude_from_indexes) | {backupKey}
1403 fixDotNames(v)
1404 elif isinstance(v, list):
1405 for x in v:
1406 if isinstance(x, dict):
1407 fixDotNames(x)
1409 # FIXME: REMOVE IN VIUR4
1410 if conf.viur2import_blobsource: # Try to fix these only when converting from ViUR2
1411 fixDotNames(skel.dbEntity)
1413 # Write the core entry back
1414 db.Put(skel.dbEntity)
1416 # Now write the blob-lock object
1417 blob_list = skel.preProcessBlobLocks(blob_list)
1418 if blob_list is None:
1419 raise ValueError("Did you forget to return the blob_list somewhere inside getReferencedBlobs()?")
1420 if None in blob_list:
1421 msg = f"None is not valid in {blob_list=}"
1422 logging.error(msg)
1423 raise ValueError(msg)
1425 if not is_add and (old_blob_lock_obj := db.Get(db.Key("viur-blob-locks", db_key.id_or_name))):
1426 removed_blobs = set(old_blob_lock_obj.get("active_blob_references", [])) - blob_list
1427 old_blob_lock_obj["active_blob_references"] = list(blob_list)
1428 if old_blob_lock_obj["old_blob_references"] is None:
1429 old_blob_lock_obj["old_blob_references"] = list(removed_blobs)
1430 else:
1431 old_blob_refs = set(old_blob_lock_obj["old_blob_references"])
1432 old_blob_refs.update(removed_blobs) # Add removed blobs
1433 old_blob_refs -= blob_list # Remove active blobs
1434 old_blob_lock_obj["old_blob_references"] = list(old_blob_refs)
1436 old_blob_lock_obj["has_old_blob_references"] = bool(old_blob_lock_obj["old_blob_references"])
1437 old_blob_lock_obj["is_stale"] = False
1438 db.Put(old_blob_lock_obj)
1439 else: # We need to create a new blob-lock-object
1440 blob_lock_obj = db.Entity(db.Key("viur-blob-locks", skel.dbEntity.key.id_or_name))
1441 blob_lock_obj["active_blob_references"] = list(blob_list)
1442 blob_lock_obj["old_blob_references"] = []
1443 blob_lock_obj["has_old_blob_references"] = False
1444 blob_lock_obj["is_stale"] = False
1445 db.Put(blob_lock_obj)
1447 return skel.dbEntity.key, skel, change_list, is_add
1449 # Parse provided key, if any, and set it to skel["key"]
1450 if key:
1451 skel["key"] = db.keyHelper(key, skel.kindName)
1453 # Run transactional function
1454 if db.IsInTransaction():
1455 key, skel, change_list, is_add = __txn_write(skel)
1456 else:
1457 key, skel, change_list, is_add = db.RunInTransaction(__txn_write, skel)
1459 for bone_name, bone in skel.items():
1460 bone.postSavedHandler(skel, bone_name, key)
1462 skel.postSavedHandler(key, skel.dbEntity)
1464 if update_relations and not is_add:
1465 if change_list and len(change_list) < 5: # Only a few bones have changed, process these individually
1466 for idx, changed_bone in enumerate(change_list):
1467 updateRelations(key, time.time() + 1, changed_bone, _countdown=10 * idx)
1468 else: # Update all inbound relations, regardless of which bones they mirror
1469 updateRelations(key, time.time() + 1, None)
1471 # Trigger the database adapter of the changes made to the entry
1472 for adapter in skel.database_adapters:
1473 adapter.write(skel, is_add, change_list)
1475 return skel
1477 @classmethod
1478 def delete(cls, skel: SkeletonInstance, key: t.Optional[KeyType] = None) -> None:
1479 """
1480 Deletes the entity associated with the current Skeleton from the data store.
1482 :param key: Allows to specify a key that is used for deletion, otherwise skel["key"] will be used.
1483 """
1485 def __txn_delete(skel: SkeletonInstance, key: db.Key):
1486 if not skel.read(key):
1487 raise ValueError("This skeleton is not in the database (anymore?)!")
1489 # Is there any relation to this Skeleton which prevents the deletion?
1490 locked_relation = (
1491 db.Query("viur-relations")
1492 .filter("dest.__key__ =", key)
1493 .filter("viur_relational_consistency =", RelationalConsistency.PreventDeletion.value)
1494 ).getEntry()
1496 if locked_relation is not None:
1497 raise errors.Locked("This entry is still referenced by other Skeletons, which prevents deleting!")
1499 # Ensure that any value lock objects remaining for this entry are being deleted
1500 viur_data = skel.dbEntity.get("viur") or {}
1502 for boneName, bone in skel.items():
1503 bone.delete(skel, boneName)
1504 if bone.unique:
1505 flushList = []
1506 for lockValue in viur_data.get(f"{boneName}_uniqueIndexValue") or []:
1507 lockKey = db.Key(f"{skel.kindName}_{boneName}_uniquePropertyIndex", lockValue)
1508 lockObj = db.Get(lockKey)
1509 if not lockObj:
1510 logging.error(f"{lockKey=} missing!")
1511 elif lockObj["references"] != key.id_or_name:
1512 logging.error(
1513 f"""{key!r} does not hold lock for {lockKey!r}""")
1514 else:
1515 flushList.append(lockObj)
1516 if flushList:
1517 db.Delete(flushList)
1519 # Delete the blob-key lock object
1520 lockObjectKey = db.Key("viur-blob-locks", key.id_or_name)
1521 lockObj = db.Get(lockObjectKey)
1523 if lockObj is not None:
1524 if lockObj["old_blob_references"] is None and lockObj["active_blob_references"] is None:
1525 db.Delete(lockObjectKey) # Nothing to do here
1526 else:
1527 if lockObj["old_blob_references"] is None:
1528 # No old stale entries, move active_blob_references -> old_blob_references
1529 lockObj["old_blob_references"] = lockObj["active_blob_references"]
1530 elif lockObj["active_blob_references"] is not None:
1531 # Append the current references to the list of old & stale references
1532 lockObj["old_blob_references"] += lockObj["active_blob_references"]
1533 lockObj["active_blob_references"] = [] # There are no active ones left
1534 lockObj["is_stale"] = True
1535 lockObj["has_old_blob_references"] = True
1536 db.Put(lockObj)
1538 db.Delete(key)
1539 processRemovedRelations(key)
1541 if key := (key or skel["key"]):
1542 key = db.keyHelper(key, skel.kindName)
1543 else:
1544 raise ValueError("This skeleton has no key!")
1546 # Full skeleton is required to have all bones!
1547 skel = skeletonByKind(skel.kindName)()
1549 if db.IsInTransaction():
1550 __txn_delete(skel, key)
1551 else:
1552 db.RunInTransaction(__txn_delete, skel, key)
1554 for boneName, bone in skel.items():
1555 bone.postDeletedHandler(skel, boneName, key)
1557 skel.postDeletedHandler(key)
1559 # Inform the custom DB Adapter
1560 for adapter in skel.database_adapters:
1561 adapter.delete(skel)
1563 @classmethod
1564 def patch(
1565 cls,
1566 skel: SkeletonInstance,
1567 values: t.Optional[dict | t.Callable[[SkeletonInstance], None]] = {},
1568 *,
1569 key: t.Optional[db.Key | int | str] = None,
1570 check: t.Optional[dict | t.Callable[[SkeletonInstance], None]] = None,
1571 create: t.Optional[bool | dict | t.Callable[[SkeletonInstance], None]] = None,
1572 update_relations: bool = True,
1573 ignore: t.Optional[t.Iterable[str]] = (),
1574 retry: int = 0,
1575 ) -> SkeletonInstance:
1576 """
1577 Performs an edit operation on a Skeleton within a transaction.
1579 The transaction performs a read, sets bones and afterwards does a write with exclusive access on the
1580 given Skeleton and its underlying database entity.
1582 All value-dicts that are being fed to this function are provided to `skel.fromClient()`. Instead of dicts,
1583 a callable can also be given that can individually modify the Skeleton that is edited.
1585 :param values: A dict of key-values to update on the entry, or a callable that is executed within
1586 the transaction.
1588 This dict allows for a special notation: Keys starting with "+" or "-" are added or substracted to the
1589 given value, which can be used for counters.
1590 :param key: A :class:`viur.core.db.Key`, string, or int; from which the data shall be fetched.
1591 If not provided, skel["key"] will be used.
1592 :param check: An optional dict of key-values or a callable to check on the Skeleton before updating.
1593 If something fails within this check, an AssertionError is being raised.
1594 :param create: Allows to specify a dict or initial callable that is executed in case the Skeleton with the
1595 given key does not exist.
1596 :param update_relations: Trigger update relations task on success. Defaults to False.
1597 :param trust: Use internal `fromClient` with trusted data (may change readonly-bones)
1598 :param retry: On ViurDatastoreError, retry for this amount of times.
1600 If the function does not raise an Exception, all went well. The function always returns the input Skeleton.
1602 Raises:
1603 ValueError: In case parameters where given wrong or incomplete.
1604 AssertionError: In case an asserted check parameter did not match.
1605 ReadFromClientException: In case a skel.fromClient() failed with a high severity.
1606 """
1608 # Transactional function
1609 def __update_txn():
1610 # Try to read the skeleton, create on demand
1611 if not skel.read(key):
1612 if create is None or create is False:
1613 raise ValueError("Creation during update is forbidden - explicitly provide `create=True` to allow.")
1615 if not (key or skel["key"]) and create in (False, None):
1616 return ValueError("No valid key provided")
1618 if key or skel["key"]:
1619 skel["key"] = db.keyHelper(key or skel["key"], skel.kindName)
1621 if isinstance(create, dict):
1622 if create and not skel.fromClient(create, amend=True, ignore=ignore):
1623 raise ReadFromClientException(skel.errors)
1624 elif callable(create):
1625 create(skel)
1626 elif create is not True:
1627 raise ValueError("'create' must either be dict or a callable.")
1629 # Handle check
1630 if isinstance(check, dict):
1631 for bone, value in check.items():
1632 if skel[bone] != value:
1633 raise AssertionError(f"{bone} contains {skel[bone]!r}, expecting {value!r}")
1635 elif callable(check):
1636 check(skel)
1638 # Set values
1639 if isinstance(values, dict):
1640 if values and not skel.fromClient(values, amend=True, ignore=ignore):
1641 raise ReadFromClientException(skel.errors)
1643 # Special-feature: "+" and "-" prefix for simple calculations
1644 # TODO: This can maybe integrated into skel.fromClient() later...
1645 for name, value in values.items():
1646 match name[0]:
1647 case "+": # Increment by value?
1648 skel[name[1:]] += value
1649 case "-": # Decrement by value?
1650 skel[name[1:]] -= value
1652 elif callable(values):
1653 values(skel)
1655 else:
1656 raise ValueError("'values' must either be dict or a callable.")
1658 return skel.write(update_relations=update_relations)
1660 if not db.IsInTransaction:
1661 # Retry loop
1662 while True:
1663 try:
1664 return db.RunInTransaction(__update_txn)
1666 except db.ViurDatastoreError as e:
1667 retry -= 1
1668 if retry < 0:
1669 raise
1671 logging.debug(f"{e}, retrying {retry} more times")
1673 time.sleep(1)
1674 else:
1675 return __update_txn()
1677 @classmethod
1678 def preProcessBlobLocks(cls, skel: SkeletonInstance, locks):
1679 """
1680 Can be overridden to modify the list of blobs referenced by this skeleton
1681 """
1682 return locks
1684 @classmethod
1685 def preProcessSerializedData(cls, skel: SkeletonInstance, entity):
1686 """
1687 Can be overridden to modify the :class:`viur.core.db.Entity` before its actually
1688 written to the data store.
1689 """
1690 return entity
1692 @classmethod
1693 def postSavedHandler(cls, skel: SkeletonInstance, key, dbObj):
1694 """
1695 Can be overridden to perform further actions after the entity has been written
1696 to the data store.
1697 """
1698 pass
1700 @classmethod
1701 def postDeletedHandler(cls, skel: SkeletonInstance, key):
1702 """
1703 Can be overridden to perform further actions after the entity has been deleted
1704 from the data store.
1705 """
1706 pass
1708 @classmethod
1709 def getCurrentSEOKeys(cls, skel: SkeletonInstance) -> None | dict[str, str]:
1710 """
1711 Should be overridden to return a dictionary of language -> SEO-Friendly key
1712 this entry should be reachable under. How theses names are derived are entirely up to the application.
1713 If the name is already in use for this module, the server will automatically append some random string
1714 to make it unique.
1715 :return:
1716 """
1717 return
1720class RelSkel(BaseSkeleton):
1721 """
1722 This is a Skeleton-like class that acts as a container for Skeletons used as a
1723 additional information data skeleton for
1724 :class:`~viur.core.bones.extendedRelationalBone.extendedRelationalBone`.
1726 It needs to be sub-classed where information about the kindName and its attributes
1727 (bones) are specified.
1729 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
1730 contained bones remains constant.
1731 """
1733 def serialize(self, parentIndexed):
1734 if self.dbEntity is None:
1735 self.dbEntity = db.Entity()
1736 for key, _bone in self.items():
1737 # if key in self.accessedValues:
1738 _bone.serialize(self, key, parentIndexed)
1739 # if "key" in self: # Write the key seperatly, as the base-bone doesn't store it
1740 # dbObj["key"] = self["key"]
1741 # FIXME: is this a good idea? Any other way to ensure only bones present in refKeys are serialized?
1742 return self.dbEntity
1744 def unserialize(self, values: db.Entity | dict):
1745 """
1746 Loads 'values' into this skeleton.
1748 :param values: dict with values we'll assign to our bones
1749 """
1750 if not isinstance(values, db.Entity):
1751 self.dbEntity = db.Entity()
1753 if values:
1754 self.dbEntity.update(values)
1755 else:
1756 self.dbEntity = values
1758 self.accessedValues = {}
1759 self.renderAccessedValues = {}
1762class RefSkel(RelSkel):
1763 @classmethod
1764 def fromSkel(cls, kindName: str, *args: list[str]) -> t.Type[RefSkel]:
1765 """
1766 Creates a ``RefSkel`` from a skeleton-class using only the bones explicitly named in ``args``.
1768 :param args: List of bone names we'll adapt
1769 :return: A new instance of RefSkel
1770 """
1771 newClass = type("RefSkelFor" + kindName, (RefSkel,), {})
1772 fromSkel = skeletonByKind(kindName)
1773 newClass.kindName = kindName
1774 bone_map = {}
1775 for arg in args:
1776 bone_map |= {k: fromSkel.__boneMap__[k] for k in fnmatch.filter(fromSkel.__boneMap__.keys(), arg)}
1777 newClass.__boneMap__ = bone_map
1778 return newClass
1780 def read(self, key: t.Optional[db.Key | str | int] = None) -> SkeletonInstance:
1781 """
1782 Read full skeleton instance referenced by the RefSkel from the database.
1784 Can be used for reading the full Skeleton from a RefSkel.
1785 The `key` parameter also allows to read another, given key from the related kind.
1787 :raise ValueError: If the entry is no longer in the database.
1788 """
1789 skel = skeletonByKind(self.kindName)()
1791 if not skel.read(key or self["key"]):
1792 raise ValueError(f"""The key {key or self["key"]!r} seems to be gone""")
1794 return skel
1797class SkelList(list):
1798 """
1799 This class is used to hold multiple skeletons together with other, commonly used information.
1801 SkelLists are returned by Skel().all()...fetch()-constructs and provide additional information
1802 about the data base query, for fetching additional entries.
1804 :ivar cursor: Holds the cursor within a query.
1805 :vartype cursor: str
1806 """
1808 __slots__ = (
1809 "baseSkel",
1810 "customQueryInfo",
1811 "getCursor",
1812 "get_orders",
1813 "renderPreparation",
1814 )
1816 def __init__(self, baseSkel=None):
1817 """
1818 :param baseSkel: The baseclass for all entries in this list
1819 """
1820 super(SkelList, self).__init__()
1821 self.baseSkel = baseSkel or {}
1822 self.getCursor = lambda: None
1823 self.get_orders = lambda: None
1824 self.renderPreparation = None
1825 self.customQueryInfo = {}
1828# Module functions
1831def skeletonByKind(kindName: str) -> t.Type[Skeleton]:
1832 """
1833 Returns the Skeleton-Class for the given kindName. That skeleton must exist, otherwise an exception is raised.
1834 :param kindName: The kindname to retreive the skeleton for
1835 :return: The skeleton-class for that kind
1836 """
1837 assert kindName in MetaBaseSkel._skelCache, f"Unknown skeleton {kindName=}"
1838 return MetaBaseSkel._skelCache[kindName]
1841def listKnownSkeletons() -> list[str]:
1842 """
1843 :return: A list of all known kindnames (all kindnames for which a skeleton is defined)
1844 """
1845 return list(MetaBaseSkel._skelCache.keys())[:]
1848def iterAllSkelClasses() -> t.Iterable[Skeleton]:
1849 """
1850 :return: An iterator that yields each Skeleton-Class once. (Only top-level skeletons are returned, so no
1851 RefSkel classes will be included)
1852 """
1853 for cls in list(MetaBaseSkel._allSkelClasses): # We'll add new classes here during setSystemInitialized()
1854 yield cls
1857### Tasks ###
1859@CallDeferred
1860def processRemovedRelations(removedKey, cursor=None):
1861 updateListQuery = (
1862 db.Query("viur-relations")
1863 .filter("dest.__key__ =", removedKey)
1864 .filter("viur_relational_consistency >", RelationalConsistency.PreventDeletion.value)
1865 )
1866 updateListQuery = updateListQuery.setCursor(cursor)
1867 updateList = updateListQuery.run(limit=5)
1869 for entry in updateList:
1870 skel = skeletonByKind(entry["viur_src_kind"])()
1872 if not skel.read(entry["src"].key):
1873 raise ValueError(f"processRemovedRelations detects inconsistency on src={entry['src'].key!r}")
1875 if entry["viur_relational_consistency"] == RelationalConsistency.SetNull.value:
1876 found = False
1878 for key, bone in skel.items():
1879 if isinstance(bone, RelationalBone):
1880 if relational_value := skel[key]:
1881 if isinstance(relational_value, dict) and relational_value["dest"]["key"] == removedKey:
1882 skel[key] = None
1883 found = True
1885 elif isinstance(relational_value, list):
1886 skel[key] = [entry for entry in relational_value if entry["dest"]["key"] != removedKey]
1887 found = True
1889 else:
1890 raise NotImplementedError(f"In {entry['src'].key!r}, no handling for {relational_value=}")
1892 if found:
1893 skel.write(update_relations=False)
1895 else:
1896 logging.critical(f"""Cascade deletion of {skel["key"]!r}""")
1897 skel.delete()
1899 if len(updateList) == 5:
1900 processRemovedRelations(removedKey, updateListQuery.getCursor())
1903@CallDeferred
1904def updateRelations(destKey: db.Key, minChangeTime: int, changedBone: t.Optional[str], cursor: t.Optional[str] = None):
1905 """
1906 This function updates Entities, which may have a copy of values from another entity which has been recently
1907 edited (updated). In ViUR, relations are implemented by copying the values from the referenced entity into the
1908 entity that's referencing them. This allows ViUR to run queries over properties of referenced entities and
1909 prevents additional db.Get's to these referenced entities if the main entity is read. However, this forces
1910 us to track changes made to entities as we might have to update these mirrored values. This is the deferred
1911 call from meth:`viur.core.skeleton.Skeleton.write()` after an update (edit) on one Entity to do exactly that.
1913 :param destKey: The database-key of the entity that has been edited
1914 :param minChangeTime: The timestamp on which the edit occurred. As we run deferred, and the entity might have
1915 been edited multiple times before we get acutally called, we can ignore entities that have been updated
1916 in the meantime as they're already up2date
1917 :param changedBone: If set, we'll update only entites that have a copy of that bone. Relations mirror only
1918 key and name by default, so we don't have to update these if only another bone has been changed.
1919 :param cursor: The database cursor for the current request as we only process five entities at once and then
1920 defer again.
1921 """
1922 logging.debug(f"Starting updateRelations for {destKey=}; {minChangeTime=}, {changedBone=}, {cursor=}")
1923 updateListQuery = (
1924 db.Query("viur-relations")
1925 .filter("dest.__key__ =", destKey)
1926 .filter("viur_delayed_update_tag <", minChangeTime)
1927 .filter("viur_relational_updateLevel =", RelationalUpdateLevel.Always.value)
1928 )
1929 if changedBone:
1930 updateListQuery.filter("viur_foreign_keys =", changedBone)
1931 if cursor:
1932 updateListQuery.setCursor(cursor)
1933 updateList = updateListQuery.run(limit=5)
1935 def updateTxn(skel, key, srcRelKey):
1936 if not skel.read(key):
1937 logging.warning(f"Cannot update stale reference to {key=} (referenced from {srcRelKey=})")
1938 return
1940 skel.refresh()
1941 skel.write(update_relations=False)
1943 for srcRel in updateList:
1944 try:
1945 skel = skeletonByKind(srcRel["viur_src_kind"])()
1946 except AssertionError:
1947 logging.info(f"""Ignoring {srcRel.key!r} which refers to unknown kind {srcRel["viur_src_kind"]!r}""")
1948 continue
1949 if db.IsInTransaction():
1950 updateTxn(skel, srcRel["src"].key, srcRel.key)
1951 else:
1952 db.RunInTransaction(updateTxn, skel, srcRel["src"].key, srcRel.key)
1953 nextCursor = updateListQuery.getCursor()
1954 if len(updateList) == 5 and nextCursor:
1955 updateRelations(destKey, minChangeTime, changedBone, nextCursor)
1958@CallableTask
1959class TaskUpdateSearchIndex(CallableTaskBase):
1960 """
1961 This tasks loads and saves *every* entity of the given module.
1962 This ensures an updated searchIndex and verifies consistency of this data.
1963 """
1964 key = "rebuildSearchIndex"
1965 name = "Rebuild search index"
1966 descr = "This task can be called to update search indexes and relational information."
1968 def canCall(self) -> bool:
1969 """Checks wherever the current user can execute this task"""
1970 user = current.user.get()
1971 return user is not None and "root" in user["access"]
1973 def dataSkel(self):
1974 modules = ["*"] + listKnownSkeletons()
1975 modules.sort()
1976 skel = BaseSkeleton().clone()
1977 skel.module = SelectBone(descr="Module", values={x: translate(x) for x in modules}, required=True)
1978 return skel
1980 def execute(self, module, *args, **kwargs):
1981 usr = current.user.get()
1982 if not usr:
1983 logging.warning("Don't know who to inform after rebuilding finished")
1984 notify = None
1985 else:
1986 notify = usr["name"]
1988 if module == "*":
1989 for module in listKnownSkeletons():
1990 logging.info("Rebuilding search index for module %r", module)
1991 self._run(module, notify)
1992 else:
1993 self._run(module, notify)
1995 @staticmethod
1996 def _run(module: str, notify: str):
1997 Skel = skeletonByKind(module)
1998 if not Skel:
1999 logging.error("TaskUpdateSearchIndex: Invalid module")
2000 return
2001 RebuildSearchIndex.startIterOnQuery(Skel().all(), {"notify": notify, "module": module})
2004class RebuildSearchIndex(QueryIter):
2005 @classmethod
2006 def handleEntry(cls, skel: SkeletonInstance, customData: dict[str, str]):
2007 skel.refresh()
2008 skel.write(update_relations=False)
2010 @classmethod
2011 def handleFinish(cls, totalCount: int, customData: dict[str, str]):
2012 QueryIter.handleFinish(totalCount, customData)
2013 if not customData["notify"]:
2014 return
2015 txt = (
2016 f"{conf.instance.project_id}: Rebuild search index finished for {customData['module']}\n\n"
2017 f"ViUR finished to rebuild the search index for module {customData['module']}.\n"
2018 f"{totalCount} records updated in total on this kind."
2019 )
2020 try:
2021 email.send_email(dests=customData["notify"], stringTemplate=txt, skel=None)
2022 except Exception as exc: # noqa; OverQuota, whatever
2023 logging.exception(f'Failed to notify {customData["notify"]}')
2026### Vacuum Relations
2028@CallableTask
2029class TaskVacuumRelations(TaskUpdateSearchIndex):
2030 """
2031 Checks entries in viur-relations and verifies that the src-kind
2032 and it's RelationalBone still exists.
2033 """
2034 key = "vacuumRelations"
2035 name = "Vacuum viur-relations (dangerous)"
2036 descr = "Drop stale inbound relations for the given kind"
2038 def execute(self, module: str, *args, **kwargs):
2039 usr = current.user.get()
2040 if not usr:
2041 logging.warning("Don't know who to inform after rebuilding finished")
2042 notify = None
2043 else:
2044 notify = usr["name"]
2045 processVacuumRelationsChunk(module.strip(), None, notify=notify)
2048@CallDeferred
2049def processVacuumRelationsChunk(
2050 module: str, cursor, count_total: int = 0, count_removed: int = 0, notify=None
2051):
2052 """
2053 Processes 25 Entries and calls the next batch
2054 """
2055 query = db.Query("viur-relations")
2056 if module != "*":
2057 query.filter("viur_src_kind =", module)
2058 query.setCursor(cursor)
2059 for relation_object in query.run(25):
2060 count_total += 1
2061 if not (src_kind := relation_object.get("viur_src_kind")):
2062 logging.critical("We got an relation-object without a src_kind!")
2063 continue
2064 if not (src_prop := relation_object.get("viur_src_property")):
2065 logging.critical("We got an relation-object without a src_prop!")
2066 continue
2067 try:
2068 skel = skeletonByKind(src_kind)()
2069 except AssertionError:
2070 # The referenced skeleton does not exist in this data model -> drop that relation object
2071 logging.info(f"Deleting {relation_object.key} which refers to unknown kind {src_kind}")
2072 db.Delete(relation_object)
2073 count_removed += 1
2074 continue
2075 if src_prop not in skel:
2076 logging.info(f"Deleting {relation_object.key} which refers to "
2077 f"non-existing RelationalBone {src_prop} of {src_kind}")
2078 db.Delete(relation_object)
2079 count_removed += 1
2080 logging.info(f"END processVacuumRelationsChunk {module}, "
2081 f"{count_total} records processed, {count_removed} removed")
2082 if new_cursor := query.getCursor():
2083 # Start processing of the next chunk
2084 processVacuumRelationsChunk(module, new_cursor, count_total, count_removed, notify)
2085 elif notify:
2086 txt = (
2087 f"{conf.instance.project_id}: Vacuum relations finished for {module}\n\n"
2088 f"ViUR finished to vacuum viur-relations for module {module}.\n"
2089 f"{count_total} records processed, "
2090 f"{count_removed} entries removed"
2091 )
2092 try:
2093 email.send_email(dests=notify, stringTemplate=txt, skel=None)
2094 except Exception as exc: # noqa; OverQuota, whatever
2095 logging.exception(f"Failed to notify {notify}")
2098# Forward our references to SkelInstance to the database (needed for queries)
2099db.config["SkeletonInstanceRef"] = SkeletonInstance
2101# DEPRECATED ATTRIBUTES HANDLING
2103__DEPRECATED_NAMES = {
2104 # stuff prior viur-core < 3.6
2105 "seoKeyBone": ("SeoKeyBone", SeoKeyBone),
2106}
2109def __getattr__(attr: str) -> object:
2110 if entry := __DEPRECATED_NAMES.get(attr):
2111 func = entry[1]
2112 msg = f"{attr} was replaced by {entry[0]}"
2113 warnings.warn(msg, DeprecationWarning, stacklevel=2)
2114 logging.warning(msg, stacklevel=2)
2115 return func
2117 return super(__import__(__name__).__class__).__getattribute__(attr)