Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/skeleton.py: 0%
823 statements
« prev ^ index » next coverage.py v7.6.3, created at 2024-10-16 22:16 +0000
1from __future__ import annotations
3import copy
4import fnmatch
5import inspect
6import logging
7import os
8import string
9import sys
10import typing as t
11import warnings
12from functools import partial
13from itertools import chain
14from time import time
16from viur.core import conf, current, db, email, errors, translate, utils
17from viur.core.bones import BaseBone, DateBone, KeyBone, RelationalBone, RelationalConsistency, RelationalUpdateLevel, \
18 SelectBone, StringBone
19from viur.core.bones.base import Compute, ComputeInterval, ComputeMethod, ReadFromClientError, \
20 ReadFromClientErrorSeverity, getSystemInitialized
21from viur.core.tasks import CallDeferred, CallableTask, CallableTaskBase, QueryIter
23_undefined = object()
24ABSTRACT_SKEL_CLS_SUFFIX = "AbstractSkel"
27class MetaBaseSkel(type):
28 """
29 This is the metaclass for Skeletons.
30 It is used to enforce several restrictions on bone names, etc.
31 """
32 _skelCache = {} # Mapping kindName -> SkelCls
33 _allSkelClasses = set() # list of all known skeleton classes (including Ref and Mail-Skels)
35 # List of reserved keywords and function names
36 __reserved_keywords = {
37 "all",
38 "bounce",
39 "clone",
40 "cursor",
41 "delete",
42 "fromClient",
43 "fromDB",
44 "get",
45 "getCurrentSEOKeys",
46 "items",
47 "keys",
48 "limit",
49 "orderby",
50 "orderdir",
51 "postDeletedHandler",
52 "postSavedHandler",
53 "preProcessBlobLocks",
54 "preProcessSerializedData",
55 "refresh",
56 "self",
57 "serialize",
58 "setBoneValue",
59 "style",
60 "structure",
61 "toDB",
62 "unserialize",
63 "values",
64 }
66 __allowed_chars = string.ascii_letters + string.digits + "_"
68 def __init__(cls, name, bases, dct):
69 cls.__boneMap__ = MetaBaseSkel.generate_bonemap(cls)
71 if not getSystemInitialized() and not cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
72 MetaBaseSkel._allSkelClasses.add(cls)
74 super(MetaBaseSkel, cls).__init__(name, bases, dct)
76 @staticmethod
77 def generate_bonemap(cls):
78 """
79 Recursively constructs a dict of bones from
80 """
81 map = {}
83 for base in cls.__bases__:
84 if "__viurBaseSkeletonMarker__" in dir(base):
85 map |= MetaBaseSkel.generate_bonemap(base)
87 for key in cls.__dict__:
88 prop = getattr(cls, key)
90 if isinstance(prop, BaseBone):
91 if not all([c in MetaBaseSkel.__allowed_chars for c in key]):
92 raise AttributeError(f"Invalid bone name: {key!r} contains invalid characters")
93 elif key in MetaBaseSkel.__reserved_keywords:
94 raise AttributeError(f"Invalid bone name: {key!r} is reserved and cannot be used")
96 map[key] = prop
98 elif prop is None and key in map: # Allow removing a bone in a subclass by setting it to None
99 del map[key]
101 return map
103 def __setattr__(self, key, value):
104 super().__setattr__(key, value)
105 if isinstance(value, BaseBone):
106 # Call BaseBone.__set_name__ manually for bones that are assigned at runtime
107 value.__set_name__(self, key)
110def skeletonByKind(kindName: str) -> t.Type[Skeleton]:
111 """
112 Returns the Skeleton-Class for the given kindName. That skeleton must exist, otherwise an exception is raised.
113 :param kindName: The kindname to retreive the skeleton for
114 :return: The skeleton-class for that kind
115 """
116 assert kindName in MetaBaseSkel._skelCache, f"Unknown skeleton {kindName=}"
117 return MetaBaseSkel._skelCache[kindName]
120def listKnownSkeletons() -> list[str]:
121 """
122 :return: A list of all known kindnames (all kindnames for which a skeleton is defined)
123 """
124 return list(MetaBaseSkel._skelCache.keys())[:]
127def iterAllSkelClasses() -> t.Iterable["Skeleton"]:
128 """
129 :return: An iterator that yields each Skeleton-Class once. (Only top-level skeletons are returned, so no
130 RefSkel classes will be included)
131 """
132 for cls in list(MetaBaseSkel._allSkelClasses): # We'll add new classes here during setSystemInitialized()
133 yield cls
136class SkeletonInstance:
137 """
138 The actual wrapper around a Skeleton-Class. An object of this class is what's actually returned when you
139 call a Skeleton-Class. With ViUR3, you don't get an instance of a Skeleton-Class any more - it's always this
140 class. This is much faster as this is a small class.
141 """
142 __slots__ = {
143 "accessedValues",
144 "boneMap",
145 "dbEntity",
146 "errors",
147 "is_cloned",
148 "renderAccessedValues",
149 "renderPreparation",
150 "skeletonCls",
151 }
153 def __init__(self, skelCls, subSkelNames=None, fullClone=False, clonedBoneMap=None):
154 if clonedBoneMap:
155 self.boneMap = clonedBoneMap
156 for k, v in self.boneMap.items():
157 v.isClonedInstance = True
159 elif subSkelNames:
160 boneList = ["key"] + list(chain(*[skelCls.subSkels.get(x, []) for x in ["*"] + subSkelNames]))
161 doesMatch = lambda name: name in boneList or any(
162 [name.startswith(x[:-1]) for x in boneList if x[-1] == "*"])
163 if fullClone:
164 self.boneMap = {k: copy.deepcopy(v) for k, v in skelCls.__boneMap__.items() if doesMatch(k)}
165 for v in self.boneMap.values():
166 v.isClonedInstance = True
167 else:
168 self.boneMap = {k: v for k, v in skelCls.__boneMap__.items() if doesMatch(k)}
170 elif fullClone:
171 self.boneMap = copy.deepcopy(skelCls.__boneMap__)
172 for v in self.boneMap.values():
173 v.isClonedInstance = True
175 else: # No Subskel, no Clone
176 self.boneMap = skelCls.__boneMap__.copy()
178 self.accessedValues = {}
179 self.dbEntity = None
180 self.errors = []
181 self.is_cloned = fullClone
182 self.renderAccessedValues = {}
183 self.renderPreparation = None
184 self.skeletonCls = skelCls
186 def items(self, yieldBoneValues: bool = False) -> t.Iterable[tuple[str, BaseBone]]:
187 if yieldBoneValues:
188 for key in self.boneMap.keys():
189 yield key, self[key]
190 else:
191 yield from self.boneMap.items()
193 def keys(self) -> t.Iterable[str]:
194 yield from self.boneMap.keys()
196 def values(self) -> t.Iterable[t.Any]:
197 yield from self.boneMap.values()
199 def __iter__(self) -> t.Iterable[str]:
200 yield from self.keys()
202 def __contains__(self, item):
203 return item in self.boneMap
205 def get(self, item, default=None):
206 if item not in self:
207 return default
209 return self[item]
211 def __setitem__(self, key, value):
212 assert self.renderPreparation is None, "Cannot modify values while rendering"
213 if isinstance(value, BaseBone):
214 raise AttributeError(f"Don't assign this bone object as skel[\"{key}\"] = ... anymore to the skeleton. "
215 f"Use skel.{key} = ... for bone to skeleton assignment!")
216 self.accessedValues[key] = value
218 def __getitem__(self, key):
219 if self.renderPreparation:
220 if key in self.renderAccessedValues:
221 return self.renderAccessedValues[key]
222 if key not in self.accessedValues:
223 boneInstance = self.boneMap.get(key, None)
224 if boneInstance:
225 if self.dbEntity is not None:
226 boneInstance.unserialize(self, key)
227 else:
228 self.accessedValues[key] = boneInstance.getDefaultValue(self)
229 if not self.renderPreparation:
230 return self.accessedValues.get(key)
231 value = self.renderPreparation(getattr(self, key), self, key, self.accessedValues.get(key))
232 self.renderAccessedValues[key] = value
233 return value
235 def __getattr__(self, item: str):
236 """
237 Get a special attribute from the SkeletonInstance
239 __getattr__ is called when an attribute access fails with an
240 AttributeError. So we know that this is not a real attribute of
241 the SkeletonInstance. But there are still a few special cases in which
242 attributes are loaded from the skeleton class.
243 """
244 if item == "boneMap":
245 return {} # There are __setAttr__ calls before __init__ has run
246 # Load attribute value from the Skeleton class
247 elif item in {"kindName", "interBoneValidations", "customDatabaseAdapter"}:
248 return getattr(self.skeletonCls, item)
249 # Load a @classmethod from the Skeleton class and bound this SkeletonInstance
250 elif item in {"fromDB", "toDB", "all", "unserialize", "serialize", "fromClient", "getCurrentSEOKeys",
251 "preProcessSerializedData", "preProcessBlobLocks", "postSavedHandler", "setBoneValue",
252 "delete", "postDeletedHandler", "refresh"}:
253 return partial(getattr(self.skeletonCls, item), self)
254 # Load a @property from the Skeleton class
255 try:
256 # Use try/except to save an if check
257 class_value = getattr(self.skeletonCls, item)
258 except AttributeError:
259 # Not inside the Skeleton class, okay at this point.
260 pass
261 else:
262 if isinstance(class_value, property):
263 # The attribute is a @property and can be called
264 # Note: `self` is this SkeletonInstance, not the Skeleton class.
265 # Therefore, you can access values inside the property method
266 # with item-access like `self["key"]`.
267 try:
268 return class_value.fget(self)
269 except AttributeError as exc:
270 # The AttributeError cannot be re-raised any further at this point.
271 # Since this would then be evaluated as an access error
272 # to the property attribute.
273 # Otherwise, it would be lost that it is an incorrect attribute access
274 # within this property (during the method call).
275 msg, *args = exc.args
276 msg = f"AttributeError: {msg}"
277 raise ValueError(msg, *args) from exc
278 # Load the bone instance from the bone map of this SkeletonInstance
279 try:
280 return self.boneMap[item]
281 except KeyError as exc:
282 raise AttributeError(f"{self.__class__.__name__!r} object has no attribute '{item}'") from exc
284 def __delattr__(self, item):
285 del self.boneMap[item]
286 if item in self.accessedValues:
287 del self.accessedValues[item]
288 if item in self.renderAccessedValues:
289 del self.renderAccessedValues[item]
291 def __setattr__(self, key, value):
292 if key in self.boneMap or isinstance(value, BaseBone):
293 if value is None:
294 del self.boneMap[key]
295 else:
296 self.boneMap[key] = value
297 elif key == "renderPreparation":
298 super().__setattr__(key, value)
299 self.renderAccessedValues.clear()
300 else:
301 super().__setattr__(key, value)
303 def __repr__(self) -> str:
304 return f"<SkeletonInstance of {self.skeletonCls.__name__} with {dict(self)}>"
306 def __str__(self) -> str:
307 return str(dict(self))
309 def __len__(self) -> int:
310 return len(self.boneMap)
312 def clone(self):
313 """
314 Clones a SkeletonInstance into a modificable, stand-alone instance.
315 This will also allow to modify the underlying data model.
316 """
317 res = SkeletonInstance(self.skeletonCls, clonedBoneMap=copy.deepcopy(self.boneMap))
318 res.accessedValues = copy.deepcopy(self.accessedValues)
319 res.dbEntity = copy.deepcopy(self.dbEntity)
320 res.is_cloned = True
321 res.renderAccessedValues = copy.deepcopy(self.renderAccessedValues)
322 return res
324 def ensure_is_cloned(self):
325 """
326 Ensured this SkeletonInstance is a stand-alone clone, which can be modified.
327 Does nothing in case it was already cloned before.
328 """
329 if not self.is_cloned:
330 return self.clone()
332 return self
334 def setEntity(self, entity: db.Entity):
335 self.dbEntity = entity
336 self.accessedValues = {}
337 self.renderAccessedValues = {}
339 def structure(self) -> dict:
340 return {
341 key: bone.structure() | {"sortindex": i}
342 for i, (key, bone) in enumerate(self.items())
343 }
345 def __deepcopy__(self, memodict):
346 res = self.clone()
347 memodict[id(self)] = res
348 return res
351class BaseSkeleton(object, metaclass=MetaBaseSkel):
352 """
353 This is a container-object holding information about one database entity.
355 It has to be sub-classed with individual information about the kindName of the entities
356 and its specific data attributes, the so called bones.
357 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
358 contained bones remains constant.
360 :ivar key: This bone stores the current database key of this entity. \
361 Assigning to this bones value is dangerous and does *not* affect the actual key its stored in.
363 :vartype key: server.bones.BaseBone
365 :ivar creationdate: The date and time where this entity has been created.
366 :vartype creationdate: server.bones.DateBone
368 :ivar changedate: The date and time of the last change to this entity.
369 :vartype changedate: server.bones.DateBone
370 """
371 __viurBaseSkeletonMarker__ = True
372 boneMap = None
374 @classmethod
375 def subSkel(cls, *name, fullClone: bool = False, **kwargs) -> SkeletonInstance:
376 """
377 Creates a new sub-skeleton as part of the current skeleton.
379 A sub-skeleton is a copy of the original skeleton, containing only a subset of its bones.
380 To define sub-skeletons, use the subSkels property of the Skeleton object.
382 By passing multiple sub-skeleton names to this function, a sub-skeleton with the union of
383 all bones of the specified sub-skeletons is returned.
385 If an entry called "*" exists in the subSkels-dictionary, the bones listed in this entry
386 will always be part of the generated sub-skeleton.
388 :param name: Name of the sub-skeleton (that's the key of the subSkels dictionary); \
389 Multiple names can be specified.
391 :return: The sub-skeleton of the specified type.
392 """
393 if not name:
394 raise ValueError("Which subSkel?")
395 return cls(subSkelNames=list(name), fullClone=fullClone)
397 @classmethod
398 def setSystemInitialized(cls):
399 for attrName in dir(cls):
400 bone = getattr(cls, attrName)
401 if isinstance(bone, BaseBone):
402 bone.setSystemInitialized()
404 @classmethod
405 def setBoneValue(
406 cls,
407 skelValues: SkeletonInstance,
408 boneName: str,
409 value: t.Any,
410 append: bool = False,
411 language: t.Optional[str] = None
412 ) -> bool:
413 """
414 Allows for setting a bones value without calling fromClient or assigning a value directly.
415 Sanity-Checks are performed; if the value is invalid, that bone flips back to its original
416 (default) value and false is returned.
418 :param boneName: The name of the bone to be modified
419 :param value: The value that should be assigned. It's type depends on the type of that bone
420 :param append: If True, the given value is appended to the values of that bone instead of
421 replacing it. Only supported on bones with multiple=True
422 :param language: Language to set
424 :return: Wherever that operation succeeded or not.
425 """
426 bone = getattr(skelValues, boneName, None)
428 if not isinstance(bone, BaseBone):
429 raise ValueError(f"{boneName!r} is no valid bone on this skeleton ({skelValues!r})")
431 if language:
432 if not bone.languages:
433 raise ValueError("The bone {boneName!r} has no language setting")
434 elif language not in bone.languages:
435 raise ValueError("The language {language!r} is not available for bone {boneName!r}")
437 if value is None:
438 if append:
439 raise ValueError("Cannot append None-value to bone {boneName!r}")
441 if language:
442 skelValues[boneName][language] = [] if bone.multiple else None
443 else:
444 skelValues[boneName] = [] if bone.multiple else None
446 return True
448 _ = skelValues[boneName] # ensure the bone is being unserialized first
449 return bone.setBoneValue(skelValues, boneName, value, append, language)
451 @classmethod
452 def fromClient(cls, skelValues: SkeletonInstance, data: dict[str, list[str] | str], amend: bool = False) -> bool:
453 """
454 Load supplied *data* into Skeleton.
456 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
457 the values retrieved from *data* are checked against the bones and their validity checks.
459 Even if this function returns False, all bones are guaranteed to be in a valid state.
460 The ones which have been read correctly are set to their valid values;
461 Bones with invalid values are set back to a safe default (None in most cases).
462 So its possible to call :func:`~viur.core.skeleton.Skeleton.toDB` afterwards even if reading
463 data with this function failed (through this might violates the assumed consistency-model).
465 :param skel: The skeleton instance to be filled.
466 :param data: Dictionary from which the data is read.
467 :param amend: Defines whether content of data may be incomplete to amend the skel,
468 which is useful for edit-actions.
470 :returns: True if all data was successfully read and complete. \
471 False otherwise (e.g. some required fields where missing or where invalid).
472 """
473 complete = True
474 skelValues.errors = []
476 for key, bone in skelValues.items():
477 if bone.readOnly:
478 continue
480 if errors := bone.fromClient(skelValues, key, data):
481 for error in errors:
482 # insert current bone name into error's fieldPath
483 error.fieldPath.insert(0, str(key))
485 # logging.debug(f"BaseSkel.fromClient {key=} {error=}")
487 incomplete = (
488 # always when something is invalid
489 error.severity == ReadFromClientErrorSeverity.Invalid
490 or (
491 # only when path is top-level
492 len(error.fieldPath) == 1
493 and (
494 # bone is generally required
495 bool(bone.required)
496 and (
497 # and value is either empty
498 error.severity == ReadFromClientErrorSeverity.Empty
499 # or when not amending, not set
500 or (not amend and error.severity == ReadFromClientErrorSeverity.NotSet)
501 )
502 )
503 )
504 )
506 # in case there are language requirements, test additionally
507 if bone.languages and isinstance(bone.required, (list, tuple)):
508 incomplete &= any([key, lang] == error.fieldPath for lang in bone.required)
510 # logging.debug(f"BaseSkel.fromClient {incomplete=} {error.severity=} {bone.required=}")
512 if incomplete:
513 complete = False
515 if conf.debug.skeleton_from_client:
516 logging.error(
517 f"""{getattr(cls, "kindName", cls.__name__)}: {".".join(error.fieldPath)}: """
518 f"""({error.severity}) {error.errorMessage}"""
519 )
521 skelValues.errors += errors
523 return complete
525 @classmethod
526 def refresh(cls, skel: SkeletonInstance):
527 """
528 Refresh the bones current content.
530 This function causes a refresh of all relational bones and their associated
531 information.
532 """
533 logging.debug(f"""Refreshing {skel["key"]=}""")
535 for key, bone in skel.items():
536 if not isinstance(bone, BaseBone):
537 continue
539 _ = skel[key] # Ensure value gets loaded
540 bone.refresh(skel, key)
542 def __new__(cls, *args, **kwargs) -> SkeletonInstance:
543 return SkeletonInstance(cls, *args, **kwargs)
546class MetaSkel(MetaBaseSkel):
547 def __init__(cls, name, bases, dct):
548 super(MetaSkel, cls).__init__(name, bases, dct)
549 relNewFileName = inspect.getfile(cls) \
550 .replace(str(conf.instance.project_base_path), "") \
551 .replace(str(conf.instance.core_base_path), "")
553 # Check if we have an abstract skeleton
554 if cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
555 # Ensure that it doesn't have a kindName
556 assert cls.kindName is _undefined or cls.kindName is None, "Abstract Skeletons can't have a kindName"
557 # Prevent any further processing by this class; it has to be sub-classed before it can be used
558 return
560 # Automatic determination of the kindName, if the class is not part of viur.core.
561 if (cls.kindName is _undefined
562 and not relNewFileName.strip(os.path.sep).startswith("viur")
563 and not "viur_doc_build" in dir(sys)):
564 if cls.__name__.endswith("Skel"):
565 cls.kindName = cls.__name__.lower()[:-4]
566 else:
567 cls.kindName = cls.__name__.lower()
568 # Try to determine which skeleton definition takes precedence
569 if cls.kindName and cls.kindName is not _undefined and cls.kindName in MetaBaseSkel._skelCache:
570 relOldFileName = inspect.getfile(MetaBaseSkel._skelCache[cls.kindName]) \
571 .replace(str(conf.instance.project_base_path), "") \
572 .replace(str(conf.instance.core_base_path), "")
573 idxOld = min(
574 [x for (x, y) in enumerate(conf.skeleton_search_path) if relOldFileName.startswith(y)] + [999])
575 idxNew = min(
576 [x for (x, y) in enumerate(conf.skeleton_search_path) if relNewFileName.startswith(y)] + [999])
577 if idxNew == 999:
578 # We could not determine a priority for this class as its from a path not listed in the config
579 raise NotImplementedError(
580 "Skeletons must be defined in a folder listed in conf.skeleton_search_path")
581 elif idxOld < idxNew: # Lower index takes precedence
582 # The currently processed skeleton has a lower priority than the one we already saw - just ignore it
583 return
584 elif idxOld > idxNew:
585 # The currently processed skeleton has a higher priority, use that from now
586 MetaBaseSkel._skelCache[cls.kindName] = cls
587 else: # They seem to be from the same Package - raise as something is messed up
588 raise ValueError(f"Duplicate definition for {cls.kindName} in {relNewFileName} and {relOldFileName}")
589 # Ensure that all skeletons are defined in folders listed in conf.skeleton_search_path
590 if (not any([relNewFileName.startswith(x) for x in conf.skeleton_search_path])
591 and not "viur_doc_build" in dir(sys)): # Do not check while documentation build
592 raise NotImplementedError(
593 f"""{relNewFileName} must be defined in a folder listed in {conf.skeleton_search_path}""")
594 if cls.kindName and cls.kindName is not _undefined:
595 MetaBaseSkel._skelCache[cls.kindName] = cls
596 # Auto-Add ViUR Search Tags Adapter if the skeleton has no adapter attached
597 if cls.customDatabaseAdapter is _undefined:
598 cls.customDatabaseAdapter = ViurTagsSearchAdapter()
601class CustomDatabaseAdapter:
602 # Set to True if we can run a fulltext search using this database
603 providesFulltextSearch: bool = False
604 # Are results returned by `meth:fulltextSearch` guaranteed to also match the databaseQuery
605 fulltextSearchGuaranteesQueryConstrains = False
606 # Indicate that we can run more types of queries than originally supported by firestore
607 providesCustomQueries: bool = False
609 def preprocessEntry(self, entry: db.Entity, skel: BaseSkeleton, changeList: list[str], isAdd: bool) -> db.Entity:
610 """
611 Can be overridden to add or alter the data of this entry before it's written to firestore.
612 Will always be called inside an transaction.
613 :param entry: The entry containing the serialized data of that skeleton
614 :param skel: The (complete) skeleton this skel.toDB() runs for
615 :param changeList: List of boneNames that are changed by this skel.toDB() call
616 :param isAdd: Is this an update or an add?
617 :return: The (maybe modified) entity
618 """
619 return entry
621 def updateEntry(self, dbObj: db.Entity, skel: BaseSkeleton, changeList: list[str], isAdd: bool) -> None:
622 """
623 Like `meth:preprocessEntry`, but runs after the transaction had completed.
624 Changes made to dbObj will be ignored.
625 :param entry: The entry containing the serialized data of that skeleton
626 :param skel: The (complete) skeleton this skel.toDB() runs for
627 :param changeList: List of boneNames that are changed by this skel.toDB() call
628 :param isAdd: Is this an update or an add?
629 """
630 return
632 def deleteEntry(self, entry: db.Entity, skel: BaseSkeleton) -> None:
633 """
634 Called, after an skeleton has been successfully deleted from firestore
635 :param entry: The db.Entity object containing an snapshot of the data that has been deleted
636 :param skel: The (complete) skeleton for which `meth:delete' had been called
637 """
638 return
640 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
641 """
642 If this database supports fulltext searches, this method has to implement them.
643 If it's a plain fulltext search engine, leave 'prop:fulltextSearchGuaranteesQueryConstrains' set to False,
644 then the server will post-process the list of entries returned from this function and drop any entry that
645 cannot be returned due to other constrains set in 'param:databaseQuery'. If you can obey *every* constrain
646 set in that Query, we can skip this post-processing and save some CPU-cycles.
647 :param queryString: the string as received from the user (no quotation or other safety checks applied!)
648 :param databaseQuery: The query containing any constrains that returned entries must also match
649 :return:
650 """
651 raise NotImplementedError
654class ViurTagsSearchAdapter(CustomDatabaseAdapter):
655 """
656 This Adapter implements a simple fulltext search on top of the datastore.
658 On skel.toDB(), all words from String-/TextBones are collected with all *min_length* postfixes and dumped
659 into the property `viurTags`. When queried, we'll run a prefix-match against this property - thus returning
660 entities with either an exact match or a match within a word.
662 Example:
663 For the word "hello" we'll write "hello", "ello" and "llo" into viurTags.
664 When queried with "hello" we'll have an exact match.
665 When queried with "hel" we'll match the prefix for "hello"
666 When queried with "ell" we'll prefix-match "ello" - this is only enabled when substring_matching is True.
668 We'll automatically add this adapter if a skeleton has no other database adapter defined.
669 """
670 providesFulltextSearch = True
671 fulltextSearchGuaranteesQueryConstrains = True
673 def __init__(self, min_length: int = 2, max_length: int = 50, substring_matching: bool = False):
674 super().__init__()
675 self.min_length = min_length
676 self.max_length = max_length
677 self.substring_matching = substring_matching
679 def _tagsFromString(self, value: str) -> set[str]:
680 """
681 Extract all words including all min_length postfixes from given string
682 """
683 res = set()
685 for tag in value.split(" "):
686 tag = "".join([x for x in tag.lower() if x in conf.search_valid_chars])
688 if len(tag) >= self.min_length:
689 res.add(tag)
691 if self.substring_matching:
692 for i in range(1, 1 + len(tag) - self.min_length):
693 res.add(tag[i:])
695 return res
697 def preprocessEntry(self, entry: db.Entity, skel: Skeleton, changeList: list[str], isAdd: bool) -> db.Entity:
698 """
699 Collect searchTags from skeleton and build viurTags
700 """
701 tags = set()
703 for boneName, bone in skel.items():
704 if bone.searchable:
705 tags = tags.union(bone.getSearchTags(skel, boneName))
707 entry["viurTags"] = list(chain(*[self._tagsFromString(x) for x in tags if len(x) <= self.max_length]))
708 return entry
710 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
711 """
712 Run a fulltext search
713 """
714 keywords = list(self._tagsFromString(queryString))[:10]
715 resultScoreMap = {}
716 resultEntryMap = {}
718 for keyword in keywords:
719 qryBase = databaseQuery.clone()
720 for entry in qryBase.filter("viurTags >=", keyword).filter("viurTags <", keyword + "\ufffd").run():
721 if not entry.key in resultScoreMap:
722 resultScoreMap[entry.key] = 1
723 else:
724 resultScoreMap[entry.key] += 1
725 if not entry.key in resultEntryMap:
726 resultEntryMap[entry.key] = entry
728 resultList = [(k, v) for k, v in resultScoreMap.items()]
729 resultList.sort(key=lambda x: x[1], reverse=True)
731 return [resultEntryMap[x[0]] for x in resultList[:databaseQuery.queries.limit]]
734class SeoKeyBone(StringBone):
735 """
736 Special kind of StringBone saving its contents as `viurCurrentSeoKeys` into the entity's `viur` dict.
737 """
739 def unserialize(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> bool:
740 try:
741 skel.accessedValues[name] = skel.dbEntity["viur"]["viurCurrentSeoKeys"]
742 except KeyError:
743 skel.accessedValues[name] = self.getDefaultValue(skel)
745 def serialize(self, skel: 'SkeletonInstance', name: str, parentIndexed: bool) -> bool:
746 # Serialize also to skel["viur"]["viurCurrentSeoKeys"], so we can use this bone in relations
747 if name in skel.accessedValues:
748 newVal = skel.accessedValues[name]
749 if not skel.dbEntity.get("viur"):
750 skel.dbEntity["viur"] = db.Entity()
751 res = db.Entity()
752 res["_viurLanguageWrapper_"] = True
753 for language in (self.languages or []):
754 if not self.indexed:
755 res.exclude_from_indexes.add(language)
756 res[language] = None
757 if language in newVal:
758 res[language] = self.singleValueSerialize(newVal[language], skel, name, parentIndexed)
759 skel.dbEntity["viur"]["viurCurrentSeoKeys"] = res
760 return True
763class Skeleton(BaseSkeleton, metaclass=MetaSkel):
764 kindName: str = _undefined # To which kind we save our data to
765 customDatabaseAdapter: CustomDatabaseAdapter | None = _undefined
766 subSkels = {} # List of pre-defined sub-skeletons of this type
767 interBoneValidations: list[
768 t.Callable[[Skeleton], list[ReadFromClientError]]] = [] # List of functions checking inter-bone dependencies
770 __seo_key_trans = str.maketrans(
771 {"<": "",
772 ">": "",
773 "\"": "",
774 "'": "",
775 "\n": "",
776 "\0": "",
777 "/": "",
778 "\\": "",
779 "?": "",
780 "&": "",
781 "#": ""
782 })
784 # The "key" bone stores the current database key of this skeleton.
785 # Warning: Assigning to this bones value now *will* set the key
786 # it gets stored in. Must be kept readOnly to avoid security-issues with add/edit.
787 key = KeyBone(
788 descr="Key"
789 )
791 name = StringBone(
792 descr="Name",
793 visible=False,
794 compute=Compute(
795 fn=lambda skel: str(skel["key"]),
796 interval=ComputeInterval(ComputeMethod.OnWrite)
797 )
798 )
800 # The date (including time) when this entry has been created
801 creationdate = DateBone(
802 descr="created at",
803 readOnly=True,
804 visible=False,
805 indexed=True,
806 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.Once)),
807 )
809 # The last date (including time) when this entry has been updated
811 changedate = DateBone(
812 descr="updated at",
813 readOnly=True,
814 visible=False,
815 indexed=True,
816 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.OnWrite)),
817 )
819 viurCurrentSeoKeys = SeoKeyBone(
820 descr="SEO-Keys",
821 readOnly=True,
822 visible=False,
823 languages=conf.i18n.available_languages
824 )
826 def __repr__(self):
827 return "<skeleton %s with data=%r>" % (self.kindName, {k: self[k] for k in self.keys()})
829 def __str__(self):
830 return str({k: self[k] for k in self.keys()})
832 def __init__(self, *args, **kwargs):
833 super(Skeleton, self).__init__(*args, **kwargs)
834 assert self.kindName and self.kindName is not _undefined, "You must set kindName on this skeleton!"
836 @classmethod
837 def all(cls, skelValues, **kwargs) -> db.Query:
838 """
839 Create a query with the current Skeletons kindName.
841 :returns: A db.Query object which allows for entity filtering and sorting.
842 """
843 return db.Query(skelValues.kindName, srcSkelClass=skelValues, **kwargs)
845 @classmethod
846 def fromClient(cls, skelValues: SkeletonInstance, data: dict[str, list[str] | str], amend: bool = False) -> bool:
847 """
848 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
849 the values retrieved from *data* are checked against the bones and their validity checks.
851 Even if this function returns False, all bones are guaranteed to be in a valid state.
852 The ones which have been read correctly are set to their valid values;
853 Bones with invalid values are set back to a safe default (None in most cases).
854 So its possible to call :func:`~viur.core.skeleton.Skeleton.toDB` afterwards even if reading
855 data with this function failed (through this might violates the assumed consistency-model).
857 :param skel: The skeleton instance to be filled.
858 :param data: Dictionary from which the data is read.
859 :param amend: Defines whether content of data may be incomplete to amend the skel,
860 which is useful for edit-actions.
862 :returns: True if all data was successfully read and complete. \
863 False otherwise (e.g. some required fields where missing or where invalid).
864 """
865 assert skelValues.renderPreparation is None, "Cannot modify values while rendering"
867 # Load data into this skeleton
868 complete = bool(data) and super().fromClient(skelValues, data, amend=amend)
870 if (
871 not data # in case data is empty
872 or (len(data) == 1 and "key" in data)
873 or (utils.parse.bool(data.get("nomissing")))
874 ):
875 skelValues.errors = []
877 # Check if all unique values are available
878 for boneName, boneInstance in skelValues.items():
879 if boneInstance.unique:
880 lockValues = boneInstance.getUniquePropertyIndexValues(skelValues, boneName)
881 for lockValue in lockValues:
882 dbObj = db.Get(db.Key(f"{skelValues.kindName}_{boneName}_uniquePropertyIndex", lockValue))
883 if dbObj and (not skelValues["key"] or dbObj["references"] != skelValues["key"].id_or_name):
884 # This value is taken (sadly, not by us)
885 complete = False
886 errorMsg = boneInstance.unique.message
887 skelValues.errors.append(
888 ReadFromClientError(ReadFromClientErrorSeverity.Invalid, errorMsg, [boneName]))
890 # Check inter-Bone dependencies
891 for checkFunc in skelValues.interBoneValidations:
892 errors = checkFunc(skelValues)
893 if errors:
894 for error in errors:
895 if error.severity.value > 1:
896 complete = False
897 if conf.debug.skeleton_from_client:
898 logging.debug(f"{cls.kindName}: {error.fieldPath}: {error.errorMessage!r}")
900 skelValues.errors.extend(errors)
902 return complete
904 @classmethod
905 def fromDB(cls, skel: SkeletonInstance, key: db.Key | int | str) -> bool:
906 """
907 Load entity with *key* from the Datastore into the Skeleton.
909 Reads all available data of entity kind *kindName* and the key *key*
910 from the Datastore into the Skeleton structure's bones. Any previous
911 data of the bones will discard.
913 To store a Skeleton object to the Datastore, see :func:`~viur.core.skeleton.Skeleton.toDB`.
915 :param key: A :class:`viur.core.DB.Key`, string, or int; from which the data shall be fetched.
917 :returns: True on success; False if the given key could not be found or can not be parsed.
919 """
920 assert skel.renderPreparation is None, "Cannot modify values while rendering"
921 try:
922 db_key = db.keyHelper(key, skel.kindName)
923 except ValueError: # This key did not parse
924 return False
926 if not (db_res := db.Get(db_key)):
927 return False
928 skel.setEntity(db_res)
929 return True
931 @classmethod
932 def toDB(cls, skel: SkeletonInstance, update_relations: bool = True, **kwargs) -> db.Key:
933 """
934 Store current Skeleton entity to the Datastore.
936 Stores the current data of this instance into the database.
937 If an *key* value is set to the object, this entity will ne updated;
938 Otherwise a new entity will be created.
940 To read a Skeleton object from the data store, see :func:`~viur.core.skeleton.Skeleton.fromDB`.
942 :param update_relations: If False, this entity won't be marked dirty;
943 This avoids from being fetched by the background task updating relations.
945 :returns: The datastore key of the entity.
946 """
947 assert skel.renderPreparation is None, "Cannot modify values while rendering"
948 # fixme: Remove in viur-core >= 4
949 if "clearUpdateTag" in kwargs:
950 msg = "clearUpdateTag was replaced by update_relations"
951 warnings.warn(msg, DeprecationWarning, stacklevel=3)
952 logging.warning(msg, stacklevel=3)
953 update_relations = not kwargs["clearUpdateTag"]
955 def __txn_update(write_skel):
956 db_key = write_skel["key"]
957 skel = write_skel.skeletonCls()
959 blob_list = set()
960 change_list = []
961 old_copy = {}
962 # Load the current values from Datastore or create a new, empty db.Entity
963 if not db_key:
964 # We'll generate the key we'll be stored under early so we can use it for locks etc
965 db_key = db.AllocateIDs(db.Key(skel.kindName))
966 db_obj = db.Entity(db_key)
967 skel.dbEntity = db_obj
968 is_add = True
969 else:
970 db_key = db.keyHelper(db_key, skel.kindName)
971 if not (db_obj := db.Get(db_key)):
972 db_obj = db.Entity(db_key)
973 skel.dbEntity = db_obj
974 is_add = True
975 else:
976 skel.setEntity(db_obj)
977 old_copy = {k: v for k, v in db_obj.items()}
978 is_add = False
980 db_obj.setdefault("viur", {})
982 # Merge values and assemble unique properties
983 # Move accessed Values from srcSkel over to skel
984 skel.accessedValues = write_skel.accessedValues
985 skel["key"] = db_key # Ensure key stays set
987 for bone_name, bone in skel.items():
988 if bone_name == "key": # Explicitly skip key on top-level - this had been set above
989 continue
991 # Allow bones to perform outstanding "magic" operations before saving to db
992 bone.performMagic(skel, bone_name, isAdd=is_add) # FIXME VIUR4: ANY MAGIC IN OUR CODE IS DEPRECATED!!!
994 if not (bone_name in skel.accessedValues or bone.compute) and bone_name not in skel.dbEntity:
995 _ = skel[bone_name] # Ensure the datastore is filled with the default value
996 if (
997 bone_name in skel.accessedValues or bone.compute # We can have a computed value on store
998 or bone_name not in skel.dbEntity # It has not been written and is not in the database
999 ):
1000 # Serialize bone into entity
1001 try:
1002 bone.serialize(skel, bone_name, True)
1003 except Exception:
1004 logging.error(f"Failed to serialize {bone_name} {bone} {skel.accessedValues[bone_name]}")
1005 raise
1007 # Obtain referenced blobs
1008 blob_list.update(bone.getReferencedBlobs(skel, bone_name))
1010 # Check if the value has actually changed
1011 if db_obj.get(bone_name) != old_copy.get(bone_name):
1012 change_list.append(bone_name)
1014 # Lock hashes from bones that must have unique values
1015 if bone.unique:
1016 # Remember old hashes for bones that must have an unique value
1017 old_unique_values = []
1019 if f"{bone_name}_uniqueIndexValue" in db_obj["viur"]:
1020 old_unique_values = db_obj["viur"][f"{bone_name}_uniqueIndexValue"]
1021 # Check if the property is unique
1022 new_unique_values = bone.getUniquePropertyIndexValues(skel, bone_name)
1023 new_lock_kind = f"{skel.kindName}_{bone_name}_uniquePropertyIndex"
1024 for new_lock_value in new_unique_values:
1025 new_lock_key = db.Key(new_lock_kind, new_lock_value)
1026 if lock_db_obj := db.Get(new_lock_key):
1028 # There's already a lock for that value, check if we hold it
1029 if lock_db_obj["references"] != db_obj.key.id_or_name:
1030 # This value has already been claimed, and not by us
1031 # TODO: Use a custom exception class which is catchable with an try/except
1032 raise ValueError(
1033 f"The unique value {skel[bone_name]!r} of bone {bone_name!r} "
1034 f"has been recently claimed!")
1035 else:
1036 # This value is locked for the first time, create a new lock-object
1037 lock_obj = db.Entity(new_lock_key)
1038 lock_obj["references"] = db_obj.key.id_or_name
1039 db.Put(lock_obj)
1040 if new_lock_value in old_unique_values:
1041 old_unique_values.remove(new_lock_value)
1042 db_obj["viur"][f"{bone_name}_uniqueIndexValue"] = new_unique_values
1044 # Remove any lock-object we're holding for values that we don't have anymore
1045 for old_unique_value in old_unique_values:
1046 # Try to delete the old lock
1048 old_lock_key = db.Key(f"{skel.kindName}_{bone_name}_uniquePropertyIndex", old_unique_value)
1049 if old_lock_obj := db.Get(old_lock_key):
1050 if old_lock_obj["references"] != db_obj.key.id_or_name:
1052 # We've been supposed to have that lock - but we don't.
1053 # Don't remove that lock as it now belongs to a different entry
1054 logging.critical("Detected Database corruption! A Value-Lock had been reassigned!")
1055 else:
1056 # It's our lock which we don't need anymore
1057 db.Delete(old_lock_key)
1058 else:
1059 logging.critical("Detected Database corruption! Could not delete stale lock-object!")
1061 # Delete legacy property (PR #1244) #TODO: Remove in ViUR4
1062 db_obj.pop("viur_incomming_relational_locks", None)
1064 # Ensure the SEO-Keys are up-to-date
1065 last_requested_seo_keys = db_obj["viur"].get("viurLastRequestedSeoKeys") or {}
1066 last_set_seo_keys = db_obj["viur"].get("viurCurrentSeoKeys") or {}
1067 # Filter garbage serialized into this field by the SeoKeyBone
1068 last_set_seo_keys = {k: v for k, v in last_set_seo_keys.items() if not k.startswith("_") and v}
1070 if not isinstance(db_obj["viur"].get("viurCurrentSeoKeys"), dict):
1071 db_obj["viur"]["viurCurrentSeoKeys"] = {}
1072 if current_seo_keys := skel.getCurrentSEOKeys():
1073 # Convert to lower-case and remove certain characters
1074 for lang, value in current_seo_keys.items():
1075 current_seo_keys[lang] = value.lower().translate(Skeleton.__seo_key_trans).strip()
1077 for language in (conf.i18n.available_languages or [conf.i18n.default_language]):
1078 if current_seo_keys and language in current_seo_keys:
1079 current_seo_key = current_seo_keys[language]
1080 if current_seo_key != last_requested_seo_keys.get(language): # This one is new or has changed
1081 new_seo_key = current_seo_keys[language]
1082 for _ in range(0, 3):
1083 entry_using_key = db.Query(skel.kindName).filter("viur.viurActiveSeoKeys =",
1084 new_seo_key).getEntry()
1085 if entry_using_key and entry_using_key.key != db_obj.key:
1086 # It's not unique; append a random string and try again
1087 new_seo_key = f"{current_seo_keys[language]}-{utils.string.random(5).lower()}"
1089 else:
1090 # We found a new SeoKey
1091 break
1092 else:
1093 raise ValueError("Could not generate an unique seo key in 3 attempts")
1094 else:
1095 new_seo_key = current_seo_key
1096 last_set_seo_keys[language] = new_seo_key
1097 else:
1098 # We'll use the database-key instead
1099 last_set_seo_keys[language] = str(db_obj.key.id_or_name)
1100 # Store the current, active key for that language
1101 db_obj["viur"]["viurCurrentSeoKeys"][language] = last_set_seo_keys[language]
1103 db_obj["viur"].setdefault("viurActiveSeoKeys", [])
1104 for language, seo_key in last_set_seo_keys.items():
1105 if db_obj["viur"]["viurCurrentSeoKeys"][language] not in db_obj["viur"]["viurActiveSeoKeys"]:
1106 # Ensure the current, active seo key is in the list of all seo keys
1107 db_obj["viur"]["viurActiveSeoKeys"].insert(0, seo_key)
1108 if str(db_obj.key.id_or_name) not in db_obj["viur"]["viurActiveSeoKeys"]:
1109 # Ensure that key is also in there
1110 db_obj["viur"]["viurActiveSeoKeys"].insert(0, str(db_obj.key.id_or_name))
1111 # Trim to the last 200 used entries
1112 db_obj["viur"]["viurActiveSeoKeys"] = db_obj["viur"]["viurActiveSeoKeys"][:200]
1113 # Store lastRequestedKeys so further updates can run more efficient
1114 db_obj["viur"]["viurLastRequestedSeoKeys"] = current_seo_keys
1116 # mark entity as "dirty" when update_relations is set, to zero otherwise.
1117 db_obj["viur"]["delayedUpdateTag"] = time() if update_relations else 0
1119 db_obj = skel.preProcessSerializedData(db_obj)
1121 # Allow the custom DB Adapter to apply last minute changes to the object
1122 if skel.customDatabaseAdapter:
1123 db_obj = skel.customDatabaseAdapter.preprocessEntry(db_obj, skel, change_list, is_add)
1125 # ViUR2 import compatibility - remove properties containing. if we have a dict with the same name
1126 def fixDotNames(entity):
1127 for k, v in list(entity.items()):
1128 if isinstance(v, dict):
1129 for k2, v2 in list(entity.items()):
1130 if k2.startswith(f"{k}."):
1131 del entity[k2]
1132 backupKey = k2.replace(".", "__")
1133 entity[backupKey] = v2
1134 entity.exclude_from_indexes = set(entity.exclude_from_indexes) | {backupKey}
1135 fixDotNames(v)
1136 elif isinstance(v, list):
1137 for x in v:
1138 if isinstance(x, dict):
1139 fixDotNames(x)
1141 if conf.viur2import_blobsource: # Try to fix these only when converting from ViUR2
1142 fixDotNames(db_obj)
1144 # Write the core entry back
1145 db.Put(db_obj)
1147 # Now write the blob-lock object
1148 blob_list = skel.preProcessBlobLocks(blob_list)
1149 if blob_list is None:
1150 raise ValueError("Did you forget to return the blob_list somewhere inside getReferencedBlobs()?")
1151 if None in blob_list:
1152 msg = f"None is not valid in {blob_list=}"
1153 logging.error(msg)
1154 raise ValueError(msg)
1156 if not is_add and (old_blob_lock_obj := db.Get(db.Key("viur-blob-locks", db_key.id_or_name))):
1157 removed_blobs = set(old_blob_lock_obj.get("active_blob_references", [])) - blob_list
1158 old_blob_lock_obj["active_blob_references"] = list(blob_list)
1159 if old_blob_lock_obj["old_blob_references"] is None:
1160 old_blob_lock_obj["old_blob_references"] = list(removed_blobs)
1161 else:
1162 old_blob_refs = set(old_blob_lock_obj["old_blob_references"])
1163 old_blob_refs.update(removed_blobs) # Add removed blobs
1164 old_blob_refs -= blob_list # Remove active blobs
1165 old_blob_lock_obj["old_blob_references"] = list(old_blob_refs)
1167 old_blob_lock_obj["has_old_blob_references"] = bool(old_blob_lock_obj["old_blob_references"])
1168 old_blob_lock_obj["is_stale"] = False
1169 db.Put(old_blob_lock_obj)
1170 else: # We need to create a new blob-lock-object
1171 blob_lock_obj = db.Entity(db.Key("viur-blob-locks", db_obj.key.id_or_name))
1172 blob_lock_obj["active_blob_references"] = list(blob_list)
1173 blob_lock_obj["old_blob_references"] = []
1174 blob_lock_obj["has_old_blob_references"] = False
1175 blob_lock_obj["is_stale"] = False
1176 db.Put(blob_lock_obj)
1178 return db_obj.key, db_obj, skel, change_list, is_add
1180 # END of __txn_update subfunction
1182 # Run our SaveTxn
1183 if db.IsInTransaction():
1184 key, db_obj, skel, change_list, is_add = __txn_update(skel)
1185 else:
1186 key, db_obj, skel, change_list, is_add = db.RunInTransaction(__txn_update, skel)
1188 for bone_name, bone in skel.items():
1189 bone.postSavedHandler(skel, bone_name, key)
1191 skel.postSavedHandler(key, db_obj)
1193 if update_relations and not is_add:
1194 if change_list and len(change_list) < 5: # Only a few bones have changed, process these individually
1195 for idx, changed_bone in enumerate(change_list):
1196 updateRelations(key, time() + 1, changed_bone, _countdown=10 * idx)
1197 else: # Update all inbound relations, regardless of which bones they mirror
1198 updateRelations(key, time() + 1, None)
1200 # Inform the custom DB Adapter of the changes made to the entry
1201 if skel.customDatabaseAdapter:
1202 skel.customDatabaseAdapter.updateEntry(db_obj, skel, change_list, is_add)
1204 return key
1206 @classmethod
1207 def preProcessBlobLocks(cls, skelValues, locks):
1208 """
1209 Can be overridden to modify the list of blobs referenced by this skeleton
1210 """
1211 return locks
1213 @classmethod
1214 def preProcessSerializedData(cls, skelValues, entity):
1215 """
1216 Can be overridden to modify the :class:`viur.core.db.Entity` before its actually
1217 written to the data store.
1218 """
1219 return entity
1221 @classmethod
1222 def postSavedHandler(cls, skelValues, key, dbObj):
1223 """
1224 Can be overridden to perform further actions after the entity has been written
1225 to the data store.
1226 """
1227 pass
1229 @classmethod
1230 def postDeletedHandler(cls, skelValues, key):
1231 """
1232 Can be overridden to perform further actions after the entity has been deleted
1233 from the data store.
1234 """
1235 pass
1237 @classmethod
1238 def getCurrentSEOKeys(cls, skelValues) -> None | dict[str, str]:
1239 """
1240 Should be overridden to return a dictionary of language -> SEO-Friendly key
1241 this entry should be reachable under. How theses names are derived are entirely up to the application.
1242 If the name is already in use for this module, the server will automatically append some random string
1243 to make it unique.
1244 :return:
1245 """
1246 return
1248 @classmethod
1249 def delete(cls, skelValues):
1250 """
1251 Deletes the entity associated with the current Skeleton from the data store.
1252 """
1254 def txnDelete(skel: SkeletonInstance) -> db.Entity:
1255 skel_key = skel["key"]
1256 entity = db.Get(skel_key) # Fetch the raw object as we might have to clear locks
1257 viur_data = entity.get("viur") or {}
1259 # Is there any relation to this Skeleton which prevents the deletion?
1260 locked_relation = (
1261 db.Query("viur-relations")
1262 .filter("dest.__key__ =", skel_key)
1263 .filter("viur_relational_consistency =", RelationalConsistency.PreventDeletion.value)
1264 ).getEntry()
1265 if locked_relation is not None:
1266 raise errors.Locked("This entry is still referenced by other Skeletons, which prevents deleting!")
1268 for boneName, bone in skel.items():
1269 # Ensure that we delete any value-lock objects remaining for this entry
1270 bone.delete(skel, boneName)
1271 if bone.unique:
1272 flushList = []
1273 for lockValue in viur_data.get(f"{boneName}_uniqueIndexValue") or []:
1274 lockKey = db.Key(f"{skel.kindName}_{boneName}_uniquePropertyIndex", lockValue)
1275 lockObj = db.Get(lockKey)
1276 if not lockObj:
1277 logging.error(f"{lockKey=} missing!")
1278 elif lockObj["references"] != entity.key.id_or_name:
1279 logging.error(
1280 f"""{skel["key"]!r} does not hold lock for {lockKey!r}""")
1281 else:
1282 flushList.append(lockObj)
1283 if flushList:
1284 db.Delete(flushList)
1286 # Delete the blob-key lock object
1287 lockObjectKey = db.Key("viur-blob-locks", entity.key.id_or_name)
1288 lockObj = db.Get(lockObjectKey)
1289 if lockObj is not None:
1290 if lockObj["old_blob_references"] is None and lockObj["active_blob_references"] is None:
1291 db.Delete(lockObjectKey) # Nothing to do here
1292 else:
1293 if lockObj["old_blob_references"] is None:
1294 # No old stale entries, move active_blob_references -> old_blob_references
1295 lockObj["old_blob_references"] = lockObj["active_blob_references"]
1296 elif lockObj["active_blob_references"] is not None:
1297 # Append the current references to the list of old & stale references
1298 lockObj["old_blob_references"] += lockObj["active_blob_references"]
1299 lockObj["active_blob_references"] = [] # There are no active ones left
1300 lockObj["is_stale"] = True
1301 lockObj["has_old_blob_references"] = True
1302 db.Put(lockObj)
1303 db.Delete(skel_key)
1304 processRemovedRelations(skel_key)
1305 return entity
1307 key = skelValues["key"]
1308 if key is None:
1309 raise ValueError("This skeleton has no key!")
1310 skel = skeletonByKind(skelValues.kindName)()
1311 if not skel.fromDB(key):
1312 raise ValueError("This skeleton is not in the database (anymore?)!")
1313 if db.IsInTransaction():
1314 dbObj = txnDelete(skel)
1315 else:
1316 dbObj = db.RunInTransaction(txnDelete, skel)
1317 for boneName, _bone in skel.items():
1318 _bone.postDeletedHandler(skel, boneName, key)
1319 skel.postDeletedHandler(key)
1320 # Inform the custom DB Adapter
1321 if skel.customDatabaseAdapter:
1322 skel.customDatabaseAdapter.deleteEntry(dbObj, skel)
1325class RelSkel(BaseSkeleton):
1326 """
1327 This is a Skeleton-like class that acts as a container for Skeletons used as a
1328 additional information data skeleton for
1329 :class:`~viur.core.bones.extendedRelationalBone.extendedRelationalBone`.
1331 It needs to be sub-classed where information about the kindName and its attributes
1332 (bones) are specified.
1334 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
1335 contained bones remains constant.
1336 """
1338 def serialize(self, parentIndexed):
1339 if self.dbEntity is None:
1340 self.dbEntity = db.Entity()
1341 for key, _bone in self.items():
1342 # if key in self.accessedValues:
1343 _bone.serialize(self, key, parentIndexed)
1344 # if "key" in self: # Write the key seperatly, as the base-bone doesn't store it
1345 # dbObj["key"] = self["key"]
1346 # FIXME: is this a good idea? Any other way to ensure only bones present in refKeys are serialized?
1347 return self.dbEntity
1349 def unserialize(self, values: db.Entity | dict):
1350 """
1351 Loads 'values' into this skeleton.
1353 :param values: dict with values we'll assign to our bones
1354 """
1355 if not isinstance(values, db.Entity):
1356 self.dbEntity = db.Entity()
1357 self.dbEntity.update(values)
1358 else:
1359 self.dbEntity = values
1361 self.accessedValues = {}
1362 self.renderAccessedValues = {}
1365class RefSkel(RelSkel):
1366 @classmethod
1367 def fromSkel(cls, kindName: str, *args: list[str]) -> t.Type[RefSkel]:
1368 """
1369 Creates a relSkel from a skeleton-class using only the bones explicitly named
1370 in \*args
1372 :param args: List of bone names we'll adapt
1373 :return: A new instance of RefSkel
1374 """
1375 newClass = type("RefSkelFor" + kindName, (RefSkel,), {})
1376 fromSkel = skeletonByKind(kindName)
1377 newClass.kindName = kindName
1378 bone_map = {}
1379 for arg in args:
1380 bone_map |= {k: fromSkel.__boneMap__[k] for k in fnmatch.filter(fromSkel.__boneMap__.keys(), arg)}
1381 newClass.__boneMap__ = bone_map
1382 return newClass
1385class SkelList(list):
1386 """
1387 This class is used to hold multiple skeletons together with other, commonly used information.
1389 SkelLists are returned by Skel().all()...fetch()-constructs and provide additional information
1390 about the data base query, for fetching additional entries.
1392 :ivar cursor: Holds the cursor within a query.
1393 :vartype cursor: str
1394 """
1396 __slots__ = (
1397 "baseSkel",
1398 "customQueryInfo",
1399 "getCursor",
1400 "get_orders",
1401 "renderPreparation",
1402 )
1404 def __init__(self, baseSkel=None):
1405 """
1406 :param baseSkel: The baseclass for all entries in this list
1407 """
1408 super(SkelList, self).__init__()
1409 self.baseSkel = baseSkel or {}
1410 self.getCursor = lambda: None
1411 self.get_orders = lambda: None
1412 self.renderPreparation = None
1413 self.customQueryInfo = {}
1416### Tasks ###
1418@CallDeferred
1419def processRemovedRelations(removedKey, cursor=None):
1420 updateListQuery = (
1421 db.Query("viur-relations")
1422 .filter("dest.__key__ =", removedKey)
1423 .filter("viur_relational_consistency >", RelationalConsistency.PreventDeletion.value)
1424 )
1425 updateListQuery = updateListQuery.setCursor(cursor)
1426 updateList = updateListQuery.run(limit=5)
1428 for entry in updateList:
1429 skel = skeletonByKind(entry["viur_src_kind"])()
1431 if not skel.fromDB(entry["src"].key):
1432 raise ValueError(f"processRemovedRelations detects inconsistency on src={entry['src'].key!r}")
1434 if entry["viur_relational_consistency"] == RelationalConsistency.SetNull.value:
1435 for key, _bone in skel.items():
1436 if isinstance(_bone, RelationalBone):
1437 relVal = skel[key]
1438 if isinstance(relVal, dict) and relVal["dest"]["key"] == removedKey:
1439 # FIXME: Should never happen: "key" not in relVal["dest"]
1440 # skel.setBoneValue(key, None)
1441 skel[key] = None
1442 elif isinstance(relVal, list):
1443 skel[key] = [x for x in relVal if x["dest"]["key"] != removedKey]
1444 else:
1445 raise NotImplementedError(f"No handling for {type(relVal)=}")
1446 skel.toDB(update_relations=False)
1448 else:
1449 logging.critical(f"""Cascade deletion of {skel["key"]!r}""")
1450 skel.delete()
1452 if len(updateList) == 5:
1453 processRemovedRelations(removedKey, updateListQuery.getCursor())
1456@CallDeferred
1457def updateRelations(destKey: db.Key, minChangeTime: int, changedBone: t.Optional[str], cursor: t.Optional[str] = None):
1458 """
1459 This function updates Entities, which may have a copy of values from another entity which has been recently
1460 edited (updated). In ViUR, relations are implemented by copying the values from the referenced entity into the
1461 entity that's referencing them. This allows ViUR to run queries over properties of referenced entities and
1462 prevents additional db.Get's to these referenced entities if the main entity is read. However, this forces
1463 us to track changes made to entities as we might have to update these mirrored values. This is the deferred
1464 call from meth:`viur.core.skeleton.Skeleton.toDB()` after an update (edit) on one Entity to do exactly that.
1466 :param destKey: The database-key of the entity that has been edited
1467 :param minChangeTime: The timestamp on which the edit occurred. As we run deferred, and the entity might have
1468 been edited multiple times before we get acutally called, we can ignore entities that have been updated
1469 in the meantime as they're already up2date
1470 :param changedBone: If set, we'll update only entites that have a copy of that bone. Relations mirror only
1471 key and name by default, so we don't have to update these if only another bone has been changed.
1472 :param cursor: The database cursor for the current request as we only process five entities at once and then
1473 defer again.
1474 """
1475 logging.debug(f"Starting updateRelations for {destKey} ; {minChangeTime=},{changedBone=}, {cursor=}")
1476 updateListQuery = (
1477 db.Query("viur-relations")
1478 .filter("dest.__key__ =", destKey)
1479 .filter("viur_delayed_update_tag <", minChangeTime)
1480 .filter("viur_relational_updateLevel =", RelationalUpdateLevel.Always.value)
1481 )
1482 if changedBone:
1483 updateListQuery.filter("viur_foreign_keys =", changedBone)
1484 if cursor:
1485 updateListQuery.setCursor(cursor)
1486 updateList = updateListQuery.run(limit=5)
1488 def updateTxn(skel, key, srcRelKey):
1489 if not skel.fromDB(key):
1490 logging.warning(f"Cannot update stale reference to {key=} (referenced from {srcRelKey=})")
1491 return
1493 skel.refresh()
1494 skel.toDB(update_relations=False)
1496 for srcRel in updateList:
1497 try:
1498 skel = skeletonByKind(srcRel["viur_src_kind"])()
1499 except AssertionError:
1500 logging.info(f"""Ignoring {srcRel.key!r} which refers to unknown kind {srcRel["viur_src_kind"]!r}""")
1501 continue
1502 if db.IsInTransaction():
1503 updateTxn(skel, srcRel["src"].key, srcRel.key)
1504 else:
1505 db.RunInTransaction(updateTxn, skel, srcRel["src"].key, srcRel.key)
1506 nextCursor = updateListQuery.getCursor()
1507 if len(updateList) == 5 and nextCursor:
1508 updateRelations(destKey, minChangeTime, changedBone, nextCursor)
1511@CallableTask
1512class TaskUpdateSearchIndex(CallableTaskBase):
1513 """
1514 This tasks loads and saves *every* entity of the given module.
1515 This ensures an updated searchIndex and verifies consistency of this data.
1516 """
1517 key = "rebuildSearchIndex"
1518 name = "Rebuild search index"
1519 descr = "This task can be called to update search indexes and relational information."
1521 def canCall(self) -> bool:
1522 """Checks wherever the current user can execute this task"""
1523 user = current.user.get()
1524 return user is not None and "root" in user["access"]
1526 def dataSkel(self):
1527 modules = ["*"] + listKnownSkeletons()
1528 modules.sort()
1529 skel = BaseSkeleton().clone()
1530 skel.module = SelectBone(descr="Module", values={x: translate(x) for x in modules}, required=True)
1531 return skel
1533 def execute(self, module, *args, **kwargs):
1534 usr = current.user.get()
1535 if not usr:
1536 logging.warning("Don't know who to inform after rebuilding finished")
1537 notify = None
1538 else:
1539 notify = usr["name"]
1541 if module == "*":
1542 for module in listKnownSkeletons():
1543 logging.info("Rebuilding search index for module %r", module)
1544 self._run(module, notify)
1545 else:
1546 self._run(module, notify)
1548 @staticmethod
1549 def _run(module: str, notify: str):
1550 Skel = skeletonByKind(module)
1551 if not Skel:
1552 logging.error("TaskUpdateSearchIndex: Invalid module")
1553 return
1554 RebuildSearchIndex.startIterOnQuery(Skel().all(), {"notify": notify, "module": module})
1557class RebuildSearchIndex(QueryIter):
1558 @classmethod
1559 def handleEntry(cls, skel: SkeletonInstance, customData: dict[str, str]):
1560 skel.refresh()
1561 skel.toDB(update_relations=False)
1563 @classmethod
1564 def handleFinish(cls, totalCount: int, customData: dict[str, str]):
1565 QueryIter.handleFinish(totalCount, customData)
1566 if not customData["notify"]:
1567 return
1568 txt = (
1569 f"{conf.instance.project_id}: Rebuild search index finished for {customData['module']}\n\n"
1570 f"ViUR finished to rebuild the search index for module {customData['module']}.\n"
1571 f"{totalCount} records updated in total on this kind."
1572 )
1573 try:
1574 email.sendEMail(dests=customData["notify"], stringTemplate=txt, skel=None)
1575 except Exception as exc: # noqa; OverQuota, whatever
1576 logging.exception(f'Failed to notify {customData["notify"]}')
1579### Vacuum Relations
1581@CallableTask
1582class TaskVacuumRelations(TaskUpdateSearchIndex):
1583 """
1584 Checks entries in viur-relations and verifies that the src-kind
1585 and it's RelationalBone still exists.
1586 """
1587 key = "vacuumRelations"
1588 name = "Vacuum viur-relations (dangerous)"
1589 descr = "Drop stale inbound relations for the given kind"
1591 def execute(self, module: str, *args, **kwargs):
1592 usr = current.user.get()
1593 if not usr:
1594 logging.warning("Don't know who to inform after rebuilding finished")
1595 notify = None
1596 else:
1597 notify = usr["name"]
1598 processVacuumRelationsChunk(module.strip(), None, notify=notify)
1601@CallDeferred
1602def processVacuumRelationsChunk(
1603 module: str, cursor, count_total: int = 0, count_removed: int = 0, notify=None
1604):
1605 """
1606 Processes 25 Entries and calls the next batch
1607 """
1608 query = db.Query("viur-relations")
1609 if module != "*":
1610 query.filter("viur_src_kind =", module)
1611 query.setCursor(cursor)
1612 for relation_object in query.run(25):
1613 count_total += 1
1614 if not (src_kind := relation_object.get("viur_src_kind")):
1615 logging.critical("We got an relation-object without a src_kind!")
1616 continue
1617 if not (src_prop := relation_object.get("viur_src_property")):
1618 logging.critical("We got an relation-object without a src_prop!")
1619 continue
1620 try:
1621 skel = skeletonByKind(src_kind)()
1622 except AssertionError:
1623 # The referenced skeleton does not exist in this data model -> drop that relation object
1624 logging.info(f"Deleting {relation_object.key} which refers to unknown kind {src_kind}")
1625 db.Delete(relation_object)
1626 count_removed += 1
1627 continue
1628 if src_prop not in skel:
1629 logging.info(f"Deleting {relation_object.key} which refers to "
1630 f"non-existing RelationalBone {src_prop} of {src_kind}")
1631 db.Delete(relation_object)
1632 count_removed += 1
1633 logging.info(f"END processVacuumRelationsChunk {module}, "
1634 f"{count_total} records processed, {count_removed} removed")
1635 if new_cursor := query.getCursor():
1636 # Start processing of the next chunk
1637 processVacuumRelationsChunk(module, new_cursor, count_total, count_removed, notify)
1638 elif notify:
1639 txt = (
1640 f"{conf.instance.project_id}: Vacuum relations finished for {module}\n\n"
1641 f"ViUR finished to vacuum viur-relations for module {module}.\n"
1642 f"{count_total} records processed, "
1643 f"{count_removed} entries removed"
1644 )
1645 try:
1646 email.sendEMail(dests=notify, stringTemplate=txt, skel=None)
1647 except Exception as exc: # noqa; OverQuota, whatever
1648 logging.exception(f"Failed to notify {notify}")
1651# Forward our references to SkelInstance to the database (needed for queries)
1652db.config["SkeletonInstanceRef"] = SkeletonInstance
1654# DEPRECATED ATTRIBUTES HANDLING
1656__DEPRECATED_NAMES = {
1657 # stuff prior viur-core < 3.6
1658 "seoKeyBone": ("SeoKeyBone", SeoKeyBone),
1659}
1662def __getattr__(attr: str) -> object:
1663 if entry := __DEPRECATED_NAMES.get(attr):
1664 func = entry[1]
1665 msg = f"{attr} was replaced by {entry[0]}"
1666 warnings.warn(msg, DeprecationWarning, stacklevel=2)
1667 logging.warning(msg, stacklevel=2)
1668 return func
1670 return super(__import__(__name__).__class__).__getattribute__(attr)