Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/skeleton.py: 0%
821 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-03 13:41 +0000
1from __future__ import annotations
3import copy
4import fnmatch
5import inspect
6import logging
7import os
8import string
9import sys
10import typing as t
11import warnings
12from functools import partial
13from itertools import chain
14from time import time
16from viur.core import conf, current, db, email, errors, translate, utils
17from viur.core.bones import BaseBone, DateBone, KeyBone, RelationalBone, RelationalConsistency, RelationalUpdateLevel, \
18 SelectBone, StringBone
19from viur.core.bones.base import Compute, ComputeInterval, ComputeMethod, ReadFromClientError, \
20 ReadFromClientErrorSeverity, getSystemInitialized
21from viur.core.tasks import CallDeferred, CallableTask, CallableTaskBase, QueryIter
23_undefined = object()
24ABSTRACT_SKEL_CLS_SUFFIX = "AbstractSkel"
27class MetaBaseSkel(type):
28 """
29 This is the metaclass for Skeletons.
30 It is used to enforce several restrictions on bone names, etc.
31 """
32 _skelCache = {} # Mapping kindName -> SkelCls
33 _allSkelClasses = set() # list of all known skeleton classes (including Ref and Mail-Skels)
35 # List of reserved keywords and function names
36 __reserved_keywords = {
37 "all",
38 "bounce",
39 "clone",
40 "cursor",
41 "delete",
42 "fromClient",
43 "fromDB",
44 "get",
45 "getCurrentSEOKeys",
46 "items",
47 "keys",
48 "limit",
49 "orderby",
50 "orderdir",
51 "postDeletedHandler",
52 "postSavedHandler",
53 "preProcessBlobLocks",
54 "preProcessSerializedData",
55 "refresh",
56 "self",
57 "serialize",
58 "setBoneValue",
59 "style",
60 "structure",
61 "toDB",
62 "unserialize",
63 "values",
64 }
66 __allowed_chars = string.ascii_letters + string.digits + "_"
68 def __init__(cls, name, bases, dct):
69 cls.__boneMap__ = MetaBaseSkel.generate_bonemap(cls)
71 if not getSystemInitialized() and not cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
72 MetaBaseSkel._allSkelClasses.add(cls)
74 super(MetaBaseSkel, cls).__init__(name, bases, dct)
76 @staticmethod
77 def generate_bonemap(cls):
78 """
79 Recursively constructs a dict of bones from
80 """
81 map = {}
83 for base in cls.__bases__:
84 if "__viurBaseSkeletonMarker__" in dir(base):
85 map |= MetaBaseSkel.generate_bonemap(base)
87 for key in cls.__dict__:
88 prop = getattr(cls, key)
90 if isinstance(prop, BaseBone):
91 if not all([c in MetaBaseSkel.__allowed_chars for c in key]):
92 raise AttributeError(f"Invalid bone name: {key!r} contains invalid characters")
93 elif key in MetaBaseSkel.__reserved_keywords:
94 raise AttributeError(f"Invalid bone name: {key!r} is reserved and cannot be used")
96 map[key] = prop
98 elif prop is None and key in map: # Allow removing a bone in a subclass by setting it to None
99 del map[key]
101 return map
103 def __setattr__(self, key, value):
104 super().__setattr__(key, value)
105 if isinstance(value, BaseBone):
106 # Call BaseBone.__set_name__ manually for bones that are assigned at runtime
107 value.__set_name__(self, key)
110def skeletonByKind(kindName: str) -> t.Type[Skeleton]:
111 """
112 Returns the Skeleton-Class for the given kindName. That skeleton must exist, otherwise an exception is raised.
113 :param kindName: The kindname to retreive the skeleton for
114 :return: The skeleton-class for that kind
115 """
116 assert kindName in MetaBaseSkel._skelCache, f"Unknown skeleton {kindName=}"
117 return MetaBaseSkel._skelCache[kindName]
120def listKnownSkeletons() -> list[str]:
121 """
122 :return: A list of all known kindnames (all kindnames for which a skeleton is defined)
123 """
124 return list(MetaBaseSkel._skelCache.keys())[:]
127def iterAllSkelClasses() -> t.Iterable["Skeleton"]:
128 """
129 :return: An iterator that yields each Skeleton-Class once. (Only top-level skeletons are returned, so no
130 RefSkel classes will be included)
131 """
132 for cls in list(MetaBaseSkel._allSkelClasses): # We'll add new classes here during setSystemInitialized()
133 yield cls
136class SkeletonInstance:
137 """
138 The actual wrapper around a Skeleton-Class. An object of this class is what's actually returned when you
139 call a Skeleton-Class. With ViUR3, you don't get an instance of a Skeleton-Class any more - it's always this
140 class. This is much faster as this is a small class.
141 """
142 __slots__ = {
143 "accessedValues",
144 "boneMap",
145 "dbEntity",
146 "errors",
147 "is_cloned",
148 "renderAccessedValues",
149 "renderPreparation",
150 "skeletonCls",
151 }
153 def __init__(self, skelCls, subSkelNames=None, fullClone=False, clonedBoneMap=None):
154 if clonedBoneMap:
155 self.boneMap = clonedBoneMap
156 for k, v in self.boneMap.items():
157 v.isClonedInstance = True
159 elif subSkelNames:
160 boneList = ["key"] + list(chain(*[skelCls.subSkels.get(x, []) for x in ["*"] + subSkelNames]))
161 doesMatch = lambda name: name in boneList or any(
162 [name.startswith(x[:-1]) for x in boneList if x[-1] == "*"])
163 if fullClone:
164 self.boneMap = {k: copy.deepcopy(v) for k, v in skelCls.__boneMap__.items() if doesMatch(k)}
165 for v in self.boneMap.values():
166 v.isClonedInstance = True
167 else:
168 self.boneMap = {k: v for k, v in skelCls.__boneMap__.items() if doesMatch(k)}
170 elif fullClone:
171 self.boneMap = copy.deepcopy(skelCls.__boneMap__)
172 for v in self.boneMap.values():
173 v.isClonedInstance = True
175 else: # No Subskel, no Clone
176 self.boneMap = skelCls.__boneMap__.copy()
178 self.accessedValues = {}
179 self.dbEntity = None
180 self.errors = []
181 self.is_cloned = fullClone
182 self.renderAccessedValues = {}
183 self.renderPreparation = None
184 self.skeletonCls = skelCls
186 def items(self, yieldBoneValues: bool = False) -> t.Iterable[tuple[str, BaseBone]]:
187 if yieldBoneValues:
188 for key in self.boneMap.keys():
189 yield key, self[key]
190 else:
191 yield from self.boneMap.items()
193 def keys(self) -> t.Iterable[str]:
194 yield from self.boneMap.keys()
196 def values(self) -> t.Iterable[t.Any]:
197 yield from self.boneMap.values()
199 def __iter__(self) -> t.Iterable[str]:
200 yield from self.keys()
202 def __contains__(self, item):
203 return item in self.boneMap
205 def get(self, item, default=None):
206 if item not in self:
207 return default
209 return self[item]
211 def __setitem__(self, key, value):
212 assert self.renderPreparation is None, "Cannot modify values while rendering"
213 if isinstance(value, BaseBone):
214 raise AttributeError(f"Don't assign this bone object as skel[\"{key}\"] = ... anymore to the skeleton. "
215 f"Use skel.{key} = ... for bone to skeleton assignment!")
216 self.accessedValues[key] = value
218 def __getitem__(self, key):
219 if self.renderPreparation:
220 if key in self.renderAccessedValues:
221 return self.renderAccessedValues[key]
222 if key not in self.accessedValues:
223 boneInstance = self.boneMap.get(key, None)
224 if boneInstance:
225 if self.dbEntity is not None:
226 boneInstance.unserialize(self, key)
227 else:
228 self.accessedValues[key] = boneInstance.getDefaultValue(self)
229 if not self.renderPreparation:
230 return self.accessedValues.get(key)
231 value = self.renderPreparation(getattr(self, key), self, key, self.accessedValues.get(key))
232 self.renderAccessedValues[key] = value
233 return value
235 def __getattr__(self, item: str):
236 """
237 Get a special attribute from the SkeletonInstance
239 __getattr__ is called when an attribute access fails with an
240 AttributeError. So we know that this is not a real attribute of
241 the SkeletonInstance. But there are still a few special cases in which
242 attributes are loaded from the skeleton class.
243 """
244 if item == "boneMap":
245 return {} # There are __setAttr__ calls before __init__ has run
246 # Load attribute value from the Skeleton class
247 elif item in {"kindName", "interBoneValidations", "customDatabaseAdapter"}:
248 return getattr(self.skeletonCls, item)
249 # Load a @classmethod from the Skeleton class and bound this SkeletonInstance
250 elif item in {"fromDB", "toDB", "all", "unserialize", "serialize", "fromClient", "getCurrentSEOKeys",
251 "preProcessSerializedData", "preProcessBlobLocks", "postSavedHandler", "setBoneValue",
252 "delete", "postDeletedHandler", "refresh"}:
253 return partial(getattr(self.skeletonCls, item), self)
254 # Load a @property from the Skeleton class
255 try:
256 # Use try/except to save an if check
257 class_value = getattr(self.skeletonCls, item)
258 except AttributeError:
259 # Not inside the Skeleton class, okay at this point.
260 pass
261 else:
262 if isinstance(class_value, property):
263 # The attribute is a @property and can be called
264 # Note: `self` is this SkeletonInstance, not the Skeleton class.
265 # Therefore, you can access values inside the property method
266 # with item-access like `self["key"]`.
267 try:
268 return class_value.fget(self)
269 except AttributeError as exc:
270 # The AttributeError cannot be re-raised any further at this point.
271 # Since this would then be evaluated as an access error
272 # to the property attribute.
273 # Otherwise, it would be lost that it is an incorrect attribute access
274 # within this property (during the method call).
275 msg, *args = exc.args
276 msg = f"AttributeError: {msg}"
277 raise ValueError(msg, *args) from exc
278 # Load the bone instance from the bone map of this SkeletonInstance
279 try:
280 return self.boneMap[item]
281 except KeyError as exc:
282 raise AttributeError(f"{self.__class__.__name__!r} object has no attribute '{item}'") from exc
284 def __delattr__(self, item):
285 del self.boneMap[item]
286 if item in self.accessedValues:
287 del self.accessedValues[item]
288 if item in self.renderAccessedValues:
289 del self.renderAccessedValues[item]
291 def __setattr__(self, key, value):
292 if key in self.boneMap or isinstance(value, BaseBone):
293 self.boneMap[key] = value
294 elif key == "renderPreparation":
295 super().__setattr__(key, value)
296 self.renderAccessedValues.clear()
297 else:
298 super().__setattr__(key, value)
300 def __repr__(self) -> str:
301 return f"<SkeletonInstance of {self.skeletonCls.__name__} with {dict(self)}>"
303 def __str__(self) -> str:
304 return str(dict(self))
306 def __len__(self) -> int:
307 return len(self.boneMap)
309 def clone(self):
310 """
311 Clones a SkeletonInstance into a modificable, stand-alone instance.
312 This will also allow to modify the underlying data model.
313 """
314 res = SkeletonInstance(self.skeletonCls, clonedBoneMap=copy.deepcopy(self.boneMap))
315 res.accessedValues = copy.deepcopy(self.accessedValues)
316 res.dbEntity = copy.deepcopy(self.dbEntity)
317 res.is_cloned = True
318 res.renderAccessedValues = copy.deepcopy(self.renderAccessedValues)
319 return res
321 def ensure_is_cloned(self):
322 """
323 Ensured this SkeletonInstance is a stand-alone clone, which can be modified.
324 Does nothing in case it was already cloned before.
325 """
326 if not self.is_cloned:
327 return self.clone()
329 return self
331 def setEntity(self, entity: db.Entity):
332 self.dbEntity = entity
333 self.accessedValues = {}
334 self.renderAccessedValues = {}
336 def structure(self) -> dict:
337 return {
338 key: bone.structure() | {"sortindex": i}
339 for i, (key, bone) in enumerate(self.items())
340 }
342 def __deepcopy__(self, memodict):
343 res = self.clone()
344 memodict[id(self)] = res
345 return res
348class BaseSkeleton(object, metaclass=MetaBaseSkel):
349 """
350 This is a container-object holding information about one database entity.
352 It has to be sub-classed with individual information about the kindName of the entities
353 and its specific data attributes, the so called bones.
354 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
355 contained bones remains constant.
357 :ivar key: This bone stores the current database key of this entity. \
358 Assigning to this bones value is dangerous and does *not* affect the actual key its stored in.
360 :vartype key: server.bones.BaseBone
362 :ivar creationdate: The date and time where this entity has been created.
363 :vartype creationdate: server.bones.DateBone
365 :ivar changedate: The date and time of the last change to this entity.
366 :vartype changedate: server.bones.DateBone
367 """
368 __viurBaseSkeletonMarker__ = True
369 boneMap = None
371 @classmethod
372 def subSkel(cls, *name, fullClone: bool = False, **kwargs) -> SkeletonInstance:
373 """
374 Creates a new sub-skeleton as part of the current skeleton.
376 A sub-skeleton is a copy of the original skeleton, containing only a subset of its bones.
377 To define sub-skeletons, use the subSkels property of the Skeleton object.
379 By passing multiple sub-skeleton names to this function, a sub-skeleton with the union of
380 all bones of the specified sub-skeletons is returned.
382 If an entry called "*" exists in the subSkels-dictionary, the bones listed in this entry
383 will always be part of the generated sub-skeleton.
385 :param name: Name of the sub-skeleton (that's the key of the subSkels dictionary); \
386 Multiple names can be specified.
388 :return: The sub-skeleton of the specified type.
389 """
390 if not name:
391 raise ValueError("Which subSkel?")
392 return cls(subSkelNames=list(name), fullClone=fullClone)
394 @classmethod
395 def setSystemInitialized(cls):
396 for attrName in dir(cls):
397 bone = getattr(cls, attrName)
398 if isinstance(bone, BaseBone):
399 bone.setSystemInitialized()
401 @classmethod
402 def setBoneValue(
403 cls,
404 skelValues: SkeletonInstance,
405 boneName: str,
406 value: t.Any,
407 append: bool = False,
408 language: t.Optional[str] = None
409 ) -> bool:
410 """
411 Allows for setting a bones value without calling fromClient or assigning a value directly.
412 Sanity-Checks are performed; if the value is invalid, that bone flips back to its original
413 (default) value and false is returned.
415 :param boneName: The name of the bone to be modified
416 :param value: The value that should be assigned. It's type depends on the type of that bone
417 :param append: If True, the given value is appended to the values of that bone instead of
418 replacing it. Only supported on bones with multiple=True
419 :param language: Language to set
421 :return: Wherever that operation succeeded or not.
422 """
423 bone = getattr(skelValues, boneName, None)
425 if not isinstance(bone, BaseBone):
426 raise ValueError(f"{boneName!r} is no valid bone on this skeleton ({skelValues!r})")
428 if language:
429 if not bone.languages:
430 raise ValueError("The bone {boneName!r} has no language setting")
431 elif language not in bone.languages:
432 raise ValueError("The language {language!r} is not available for bone {boneName!r}")
434 if value is None:
435 if append:
436 raise ValueError("Cannot append None-value to bone {boneName!r}")
438 if language:
439 skelValues[boneName][language] = [] if bone.multiple else None
440 else:
441 skelValues[boneName] = [] if bone.multiple else None
443 return True
445 _ = skelValues[boneName] # ensure the bone is being unserialized first
446 return bone.setBoneValue(skelValues, boneName, value, append, language)
448 @classmethod
449 def fromClient(cls, skelValues: SkeletonInstance, data: dict[str, list[str] | str], amend: bool = False) -> bool:
450 """
451 Load supplied *data* into Skeleton.
453 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
454 the values retrieved from *data* are checked against the bones and their validity checks.
456 Even if this function returns False, all bones are guaranteed to be in a valid state.
457 The ones which have been read correctly are set to their valid values;
458 Bones with invalid values are set back to a safe default (None in most cases).
459 So its possible to call :func:`~viur.core.skeleton.Skeleton.toDB` afterwards even if reading
460 data with this function failed (through this might violates the assumed consistency-model).
462 :param skel: The skeleton instance to be filled.
463 :param data: Dictionary from which the data is read.
464 :param amend: Defines whether content of data may be incomplete to amend the skel,
465 which is useful for edit-actions.
467 :returns: True if all data was successfully read and complete. \
468 False otherwise (e.g. some required fields where missing or where invalid).
469 """
470 complete = True
471 skelValues.errors = []
473 for key, bone in skelValues.items():
474 if bone.readOnly:
475 continue
477 if errors := bone.fromClient(skelValues, key, data):
478 for error in errors:
479 # insert current bone name into error's fieldPath
480 error.fieldPath.insert(0, str(key))
482 # logging.debug(f"BaseSkel.fromClient {key=} {error=}")
484 incomplete = (
485 # always when something is invalid
486 error.severity == ReadFromClientErrorSeverity.Invalid
487 or (
488 # only when path is top-level
489 len(error.fieldPath) == 1
490 and (
491 # bone is generally required
492 bool(bone.required)
493 and (
494 # and value is either empty
495 error.severity == ReadFromClientErrorSeverity.Empty
496 # or when not amending, not set
497 or (not amend and error.severity == ReadFromClientErrorSeverity.NotSet)
498 )
499 )
500 )
501 )
503 # in case there are language requirements, test additionally
504 if bone.languages and isinstance(bone.required, (list, tuple)):
505 incomplete &= any([key, lang] == error.fieldPath for lang in bone.required)
507 # logging.debug(f"BaseSkel.fromClient {incomplete=} {error.severity=} {bone.required=}")
509 if incomplete:
510 complete = False
512 if conf.debug.skeleton_from_client:
513 logging.error(
514 f"""{getattr(cls, "kindName", cls.__name__)}: {".".join(error.fieldPath)}: """
515 f"""({error.severity}) {error.errorMessage}"""
516 )
518 skelValues.errors += errors
520 return complete
522 @classmethod
523 def refresh(cls, skel: SkeletonInstance):
524 """
525 Refresh the bones current content.
527 This function causes a refresh of all relational bones and their associated
528 information.
529 """
530 logging.debug(f"""Refreshing {skel["key"]=}""")
532 for key, bone in skel.items():
533 if not isinstance(bone, BaseBone):
534 continue
536 _ = skel[key] # Ensure value gets loaded
537 bone.refresh(skel, key)
539 def __new__(cls, *args, **kwargs) -> SkeletonInstance:
540 return SkeletonInstance(cls, *args, **kwargs)
543class MetaSkel(MetaBaseSkel):
544 def __init__(cls, name, bases, dct):
545 super(MetaSkel, cls).__init__(name, bases, dct)
546 relNewFileName = inspect.getfile(cls) \
547 .replace(str(conf.instance.project_base_path), "") \
548 .replace(str(conf.instance.core_base_path), "")
550 # Check if we have an abstract skeleton
551 if cls.__name__.endswith(ABSTRACT_SKEL_CLS_SUFFIX):
552 # Ensure that it doesn't have a kindName
553 assert cls.kindName is _undefined or cls.kindName is None, "Abstract Skeletons can't have a kindName"
554 # Prevent any further processing by this class; it has to be sub-classed before it can be used
555 return
557 # Automatic determination of the kindName, if the class is not part of viur.core.
558 if (cls.kindName is _undefined
559 and not relNewFileName.strip(os.path.sep).startswith("viur")
560 and not "viur_doc_build" in dir(sys)):
561 if cls.__name__.endswith("Skel"):
562 cls.kindName = cls.__name__.lower()[:-4]
563 else:
564 cls.kindName = cls.__name__.lower()
565 # Try to determine which skeleton definition takes precedence
566 if cls.kindName and cls.kindName is not _undefined and cls.kindName in MetaBaseSkel._skelCache:
567 relOldFileName = inspect.getfile(MetaBaseSkel._skelCache[cls.kindName]) \
568 .replace(str(conf.instance.project_base_path), "") \
569 .replace(str(conf.instance.core_base_path), "")
570 idxOld = min(
571 [x for (x, y) in enumerate(conf.skeleton_search_path) if relOldFileName.startswith(y)] + [999])
572 idxNew = min(
573 [x for (x, y) in enumerate(conf.skeleton_search_path) if relNewFileName.startswith(y)] + [999])
574 if idxNew == 999:
575 # We could not determine a priority for this class as its from a path not listed in the config
576 raise NotImplementedError(
577 "Skeletons must be defined in a folder listed in conf.skeleton_search_path")
578 elif idxOld < idxNew: # Lower index takes precedence
579 # The currently processed skeleton has a lower priority than the one we already saw - just ignore it
580 return
581 elif idxOld > idxNew:
582 # The currently processed skeleton has a higher priority, use that from now
583 MetaBaseSkel._skelCache[cls.kindName] = cls
584 else: # They seem to be from the same Package - raise as something is messed up
585 raise ValueError(f"Duplicate definition for {cls.kindName} in {relNewFileName} and {relOldFileName}")
586 # Ensure that all skeletons are defined in folders listed in conf.skeleton_search_path
587 if (not any([relNewFileName.startswith(x) for x in conf.skeleton_search_path])
588 and not "viur_doc_build" in dir(sys)): # Do not check while documentation build
589 raise NotImplementedError(
590 f"""{relNewFileName} must be defined in a folder listed in {conf.skeleton_search_path}""")
591 if cls.kindName and cls.kindName is not _undefined:
592 MetaBaseSkel._skelCache[cls.kindName] = cls
593 # Auto-Add ViUR Search Tags Adapter if the skeleton has no adapter attached
594 if cls.customDatabaseAdapter is _undefined:
595 cls.customDatabaseAdapter = ViurTagsSearchAdapter()
598class CustomDatabaseAdapter:
599 # Set to True if we can run a fulltext search using this database
600 providesFulltextSearch: bool = False
601 # Are results returned by `meth:fulltextSearch` guaranteed to also match the databaseQuery
602 fulltextSearchGuaranteesQueryConstrains = False
603 # Indicate that we can run more types of queries than originally supported by firestore
604 providesCustomQueries: bool = False
606 def preprocessEntry(self, entry: db.Entity, skel: BaseSkeleton, changeList: list[str], isAdd: bool) -> db.Entity:
607 """
608 Can be overridden to add or alter the data of this entry before it's written to firestore.
609 Will always be called inside an transaction.
610 :param entry: The entry containing the serialized data of that skeleton
611 :param skel: The (complete) skeleton this skel.toDB() runs for
612 :param changeList: List of boneNames that are changed by this skel.toDB() call
613 :param isAdd: Is this an update or an add?
614 :return: The (maybe modified) entity
615 """
616 return entry
618 def updateEntry(self, dbObj: db.Entity, skel: BaseSkeleton, changeList: list[str], isAdd: bool) -> None:
619 """
620 Like `meth:preprocessEntry`, but runs after the transaction had completed.
621 Changes made to dbObj will be ignored.
622 :param entry: The entry containing the serialized data of that skeleton
623 :param skel: The (complete) skeleton this skel.toDB() runs for
624 :param changeList: List of boneNames that are changed by this skel.toDB() call
625 :param isAdd: Is this an update or an add?
626 """
627 return
629 def deleteEntry(self, entry: db.Entity, skel: BaseSkeleton) -> None:
630 """
631 Called, after an skeleton has been successfully deleted from firestore
632 :param entry: The db.Entity object containing an snapshot of the data that has been deleted
633 :param skel: The (complete) skeleton for which `meth:delete' had been called
634 """
635 return
637 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
638 """
639 If this database supports fulltext searches, this method has to implement them.
640 If it's a plain fulltext search engine, leave 'prop:fulltextSearchGuaranteesQueryConstrains' set to False,
641 then the server will post-process the list of entries returned from this function and drop any entry that
642 cannot be returned due to other constrains set in 'param:databaseQuery'. If you can obey *every* constrain
643 set in that Query, we can skip this post-processing and save some CPU-cycles.
644 :param queryString: the string as received from the user (no quotation or other safety checks applied!)
645 :param databaseQuery: The query containing any constrains that returned entries must also match
646 :return:
647 """
648 raise NotImplementedError
651class ViurTagsSearchAdapter(CustomDatabaseAdapter):
652 """
653 This Adapter implements a simple fulltext search on top of the datastore.
655 On skel.toDB(), all words from String-/TextBones are collected with all *min_length* postfixes and dumped
656 into the property `viurTags`. When queried, we'll run a prefix-match against this property - thus returning
657 entities with either an exact match or a match within a word.
659 Example:
660 For the word "hello" we'll write "hello", "ello" and "llo" into viurTags.
661 When queried with "hello" we'll have an exact match.
662 When queried with "hel" we'll match the prefix for "hello"
663 When queried with "ell" we'll prefix-match "ello" - this is only enabled when substring_matching is True.
665 We'll automatically add this adapter if a skeleton has no other database adapter defined.
666 """
667 providesFulltextSearch = True
668 fulltextSearchGuaranteesQueryConstrains = True
670 def __init__(self, min_length: int = 2, max_length: int = 50, substring_matching: bool = False):
671 super().__init__()
672 self.min_length = min_length
673 self.max_length = max_length
674 self.substring_matching = substring_matching
676 def _tagsFromString(self, value: str) -> set[str]:
677 """
678 Extract all words including all min_length postfixes from given string
679 """
680 res = set()
682 for tag in value.split(" "):
683 tag = "".join([x for x in tag.lower() if x in conf.search_valid_chars])
685 if len(tag) >= self.min_length:
686 res.add(tag)
688 if self.substring_matching:
689 for i in range(1, 1 + len(tag) - self.min_length):
690 res.add(tag[i:])
692 return res
694 def preprocessEntry(self, entry: db.Entity, skel: Skeleton, changeList: list[str], isAdd: bool) -> db.Entity:
695 """
696 Collect searchTags from skeleton and build viurTags
697 """
698 tags = set()
700 for boneName, bone in skel.items():
701 if bone.searchable:
702 tags = tags.union(bone.getSearchTags(skel, boneName))
704 entry["viurTags"] = list(chain(*[self._tagsFromString(x) for x in tags if len(x) <= self.max_length]))
705 return entry
707 def fulltextSearch(self, queryString: str, databaseQuery: db.Query) -> list[db.Entity]:
708 """
709 Run a fulltext search
710 """
711 keywords = list(self._tagsFromString(queryString))[:10]
712 resultScoreMap = {}
713 resultEntryMap = {}
715 for keyword in keywords:
716 qryBase = databaseQuery.clone()
717 for entry in qryBase.filter("viurTags >=", keyword).filter("viurTags <", keyword + "\ufffd").run():
718 if not entry.key in resultScoreMap:
719 resultScoreMap[entry.key] = 1
720 else:
721 resultScoreMap[entry.key] += 1
722 if not entry.key in resultEntryMap:
723 resultEntryMap[entry.key] = entry
725 resultList = [(k, v) for k, v in resultScoreMap.items()]
726 resultList.sort(key=lambda x: x[1], reverse=True)
728 return [resultEntryMap[x[0]] for x in resultList[:databaseQuery.queries.limit]]
731class SeoKeyBone(StringBone):
732 """
733 Special kind of StringBone saving its contents as `viurCurrentSeoKeys` into the entity's `viur` dict.
734 """
736 def unserialize(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> bool:
737 try:
738 skel.accessedValues[name] = skel.dbEntity["viur"]["viurCurrentSeoKeys"]
739 except KeyError:
740 skel.accessedValues[name] = self.getDefaultValue(skel)
742 def serialize(self, skel: 'SkeletonInstance', name: str, parentIndexed: bool) -> bool:
743 # Serialize also to skel["viur"]["viurCurrentSeoKeys"], so we can use this bone in relations
744 if name in skel.accessedValues:
745 newVal = skel.accessedValues[name]
746 if not skel.dbEntity.get("viur"):
747 skel.dbEntity["viur"] = db.Entity()
748 res = db.Entity()
749 res["_viurLanguageWrapper_"] = True
750 for language in (self.languages or []):
751 if not self.indexed:
752 res.exclude_from_indexes.add(language)
753 res[language] = None
754 if language in newVal:
755 res[language] = self.singleValueSerialize(newVal[language], skel, name, parentIndexed)
756 skel.dbEntity["viur"]["viurCurrentSeoKeys"] = res
757 return True
760class Skeleton(BaseSkeleton, metaclass=MetaSkel):
761 kindName: str = _undefined # To which kind we save our data to
762 customDatabaseAdapter: CustomDatabaseAdapter | None = _undefined
763 subSkels = {} # List of pre-defined sub-skeletons of this type
764 interBoneValidations: list[
765 t.Callable[[Skeleton], list[ReadFromClientError]]] = [] # List of functions checking inter-bone dependencies
767 __seo_key_trans = str.maketrans(
768 {"<": "",
769 ">": "",
770 "\"": "",
771 "'": "",
772 "\n": "",
773 "\0": "",
774 "/": "",
775 "\\": "",
776 "?": "",
777 "&": "",
778 "#": ""
779 })
781 # The "key" bone stores the current database key of this skeleton.
782 # Warning: Assigning to this bones value now *will* set the key
783 # it gets stored in. Must be kept readOnly to avoid security-issues with add/edit.
784 key = KeyBone(
785 descr="Key"
786 )
788 name = StringBone(
789 descr="Name",
790 visible=False,
791 compute=Compute(
792 fn=lambda skel: str(skel["key"]),
793 interval=ComputeInterval(ComputeMethod.OnWrite)
794 )
795 )
797 # The date (including time) when this entry has been created
798 creationdate = DateBone(
799 descr="created at",
800 readOnly=True,
801 visible=False,
802 indexed=True,
803 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.Once)),
804 )
806 # The last date (including time) when this entry has been updated
808 changedate = DateBone(
809 descr="updated at",
810 readOnly=True,
811 visible=False,
812 indexed=True,
813 compute=Compute(fn=utils.utcNow, interval=ComputeInterval(ComputeMethod.OnWrite)),
814 )
816 viurCurrentSeoKeys = SeoKeyBone(
817 descr="SEO-Keys",
818 readOnly=True,
819 visible=False,
820 languages=conf.i18n.available_languages
821 )
823 def __repr__(self):
824 return "<skeleton %s with data=%r>" % (self.kindName, {k: self[k] for k in self.keys()})
826 def __str__(self):
827 return str({k: self[k] for k in self.keys()})
829 def __init__(self, *args, **kwargs):
830 super(Skeleton, self).__init__(*args, **kwargs)
831 assert self.kindName and self.kindName is not _undefined, "You must set kindName on this skeleton!"
833 @classmethod
834 def all(cls, skelValues, **kwargs) -> db.Query:
835 """
836 Create a query with the current Skeletons kindName.
838 :returns: A db.Query object which allows for entity filtering and sorting.
839 """
840 return db.Query(skelValues.kindName, srcSkelClass=skelValues, **kwargs)
842 @classmethod
843 def fromClient(cls, skelValues: SkeletonInstance, data: dict[str, list[str] | str], amend: bool = False) -> bool:
844 """
845 This function works similar to :func:`~viur.core.skeleton.Skeleton.setValues`, except that
846 the values retrieved from *data* are checked against the bones and their validity checks.
848 Even if this function returns False, all bones are guaranteed to be in a valid state.
849 The ones which have been read correctly are set to their valid values;
850 Bones with invalid values are set back to a safe default (None in most cases).
851 So its possible to call :func:`~viur.core.skeleton.Skeleton.toDB` afterwards even if reading
852 data with this function failed (through this might violates the assumed consistency-model).
854 :param skel: The skeleton instance to be filled.
855 :param data: Dictionary from which the data is read.
856 :param amend: Defines whether content of data may be incomplete to amend the skel,
857 which is useful for edit-actions.
859 :returns: True if all data was successfully read and complete. \
860 False otherwise (e.g. some required fields where missing or where invalid).
861 """
862 assert skelValues.renderPreparation is None, "Cannot modify values while rendering"
864 # Load data into this skeleton
865 complete = bool(data) and super().fromClient(skelValues, data, amend=amend)
867 if (
868 not data # in case data is empty
869 or (len(data) == 1 and "key" in data)
870 or (utils.parse.bool(data.get("nomissing")))
871 ):
872 skelValues.errors = []
874 # Check if all unique values are available
875 for boneName, boneInstance in skelValues.items():
876 if boneInstance.unique:
877 lockValues = boneInstance.getUniquePropertyIndexValues(skelValues, boneName)
878 for lockValue in lockValues:
879 dbObj = db.Get(db.Key(f"{skelValues.kindName}_{boneName}_uniquePropertyIndex", lockValue))
880 if dbObj and (not skelValues["key"] or dbObj["references"] != skelValues["key"].id_or_name):
881 # This value is taken (sadly, not by us)
882 complete = False
883 errorMsg = boneInstance.unique.message
884 skelValues.errors.append(
885 ReadFromClientError(ReadFromClientErrorSeverity.Invalid, errorMsg, [boneName]))
887 # Check inter-Bone dependencies
888 for checkFunc in skelValues.interBoneValidations:
889 errors = checkFunc(skelValues)
890 if errors:
891 for error in errors:
892 if error.severity.value > 1:
893 complete = False
894 if conf.debug.skeleton_from_client:
895 logging.debug(f"{cls.kindName}: {error.fieldPath}: {error.errorMessage!r}")
897 skelValues.errors.extend(errors)
899 return complete
901 @classmethod
902 def fromDB(cls, skel: SkeletonInstance, key: db.Key | int | str) -> bool:
903 """
904 Load entity with *key* from the Datastore into the Skeleton.
906 Reads all available data of entity kind *kindName* and the key *key*
907 from the Datastore into the Skeleton structure's bones. Any previous
908 data of the bones will discard.
910 To store a Skeleton object to the Datastore, see :func:`~viur.core.skeleton.Skeleton.toDB`.
912 :param key: A :class:`viur.core.DB.Key`, string, or int; from which the data shall be fetched.
914 :returns: True on success; False if the given key could not be found or can not be parsed.
916 """
917 assert skel.renderPreparation is None, "Cannot modify values while rendering"
918 try:
919 db_key = db.keyHelper(key, skel.kindName)
920 except ValueError: # This key did not parse
921 return False
923 if not (db_res := db.Get(db_key)):
924 return False
925 skel.setEntity(db_res)
926 return True
928 @classmethod
929 def toDB(cls, skel: SkeletonInstance, update_relations: bool = True, **kwargs) -> db.Key:
930 """
931 Store current Skeleton entity to the Datastore.
933 Stores the current data of this instance into the database.
934 If an *key* value is set to the object, this entity will ne updated;
935 Otherwise a new entity will be created.
937 To read a Skeleton object from the data store, see :func:`~viur.core.skeleton.Skeleton.fromDB`.
939 :param update_relations: If False, this entity won't be marked dirty;
940 This avoids from being fetched by the background task updating relations.
942 :returns: The datastore key of the entity.
943 """
944 assert skel.renderPreparation is None, "Cannot modify values while rendering"
945 # fixme: Remove in viur-core >= 4
946 if "clearUpdateTag" in kwargs:
947 msg = "clearUpdateTag was replaced by update_relations"
948 warnings.warn(msg, DeprecationWarning, stacklevel=3)
949 logging.warning(msg, stacklevel=3)
950 update_relations = not kwargs["clearUpdateTag"]
952 def __txn_update(write_skel):
953 db_key = write_skel["key"]
954 skel = write_skel.skeletonCls()
956 blob_list = set()
957 change_list = []
958 old_copy = {}
959 # Load the current values from Datastore or create a new, empty db.Entity
960 if not db_key:
961 # We'll generate the key we'll be stored under early so we can use it for locks etc
962 db_key = db.AllocateIDs(db.Key(skel.kindName))
963 db_obj = db.Entity(db_key)
964 skel.dbEntity = db_obj
965 is_add = True
966 else:
967 db_key = db.keyHelper(db_key, skel.kindName)
968 if not (db_obj := db.Get(db_key)):
969 db_obj = db.Entity(db_key)
970 skel.dbEntity = db_obj
971 is_add = True
972 else:
973 skel.setEntity(db_obj)
974 old_copy = {k: v for k, v in db_obj.items()}
975 is_add = False
977 db_obj.setdefault("viur", {})
979 # Merge values and assemble unique properties
980 # Move accessed Values from srcSkel over to skel
981 skel.accessedValues = write_skel.accessedValues
982 skel["key"] = db_key # Ensure key stays set
984 for bone_name, bone in skel.items():
985 if bone_name == "key": # Explicitly skip key on top-level - this had been set above
986 continue
988 # Allow bones to perform outstanding "magic" operations before saving to db
989 bone.performMagic(skel, bone_name, isAdd=is_add) # FIXME VIUR4: ANY MAGIC IN OUR CODE IS DEPRECATED!!!
991 if not (bone_name in skel.accessedValues or bone.compute) and bone_name not in skel.dbEntity:
992 _ = skel[bone_name] # Ensure the datastore is filled with the default value
993 if (
994 bone_name in skel.accessedValues or bone.compute # We can have a computed value on store
995 or bone_name not in skel.dbEntity # It has not been written and is not in the database
996 ):
997 # Serialize bone into entity
998 try:
999 bone.serialize(skel, bone_name, True)
1000 except Exception:
1001 logging.error(f"Failed to serialize {bone_name} {bone} {skel.accessedValues[bone_name]}")
1002 raise
1004 # Obtain referenced blobs
1005 blob_list.update(bone.getReferencedBlobs(skel, bone_name))
1007 # Check if the value has actually changed
1008 if db_obj.get(bone_name) != old_copy.get(bone_name):
1009 change_list.append(bone_name)
1011 # Lock hashes from bones that must have unique values
1012 if bone.unique:
1013 # Remember old hashes for bones that must have an unique value
1014 old_unique_values = []
1016 if f"{bone_name}_uniqueIndexValue" in db_obj["viur"]:
1017 old_unique_values = db_obj["viur"][f"{bone_name}_uniqueIndexValue"]
1018 # Check if the property is unique
1019 new_unique_values = bone.getUniquePropertyIndexValues(skel, bone_name)
1020 new_lock_kind = f"{skel.kindName}_{bone_name}_uniquePropertyIndex"
1021 for new_lock_value in new_unique_values:
1022 new_lock_key = db.Key(new_lock_kind, new_lock_value)
1023 if lock_db_obj := db.Get(new_lock_key):
1025 # There's already a lock for that value, check if we hold it
1026 if lock_db_obj["references"] != db_obj.key.id_or_name:
1027 # This value has already been claimed, and not by us
1028 # TODO: Use a custom exception class which is catchable with an try/except
1029 raise ValueError(
1030 f"The unique value {skel[bone_name]!r} of bone {bone_name!r} "
1031 f"has been recently claimed!")
1032 else:
1033 # This value is locked for the first time, create a new lock-object
1034 lock_obj = db.Entity(new_lock_key)
1035 lock_obj["references"] = db_obj.key.id_or_name
1036 db.Put(lock_obj)
1037 if new_lock_value in old_unique_values:
1038 old_unique_values.remove(new_lock_value)
1039 db_obj["viur"][f"{bone_name}_uniqueIndexValue"] = new_unique_values
1041 # Remove any lock-object we're holding for values that we don't have anymore
1042 for old_unique_value in old_unique_values:
1043 # Try to delete the old lock
1045 old_lock_key = db.Key(f"{skel.kindName}_{bone_name}_uniquePropertyIndex", old_unique_value)
1046 if old_lock_obj := db.Get(old_lock_key):
1047 if old_lock_obj["references"] != db_obj.key.id_or_name:
1049 # We've been supposed to have that lock - but we don't.
1050 # Don't remove that lock as it now belongs to a different entry
1051 logging.critical("Detected Database corruption! A Value-Lock had been reassigned!")
1052 else:
1053 # It's our lock which we don't need anymore
1054 db.Delete(old_lock_key)
1055 else:
1056 logging.critical("Detected Database corruption! Could not delete stale lock-object!")
1058 # Delete legacy property (PR #1244) #TODO: Remove in ViUR4
1059 db_obj.pop("viur_incomming_relational_locks", None)
1061 # Ensure the SEO-Keys are up-to-date
1062 last_requested_seo_keys = db_obj["viur"].get("viurLastRequestedSeoKeys") or {}
1063 last_set_seo_keys = db_obj["viur"].get("viurCurrentSeoKeys") or {}
1064 # Filter garbage serialized into this field by the SeoKeyBone
1065 last_set_seo_keys = {k: v for k, v in last_set_seo_keys.items() if not k.startswith("_") and v}
1067 if not isinstance(db_obj["viur"].get("viurCurrentSeoKeys"), dict):
1068 db_obj["viur"]["viurCurrentSeoKeys"] = {}
1069 if current_seo_keys := skel.getCurrentSEOKeys():
1070 # Convert to lower-case and remove certain characters
1071 for lang, value in current_seo_keys.items():
1072 current_seo_keys[lang] = value.lower().translate(Skeleton.__seo_key_trans).strip()
1074 for language in (conf.i18n.available_languages or [conf.i18n.default_language]):
1075 if current_seo_keys and language in current_seo_keys:
1076 current_seo_key = current_seo_keys[language]
1077 if current_seo_key != last_requested_seo_keys.get(language): # This one is new or has changed
1078 new_seo_key = current_seo_keys[language]
1079 for _ in range(0, 3):
1080 entry_using_key = db.Query(skel.kindName).filter("viur.viurActiveSeoKeys =",
1081 new_seo_key).getEntry()
1082 if entry_using_key and entry_using_key.key != db_obj.key:
1083 # It's not unique; append a random string and try again
1084 new_seo_key = f"{current_seo_keys[language]}-{utils.string.random(5).lower()}"
1086 else:
1087 # We found a new SeoKey
1088 break
1089 else:
1090 raise ValueError("Could not generate an unique seo key in 3 attempts")
1091 else:
1092 new_seo_key = current_seo_key
1093 last_set_seo_keys[language] = new_seo_key
1094 else:
1095 # We'll use the database-key instead
1096 last_set_seo_keys[language] = str(db_obj.key.id_or_name)
1097 # Store the current, active key for that language
1098 db_obj["viur"]["viurCurrentSeoKeys"][language] = last_set_seo_keys[language]
1100 db_obj["viur"].setdefault("viurActiveSeoKeys", [])
1101 for language, seo_key in last_set_seo_keys.items():
1102 if db_obj["viur"]["viurCurrentSeoKeys"][language] not in db_obj["viur"]["viurActiveSeoKeys"]:
1103 # Ensure the current, active seo key is in the list of all seo keys
1104 db_obj["viur"]["viurActiveSeoKeys"].insert(0, seo_key)
1105 if str(db_obj.key.id_or_name) not in db_obj["viur"]["viurActiveSeoKeys"]:
1106 # Ensure that key is also in there
1107 db_obj["viur"]["viurActiveSeoKeys"].insert(0, str(db_obj.key.id_or_name))
1108 # Trim to the last 200 used entries
1109 db_obj["viur"]["viurActiveSeoKeys"] = db_obj["viur"]["viurActiveSeoKeys"][:200]
1110 # Store lastRequestedKeys so further updates can run more efficient
1111 db_obj["viur"]["viurLastRequestedSeoKeys"] = current_seo_keys
1113 # mark entity as "dirty" when update_relations is set, to zero otherwise.
1114 db_obj["viur"]["delayedUpdateTag"] = time() if update_relations else 0
1116 db_obj = skel.preProcessSerializedData(db_obj)
1118 # Allow the custom DB Adapter to apply last minute changes to the object
1119 if skel.customDatabaseAdapter:
1120 db_obj = skel.customDatabaseAdapter.preprocessEntry(db_obj, skel, change_list, is_add)
1122 # ViUR2 import compatibility - remove properties containing. if we have a dict with the same name
1123 def fixDotNames(entity):
1124 for k, v in list(entity.items()):
1125 if isinstance(v, dict):
1126 for k2, v2 in list(entity.items()):
1127 if k2.startswith(f"{k}."):
1128 del entity[k2]
1129 backupKey = k2.replace(".", "__")
1130 entity[backupKey] = v2
1131 entity.exclude_from_indexes = set(entity.exclude_from_indexes) | {backupKey}
1132 fixDotNames(v)
1133 elif isinstance(v, list):
1134 for x in v:
1135 if isinstance(x, dict):
1136 fixDotNames(x)
1138 if conf.viur2import_blobsource: # Try to fix these only when converting from ViUR2
1139 fixDotNames(db_obj)
1141 # Write the core entry back
1142 db.Put(db_obj)
1144 # Now write the blob-lock object
1145 blob_list = skel.preProcessBlobLocks(blob_list)
1146 if blob_list is None:
1147 raise ValueError("Did you forget to return the blob_list somewhere inside getReferencedBlobs()?")
1148 if None in blob_list:
1149 msg = f"None is not valid in {blob_list=}"
1150 logging.error(msg)
1151 raise ValueError(msg)
1153 if not is_add and (old_blob_lock_obj := db.Get(db.Key("viur-blob-locks", db_key.id_or_name))):
1154 removed_blobs = set(old_blob_lock_obj.get("active_blob_references", [])) - blob_list
1155 old_blob_lock_obj["active_blob_references"] = list(blob_list)
1156 if old_blob_lock_obj["old_blob_references"] is None:
1157 old_blob_lock_obj["old_blob_references"] = list(removed_blobs)
1158 else:
1159 old_blob_refs = set(old_blob_lock_obj["old_blob_references"])
1160 old_blob_refs.update(removed_blobs) # Add removed blobs
1161 old_blob_refs -= blob_list # Remove active blobs
1162 old_blob_lock_obj["old_blob_references"] = list(old_blob_refs)
1164 old_blob_lock_obj["has_old_blob_references"] = bool(old_blob_lock_obj["old_blob_references"])
1165 old_blob_lock_obj["is_stale"] = False
1166 db.Put(old_blob_lock_obj)
1167 else: # We need to create a new blob-lock-object
1168 blob_lock_obj = db.Entity(db.Key("viur-blob-locks", db_obj.key.id_or_name))
1169 blob_lock_obj["active_blob_references"] = list(blob_list)
1170 blob_lock_obj["old_blob_references"] = []
1171 blob_lock_obj["has_old_blob_references"] = False
1172 blob_lock_obj["is_stale"] = False
1173 db.Put(blob_lock_obj)
1175 return db_obj.key, db_obj, skel, change_list, is_add
1177 # END of __txn_update subfunction
1179 # Run our SaveTxn
1180 if db.IsInTransaction():
1181 key, db_obj, skel, change_list, is_add = __txn_update(skel)
1182 else:
1183 key, db_obj, skel, change_list, is_add = db.RunInTransaction(__txn_update, skel)
1185 for bone_name, bone in skel.items():
1186 bone.postSavedHandler(skel, bone_name, key)
1188 skel.postSavedHandler(key, db_obj)
1190 if update_relations and not is_add:
1191 if change_list and len(change_list) < 5: # Only a few bones have changed, process these individually
1192 for idx, changed_bone in enumerate(change_list):
1193 updateRelations(key, time() + 1, changed_bone, _countdown=10 * idx)
1194 else: # Update all inbound relations, regardless of which bones they mirror
1195 updateRelations(key, time() + 1, None)
1197 # Inform the custom DB Adapter of the changes made to the entry
1198 if skel.customDatabaseAdapter:
1199 skel.customDatabaseAdapter.updateEntry(db_obj, skel, change_list, is_add)
1201 return key
1203 @classmethod
1204 def preProcessBlobLocks(cls, skelValues, locks):
1205 """
1206 Can be overridden to modify the list of blobs referenced by this skeleton
1207 """
1208 return locks
1210 @classmethod
1211 def preProcessSerializedData(cls, skelValues, entity):
1212 """
1213 Can be overridden to modify the :class:`viur.core.db.Entity` before its actually
1214 written to the data store.
1215 """
1216 return entity
1218 @classmethod
1219 def postSavedHandler(cls, skelValues, key, dbObj):
1220 """
1221 Can be overridden to perform further actions after the entity has been written
1222 to the data store.
1223 """
1224 pass
1226 @classmethod
1227 def postDeletedHandler(cls, skelValues, key):
1228 """
1229 Can be overridden to perform further actions after the entity has been deleted
1230 from the data store.
1231 """
1232 pass
1234 @classmethod
1235 def getCurrentSEOKeys(cls, skelValues) -> None | dict[str, str]:
1236 """
1237 Should be overridden to return a dictionary of language -> SEO-Friendly key
1238 this entry should be reachable under. How theses names are derived are entirely up to the application.
1239 If the name is already in use for this module, the server will automatically append some random string
1240 to make it unique.
1241 :return:
1242 """
1243 return
1245 @classmethod
1246 def delete(cls, skelValues):
1247 """
1248 Deletes the entity associated with the current Skeleton from the data store.
1249 """
1251 def txnDelete(skel: SkeletonInstance) -> db.Entity:
1252 skel_key = skel["key"]
1253 entity = db.Get(skel_key) # Fetch the raw object as we might have to clear locks
1254 viur_data = entity.get("viur") or {}
1256 # Is there any relation to this Skeleton which prevents the deletion?
1257 locked_relation = (
1258 db.Query("viur-relations")
1259 .filter("dest.__key__ =", skel_key)
1260 .filter("viur_relational_consistency =", RelationalConsistency.PreventDeletion.value)
1261 ).getEntry()
1262 if locked_relation is not None:
1263 raise errors.Locked("This entry is still referenced by other Skeletons, which prevents deleting!")
1265 for boneName, bone in skel.items():
1266 # Ensure that we delete any value-lock objects remaining for this entry
1267 bone.delete(skel, boneName)
1268 if bone.unique:
1269 flushList = []
1270 for lockValue in viur_data.get(f"{boneName}_uniqueIndexValue") or []:
1271 lockKey = db.Key(f"{skel.kindName}_{boneName}_uniquePropertyIndex", lockValue)
1272 lockObj = db.Get(lockKey)
1273 if not lockObj:
1274 logging.error(f"{lockKey=} missing!")
1275 elif lockObj["references"] != entity.key.id_or_name:
1276 logging.error(
1277 f"""{skel["key"]!r} does not hold lock for {lockKey!r}""")
1278 else:
1279 flushList.append(lockObj)
1280 if flushList:
1281 db.Delete(flushList)
1283 # Delete the blob-key lock object
1284 lockObjectKey = db.Key("viur-blob-locks", entity.key.id_or_name)
1285 lockObj = db.Get(lockObjectKey)
1286 if lockObj is not None:
1287 if lockObj["old_blob_references"] is None and lockObj["active_blob_references"] is None:
1288 db.Delete(lockObjectKey) # Nothing to do here
1289 else:
1290 if lockObj["old_blob_references"] is None:
1291 # No old stale entries, move active_blob_references -> old_blob_references
1292 lockObj["old_blob_references"] = lockObj["active_blob_references"]
1293 elif lockObj["active_blob_references"] is not None:
1294 # Append the current references to the list of old & stale references
1295 lockObj["old_blob_references"] += lockObj["active_blob_references"]
1296 lockObj["active_blob_references"] = [] # There are no active ones left
1297 lockObj["is_stale"] = True
1298 lockObj["has_old_blob_references"] = True
1299 db.Put(lockObj)
1300 db.Delete(skel_key)
1301 processRemovedRelations(skel_key)
1302 return entity
1304 key = skelValues["key"]
1305 if key is None:
1306 raise ValueError("This skeleton has no key!")
1307 skel = skeletonByKind(skelValues.kindName)()
1308 if not skel.fromDB(key):
1309 raise ValueError("This skeleton is not in the database (anymore?)!")
1310 if db.IsInTransaction():
1311 dbObj = txnDelete(skel)
1312 else:
1313 dbObj = db.RunInTransaction(txnDelete, skel)
1314 for boneName, _bone in skel.items():
1315 _bone.postDeletedHandler(skel, boneName, key)
1316 skel.postDeletedHandler(key)
1317 # Inform the custom DB Adapter
1318 if skel.customDatabaseAdapter:
1319 skel.customDatabaseAdapter.deleteEntry(dbObj, skel)
1322class RelSkel(BaseSkeleton):
1323 """
1324 This is a Skeleton-like class that acts as a container for Skeletons used as a
1325 additional information data skeleton for
1326 :class:`~viur.core.bones.extendedRelationalBone.extendedRelationalBone`.
1328 It needs to be sub-classed where information about the kindName and its attributes
1329 (bones) are specified.
1331 The Skeleton stores its bones in an :class:`OrderedDict`-Instance, so the definition order of the
1332 contained bones remains constant.
1333 """
1335 def serialize(self, parentIndexed):
1336 if self.dbEntity is None:
1337 self.dbEntity = db.Entity()
1338 for key, _bone in self.items():
1339 # if key in self.accessedValues:
1340 _bone.serialize(self, key, parentIndexed)
1341 # if "key" in self: # Write the key seperatly, as the base-bone doesn't store it
1342 # dbObj["key"] = self["key"]
1343 # FIXME: is this a good idea? Any other way to ensure only bones present in refKeys are serialized?
1344 return self.dbEntity
1346 def unserialize(self, values: db.Entity | dict):
1347 """
1348 Loads 'values' into this skeleton.
1350 :param values: dict with values we'll assign to our bones
1351 """
1352 if not isinstance(values, db.Entity):
1353 self.dbEntity = db.Entity()
1354 self.dbEntity.update(values)
1355 else:
1356 self.dbEntity = values
1358 self.accessedValues = {}
1359 self.renderAccessedValues = {}
1362class RefSkel(RelSkel):
1363 @classmethod
1364 def fromSkel(cls, kindName: str, *args: list[str]) -> t.Type[RefSkel]:
1365 """
1366 Creates a relSkel from a skeleton-class using only the bones explicitly named
1367 in \*args
1369 :param args: List of bone names we'll adapt
1370 :return: A new instance of RefSkel
1371 """
1372 newClass = type("RefSkelFor" + kindName, (RefSkel,), {})
1373 fromSkel = skeletonByKind(kindName)
1374 newClass.kindName = kindName
1375 bone_map = {}
1376 for arg in args:
1377 bone_map |= {k: fromSkel.__boneMap__[k] for k in fnmatch.filter(fromSkel.__boneMap__.keys(), arg)}
1378 newClass.__boneMap__ = bone_map
1379 return newClass
1382class SkelList(list):
1383 """
1384 This class is used to hold multiple skeletons together with other, commonly used information.
1386 SkelLists are returned by Skel().all()...fetch()-constructs and provide additional information
1387 about the data base query, for fetching additional entries.
1389 :ivar cursor: Holds the cursor within a query.
1390 :vartype cursor: str
1391 """
1393 __slots__ = (
1394 "baseSkel",
1395 "customQueryInfo",
1396 "getCursor",
1397 "get_orders",
1398 "renderPreparation",
1399 )
1401 def __init__(self, baseSkel=None):
1402 """
1403 :param baseSkel: The baseclass for all entries in this list
1404 """
1405 super(SkelList, self).__init__()
1406 self.baseSkel = baseSkel or {}
1407 self.getCursor = lambda: None
1408 self.get_orders = lambda: None
1409 self.renderPreparation = None
1410 self.customQueryInfo = {}
1413### Tasks ###
1415@CallDeferred
1416def processRemovedRelations(removedKey, cursor=None):
1417 updateListQuery = (
1418 db.Query("viur-relations")
1419 .filter("dest.__key__ =", removedKey)
1420 .filter("viur_relational_consistency >", RelationalConsistency.PreventDeletion.value)
1421 )
1422 updateListQuery = updateListQuery.setCursor(cursor)
1423 updateList = updateListQuery.run(limit=5)
1425 for entry in updateList:
1426 skel = skeletonByKind(entry["viur_src_kind"])()
1428 if not skel.fromDB(entry["src"].key):
1429 raise ValueError(f"processRemovedRelations detects inconsistency on src={entry['src'].key!r}")
1431 if entry["viur_relational_consistency"] == RelationalConsistency.SetNull.value:
1432 for key, _bone in skel.items():
1433 if isinstance(_bone, RelationalBone):
1434 relVal = skel[key]
1435 if isinstance(relVal, dict) and relVal["dest"]["key"] == removedKey:
1436 # FIXME: Should never happen: "key" not in relVal["dest"]
1437 # skel.setBoneValue(key, None)
1438 skel[key] = None
1439 elif isinstance(relVal, list):
1440 skel[key] = [x for x in relVal if x["dest"]["key"] != removedKey]
1441 else:
1442 raise NotImplementedError(f"No handling for {type(relVal)=}")
1443 skel.toDB(update_relations=False)
1445 else:
1446 logging.critical(f"""Cascade deletion of {skel["key"]!r}""")
1447 skel.delete()
1449 if len(updateList) == 5:
1450 processRemovedRelations(removedKey, updateListQuery.getCursor())
1453@CallDeferred
1454def updateRelations(destKey: db.Key, minChangeTime: int, changedBone: t.Optional[str], cursor: t.Optional[str] = None):
1455 """
1456 This function updates Entities, which may have a copy of values from another entity which has been recently
1457 edited (updated). In ViUR, relations are implemented by copying the values from the referenced entity into the
1458 entity that's referencing them. This allows ViUR to run queries over properties of referenced entities and
1459 prevents additional db.Get's to these referenced entities if the main entity is read. However, this forces
1460 us to track changes made to entities as we might have to update these mirrored values. This is the deferred
1461 call from meth:`viur.core.skeleton.Skeleton.toDB()` after an update (edit) on one Entity to do exactly that.
1463 :param destKey: The database-key of the entity that has been edited
1464 :param minChangeTime: The timestamp on which the edit occurred. As we run deferred, and the entity might have
1465 been edited multiple times before we get acutally called, we can ignore entities that have been updated
1466 in the meantime as they're already up2date
1467 :param changedBone: If set, we'll update only entites that have a copy of that bone. Relations mirror only
1468 key and name by default, so we don't have to update these if only another bone has been changed.
1469 :param cursor: The database cursor for the current request as we only process five entities at once and then
1470 defer again.
1471 """
1472 logging.debug(f"Starting updateRelations for {destKey} ; {minChangeTime=},{changedBone=}, {cursor=}")
1473 updateListQuery = (
1474 db.Query("viur-relations")
1475 .filter("dest.__key__ =", destKey)
1476 .filter("viur_delayed_update_tag <", minChangeTime)
1477 .filter("viur_relational_updateLevel =", RelationalUpdateLevel.Always.value)
1478 )
1479 if changedBone:
1480 updateListQuery.filter("viur_foreign_keys =", changedBone)
1481 if cursor:
1482 updateListQuery.setCursor(cursor)
1483 updateList = updateListQuery.run(limit=5)
1485 def updateTxn(skel, key, srcRelKey):
1486 if not skel.fromDB(key):
1487 logging.warning(f"Cannot update stale reference to {key=} (referenced from {srcRelKey=})")
1488 return
1490 skel.refresh()
1491 skel.toDB(update_relations=False)
1493 for srcRel in updateList:
1494 try:
1495 skel = skeletonByKind(srcRel["viur_src_kind"])()
1496 except AssertionError:
1497 logging.info(f"""Ignoring {srcRel.key!r} which refers to unknown kind {srcRel["viur_src_kind"]!r}""")
1498 continue
1499 if db.IsInTransaction():
1500 updateTxn(skel, srcRel["src"].key, srcRel.key)
1501 else:
1502 db.RunInTransaction(updateTxn, skel, srcRel["src"].key, srcRel.key)
1503 nextCursor = updateListQuery.getCursor()
1504 if len(updateList) == 5 and nextCursor:
1505 updateRelations(destKey, minChangeTime, changedBone, nextCursor)
1508@CallableTask
1509class TaskUpdateSearchIndex(CallableTaskBase):
1510 """
1511 This tasks loads and saves *every* entity of the given module.
1512 This ensures an updated searchIndex and verifies consistency of this data.
1513 """
1514 key = "rebuildSearchIndex"
1515 name = "Rebuild search index"
1516 descr = "This task can be called to update search indexes and relational information."
1518 def canCall(self) -> bool:
1519 """Checks wherever the current user can execute this task"""
1520 user = current.user.get()
1521 return user is not None and "root" in user["access"]
1523 def dataSkel(self):
1524 modules = ["*"] + listKnownSkeletons()
1525 modules.sort()
1526 skel = BaseSkeleton().clone()
1527 skel.module = SelectBone(descr="Module", values={x: translate(x) for x in modules}, required=True)
1528 return skel
1530 def execute(self, module, *args, **kwargs):
1531 usr = current.user.get()
1532 if not usr:
1533 logging.warning("Don't know who to inform after rebuilding finished")
1534 notify = None
1535 else:
1536 notify = usr["name"]
1538 if module == "*":
1539 for module in listKnownSkeletons():
1540 logging.info("Rebuilding search index for module %r", module)
1541 self._run(module, notify)
1542 else:
1543 self._run(module, notify)
1545 @staticmethod
1546 def _run(module: str, notify: str):
1547 Skel = skeletonByKind(module)
1548 if not Skel:
1549 logging.error("TaskUpdateSearchIndex: Invalid module")
1550 return
1551 RebuildSearchIndex.startIterOnQuery(Skel().all(), {"notify": notify, "module": module})
1554class RebuildSearchIndex(QueryIter):
1555 @classmethod
1556 def handleEntry(cls, skel: SkeletonInstance, customData: dict[str, str]):
1557 skel.refresh()
1558 skel.toDB(update_relations=False)
1560 @classmethod
1561 def handleFinish(cls, totalCount: int, customData: dict[str, str]):
1562 QueryIter.handleFinish(totalCount, customData)
1563 if not customData["notify"]:
1564 return
1565 txt = (
1566 f"{conf.instance.project_id}: Rebuild search index finished for {customData['module']}\n\n"
1567 f"ViUR finished to rebuild the search index for module {customData['module']}.\n"
1568 f"{totalCount} records updated in total on this kind."
1569 )
1570 try:
1571 email.sendEMail(dests=customData["notify"], stringTemplate=txt, skel=None)
1572 except Exception as exc: # noqa; OverQuota, whatever
1573 logging.exception(f'Failed to notify {customData["notify"]}')
1576### Vacuum Relations
1578@CallableTask
1579class TaskVacuumRelations(TaskUpdateSearchIndex):
1580 """
1581 Checks entries in viur-relations and verifies that the src-kind
1582 and it's RelationalBone still exists.
1583 """
1584 key = "vacuumRelations"
1585 name = "Vacuum viur-relations (dangerous)"
1586 descr = "Drop stale inbound relations for the given kind"
1588 def execute(self, module: str, *args, **kwargs):
1589 usr = current.user.get()
1590 if not usr:
1591 logging.warning("Don't know who to inform after rebuilding finished")
1592 notify = None
1593 else:
1594 notify = usr["name"]
1595 processVacuumRelationsChunk(module.strip(), None, notify=notify)
1598@CallDeferred
1599def processVacuumRelationsChunk(
1600 module: str, cursor, count_total: int = 0, count_removed: int = 0, notify=None
1601):
1602 """
1603 Processes 25 Entries and calls the next batch
1604 """
1605 query = db.Query("viur-relations")
1606 if module != "*":
1607 query.filter("viur_src_kind =", module)
1608 query.setCursor(cursor)
1609 for relation_object in query.run(25):
1610 count_total += 1
1611 if not (src_kind := relation_object.get("viur_src_kind")):
1612 logging.critical("We got an relation-object without a src_kind!")
1613 continue
1614 if not (src_prop := relation_object.get("viur_src_property")):
1615 logging.critical("We got an relation-object without a src_prop!")
1616 continue
1617 try:
1618 skel = skeletonByKind(src_kind)()
1619 except AssertionError:
1620 # The referenced skeleton does not exist in this data model -> drop that relation object
1621 logging.info(f"Deleting {relation_object.key} which refers to unknown kind {src_kind}")
1622 db.Delete(relation_object)
1623 count_removed += 1
1624 continue
1625 if src_prop not in skel:
1626 logging.info(f"Deleting {relation_object.key} which refers to "
1627 f"non-existing RelationalBone {src_prop} of {src_kind}")
1628 db.Delete(relation_object)
1629 count_removed += 1
1630 logging.info(f"END processVacuumRelationsChunk {module}, "
1631 f"{count_total} records processed, {count_removed} removed")
1632 if new_cursor := query.getCursor():
1633 # Start processing of the next chunk
1634 processVacuumRelationsChunk(module, new_cursor, count_total, count_removed, notify)
1635 elif notify:
1636 txt = (
1637 f"{conf.instance.project_id}: Vacuum relations finished for {module}\n\n"
1638 f"ViUR finished to vacuum viur-relations for module {module}.\n"
1639 f"{count_total} records processed, "
1640 f"{count_removed} entries removed"
1641 )
1642 try:
1643 email.sendEMail(dests=notify, stringTemplate=txt, skel=None)
1644 except Exception as exc: # noqa; OverQuota, whatever
1645 logging.exception(f"Failed to notify {notify}")
1648# Forward our references to SkelInstance to the database (needed for queries)
1649db.config["SkeletonInstanceRef"] = SkeletonInstance
1651# DEPRECATED ATTRIBUTES HANDLING
1653__DEPRECATED_NAMES = {
1654 # stuff prior viur-core < 3.6
1655 "seoKeyBone": ("SeoKeyBone", SeoKeyBone),
1656}
1659def __getattr__(attr: str) -> object:
1660 if entry := __DEPRECATED_NAMES.get(attr):
1661 func = entry[1]
1662 msg = f"{attr} was replaced by {entry[0]}"
1663 warnings.warn(msg, DeprecationWarning, stacklevel=2)
1664 logging.warning(msg, stacklevel=2)
1665 return func
1667 return super(__import__(__name__).__class__).__getattribute__(attr)