Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/bones/file.py: 9%
150 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-02-07 19:28 +0000
1"""
2The FileBone is a subclass of the TreeLeafBone class, which is a relational bone that can reference
3another entity's fields. FileBone provides additional file-specific properties and methods, such as
4managing file derivatives, handling file size and mime type restrictions, and refreshing file
5metadata.
6"""
8from hashlib import sha256
9from time import time
10import typing as t
11from viur.core import conf, db
12from viur.core.bones.treeleaf import TreeLeafBone
13from viur.core.tasks import CallDeferred
15import logging
18@CallDeferred
19def ensureDerived(key: db.Key, srcKey, deriveMap: dict[str, t.Any], refreshKey: db.Key = None):
20 r"""
21 The function is a deferred function that ensures all pending thumbnails or other derived files
22 are built. It takes the following parameters:
24 :param db.key key: The database key of the file-object that needs to have its derivation map
25 updated.
26 :param str srcKey: A prefix for a stable key to prevent rebuilding derived files repeatedly.
27 :param dict[str,Any] deriveMap: A list of DeriveDicts that need to be built or updated.
28 :param db.Key refreshKey: If set, the function fetches and refreshes the skeleton after
29 building new derived files.
31 The function works by fetching the skeleton of the file-object, checking if it has any derived
32 files, and updating the derivation map accordingly. It iterates through the deriveMap items and
33 calls the appropriate deriver function. If the deriver function returns a result, the function
34 creates a new or updated resultDict and merges it into the file-object's metadata. Finally,
35 the updated results are written back to the database and the updateRelations function is called
36 to ensure proper relations are maintained.
37 """
38 from viur.core.skeleton import skeletonByKind, updateRelations
39 deriveFuncMap = conf.file_derivations
40 skel = skeletonByKind("file")()
41 if not skel.read(key):
42 logging.info("File-Entry went missing in ensureDerived")
43 return
44 if not skel["derived"]:
45 logging.info("No Derives for this file")
46 skel["derived"] = {}
47 skel["derived"]["deriveStatus"] = skel["derived"].get("deriveStatus") or {}
48 skel["derived"]["files"] = skel["derived"].get("files") or {}
49 resDict = {} # Will contain new or updated resultDicts that will be merged into our file
50 for calleeKey, params in deriveMap.items():
51 fullSrcKey = f"{srcKey}_{calleeKey}"
52 paramsHash = sha256(str(params).encode("UTF-8")).hexdigest() # Hash over given params (dict?)
53 if skel["derived"]["deriveStatus"].get(fullSrcKey) != paramsHash:
54 if calleeKey not in deriveFuncMap:
55 logging.warning(f"File-Deriver {calleeKey} not found - skipping!")
56 continue
57 callee = deriveFuncMap[calleeKey]
58 callRes = callee(skel, skel["derived"]["files"], params)
59 if callRes:
60 assert isinstance(callRes, list), "Old (non-list) return value from deriveFunc"
61 resDict[fullSrcKey] = {"version": paramsHash, "files": {}}
62 for fileName, size, mimetype, customData in callRes:
63 resDict[fullSrcKey]["files"][fileName] = {
64 "size": size,
65 "mimetype": mimetype,
66 "customData": customData
67 }
69 def updateTxn(key, resDict):
70 obj = db.Get(key)
71 if not obj: # File-object got deleted during building of our derives
72 return
73 obj["derived"] = obj.get("derived") or {}
74 obj["derived"]["deriveStatus"] = obj["derived"].get("deriveStatus") or {}
75 obj["derived"]["files"] = obj["derived"].get("files") or {}
76 for k, v in resDict.items():
77 obj["derived"]["deriveStatus"][k] = v["version"]
78 for fileName, fileDict in v["files"].items():
79 obj["derived"]["files"][fileName] = fileDict
80 db.Put(obj)
82 if resDict: # Write updated results back and queue updateRelationsTask
83 db.RunInTransaction(updateTxn, key, resDict)
84 # Queue that updateRelations call at least 30 seconds into the future, so that other ensureDerived calls from
85 # the same FileBone have the chance to finish, otherwise that updateRelations Task will call postSavedHandler
86 # on that FileBone again - re-queueing any ensureDerivedCalls that have not finished yet.
87 updateRelations(key, time() + 1, "derived", _countdown=30)
88 if refreshKey:
89 def refreshTxn():
90 skel = skeletonByKind(refreshKey.kind)()
91 if not skel.read(refreshKey):
92 return
93 skel.refresh()
94 skel.write(update_relations=False)
96 db.RunInTransaction(refreshTxn)
99class FileBone(TreeLeafBone):
100 r"""
101 A FileBone is a custom bone class that inherits from the TreeLeafBone class, and is used to store and manage
102 file references in a ViUR application.
104 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
105 :param maxFileSize:
106 The maximum filesize accepted by this bone in bytes. None means no limit.
107 This will always be checked against the original file uploaded - not any of it's derivatives.
109 :param derive: A set of functions used to derive other files from the referenced ones. Used fe.
110 to create thumbnails / images for srcmaps from hires uploads. If set, must be a dictionary from string
111 (a key from conf.file_derivations) to the parameters passed to that function. The parameters can be
112 any type (including None) that can be json-serialized.
114 .. code-block:: python
116 # Example
117 derive = { "thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
119 :param validMimeTypes:
120 A list of Mimetypes that can be selected in this bone (or None for any) Wildcards ("image\/*") are supported.
122 .. code-block:: python
124 # Example
125 validMimeTypes=["application/pdf", "image/*"]
127 """
129 kind = "file"
130 """The kind of this bone is 'file'"""
132 type = "relational.tree.leaf.file"
133 """The type of this bone is 'relational.tree.leaf.file'."""
135 def __init__(
136 self,
137 *,
138 derive: None | dict[str, t.Any] = None,
139 maxFileSize: None | int = None,
140 validMimeTypes: None | list[str] = None,
141 refKeys: t.Optional[t.Iterable[str]] = (
142 "name",
143 "mimetype",
144 "size",
145 "width",
146 "height",
147 "derived",
148 "public",
149 "serving_url",
150 ),
151 public: bool = False,
152 **kwargs
153 ):
154 r"""
155 Initializes a new Filebone. All properties inherited by RelationalBone are supported.
157 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
158 :param maxFileSize: The maximum filesize accepted by this bone in bytes. None means no limit.
159 This will always be checked against the original file uploaded - not any of it's derivatives.
160 :param derive: A set of functions used to derive other files from the referenced ones.
161 Used to create thumbnails and images for srcmaps from hires uploads.
162 If set, must be a dictionary from string (a key from) conf.file_derivations) to the parameters passed to
163 that function. The parameters can be any type (including None) that can be json-serialized.
165 .. code-block:: python
167 # Example
168 derive = {"thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
170 :param validMimeTypes:
171 A list of Mimetypes that can be selected in this bone (or None for any).
172 Wildcards `('image\*')` are supported.
174 .. code-block:: python
176 #Example
177 validMimeTypes=["application/pdf", "image/*"]
179 """
180 super().__init__(refKeys=refKeys, **kwargs)
182 self.refKeys.add("dlkey")
183 self.derive = derive
184 self.public = public
185 self.validMimeTypes = validMimeTypes
186 self.maxFileSize = maxFileSize
188 def isInvalid(self, value):
189 """
190 Checks if the provided value is invalid for this bone based on its MIME type and file size.
192 :param dict value: The value to check for validity.
193 :returns: None if the value is valid, or an error message if it is invalid.
194 """
195 if self.validMimeTypes:
196 mimeType = value["dest"]["mimetype"]
197 for checkMT in self.validMimeTypes:
198 checkMT = checkMT.lower()
199 if checkMT == mimeType or checkMT.endswith("*") and mimeType.startswith(checkMT[:-1]):
200 break
201 else:
202 return "Invalid filetype selected"
203 if self.maxFileSize:
204 if value["dest"]["size"] > self.maxFileSize:
205 return "File too large."
207 if value["dest"]["public"] != self.public:
208 return f"Only files marked public={self.public!r} are allowed."
210 return None
212 def postSavedHandler(self, skel, boneName, key):
213 """
214 Handles post-save processing for the FileBone, including ensuring derived files are built.
216 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
217 :param str boneName: The name of the bone.
218 :param db.Key key: The datastore key of the skeleton.
220 This method first calls the postSavedHandler of its superclass. Then, it checks if the
221 derive attribute is set and if there are any values in the skeleton for the given bone. If
222 so, it handles the creation of derived files based on the provided configuration.
224 If the values are stored as a dictionary without a "dest" key, it assumes a multi-language
225 setup and iterates over each language to handle the derived files. Otherwise, it handles
226 the derived files directly.
227 """
228 super().postSavedHandler(skel, boneName, key)
229 from viur.core.skeleton import RelSkel, Skeleton
231 if issubclass(skel.skeletonCls, Skeleton):
232 prefix = f"{skel.kindName}_{boneName}"
233 elif issubclass(skel.skeletonCls, RelSkel): # RelSkel is just a container and has no kindname
234 prefix = f"{skel.skeletonCls.__name__}_{boneName}"
235 else:
236 raise NotImplementedError(f"Cannot handle {skel.skeletonCls=}")
238 def handleDerives(values):
239 if isinstance(values, dict):
240 values = [values]
241 for val in (values or ()): # Ensure derives getting build for each file referenced in this relation
242 ensureDerived(val["dest"]["key"], prefix, self.derive)
244 values = skel[boneName]
245 if self.derive and values:
246 if isinstance(values, dict) and "dest" not in values: # multi lang
247 for lang in values:
248 handleDerives(values[lang])
249 else:
250 handleDerives(values)
252 def getReferencedBlobs(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> set[str]:
253 r"""
254 Retrieves the referenced blobs in the FileBone.
256 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
257 :param str name: The name of the bone.
258 :return: A set of download keys for the referenced blobs.
259 :rtype: Set[str]
261 This method iterates over the bone values for the given skeleton and bone name. It skips
262 values that are None. For each non-None value, it adds the download key of the referenced
263 blob to a set. Finally, it returns the set of unique download keys for the referenced blobs.
264 """
265 result = set()
266 for idx, lang, value in self.iter_bone_value(skel, name):
267 if value is None:
268 continue
269 result.add(value["dest"]["dlkey"])
270 return result
272 def refresh(self, skel, boneName):
273 r"""
274 Refreshes the FileBone by recreating file entries if needed and importing blobs from ViUR 2.
276 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
277 :param str boneName: The name of the bone.
279 This method defines an inner function, recreateFileEntryIfNeeded(val), which is responsible
280 for recreating the weak file entry referenced by the relation in val if it doesn't exist
281 (e.g., if it was deleted by ViUR 2). It initializes a new skeleton for the "file" kind and
282 checks if the file object already exists. If not, it recreates the file entry with the
283 appropriate properties and saves it to the database.
285 The main part of the refresh method calls the superclass's refresh method and checks if the
286 configuration contains a ViUR 2 import blob source. If it does, it iterates through the file
287 references in the bone value, imports the blobs from ViUR 2, and recreates the file entries if
288 needed using the inner function.
289 """
290 super().refresh(skel, boneName)
292 for _, _, value in self.iter_bone_value(skel, boneName):
293 # Patch any empty serving_url when public file
294 if (
295 value
296 and (value := value["dest"])
297 and value["public"]
298 and value["mimetype"]
299 and value["mimetype"].startswith("image/")
300 and not value["serving_url"]
301 ):
302 logging.info(f"Patching public image with empty serving_url {value['key']!r} ({value['name']!r})")
303 try:
304 file_skel = value.read()
305 except ValueError:
306 continue
308 file_skel.patch(lambda skel: skel.refresh(), update_relations=False)
309 value["serving_url"] = file_skel["serving_url"]
311 # FIXME: REMOVE THIS WITH VIUR4
312 if conf.viur2import_blobsource:
313 from viur.core.modules.file import importBlobFromViur2
314 from viur.core.skeleton import skeletonByKind
316 def recreateFileEntryIfNeeded(val):
317 # Recreate the (weak) filenetry referenced by the relation *val*. (ViUR2 might have deleted them)
318 skel = skeletonByKind("file")()
319 if skel.read(val["key"]): # This file-object exist, no need to recreate it
320 return
321 skel["key"] = val["key"]
322 skel["name"] = val["name"]
323 skel["mimetype"] = val["mimetype"]
324 skel["dlkey"] = val["dlkey"]
325 skel["size"] = val["size"]
326 skel["width"] = val["width"]
327 skel["height"] = val["height"]
328 skel["weak"] = True
329 skel["pending"] = False
330 skel.write()
332 # Just ensure the file get's imported as it may not have an file entry
333 val = skel[boneName]
334 if isinstance(val, list):
335 for x in val:
336 importBlobFromViur2(x["dest"]["dlkey"], x["dest"]["name"])
337 recreateFileEntryIfNeeded(x["dest"])
338 elif isinstance(val, dict):
339 if not "dest" in val:
340 return
341 importBlobFromViur2(val["dest"]["dlkey"], val["dest"]["name"])
342 recreateFileEntryIfNeeded(val["dest"])
344 def structure(self) -> dict:
345 return super().structure() | {
346 "valid_mime_types": self.validMimeTypes,
347 "public": self.public,
348 }