Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/bones/file.py: 11%
132 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-03 13:41 +0000
1"""
2The FileBone is a subclass of the TreeLeafBone class, which is a relational bone that can reference
3another entity's fields. FileBone provides additional file-specific properties and methods, such as
4managing file derivatives, handling file size and mime type restrictions, and refreshing file
5metadata.
6"""
8from hashlib import sha256
9from time import time
10import typing as t
11from viur.core import conf, db
12from viur.core.bones.treeleaf import TreeLeafBone
13from viur.core.tasks import CallDeferred
15import logging
18@CallDeferred
19def ensureDerived(key: db.Key, srcKey, deriveMap: dict[str, t.Any], refreshKey: db.Key = None):
20 r"""
21 The function is a deferred function that ensures all pending thumbnails or other derived files
22 are built. It takes the following parameters:
24 :param db.key key: The database key of the file-object that needs to have its derivation map
25 updated.
26 :param str srcKey: A prefix for a stable key to prevent rebuilding derived files repeatedly.
27 :param dict[str,Any] deriveMap: A list of DeriveDicts that need to be built or updated.
28 :param db.Key refreshKey: If set, the function fetches and refreshes the skeleton after
29 building new derived files.
31 The function works by fetching the skeleton of the file-object, checking if it has any derived
32 files, and updating the derivation map accordingly. It iterates through the deriveMap items and
33 calls the appropriate deriver function. If the deriver function returns a result, the function
34 creates a new or updated resultDict and merges it into the file-object's metadata. Finally,
35 the updated results are written back to the database and the updateRelations function is called
36 to ensure proper relations are maintained.
37 """
38 from viur.core.skeleton import skeletonByKind, updateRelations
39 deriveFuncMap = conf.file_derivations
40 skel = skeletonByKind("file")()
41 if not skel.fromDB(key):
42 logging.info("File-Entry went missing in ensureDerived")
43 return
44 if not skel["derived"]:
45 logging.info("No Derives for this file")
46 skel["derived"] = {}
47 skel["derived"]["deriveStatus"] = skel["derived"].get("deriveStatus") or {}
48 skel["derived"]["files"] = skel["derived"].get("files") or {}
49 resDict = {} # Will contain new or updated resultDicts that will be merged into our file
50 for calleeKey, params in deriveMap.items():
51 fullSrcKey = f"{srcKey}_{calleeKey}"
52 paramsHash = sha256(str(params).encode("UTF-8")).hexdigest() # Hash over given params (dict?)
53 if skel["derived"]["deriveStatus"].get(fullSrcKey) != paramsHash:
54 if calleeKey not in deriveFuncMap:
55 logging.warning(f"File-Deriver {calleeKey} not found - skipping!")
56 continue
57 callee = deriveFuncMap[calleeKey]
58 callRes = callee(skel, skel["derived"]["files"], params)
59 if callRes:
60 assert isinstance(callRes, list), "Old (non-list) return value from deriveFunc"
61 resDict[fullSrcKey] = {"version": paramsHash, "files": {}}
62 for fileName, size, mimetype, customData in callRes:
63 resDict[fullSrcKey]["files"][fileName] = {
64 "size": size,
65 "mimetype": mimetype,
66 "customData": customData
67 }
69 def updateTxn(key, resDict):
70 obj = db.Get(key)
71 if not obj: # File-object got deleted during building of our derives
72 return
73 obj["derived"] = obj.get("derived") or {}
74 obj["derived"]["deriveStatus"] = obj["derived"].get("deriveStatus") or {}
75 obj["derived"]["files"] = obj["derived"].get("files") or {}
76 for k, v in resDict.items():
77 obj["derived"]["deriveStatus"][k] = v["version"]
78 for fileName, fileDict in v["files"].items():
79 obj["derived"]["files"][fileName] = fileDict
80 db.Put(obj)
82 if resDict: # Write updated results back and queue updateRelationsTask
83 db.RunInTransaction(updateTxn, key, resDict)
84 # Queue that updateRelations call at least 30 seconds into the future, so that other ensureDerived calls from
85 # the same FileBone have the chance to finish, otherwise that updateRelations Task will call postSavedHandler
86 # on that FileBone again - re-queueing any ensureDerivedCalls that have not finished yet.
87 updateRelations(key, time() + 1, "derived", _countdown=30)
88 if refreshKey:
89 def refreshTxn():
90 skel = skeletonByKind(refreshKey.kind)()
91 if not skel.fromDB(refreshKey):
92 return
93 skel.refresh()
94 skel.toDB(update_relations=False)
96 db.RunInTransaction(refreshTxn)
99class FileBone(TreeLeafBone):
100 r"""
101 A FileBone is a custom bone class that inherits from the TreeLeafBone class, and is used to store and manage
102 file references in a ViUR application.
104 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
105 :param maxFileSize:
106 The maximum filesize accepted by this bone in bytes. None means no limit.
107 This will always be checked against the original file uploaded - not any of it's derivatives.
109 :param derive: A set of functions used to derive other files from the referenced ones. Used fe.
110 to create thumbnails / images for srcmaps from hires uploads. If set, must be a dictionary from string
111 (a key from conf.file_derivations) to the parameters passed to that function. The parameters can be
112 any type (including None) that can be json-serialized.
114 .. code-block:: python
116 # Example
117 derive = { "thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
119 :param validMimeTypes:
120 A list of Mimetypes that can be selected in this bone (or None for any) Wildcards ("image\/*") are supported.
122 .. code-block:: python
124 # Example
125 validMimeTypes=["application/pdf", "image/*"]
127 """
129 kind = "file"
130 """The kind of this bone is 'file'"""
131 type = "relational.tree.leaf.file"
132 """The type of this bone is 'relational.tree.leaf.file'."""
134 def __init__(
135 self,
136 *,
137 derive: None | dict[str, t.Any] = None,
138 maxFileSize: None | int = None,
139 validMimeTypes: None | list[str] = None,
140 refKeys: t.Optional[t.Iterable[str]] = ("name", "mimetype", "size", "width", "height", "derived"),
141 **kwargs
142 ):
143 r"""
144 Initializes a new Filebone. All properties inherited by RelationalBone are supported.
146 :param format: Hint for the UI how to display a file entry (defaults to it's filename)
147 :param maxFileSize: The maximum filesize accepted by this bone in bytes. None means no limit.
148 This will always be checked against the original file uploaded - not any of it's derivatives.
149 :param derive: A set of functions used to derive other files from the referenced ones.
150 Used to create thumbnails and images for srcmaps from hires uploads.
151 If set, must be a dictionary from string (a key from) conf.file_derivations) to the parameters passed to
152 that function. The parameters can be any type (including None) that can be json-serialized.
154 .. code-block:: python
156 # Example
157 derive = {"thumbnail": [{"width": 111}, {"width": 555, "height": 666}]}
159 :param validMimeTypes:
160 A list of Mimetypes that can be selected in this bone (or None for any).
161 Wildcards `('image\*')` are supported.
163 .. code-block:: python
165 #Example
166 validMimeTypes=["application/pdf", "image/*"]
168 """
169 super().__init__(refKeys=refKeys, **kwargs)
171 self.refKeys.add("dlkey")
172 self.derive = derive
173 self.validMimeTypes = validMimeTypes
174 self.maxFileSize = maxFileSize
176 def isInvalid(self, value):
177 """
178 Checks if the provided value is invalid for this bone based on its MIME type and file size.
180 :param dict value: The value to check for validity.
181 :returns: None if the value is valid, or an error message if it is invalid.
182 """
183 if self.validMimeTypes:
184 mimeType = value["dest"]["mimetype"]
185 for checkMT in self.validMimeTypes:
186 checkMT = checkMT.lower()
187 if checkMT == mimeType or checkMT.endswith("*") and mimeType.startswith(checkMT[:-1]):
188 break
189 else:
190 return "Invalid filetype selected"
191 if self.maxFileSize:
192 if value["dest"]["size"] > self.maxFileSize:
193 return "File too large."
194 return None
196 def postSavedHandler(self, skel, boneName, key):
197 """
198 Handles post-save processing for the FileBone, including ensuring derived files are built.
200 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
201 :param str boneName: The name of the bone.
202 :param db.Key key: The datastore key of the skeleton.
204 This method first calls the postSavedHandler of its superclass. Then, it checks if the
205 derive attribute is set and if there are any values in the skeleton for the given bone. If
206 so, it handles the creation of derived files based on the provided configuration.
208 If the values are stored as a dictionary without a "dest" key, it assumes a multi-language
209 setup and iterates over each language to handle the derived files. Otherwise, it handles
210 the derived files directly.
211 """
212 super().postSavedHandler(skel, boneName, key)
214 def handleDerives(values):
215 if isinstance(values, dict):
216 values = [values]
217 for val in values: # Ensure derives getting build for each file referenced in this relation
218 ensureDerived(val["dest"]["key"], f"{skel.kindName}_{boneName}", self.derive)
220 values = skel[boneName]
221 if self.derive and values:
222 if isinstance(values, dict) and "dest" not in values: # multi lang
223 for lang in values:
224 handleDerives(values[lang])
225 else:
226 handleDerives(values)
228 def getReferencedBlobs(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str) -> set[str]:
229 r"""
230 Retrieves the referenced blobs in the FileBone.
232 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
233 :param str name: The name of the bone.
234 :return: A set of download keys for the referenced blobs.
235 :rtype: Set[str]
237 This method iterates over the bone values for the given skeleton and bone name. It skips
238 values that are None. For each non-None value, it adds the download key of the referenced
239 blob to a set. Finally, it returns the set of unique download keys for the referenced blobs.
240 """
241 result = set()
242 for idx, lang, value in self.iter_bone_value(skel, name):
243 if value is None:
244 continue
245 result.add(value["dest"]["dlkey"])
246 return result
248 def refresh(self, skel, boneName):
249 r"""
250 Refreshes the FileBone by recreating file entries if needed and importing blobs from ViUR 2.
252 :param SkeletonInstance skel: The skeleton instance this bone belongs to.
253 :param str boneName: The name of the bone.
255 This method defines an inner function, recreateFileEntryIfNeeded(val), which is responsible
256 for recreating the weak file entry referenced by the relation in val if it doesn't exist
257 (e.g., if it was deleted by ViUR 2). It initializes a new skeleton for the "file" kind and
258 checks if the file object already exists. If not, it recreates the file entry with the
259 appropriate properties and saves it to the database.
261 The main part of the refresh method calls the superclass's refresh method and checks if the
262 configuration contains a ViUR 2 import blob source. If it does, it iterates through the file
263 references in the bone value, imports the blobs from ViUR 2, and recreates the file entries if
264 needed using the inner function.
265 """
266 from viur.core.skeleton import skeletonByKind
268 def recreateFileEntryIfNeeded(val):
269 # Recreate the (weak) filenetry referenced by the relation *val*. (ViUR2 might have deleted them)
270 skel = skeletonByKind("file")()
271 if skel.fromDB(val["key"]): # This file-object exist, no need to recreate it
272 return
273 skel["key"] = val["key"]
274 skel["name"] = val["name"]
275 skel["mimetype"] = val["mimetype"]
276 skel["dlkey"] = val["dlkey"]
277 skel["size"] = val["size"]
278 skel["width"] = val["width"]
279 skel["height"] = val["height"]
280 skel["weak"] = True
281 skel["pending"] = False
282 k = skel.toDB()
284 from viur.core.modules.file import importBlobFromViur2
285 super().refresh(skel, boneName)
286 if conf.viur2import_blobsource:
287 # Just ensure the file get's imported as it may not have an file entry
288 val = skel[boneName]
289 if isinstance(val, list):
290 for x in val:
291 importBlobFromViur2(x["dest"]["dlkey"], x["dest"]["name"])
292 recreateFileEntryIfNeeded(x["dest"])
293 elif isinstance(val, dict):
294 if not "dest" in val:
295 return
296 importBlobFromViur2(val["dest"]["dlkey"], val["dest"]["name"])
297 recreateFileEntryIfNeeded(val["dest"])
299 def structure(self) -> dict:
300 return super().structure() | {
301 "valid_mime_types": self.validMimeTypes
302 }