Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/cache.py: 0%
139 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-09-03 13:41 +0000
1import logging
2import os
3from datetime import timedelta
4from functools import wraps
5from hashlib import sha512
6import typing as t
8from viur.core import Method, current, db, tasks, utils
9from viur.core.config import conf
11"""
12 This module implements a cache that can be used to serve entire requests or cache the output of any function
13 (as long it's result can be stored in datastore). The intended use is to wrap functions that can be called from
14 the outside (@exposed) with the @enableCache decorator. This will enable the cache provided in this module for that
15 function, intercepting all calls to this function and serve a cached response instead of calling the function if
16 possible. Authenticated users with "root" access can always bypass this cache by sending the X-Viur-Disable-Cache
17 http Header along with their requests. Entities in this cache will expire if
18 - Their TTL is exceeded
19 - They're explicitly removed from the cache by calling :meth:`viur.core.cache.flushCache` using their path
20 - A Datastore entity that has been accessed using db.Get() from within the cached function has been modified
21 - The wrapped function has run a query over a kind in which an entity has been added/edited/deleted
23 ..Warning: As this cache is intended to be used with exposed functions, it will not only store the result of the
24 wrapped function, but will also store and restore the Content-Type http header. This can cause unexpected
25 behaviour if it's used to cache the result of non top-level functions, as calls to these functions now may
26 cause this header to be rewritten.
27"""
29viurCacheName = "viur-cache"
32def keyFromArgs(f: t.Callable, userSensitive: int, languageSensitive: bool, evaluatedArgs: list[str], path: str,
33 args: tuple, kwargs: dict) -> str:
34 """
35 Utility function to derive a unique but stable string-key that can be used in a datastore-key
36 for the given wrapped function f, the parameter *args and **kwargs it has been called with,
37 the path-components from the url that point to the outer @exposed function which handles this request
38 as well as the configuration (userSensitive, languageSensitive and evaluatedArgs) provided to the @enableCache
39 decorator. To derive the key, we'll first map all positional arguments to their keyword equivalent, construct
40 a dict with the parameters having an effect on the result, merge the context variables (language, session state)
41 in, sort this dict by key, cast it to string and return it's sha512 hash.
43 :param f: Callable which is inspected for its signature
44 (we need to figure out what positional arguments map to which key argument)
45 :param userSensitive: Signals wherever the output of f depends on the current user.
46 0 means independent of wherever the user is a guest or known, all will get the same content.
47 1 means cache only for guests, no cache will be performed if the user is logged-in.
48 2 means cache in two groups, one for guests and one for all users
49 3 will cache the result of that function for each individual users separately.
50 :param evaluatedArgs: List of keyword-arguments having influence to the output generated by
51 that function. This list *must* complete! Parameters not named here are ignored!
52 :param path: Path to the function called but without parameters (ie. "/page/view")
53 :returns: The unique key derived
54 """
55 res = {}
56 argsOrder = list(f.__code__.co_varnames)[1: f.__code__.co_argcount]
57 # Map default values in
58 reversedArgsOrder = argsOrder[:: -1]
59 for defaultValue in list(f.__defaults__ or [])[:: -1]:
60 res[reversedArgsOrder.pop(0)] = defaultValue
61 del reversedArgsOrder
62 # Map args in
63 setArgs = [] # Store a list of args already set by *args
64 for idx in range(0, min(len(args), len(argsOrder))):
65 if argsOrder[idx] in evaluatedArgs:
66 setArgs.append(argsOrder[idx])
67 res[argsOrder[idx]] = args[idx]
68 # Last, we map the kwargs in
69 for k, v in kwargs.items():
70 if k in evaluatedArgs:
71 if k in setArgs:
72 raise AssertionError(f"Got duplicate arguments for {k}")
73 res[k] = v
74 if userSensitive:
75 user = current.user.get()
76 if userSensitive == 1 and user: # We dont cache requests for each user separately
77 return None
78 elif userSensitive == 2:
79 if user:
80 res["__user"] = "__ISUSER"
81 else:
82 res["__user"] = None
83 elif userSensitive == 3:
84 if user:
85 res["__user"] = user["key"]
86 else:
87 res["__user"] = None
88 if languageSensitive:
89 res["__lang"] = current.language.get()
90 if conf.cache_environment_key:
91 try:
92 res["_cacheEnvironment"] = conf.cache_environment_key()
93 except RuntimeError:
94 return None
95 res["__path"] = path # Different path might have different output (html,xml,..)
96 try:
97 appVersion = os.getenv("GAE_VERSION")
98 except:
99 logging.error("Could not determine the current application version! Caching might produce unexpected results!")
100 appVersion = ""
101 res["__appVersion"] = appVersion
102 # Last check, that every parameter is satisfied:
103 if not all([x in res.keys() for x in argsOrder]):
104 # we have too few parameters for this function; that wont work
105 return None
106 res = list(res.items()) # flatten our dict to a list
107 res.sort(key=lambda x: x[0]) # sort by keys
108 mysha512 = sha512()
109 mysha512.update(str(res).encode("UTF8"))
110 return mysha512.hexdigest()
113def wrapCallable(f, urls: list[str], userSensitive: int, languageSensitive: bool,
114 evaluatedArgs: list[str], maxCacheTime: int | timedelta):
115 """
116 Does the actual work of wrapping a callable.
117 Use the decorator enableCache instead of calling this directly.
118 """
119 method = None
120 if isinstance(f, Method):
121 # Wrapping an (exposed) Method; continue with Method._func
122 method = f
123 f = f._func
125 @wraps(f)
126 def wrapF(self, *args, **kwargs) -> str | bytes:
127 currReq = current.request.get()
128 if conf.debug.disable_cache or currReq.disableCache:
129 # Caching disabled
130 if conf.debug.disable_cache:
131 logging.debug("Caching is disabled by config")
132 return f(self, *args, **kwargs)
133 # How many arguments are part of the way to the function called (and how many are just *args)
134 offset = -len(currReq.args) or len(currReq.path_list)
135 path = "/" + "/".join(currReq.path_list[: offset])
136 if not path in urls:
137 # This path (possibly a sub-render) should not be cached
138 logging.info(f"No caching for {path}")
139 return f(self, *args, **kwargs)
140 key = keyFromArgs(f, userSensitive, languageSensitive, evaluatedArgs, path, args, kwargs)
141 if not key:
142 # Something is wrong (possibly the parameter-count)
143 # Let's call f, but we knew already that this will clash
144 return f(self, *args, **kwargs)
145 dbRes = db.Get(db.Key(viurCacheName, key))
146 if dbRes is not None:
147 if (
148 not maxCacheTime or dbRes["creationtime"] > utils.utcNow()
149 - utils.parse.timedelta(maxCacheTime)
150 ):
151 # We store it unlimited or the cache is fresh enough
152 logging.debug("This request was served from cache.")
153 currReq.response.headers['Content-Type'] = dbRes["content-type"]
154 return dbRes["data"]
155 # If we made it this far, the request wasn't cached or too old; we need to rebuild it
156 oldAccessLog = db.startDataAccessLog()
157 try:
158 res = f(self, *args, **kwargs)
159 finally:
160 accessedEntries = db.endDataAccessLog(oldAccessLog)
161 dbEntity = db.Entity(db.Key(viurCacheName, key))
162 dbEntity["data"] = res
163 dbEntity["creationtime"] = utils.utcNow()
164 dbEntity["path"] = path
165 dbEntity["content-type"] = currReq.response.headers['Content-Type']
166 dbEntity["accessedEntries"] = list(accessedEntries)
167 dbEntity.exclude_from_indexes = {"data", "content-type"} # save two DB-writes.
168 db.Put(dbEntity)
169 logging.debug("This request was a cache-miss. Cache has been updated.")
170 return res
172 if method is None:
173 return wrapF
174 else:
175 method._func = wrapF
176 return method
179def enableCache(urls: list[str], userSensitive: int = 0, languageSensitive: bool = False,
180 evaluatedArgs: list[str] | None = None, maxCacheTime: int | None = None):
181 """
182 Decorator to wrap this cache around a function. In order for this to function correctly, you must provide
183 additional information so ViUR can determine in which situations it's possible to re-use an already cached
184 result and when to call the wrapped function instead.
186 ..Warning: It's not possible to cache the result of a function relying on reading/modifying
187 the environment (ie. setting custom http-headers). The only exception is the content-type header which
188 will be stored along with the cached response.
190 :param urls: A list of urls for this function, for which the cache should be enabled.
191 A function can have several urls (eg. /page/view or /pdf/page/view), and it
192 might should not be cached under all urls (eg. /admin/page/view).
193 :param userSensitive: Signals wherever the output of f depends on the current user.
194 0 means independent of wherever the user is a guest or known, all will get the same content.
195 1 means cache only for guests, no cache will be performed if the user is logged-in.
196 2 means cache in two groups, one for guests and one for all users
197 3 will cache the result of that function for each individual users separately.
198 :param languageSensitive: If true, signals that the output of f might got translated.
199 If true, the result of that function is cached separately for each language.
200 :param evaluatedArgs: List of keyword-arguments having influence to the output generated by
201 that function. This list *must* be complete! Parameters not named here are ignored!
202 Warning: Double-check this list! F.e. if that function generates a list of entries and
203 you miss the parameter "order" here, it would be impossible to sort the list.
204 It would always have the ordering it had when the cache-entry was created.
205 :param maxCacheTime: Specifies the maximum time an entry stays in the cache in seconds.
206 Note: Its not erased from the db after that time, but it won't be served anymore.
207 If None, the cache stays valid forever (until manually erased by calling flushCache.
208 """
209 if evaluatedArgs is None:
210 evaluatedArgs = []
211 assert not any([x.startswith("_") for x in evaluatedArgs]), "A evaluated Parameter cannot start with an underscore!"
212 return lambda f: wrapCallable(f, urls, userSensitive, languageSensitive, evaluatedArgs, maxCacheTime)
215@tasks.CallDeferred
216def flushCache(prefix: str = None, key: db.Key | None = None, kind: str | None = None):
217 """
218 Flushes the cache. Its possible the flush only a part of the cache by specifying
219 the path-prefix. The path is equal to the url that caused it to be cached (eg /page/view) and must be one
220 listed in the 'url' param of :meth:`viur.core.cache.enableCache`.
222 :param prefix: Path or prefix that should be flushed.
223 :param key: Flush all cache entries which may contain this key. Also flushes entries
224 which executed a query over that kind.
225 :param kind: Flush all cache entries which executed a query over that kind.
227 Examples:
228 - "/" would flush the main page (and only that),
229 - "/*" everything from the cache, "/page/*" everything from the page-module (default render),
230 - and "/page/view/*" only that specific subset of the page-module.
231 """
232 if prefix is None and key is None and kind is None:
233 prefix = "/*"
234 if prefix is not None:
235 items = db.Query(viurCacheName).filter("path =", prefix.rstrip("*")).iter()
236 for item in items:
237 db.Delete(item)
238 if prefix.endswith("*"):
239 items = db.Query(viurCacheName) \
240 .filter("path >", prefix.rstrip("*")) \
241 .filter("path <", prefix.rstrip("*") + u"\ufffd") \
242 .iter()
243 for item in items:
244 db.Delete(item)
245 logging.debug(f"Flushing cache succeeded. Everything matching {prefix=} is gone.")
246 if key is not None:
247 items = db.Query(viurCacheName).filter("accessedEntries =", key).iter()
248 for item in items:
249 logging.info(f"""Deleted cache entry {item["path"]!r}""")
250 db.Delete(item.key)
251 if not isinstance(key, db.Key):
252 key = db.Key.from_legacy_urlsafe(key) # hopefully is a string
253 items = db.Query(viurCacheName).filter("accessedEntries =", key.kind).iter()
254 for item in items:
255 logging.info(f"""Deleted cache entry {item["path"]!r}""")
256 db.Delete(item.key)
257 if kind is not None:
258 items = db.Query(viurCacheName).filter("accessedEntries =", kind).iter()
259 for item in items:
260 logging.info(f"""Deleted cache entry {item["path"]!r}""")
261 db.Delete(item.key)
264__all__ = ["enableCache", "flushCache"]