From 27de977bfefe8844fb610f1988a7adbd63298bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn-Michael=20Miehe?= <40151420+ldericher@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:16:25 +0100 Subject: [PATCH] fix redis keying and de/serializing --- api/ovdashboard_api/core/dav/caldav.py | 24 +++++---------- api/ovdashboard_api/core/dav/helpers.py | 40 +++++++++++++++---------- api/ovdashboard_api/core/dav/webdav.py | 28 ++++------------- 3 files changed, 38 insertions(+), 54 deletions(-) diff --git a/api/ovdashboard_api/core/dav/caldav.py b/api/ovdashboard_api/core/dav/caldav.py index d85413f..3a0acb7 100644 --- a/api/ovdashboard_api/core/dav/caldav.py +++ b/api/ovdashboard_api/core/dav/caldav.py @@ -1,17 +1,16 @@ -import functools import logging from datetime import datetime, timedelta from typing import cast from asyncify import asyncify -from cachetools import TTLCache, cachedmethod +from cachetools import cachedmethod from caldav import Calendar, DAVClient, Event, Principal from vobject.base import Component, toVName from ..calevent import CalEvent from ..config import Config from ..settings import SETTINGS -from .webdav import davkey +from .helpers import REDIS, RedisCache, davkey _logger = logging.getLogger(__name__) @@ -23,9 +22,9 @@ class CalDAV: password=SETTINGS.caldav.password, ) - _cache = TTLCache( + _cache = RedisCache( + cache=REDIS, ttl=SETTINGS.caldav.cache_ttl, - maxsize=SETTINGS.caldav.cache_size, ) @classmethod @@ -40,10 +39,7 @@ class CalDAV: @classmethod @property @asyncify - @cachedmethod( - cache=lambda cls: cls._cache, - key=functools.partial(davkey, "calendars"), - ) + @cachedmethod(cache=lambda cls: cls._cache, key=davkey("calendars")) def calendars(cls) -> list[str]: """ Asynchroneously lists all calendars using the main WebDAV client. @@ -54,10 +50,7 @@ class CalDAV: @classmethod @asyncify - @cachedmethod( - cache=lambda cls: cls._cache, - key=functools.partial(davkey, "get_calendar"), - ) + @cachedmethod(cache=lambda cls: cls._cache, key=davkey("get_calendar")) def get_calendar(cls, calendar_name: str) -> Calendar: """ Get a calendar by name using the CalDAV principal object. @@ -67,10 +60,7 @@ class CalDAV: @classmethod @asyncify - @cachedmethod( - cache=lambda cls: cls._cache, - key=functools.partial(davkey, "get_events"), - ) + @cachedmethod(cache=lambda cls: cls._cache, key=davkey("get_events", slice(1, 2))) def get_events(cls, calendar_name: str, cfg: Config) -> list[CalEvent]: """ Get a sorted list of events by CalDAV calendar name. diff --git a/api/ovdashboard_api/core/dav/helpers.py b/api/ovdashboard_api/core/dav/helpers.py index e15617d..92fee55 100644 --- a/api/ovdashboard_api/core/dav/helpers.py +++ b/api/ovdashboard_api/core/dav/helpers.py @@ -1,18 +1,28 @@ -from json import JSONDecodeError +import pickle +from typing import Callable, Hashable import requests from cachetools.keys import hashkey from CacheToolsUtils import RedisCache as __RedisCache +from redis import Redis from redis.commands.core import ResponseT from redis.typing import EncodableT from webdav3.client import Client as __WebDAVclient +from ..settings import SETTINGS -def davkey(name, _, *args, **kwargs): - """Return a cache key for use with cached methods.""" - key = hashkey(name, *args, **kwargs) - return hashkey(*(str(key_item) for key_item in key)) +def davkey( + name: str, + slice: slice = slice(1, None), +) -> Callable[..., tuple[Hashable, ...]]: + def func(*args, **kwargs) -> tuple[Hashable, ...]: + """Return a cache key for use with cached methods.""" + + key = hashkey(name, *args[slice], **kwargs) + return hashkey(*(str(key_item) for key_item in key)) + + return func class WebDAVclient(__WebDAVclient): @@ -39,16 +49,16 @@ class RedisCache(__RedisCache): """ def _serialize(self, s) -> EncodableT: - if isinstance(s, bytes): - return s - - else: - return super()._serialize(s) + return pickle.dumps(s) def _deserialize(self, s: ResponseT): - try: - return super()._deserialize(s) + assert isinstance(s, bytes) + return pickle.loads(s) - except (UnicodeDecodeError, JSONDecodeError): - assert isinstance(s, bytes) - return s + +REDIS = Redis( + host=SETTINGS.redis.host, + port=SETTINGS.redis.port, + db=SETTINGS.redis.db, + protocol=SETTINGS.redis.protocol, +) diff --git a/api/ovdashboard_api/core/dav/webdav.py b/api/ovdashboard_api/core/dav/webdav.py index 4c316a5..c195674 100644 --- a/api/ovdashboard_api/core/dav/webdav.py +++ b/api/ovdashboard_api/core/dav/webdav.py @@ -1,14 +1,12 @@ -import functools import logging import re from io import BytesIO from asyncify import asyncify from cachetools import cachedmethod -from redis import Redis from ..settings import SETTINGS -from .helpers import RedisCache, WebDAVclient, davkey +from .helpers import REDIS, RedisCache, WebDAVclient, davkey _logger = logging.getLogger(__name__) @@ -23,21 +21,13 @@ class WebDAV: ) _cache = RedisCache( - cache=Redis( - host=SETTINGS.redis.host, - port=SETTINGS.redis.port, - db=SETTINGS.redis.db, - protocol=SETTINGS.redis.protocol, - ), + cache=REDIS, ttl=SETTINGS.webdav.cache_ttl, ) @classmethod @asyncify - @cachedmethod( - cache=lambda cls: cls._cache, - key=functools.partial(davkey, "list_files"), - ) + @cachedmethod(cache=lambda cls: cls._cache, key=davkey("list_files")) def list_files( cls, directory: str = "", @@ -55,10 +45,7 @@ class WebDAV: @classmethod @asyncify - @cachedmethod( - cache=lambda cls: cls._cache, - key=functools.partial(davkey, "exists"), - ) + @cachedmethod(cache=lambda cls: cls._cache, key=davkey("exists")) def exists(cls, path: str) -> bool: """ `True` iff there is a WebDAV resource at `path` @@ -69,10 +56,7 @@ class WebDAV: @classmethod @asyncify - @cachedmethod( - cache=lambda cls: cls._cache, - key=functools.partial(davkey, "read_bytes"), - ) + @cachedmethod(cache=lambda cls: cls._cache, key=davkey("read_bytes")) def read_bytes(cls, path: str) -> bytes: """ Load WebDAV file from `path` as bytes @@ -105,7 +89,7 @@ class WebDAV: cls._webdav_client.upload_to(buffer, path) # invalidate cache entry - cls._cache.pop(davkey("read_bytes", None, path)) + cls._cache.pop(davkey("read_bytes")(path)) @classmethod async def write_str(cls, path: str, content: str, encoding="utf-8") -> None: