tests versuch 2

venv/lib/python3.11/site-packages/cachelib/__init__.py

@@ -0,0 +1,20 @@
+from cachelib.base import BaseCache
+from cachelib.base import NullCache
+from cachelib.dynamodb import DynamoDbCache
+from cachelib.file import FileSystemCache
+from cachelib.memcached import MemcachedCache
+from cachelib.redis import RedisCache
+from cachelib.simple import SimpleCache
+from cachelib.uwsgi import UWSGICache
+
+__all__ = [
+    "BaseCache",
+    "NullCache",
+    "SimpleCache",
+    "FileSystemCache",
+    "MemcachedCache",
+    "RedisCache",
+    "UWSGICache",
+    "DynamoDbCache",
+]
+__version__ = "0.10.2"

venv/lib/python3.11/site-packages/cachelib/base.py

@@ -0,0 +1,185 @@
+import typing as _t
+
+
+class BaseCache:
+    """Baseclass for the cache systems.  All the cache systems implement this
+    API or a superset of it.
+
+    :param default_timeout: the default timeout (in seconds) that is used if
+                            no timeout is specified on :meth:`set`. A timeout
+                            of 0 indicates that the cache never expires.
+    """
+
+    def __init__(self, default_timeout: int = 300):
+        self.default_timeout = default_timeout
+
+    def _normalize_timeout(self, timeout: _t.Optional[int]) -> int:
+        if timeout is None:
+            timeout = self.default_timeout
+        return timeout
+
+    def get(self, key: str) -> _t.Any:
+        """Look up key in the cache and return the value for it.
+
+        :param key: the key to be looked up.
+        :returns: The value if it exists and is readable, else ``None``.
+        """
+        return None
+
+    def delete(self, key: str) -> bool:
+        """Delete `key` from the cache.
+
+        :param key: the key to delete.
+        :returns: Whether the key existed and has been deleted.
+        :rtype: boolean
+        """
+        return True
+
+    def get_many(self, *keys: str) -> _t.List[_t.Any]:
+        """Returns a list of values for the given keys.
+        For each key an item in the list is created::
+
+            foo, bar = cache.get_many("foo", "bar")
+
+        Has the same error handling as :meth:`get`.
+
+        :param keys: The function accepts multiple keys as positional
+                     arguments.
+        """
+        return [self.get(k) for k in keys]
+
+    def get_dict(self, *keys: str) -> _t.Dict[str, _t.Any]:
+        """Like :meth:`get_many` but return a dict::
+
+            d = cache.get_dict("foo", "bar")
+            foo = d["foo"]
+            bar = d["bar"]
+
+        :param keys: The function accepts multiple keys as positional
+                     arguments.
+        """
+        return dict(zip(keys, self.get_many(*keys)))  # noqa: B905
+
+    def set(
+        self, key: str, value: _t.Any, timeout: _t.Optional[int] = None
+    ) -> _t.Optional[bool]:
+        """Add a new key/value to the cache (overwrites value, if key already
+        exists in the cache).
+
+        :param key: the key to set
+        :param value: the value for the key
+        :param timeout: the cache timeout for the key in seconds (if not
+                        specified, it uses the default timeout). A timeout of
+                        0 indicates that the cache never expires.
+        :returns: ``True`` if key has been updated, ``False`` for backend
+                  errors. Pickling errors, however, will raise a subclass of
+                  ``pickle.PickleError``.
+        :rtype: boolean
+        """
+        return True
+
+    def add(self, key: str, value: _t.Any, timeout: _t.Optional[int] = None) -> bool:
+        """Works like :meth:`set` but does not overwrite the values of already
+        existing keys.
+
+        :param key: the key to set
+        :param value: the value for the key
+        :param timeout: the cache timeout for the key in seconds (if not
+                        specified, it uses the default timeout). A timeout of
+                        0 indicates that the cache never expires.
+        :returns: Same as :meth:`set`, but also ``False`` for already
+                  existing keys.
+        :rtype: boolean
+        """
+        return True
+
+    def set_many(
+        self, mapping: _t.Dict[str, _t.Any], timeout: _t.Optional[int] = None
+    ) -> _t.List[_t.Any]:
+        """Sets multiple keys and values from a mapping.
+
+        :param mapping: a mapping with the keys/values to set.
+        :param timeout: the cache timeout for the key in seconds (if not
+                        specified, it uses the default timeout). A timeout of
+                        0 indicates that the cache never expires.
+        :returns: A list containing all keys sucessfuly set
+        :rtype: boolean
+        """
+        set_keys = []
+        for key, value in mapping.items():
+            if self.set(key, value, timeout):
+                set_keys.append(key)
+        return set_keys
+
+    def delete_many(self, *keys: str) -> _t.List[_t.Any]:
+        """Deletes multiple keys at once.
+
+        :param keys: The function accepts multiple keys as positional
+                     arguments.
+        :returns: A list containing all sucessfuly deleted keys
+        :rtype: boolean
+        """
+        deleted_keys = []
+        for key in keys:
+            if self.delete(key):
+                deleted_keys.append(key)
+        return deleted_keys
+
+    def has(self, key: str) -> bool:
+        """Checks if a key exists in the cache without returning it. This is a
+        cheap operation that bypasses loading the actual data on the backend.
+
+        :param key: the key to check
+        """
+        raise NotImplementedError(
+            "%s doesn't have an efficient implementation of `has`. That "
+            "means it is impossible to check whether a key exists without "
+            "fully loading the key's data. Consider using `self.get` "
+            "explicitly if you don't care about performance."
+        )
+
+    def clear(self) -> bool:
+        """Clears the cache.  Keep in mind that not all caches support
+        completely clearing the cache.
+
+        :returns: Whether the cache has been cleared.
+        :rtype: boolean
+        """
+        return True
+
+    def inc(self, key: str, delta: int = 1) -> _t.Optional[int]:
+        """Increments the value of a key by `delta`.  If the key does
+        not yet exist it is initialized with `delta`.
+
+        For supporting caches this is an atomic operation.
+
+        :param key: the key to increment.
+        :param delta: the delta to add.
+        :returns: The new value or ``None`` for backend errors.
+        """
+        value = (self.get(key) or 0) + delta
+        return value if self.set(key, value) else None
+
+    def dec(self, key: str, delta: int = 1) -> _t.Optional[int]:
+        """Decrements the value of a key by `delta`.  If the key does
+        not yet exist it is initialized with `-delta`.
+
+        For supporting caches this is an atomic operation.
+
+        :param key: the key to increment.
+        :param delta: the delta to subtract.
+        :returns: The new value or `None` for backend errors.
+        """
+        value = (self.get(key) or 0) - delta
+        return value if self.set(key, value) else None
+
+
+class NullCache(BaseCache):
+    """A cache that doesn't cache.  This can be useful for unit testing.
+
+    :param default_timeout: a dummy parameter that is ignored but exists
+                            for API compatibility with other caches.
+    """
+
+    def has(self, key: str) -> bool:
+        return False

venv/lib/python3.11/site-packages/cachelib/dynamodb.py

@@ -0,0 +1,227 @@
+import datetime
+import typing as _t
+
+from cachelib.base import BaseCache
+from cachelib.serializers import DynamoDbSerializer
+
+CREATED_AT_FIELD = "created_at"
+RESPONSE_FIELD = "response"
+
+
+class DynamoDbCache(BaseCache):
+    """
+    Implementation of cachelib.BaseCache that uses an AWS DynamoDb table
+    as the backend.
+
+    Your server process will require dynamodb:GetItem and dynamodb:PutItem
+    IAM permissions on the cache table.
+
+    Limitations: DynamoDB table items are limited to 400 KB in size.  Since
+    this class stores cached items in a table, the max size of a cache entry
+    will be slightly less than 400 KB, since the cache key and expiration
+    time fields are also part of the item.
+
+    :param table_name: The name of the DynamoDB table to use
+    :param default_timeout: Set the timeout in seconds after which cache entries
+                            expire
+    :param key_field: The name of the hash_key attribute in the DynamoDb
+                      table. This must be a string attribute.
+    :param expiration_time_field: The name of the table attribute to store the
+                                  expiration time in.  This will be an int
+                                  attribute. The timestamp will be stored as
+                                  seconds past the epoch.  If you configure
+                                  this as the TTL field, then DynamoDB will
+                                  automatically delete expired entries.
+    :param key_prefix: A prefix that should be added to all keys.
+
+    """
+
+    serializer = DynamoDbSerializer()
+
+    def __init__(
+        self,
+        table_name: _t.Optional[str] = "python-cache",
+        default_timeout: int = 300,
+        key_field: _t.Optional[str] = "cache_key",
+        expiration_time_field: _t.Optional[str] = "expiration_time",
+        key_prefix: _t.Optional[str] = None,
+        **kwargs: _t.Any
+    ):
+        super().__init__(default_timeout)
+
+        try:
+            import boto3  # type: ignore
+        except ImportError as err:
+            raise RuntimeError("no boto3 module found") from err
+
+        self._table_name = table_name
+        self._key_field = key_field
+        self._expiration_time_field = expiration_time_field
+        self.key_prefix = key_prefix or ""
+        self._dynamo = boto3.resource("dynamodb", **kwargs)
+        self._attr = boto3.dynamodb.conditions.Attr
+
+        try:
+            self._table = self._dynamo.Table(table_name)
+            self._table.load()
+            # catch this exception (triggered if the table doesn't exist)
+        except Exception:
+            table = self._dynamo.create_table(
+                AttributeDefinitions=[
+                    {"AttributeName": key_field, "AttributeType": "S"}
+                ],
+                TableName=table_name,
+                KeySchema=[
+                    {"AttributeName": key_field, "KeyType": "HASH"},
+                ],
+                BillingMode="PAY_PER_REQUEST",
+            )
+            table.wait_until_exists()
+            dynamo = boto3.client("dynamodb", **kwargs)
+            dynamo.update_time_to_live(
+                TableName=table_name,
+                TimeToLiveSpecification={
+                    "Enabled": True,
+                    "AttributeName": expiration_time_field,
+                },
+            )
+            self._table = self._dynamo.Table(table_name)
+            self._table.load()
+
+    def _utcnow(self) -> _t.Any:
+        """Return a tz-aware UTC datetime representing the current time"""
+        return datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
+
+    def _get_item(self, key: str, attributes: _t.Optional[list] = None) -> _t.Any:
+        """
+        Get an item from the cache table, optionally limiting the returned
+        attributes.
+
+        :param key: The cache key of the item to fetch
+
+        :param attributes: An optional list of attributes to fetch.  If not
+                           given, all attributes are fetched.  The
+                           expiration_time field will always be added to the
+                           list of fetched attributes.
+        :return: The table item for key if it exists and is not expired, else
+                 None
+        """
+        kwargs = {}
+        if attributes:
+            if self._expiration_time_field not in attributes:
+                attributes = list(attributes) + [self._expiration_time_field]
+            kwargs = dict(ProjectionExpression=",".join(attributes))
+
+        response = self._table.get_item(Key={self._key_field: key}, **kwargs)
+        cache_item = response.get("Item")
+
+        if cache_item:
+            now = int(self._utcnow().timestamp())
+            if cache_item.get(self._expiration_time_field, now + 100) > now:
+                return cache_item
+
+        return None
+
+    def get(self, key: str) -> _t.Any:
+        """
+        Get a cache item
+
+        :param key: The cache key of the item to fetch
+        :return: cache value if not expired, else None
+        """
+        cache_item = self._get_item(self.key_prefix + key)
+        if cache_item:
+            response = cache_item[RESPONSE_FIELD]
+            value = self.serializer.loads(response)
+            return value
+        return None
+
+    def delete(self, key: str) -> bool:
+        """
+        Deletes an item from the cache.  This is a no-op if the item doesn't
+        exist
+
+        :param key: Key of the item to delete.
+        :return: True if the key existed and was deleted
+        """
+        try:
+
+            self._table.delete_item(
+                Key={self._key_field: self.key_prefix + key},
+                ConditionExpression=self._attr(self._key_field).exists(),
+            )
+            return True
+        except self._dynamo.meta.client.exceptions.ConditionalCheckFailedException:
+            return False
+
+    def _set(
+        self,
+        key: str,
+        value: _t.Any,
+        timeout: _t.Optional[int] = None,
+        overwrite: _t.Optional[bool] = True,
+    ) -> _t.Any:
+        """
+        Store a cache item, with the option to not overwrite existing items
+
+        :param key: Cache key to use
+        :param value: a serializable object
+        :param timeout: The timeout in seconds for the cached item, to override
+                        the default
+        :param overwrite: If true, overwrite any existing cache item with key.
+                          If false, the new value will only be stored if no
+                          non-expired cache item exists with key.
+        :return: True if the new item was stored.
+        """
+        timeout = self._normalize_timeout(timeout)
+        now = self._utcnow()
+
+        kwargs = {}
+        if not overwrite:
+            # Cause the put to fail if a non-expired item with this key
+            # already exists
+
+            cond = self._attr(self._key_field).not_exists() | self._attr(
+                self._expiration_time_field
+            ).lte(int(now.timestamp()))
+            kwargs = dict(ConditionExpression=cond)
+
+        try:
+            dump = self.serializer.dumps(value)
+            item = {
+                self._key_field: key,
+                CREATED_AT_FIELD: now.isoformat(),
+                RESPONSE_FIELD: dump,
+            }
+            if timeout > 0:
+                expiration_time = now + datetime.timedelta(seconds=timeout)
+                item[self._expiration_time_field] = int(expiration_time.timestamp())
+            self._table.put_item(Item=item, **kwargs)
+            return True
+        except Exception:
+            return False
+
+    def set(self, key: str, value: _t.Any, timeout: _t.Optional[int] = None) -> _t.Any:
+        return self._set(self.key_prefix + key, value, timeout=timeout, overwrite=True)
+
+    def add(self, key: str, value: _t.Any, timeout: _t.Optional[int] = None) -> _t.Any:
+        return self._set(self.key_prefix + key, value, timeout=timeout, overwrite=False)
+
+    def has(self, key: str) -> bool:
+        return (
+            self._get_item(self.key_prefix + key, [self._expiration_time_field])
+            is not None
+        )
+
+    def clear(self) -> bool:
+        paginator = self._dynamo.meta.client.get_paginator("scan")
+        pages = paginator.paginate(
+            TableName=self._table_name, ProjectionExpression=self._key_field
+        )
+
+        with self._table.batch_writer() as batch:
+            for page in pages:
+                for item in page["Items"]:
+                    batch.delete_item(Key=item)
+
+        return True

venv/lib/python3.11/site-packages/cachelib/file.py

@@ -0,0 +1,336 @@
+import errno
+import logging
+import os
+import platform
+import stat
+import struct
+import tempfile
+import typing as _t
+from contextlib import contextmanager
+from hashlib import md5
+from pathlib import Path
+from time import sleep
+from time import time
+
+from cachelib.base import BaseCache
+from cachelib.serializers import FileSystemSerializer
+
+
+class FileSystemCache(BaseCache):
+    """A cache that stores the items on the file system.  This cache depends
+    on being the only user of the `cache_dir`.  Make absolutely sure that
+    nobody but this cache stores files there or otherwise the cache will
+    randomly delete files therein.
+
+    :param cache_dir: the directory where cache files are stored.
+    :param threshold: the maximum number of items the cache stores before
+                      it starts deleting some. A threshold value of 0
+                      indicates no threshold.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`. A timeout of
+                            0 indicates that the cache never expires.
+    :param mode: the file mode wanted for the cache files, default 0600
+    :param hash_method: Default hashlib.md5. The hash method used to
+                        generate the filename for cached results.
+    """
+
+    #: used for temporary files by the FileSystemCache
+    _fs_transaction_suffix = ".__wz_cache"
+    #: keep amount of files in a cache element
+    _fs_count_file = "__wz_cache_count"
+
+    serializer = FileSystemSerializer()
+
+    def __init__(
+        self,
+        cache_dir: str,
+        threshold: int = 500,
+        default_timeout: int = 300,
+        mode: _t.Optional[int] = None,
+        hash_method: _t.Any = md5,
+    ):
+        BaseCache.__init__(self, default_timeout)
+        self._path = cache_dir
+        self._threshold = threshold
+        self._hash_method = hash_method
+
+        # Mode set by user takes precedence. If no mode has
+        # been given, we need to set the correct default based
+        # on user platform.
+        self._mode = mode
+        if self._mode is None:
+            self._mode = self._get_compatible_platform_mode()
+
+        try:
+            os.makedirs(self._path)
+        except OSError as ex:
+            if ex.errno != errno.EEXIST:
+                raise
+
+        # If there are many files and a zero threshold,
+        # the list_dir can slow initialisation massively
+        if self._threshold != 0:
+            self._update_count(value=len(list(self._list_dir())))
+
+    def _get_compatible_platform_mode(self) -> int:
+        mode = 0o600  # nix systems
+        if platform.system() == "Windows":
+            mode = stat.S_IWRITE
+        return mode
+
+    @property
+    def _file_count(self) -> int:
+        return self.get(self._fs_count_file) or 0
+
+    def _update_count(
+        self, delta: _t.Optional[int] = None, value: _t.Optional[int] = None
+    ) -> None:
+        # If we have no threshold, don't count files
+        if self._threshold == 0:
+            return
+        if delta:
+            new_count = self._file_count + delta
+        else:
+            new_count = value or 0
+        self.set(self._fs_count_file, new_count, mgmt_element=True)
+
+    def _normalize_timeout(self, timeout: _t.Optional[int]) -> int:
+        timeout = BaseCache._normalize_timeout(self, timeout)
+        if timeout != 0:
+            timeout = int(time()) + timeout
+        return int(timeout)
+
+    def _is_mgmt(self, name: str) -> bool:
+        fshash = self._get_filename(self._fs_count_file).split(os.sep)[-1]
+        return name == fshash or name.endswith(self._fs_transaction_suffix)
+
+    def _list_dir(self) -> _t.Generator[str, None, None]:
+        """return a list of (fully qualified) cache filenames"""
+        return (
+            os.path.join(self._path, fn)
+            for fn in os.listdir(self._path)
+            if not self._is_mgmt(fn)
+        )
+
+    def _over_threshold(self) -> bool:
+        return self._threshold != 0 and self._file_count > self._threshold
+
+    def _remove_expired(self, now: float) -> None:
+        for fname in self._list_dir():
+            try:
+                with self._safe_stream_open(fname, "rb") as f:
+                    expires = struct.unpack("I", f.read(4))[0]
+                if expires != 0 and expires < now:
+                    os.remove(fname)
+                    self._update_count(delta=-1)
+            except FileNotFoundError:
+                pass
+            except (OSError, EOFError, struct.error):
+                logging.warning(
+                    "Exception raised while handling cache file '%s'",
+                    fname,
+                    exc_info=True,
+                )
+
+    def _remove_older(self) -> bool:
+        exp_fname_tuples = []
+        for fname in self._list_dir():
+            try:
+                with self._safe_stream_open(fname, "rb") as f:
+                    timestamp = struct.unpack("I", f.read(4))[0]
+                    exp_fname_tuples.append((timestamp, fname))
+            except FileNotFoundError:
+                pass
+            except (OSError, EOFError, struct.error):
+                logging.warning(
+                    "Exception raised while handling cache file '%s'",
+                    fname,
+                    exc_info=True,
+                )
+        fname_sorted = (
+            fname
+            for _, fname in sorted(
+                exp_fname_tuples, key=lambda item: item[0]  # type: ignore
+            )
+        )
+        for fname in fname_sorted:
+            try:
+                os.remove(fname)
+                self._update_count(delta=-1)
+            except FileNotFoundError:
+                pass
+            except OSError:
+                logging.warning(
+                    "Exception raised while handling cache file '%s'",
+                    fname,
+                    exc_info=True,
+                )
+                return False
+            if not self._over_threshold():
+                break
+        return True
+
+    def _prune(self) -> None:
+        if self._over_threshold():
+            now = time()
+            self._remove_expired(now)
+        # if still over threshold
+        if self._over_threshold():
+            self._remove_older()
+
+    def clear(self) -> bool:
+        for i, fname in enumerate(self._list_dir()):
+            try:
+                os.remove(fname)
+            except FileNotFoundError:
+                pass
+            except OSError:
+                logging.warning(
+                    "Exception raised while handling cache file '%s'",
+                    fname,
+                    exc_info=True,
+                )
+                self._update_count(delta=-i)
+                return False
+        self._update_count(value=0)
+        return True
+
+    def _get_filename(self, key: str) -> str:
+        if isinstance(key, str):
+            bkey = key.encode("utf-8")  # XXX unicode review
+            bkey_hash = self._hash_method(bkey).hexdigest()
+        else:
+            raise TypeError(f"Key must be a string, received type {type(key)}")
+        return os.path.join(self._path, bkey_hash)
+
+    def get(self, key: str) -> _t.Any:
+        filename = self._get_filename(key)
+        try:
+            with self._safe_stream_open(filename, "rb") as f:
+                pickle_time = struct.unpack("I", f.read(4))[0]
+                if pickle_time == 0 or pickle_time >= time():
+                    return self.serializer.load(f)
+        except FileNotFoundError:
+            pass
+        except (OSError, EOFError, struct.error):
+            logging.warning(
+                "Exception raised while handling cache file '%s'",
+                filename,
+                exc_info=True,
+            )
+        return None
+
+    def add(self, key: str, value: _t.Any, timeout: _t.Optional[int] = None) -> bool:
+        filename = self._get_filename(key)
+        if not os.path.exists(filename):
+            return self.set(key, value, timeout)
+        return False
+
+    def set(
+        self,
+        key: str,
+        value: _t.Any,
+        timeout: _t.Optional[int] = None,
+        mgmt_element: bool = False,
+    ) -> bool:
+        # Management elements have no timeout
+        if mgmt_element:
+            timeout = 0
+        # Don't prune on management element update, to avoid loop
+        else:
+            self._prune()
+
+        timeout = self._normalize_timeout(timeout)
+        filename = self._get_filename(key)
+        overwrite = os.path.isfile(filename)
+
+        try:
+            fd, tmp = tempfile.mkstemp(
+                suffix=self._fs_transaction_suffix, dir=self._path
+            )
+            with os.fdopen(fd, "wb") as f:
+                f.write(struct.pack("I", timeout))
+                self.serializer.dump(value, f)
+
+            self._run_safely(os.replace, tmp, filename)
+            self._run_safely(os.chmod, filename, self._mode)
+
+            fsize = Path(filename).stat().st_size
+        except OSError:
+            logging.warning(
+                "Exception raised while handling cache file '%s'",
+                filename,
+                exc_info=True,
+            )
+            return False
+        else:
+            # Management elements should not count towards threshold
+            if not overwrite and not mgmt_element:
+                self._update_count(delta=1)
+            return fsize > 0  # function should fail if file is empty
+
+    def delete(self, key: str, mgmt_element: bool = False) -> bool:
+        try:
+            os.remove(self._get_filename(key))
+        except FileNotFoundError:  # if file doesn't exist we consider it deleted
+            return True
+        except OSError:
+            logging.warning("Exception raised while handling cache file", exc_info=True)
+            return False
+        else:
+            # Management elements should not count towards threshold
+            if not mgmt_element:
+                self._update_count(delta=-1)
+            return True
+
+    def has(self, key: str) -> bool:
+        filename = self._get_filename(key)
+        try:
+            with self._safe_stream_open(filename, "rb") as f:
+                pickle_time = struct.unpack("I", f.read(4))[0]
+                if pickle_time == 0 or pickle_time >= time():
+                    return True
+                else:
+                    return False
+        except FileNotFoundError:  # if there is no file there is no key
+            return False
+        except (OSError, EOFError, struct.error):
+            logging.warning(
+                "Exception raised while handling cache file '%s'",
+                filename,
+                exc_info=True,
+            )
+            return False
+
+    def _run_safely(self, fn: _t.Callable, *args: _t.Any, **kwargs: _t.Any) -> _t.Any:
+        """On Windows os.replace, os.chmod and open can yield
+        permission errors if executed by two different processes."""
+        if platform.system() == "Windows":
+            output = None
+            wait_step = 0.001
+            max_sleep_time = 10.0
+            total_sleep_time = 0.0
+
+            while total_sleep_time < max_sleep_time:
+                try:
+                    output = fn(*args, **kwargs)
+                except PermissionError:
+                    sleep(wait_step)
+                    total_sleep_time += wait_step
+                    wait_step *= 2
+                else:
+                    break
+        else:
+            output = fn(*args, **kwargs)
+
+        return output
+
+    @contextmanager
+    def _safe_stream_open(self, path: str, mode: str) -> _t.Generator:
+        fs = self._run_safely(open, path, mode)
+        if fs is None:
+            raise OSError
+        try:
+            yield fs
+        finally:
+            fs.close()

venv/lib/python3.11/site-packages/cachelib/memcached.py

@@ -0,0 +1,197 @@
+import re
+import typing as _t
+from time import time
+
+from cachelib.base import BaseCache
+
+
+_test_memcached_key = re.compile(r"[^\x00-\x21\xff]{1,250}$").match
+
+
+class MemcachedCache(BaseCache):
+
+    """A cache that uses memcached as backend.
+
+    The first argument can either be an object that resembles the API of a
+    :class:`memcache.Client` or a tuple/list of server addresses. In the
+    event that a tuple/list is passed, Werkzeug tries to import the best
+    available memcache library.
+
+    This cache looks into the following packages/modules to find bindings for
+    memcached:
+
+        - ``pylibmc``
+        - ``google.appengine.api.memcached``
+        - ``memcached``
+        - ``libmc``
+
+    Implementation notes:  This cache backend works around some limitations in
+    memcached to simplify the interface.  For example unicode keys are encoded
+    to utf-8 on the fly.  Methods such as :meth:`~BaseCache.get_dict` return
+    the keys in the same format as passed.  Furthermore all get methods
+    silently ignore key errors to not cause problems when untrusted user data
+    is passed to the get methods which is often the case in web applications.
+
+    :param servers: a list or tuple of server addresses or alternatively
+                    a :class:`memcache.Client` or a compatible client.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`. A timeout of
+                            0 indicates that the cache never expires.
+    :param key_prefix: a prefix that is added before all keys.  This makes it
+                       possible to use the same memcached server for different
+                       applications.  Keep in mind that
+                       :meth:`~BaseCache.clear` will also clear keys with a
+                       different prefix.
+    """
+
+    def __init__(
+        self,
+        servers: _t.Any = None,
+        default_timeout: int = 300,
+        key_prefix: _t.Optional[str] = None,
+    ):
+        BaseCache.__init__(self, default_timeout)
+        if servers is None or isinstance(servers, (list, tuple)):
+            if servers is None:
+                servers = ["127.0.0.1:11211"]
+            self._client = self.import_preferred_memcache_lib(servers)
+            if self._client is None:
+                raise RuntimeError("no memcache module found")
+        else:
+            # NOTE: servers is actually an already initialized memcache
+            # client.
+            self._client = servers
+
+        self.key_prefix = key_prefix
+
+    def _normalize_key(self, key: str) -> str:
+        if self.key_prefix:
+            key = self.key_prefix + key
+        return key
+
+    def _normalize_timeout(self, timeout: _t.Optional[int]) -> int:
+        timeout = BaseCache._normalize_timeout(self, timeout)
+        if timeout > 0:
+            timeout = int(time()) + timeout
+        return timeout
+
+    def get(self, key: str) -> _t.Any:
+        key = self._normalize_key(key)
+        # memcached doesn't support keys longer than that.  Because often
+        # checks for so long keys can occur because it's tested from user
+        # submitted data etc we fail silently for getting.
+        if _test_memcached_key(key):
+            return self._client.get(key)
+
+    def get_dict(self, *keys: str) -> _t.Dict[str, _t.Any]:
+        key_mapping = {}
+        for key in keys:
+            encoded_key = self._normalize_key(key)
+            if _test_memcached_key(key):
+                key_mapping[encoded_key] = key
+        _keys = list(key_mapping)
+        d = rv = self._client.get_multi(_keys)  # type: _t.Dict[str, _t.Any]
+        if self.key_prefix:
+            rv = {}
+            for key, value in d.items():
+                rv[key_mapping[key]] = value
+        if len(rv) < len(keys):
+            for key in keys:
+                if key not in rv:
+                    rv[key] = None
+        return rv
+
+    def add(self, key: str, value: _t.Any, timeout: _t.Optional[int] = None) -> bool:
+        key = self._normalize_key(key)
+        timeout = self._normalize_timeout(timeout)
+        return bool(self._client.add(key, value, timeout))
+
+    def set(
+        self, key: str, value: _t.Any, timeout: _t.Optional[int] = None
+    ) -> _t.Optional[bool]:
+        key = self._normalize_key(key)
+        timeout = self._normalize_timeout(timeout)
+        return bool(self._client.set(key, value, timeout))
+
+    def get_many(self, *keys: str) -> _t.List[_t.Any]:
+        d = self.get_dict(*keys)
+        return [d[key] for key in keys]
+
+    def set_many(
+        self, mapping: _t.Dict[str, _t.Any], timeout: _t.Optional[int] = None
+    ) -> _t.List[_t.Any]:
+        new_mapping = {}
+        for key, value in mapping.items():
+            key = self._normalize_key(key)
+            new_mapping[key] = value
+
+        timeout = self._normalize_timeout(timeout)
+        failed_keys = self._client.set_multi(
+            new_mapping, timeout
+        )  # type: _t.List[_t.Any]
+        k_normkey = zip(mapping.keys(), new_mapping.keys())  # noqa: B905
+        return [k for k, nkey in k_normkey if nkey not in failed_keys]
+
+    def delete(self, key: str) -> bool:
+        key = self._normalize_key(key)
+        if _test_memcached_key(key):
+            return bool(self._client.delete(key))
+        return False
+
+    def delete_many(self, *keys: str) -> _t.List[_t.Any]:
+        new_keys = []
+        for key in keys:
+            key = self._normalize_key(key)
+            if _test_memcached_key(key):
+                new_keys.append(key)
+        self._client.delete_multi(new_keys)
+        return [k for k in new_keys if not self.has(k)]
+
+    def has(self, key: str) -> bool:
+        key = self._normalize_key(key)
+        if _test_memcached_key(key):
+            return bool(self._client.append(key, ""))
+        return False
+
+    def clear(self) -> bool:
+        return bool(self._client.flush_all())
+
+    def inc(self, key: str, delta: int = 1) -> _t.Optional[int]:
+        key = self._normalize_key(key)
+        value = (self._client.get(key) or 0) + delta
+        return value if self.set(key, value) else None
+
+    def dec(self, key: str, delta: int = 1) -> _t.Optional[int]:
+        key = self._normalize_key(key)
+        value = (self._client.get(key) or 0) - delta
+        return value if self.set(key, value) else None
+
+    def import_preferred_memcache_lib(self, servers: _t.Any) -> _t.Any:
+        """Returns an initialized memcache client.  Used by the constructor."""
+        try:
+            import pylibmc  # type: ignore
+        except ImportError:
+            pass
+        else:
+            return pylibmc.Client(servers)
+
+        try:
+            from google.appengine.api import memcache  # type: ignore
+        except ImportError:
+            pass
+        else:
+            return memcache.Client()
+
+        try:
+            import memcache  # type: ignore
+        except ImportError:
+            pass
+        else:
+            return memcache.Client(servers)
+
+        try:
+            import libmc  # type: ignore
+        except ImportError:
+            pass
+        else:
+            return libmc.Client(servers)

venv/lib/python3.11/site-packages/cachelib/redis.py

@@ -0,0 +1,149 @@
+import typing as _t
+
+from cachelib.base import BaseCache
+from cachelib.serializers import RedisSerializer
+
+
+class RedisCache(BaseCache):
+    """Uses the Redis key-value store as a cache backend.
+
+    The first argument can be either a string denoting address of the Redis
+    server or an object resembling an instance of a redis.Redis class.
+
+    Note: Python Redis API already takes care of encoding unicode strings on
+    the fly.
+
+    :param host: address of the Redis server or an object which API is
+                 compatible with the official Python Redis client (redis-py).
+    :param port: port number on which Redis server listens for connections.
+    :param password: password authentication for the Redis server.
+    :param db: db (zero-based numeric index) on Redis Server to connect.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`. A timeout of
+                            0 indicates that the cache never expires.
+    :param key_prefix: A prefix that should be added to all keys.
+
+    Any additional keyword arguments will be passed to ``redis.Redis``.
+    """
+
+    _read_client: _t.Any = None
+    _write_client: _t.Any = None
+    serializer = RedisSerializer()
+
+    def __init__(
+        self,
+        host: _t.Any = "localhost",
+        port: int = 6379,
+        password: _t.Optional[str] = None,
+        db: int = 0,
+        default_timeout: int = 300,
+        key_prefix: _t.Optional[str] = None,
+        **kwargs: _t.Any
+    ):
+        BaseCache.__init__(self, default_timeout)
+        if host is None:
+            raise ValueError("RedisCache host parameter may not be None")
+        if isinstance(host, str):
+            try:
+                import redis
+            except ImportError as err:
+                raise RuntimeError("no redis module found") from err
+            if kwargs.get("decode_responses", None):
+                raise ValueError("decode_responses is not supported by RedisCache.")
+            self._write_client = self._read_client = redis.Redis(
+                host=host, port=port, password=password, db=db, **kwargs
+            )
+        else:
+            self._read_client = self._write_client = host
+        self.key_prefix = key_prefix or ""
+
+    def _normalize_timeout(self, timeout: _t.Optional[int]) -> int:
+        """Normalize timeout by setting it to default of 300 if
+        not defined (None) or -1 if explicitly set to zero.
+
+        :param timeout: timeout to normalize.
+        """
+        timeout = BaseCache._normalize_timeout(self, timeout)
+        if timeout == 0:
+            timeout = -1
+        return timeout
+
+    def get(self, key: str) -> _t.Any:
+        return self.serializer.loads(self._read_client.get(self.key_prefix + key))
+
+    def get_many(self, *keys: str) -> _t.List[_t.Any]:
+        if self.key_prefix:
+            prefixed_keys = [self.key_prefix + key for key in keys]
+        else:
+            prefixed_keys = list(keys)
+        return [self.serializer.loads(x) for x in self._read_client.mget(prefixed_keys)]
+
+    def set(self, key: str, value: _t.Any, timeout: _t.Optional[int] = None) -> _t.Any:
+        timeout = self._normalize_timeout(timeout)
+        dump = self.serializer.dumps(value)
+        if timeout == -1:
+            result = self._write_client.set(name=self.key_prefix + key, value=dump)
+        else:
+            result = self._write_client.setex(
+                name=self.key_prefix + key, value=dump, time=timeout
+            )
+        return result
+
+    def add(self, key: str, value: _t.Any, timeout: _t.Optional[int] = None) -> _t.Any:
+        timeout = self._normalize_timeout(timeout)
+        dump = self.serializer.dumps(value)
+        created = self._write_client.setnx(name=self.key_prefix + key, value=dump)
+        # handle case where timeout is explicitly set to zero
+        if created and timeout != -1:
+            self._write_client.expire(name=self.key_prefix + key, time=timeout)
+        return created
+
+    def set_many(
+        self, mapping: _t.Dict[str, _t.Any], timeout: _t.Optional[int] = None
+    ) -> _t.List[_t.Any]:
+        timeout = self._normalize_timeout(timeout)
+        # Use transaction=False to batch without calling redis MULTI
+        # which is not supported by twemproxy
+        pipe = self._write_client.pipeline(transaction=False)
+
+        for key, value in mapping.items():
+            dump = self.serializer.dumps(value)
+            if timeout == -1:
+                pipe.set(name=self.key_prefix + key, value=dump)
+            else:
+                pipe.setex(name=self.key_prefix + key, value=dump, time=timeout)
+        results = pipe.execute()
+        res = zip(mapping.keys(), results)  # noqa: B905
+        return [k for k, was_set in res if was_set]
+
+    def delete(self, key: str) -> bool:
+        return bool(self._write_client.delete(self.key_prefix + key))
+
+    def delete_many(self, *keys: str) -> _t.List[_t.Any]:
+        if not keys:
+            return []
+        if self.key_prefix:
+            prefixed_keys = [self.key_prefix + key for key in keys]
+        else:
+            prefixed_keys = [k for k in keys]
+        self._write_client.delete(*prefixed_keys)
+        return [k for k in prefixed_keys if not self.has(k)]
+
+    def has(self, key: str) -> bool:
+        return bool(self._read_client.exists(self.key_prefix + key))
+
+    def clear(self) -> bool:
+        status = 0
+        if self.key_prefix:
+            keys = self._read_client.keys(self.key_prefix + "*")
+            if keys:
+                status = self._write_client.delete(*keys)
+        else:
+            status = self._write_client.flushdb()
+        return bool(status)
+
+    def inc(self, key: str, delta: int = 1) -> _t.Any:
+        return self._write_client.incr(name=self.key_prefix + key, amount=delta)
+
+    def dec(self, key: str, delta: int = 1) -> _t.Any:
+        return self._write_client.incr(name=self.key_prefix + key, amount=-delta)

venv/lib/python3.11/site-packages/cachelib/serializers.py

@@ -0,0 +1,114 @@
+import logging
+import pickle
+import typing as _t
+
+
+class BaseSerializer:
+    """This is the base interface for all default serializers.
+
+    BaseSerializer.load and BaseSerializer.dump will
+    default to pickle.load and pickle.dump. This is currently
+    used only by FileSystemCache which dumps/loads to/from a file stream.
+    """
+
+    def _warn(self, e: pickle.PickleError) -> None:
+        logging.warning(
+            f"An exception has been raised during a pickling operation: {e}"
+        )
+
+    def dump(
+        self, value: int, f: _t.IO, protocol: int = pickle.HIGHEST_PROTOCOL
+    ) -> None:
+        try:
+            pickle.dump(value, f, protocol)
+        except (pickle.PickleError, pickle.PicklingError) as e:
+            self._warn(e)
+
+    def load(self, f: _t.BinaryIO) -> _t.Any:
+        try:
+            data = pickle.load(f)
+        except pickle.PickleError as e:
+            self._warn(e)
+            return None
+        else:
+            return data
+
+    """BaseSerializer.loads and BaseSerializer.dumps
+    work on top of pickle.loads and pickle.dumps. Dumping/loading
+    strings and byte strings is the default for most cache types.
+    """
+
+    def dumps(self, value: _t.Any, protocol: int = pickle.HIGHEST_PROTOCOL) -> bytes:
+        try:
+            serialized = pickle.dumps(value, protocol)
+        except (pickle.PickleError, pickle.PicklingError) as e:
+            self._warn(e)
+        return serialized
+
+    def loads(self, bvalue: bytes) -> _t.Any:
+        try:
+            data = pickle.loads(bvalue)
+        except pickle.PickleError as e:
+            self._warn(e)
+            return None
+        else:
+            return data
+
+
+"""Default serializers for each cache type.
+
+The following classes can be used to further customize
+serialiation behaviour. Alternatively, any serializer can be
+overriden in order to use a custom serializer with a different
+strategy altogether.
+"""
+
+
+class UWSGISerializer(BaseSerializer):
+    """Default serializer for UWSGICache."""
+
+
+class SimpleSerializer(BaseSerializer):
+    """Default serializer for SimpleCache."""
+
+
+class FileSystemSerializer(BaseSerializer):
+    """Default serializer for FileSystemCache."""
+
+
+class RedisSerializer(BaseSerializer):
+    """Default serializer for RedisCache."""
+
+    def dumps(self, value: _t.Any, protocol: int = pickle.HIGHEST_PROTOCOL) -> bytes:
+        """Dumps an object into a string for redis. By default it serializes
+        integers as regular string and pickle dumps everything else.
+        """
+        return b"!" + pickle.dumps(value, protocol)
+
+    def loads(self, value: _t.Optional[bytes]) -> _t.Any:
+        """The reversal of :meth:`dump_object`. This might be called with
+        None.
+        """
+        if value is None:
+            return None
+        if value.startswith(b"!"):
+            try:
+                return pickle.loads(value[1:])
+            except pickle.PickleError:
+                return None
+        try:
+            return int(value)
+        except ValueError:
+            # before 0.8 we did not have serialization. Still support that.
+            return value
+
+
+class DynamoDbSerializer(RedisSerializer):
+    """Default serializer for DynamoDbCache."""
+
+    def loads(self, value: _t.Any) -> _t.Any:
+        """The reversal of :meth:`dump_object`. This might be called with
+        None.
+        """
+        value = value.value
+        return super().loads(value)

venv/lib/python3.11/site-packages/cachelib/simple.py

@@ -0,0 +1,104 @@
+import typing as _t
+from time import time
+
+from cachelib.base import BaseCache
+from cachelib.serializers import SimpleSerializer
+
+
+class SimpleCache(BaseCache):
+
+    """Simple memory cache for single process environments.  This class exists
+    mainly for the development server and is not 100% thread safe.  It tries
+    to use as many atomic operations as possible and no locks for simplicity
+    but it could happen under heavy load that keys are added multiple times.
+
+    :param threshold: the maximum number of items the cache stores before
+                      it starts deleting some.
+    :param default_timeout: the default timeout that is used if no timeout is
+                            specified on :meth:`~BaseCache.set`. A timeout of
+                            0 indicates that the cache never expires.
+    """
+
+    serializer = SimpleSerializer()
+
+    def __init__(
+        self,
+        threshold: int = 500,
+        default_timeout: int = 300,
+    ):
+        BaseCache.__init__(self, default_timeout)
+        self._cache: _t.Dict[str, _t.Any] = {}
+        self._threshold = threshold or 500  # threshold = 0
+
+    def _over_threshold(self) -> bool:
+        return len(self._cache) > self._threshold
+
+    def _remove_expired(self, now: float) -> None:
+        toremove = [k for k, (expires, _) in self._cache.items() if expires < now]
+        for k in toremove:
+            self._cache.pop(k, None)
+
+    def _remove_older(self) -> None:
+        k_ordered = (
+            k
+            for k, v in sorted(
+                self._cache.items(), key=lambda item: item[1][0]  # type: ignore
+            )
+        )
+        for k in k_ordered:
+            self._cache.pop(k, None)
+            if not self._over_threshold():
+                break
+
+    def _prune(self) -> None:
+        if self._over_threshold():
+            now = time()
+            self._remove_expired(now)
+        # remove older items if still over threshold
+        if self._over_threshold():
+            self._remove_older()
+
+    def _normalize_timeout(self, timeout: _t.Optional[int]) -> int:
+        timeout = BaseCache._normalize_timeout(self, timeout)
+        if timeout > 0:
+            timeout = int(time()) + timeout
+        return timeout
+
+    def get(self, key: str) -> _t.Any:
+        try:
+            expires, value = self._cache[key]
+            if expires == 0 or expires > time():
+                return self.serializer.loads(value)
+        except KeyError:
+            return None
+
+    def set(
+        self, key: str, value: _t.Any, timeout: _t.Optional[int] = None
+    ) -> _t.Optional[bool]:
+        expires = self._normalize_timeout(timeout)
+        self._prune()
+        self._cache[key] = (expires, self.serializer.dumps(value))
+        return True
+
+    def add(self, key: str, value: _t.Any, timeout: _t.Optional[int] = None) -> bool:
+        expires = self._normalize_timeout(timeout)
+        self._prune()
+        item = (expires, self.serializer.dumps(value))
+        if key in self._cache:
+            return False
+        self._cache.setdefault(key, item)
+        return True
+
+    def delete(self, key: str) -> bool:
+        return self._cache.pop(key, None) is not None
+
+    def has(self, key: str) -> bool:
+        try:
+            expires, value = self._cache[key]
+            return bool(expires == 0 or expires > time())
+        except KeyError:
+            return False
+
+    def clear(self) -> bool:
+        self._cache.clear()
+        return not bool(self._cache)

venv/lib/python3.11/site-packages/cachelib/uwsgi.py

@@ -0,0 +1,83 @@
+import platform
+import typing as _t
+
+from cachelib.base import BaseCache
+from cachelib.serializers import UWSGISerializer
+
+
+class UWSGICache(BaseCache):
+    """Implements the cache using uWSGI's caching framework.
+
+    .. note::
+        This class cannot be used when running under PyPy, because the uWSGI
+        API implementation for PyPy is lacking the needed functionality.
+
+    :param default_timeout: The default timeout in seconds.
+    :param cache: The name of the caching instance to connect to, for
+        example: mycache@localhost:3031, defaults to an empty string, which
+        means uWSGI will cache in the local instance. If the cache is in the
+        same instance as the werkzeug app, you only have to provide the name of
+        the cache.
+    """
+
+    serializer = UWSGISerializer()
+
+    def __init__(
+        self,
+        default_timeout: int = 300,
+        cache: str = "",
+    ):
+        BaseCache.__init__(self, default_timeout)
+
+        if platform.python_implementation() == "PyPy":
+            raise RuntimeError(
+                "uWSGI caching does not work under PyPy, see "
+                "the docs for more details."
+            )
+
+        try:
+            import uwsgi  # type: ignore
+
+            self._uwsgi = uwsgi
+        except ImportError as err:
+            raise RuntimeError(
+                "uWSGI could not be imported, are you running under uWSGI?"
+            ) from err
+
+        self.cache = cache
+
+    def get(self, key: str) -> _t.Any:
+        rv = self._uwsgi.cache_get(key, self.cache)
+        if rv is None:
+            return
+        return self.serializer.loads(rv)
+
+    def delete(self, key: str) -> bool:
+        return bool(self._uwsgi.cache_del(key, self.cache))
+
+    def set(
+        self, key: str, value: _t.Any, timeout: _t.Optional[int] = None
+    ) -> _t.Optional[bool]:
+        result = self._uwsgi.cache_update(
+            key,
+            self.serializer.dumps(value),
+            self._normalize_timeout(timeout),
+            self.cache,
+        )  # type: bool
+        return result
+
+    def add(self, key: str, value: _t.Any, timeout: _t.Optional[int] = None) -> bool:
+        return bool(
+            self._uwsgi.cache_set(
+                key,
+                self.serializer.dumps(value),
+                self._normalize_timeout(timeout),
+                self.cache,
+            )
+        )
+
+    def clear(self) -> bool:
+        return bool(self._uwsgi.cache_clear(self.cache))
+
+    def has(self, key: str) -> bool:
+        return self._uwsgi.cache_exists(key, self.cache) is not None