From 2c302a2c16bce393f6002331cb703a805bc92f77 Mon Sep 17 00:00:00 2001 From: Alexandre Teles Date: Sun, 4 Sep 2022 02:32:25 -0300 Subject: [PATCH] implements a crude internal cache --- config.toml | 5 ++++ main.py | 4 +-- modules/InternalCache.py | 38 +++++++++++++++++++++++++++ modules/Releases.py | 55 ++++++++++++++++++++++++---------------- poetry.lock | 18 ++++++------- pyproject.toml | 2 ++ requirements.txt | 6 ++--- 7 files changed, 92 insertions(+), 36 deletions(-) create mode 100644 modules/InternalCache.py diff --git a/config.toml b/config.toml index 4327c3a..c4460ec 100644 --- a/config.toml +++ b/config.toml @@ -46,6 +46,11 @@ limit = "15/minute" [cache] expire = 60 +database = 0 + +[internal-cache] +expire = 300 +database = 1 [app] diff --git a/main.py b/main.py index 63cc660..cf4e93e 100755 --- a/main.py +++ b/main.py @@ -26,7 +26,7 @@ config: dict = toml.load("config.toml") redis_config: dict[ str, str | int ] = { "url": f"redis://{os.environ['REDIS_URL']}", "port": os.environ['REDIS_PORT'], - "collection": 0 + "database": config['cache']['database'], } # Create releases instance @@ -102,7 +102,7 @@ async def contributors(request: Request, response: Response) -> dict: @app.on_event("startup") async def startup() -> None: - redis_url = f"{redis_config['url']}:{redis_config['port']}/{redis_config['collection']}" + redis_url = f"{redis_config['url']}:{redis_config['port']}/{redis_config['database']}" redis = aioredis.from_url(redis_url, encoding="utf8", decode_responses=True) FastAPICache.init(RedisBackend(redis), prefix="fastapi-cache") diff --git a/modules/InternalCache.py b/modules/InternalCache.py new file mode 100644 index 0000000..a1994a9 --- /dev/null +++ b/modules/InternalCache.py @@ -0,0 +1,38 @@ +import os +import toml +import msgpack +import aioredis + +# Load config + +config: dict = toml.load("config.toml") + +# Redis connection parameters + +redis_config: dict[ str, str | int ] = { + "url": f"redis://{os.environ['REDIS_URL']}", + "port": os.environ['REDIS_PORT'], + "database": config['internal-cache']['database'], +} + +class InternalCache: + """Implements an internal cache for ReVanced Releases API.""" + + redis_url = f"{redis_config['url']}:{redis_config['port']}/{redis_config['database']}" + redis = aioredis.from_url(redis_url, encoding="utf-8", decode_responses=True) + + async def store(self, key: str, value: dict) -> None: + await self.redis.set(key, msgpack.packb(value), ex=config['internal-cache']['expire']) + + async def delete(self, key: str) -> None: + await self.redis.delete(key) + + async def update(self, key: str, value: dict) -> None: + await self.redis.set(key, msgpack.packb(value), ex=config['internal-cache']['expire']) + + async def get(self, key: str) -> dict: + return msgpack.unpackb(await self.redis.get(key)) + + + + \ No newline at end of file diff --git a/modules/Releases.py b/modules/Releases.py index aa1b504..d10fa74 100644 --- a/modules/Releases.py +++ b/modules/Releases.py @@ -3,9 +3,12 @@ import orjson import httpx_cache from typing import Dict, List from base64 import b64decode +from modules.InternalCache import InternalCache class Releases: + InternalCache = InternalCache() + """Implements the methods required to get the latest releases and patches from revanced repositories.""" headers = {'Accept': "application/vnd.github+json", @@ -51,12 +54,17 @@ class Releases: releases: Dict[str, List] = {} releases['tools'] = [] - async with httpx_cache.AsyncClient(headers=self.headers, http2=True) as client: - for repository in repositories: - files = await self._get_release(client, repository) - if files: - for file in files: - releases['tools'].append(file) + try: + cached_releases = await self.InternalCache.get("releases") + return cached_releases + except: + async with httpx_cache.AsyncClient(headers=self.headers, http2=True) as client: + for repository in repositories: + files = await self._get_release(client, repository) + if files: + for file in files: + releases['tools'].append(file) + await self.InternalCache.store('releases', releases) return releases @@ -80,17 +88,15 @@ class Releases: Returns: dict: Patches available for a given app """ + try: + cached_patches = await self.InternalCache.get("patches") + return cached_patches + except: + async with httpx_cache.AsyncClient(headers=self.headers, http2=True) as client: + patches = await self._get_patches_json(client) + await self.InternalCache.store('patches', patches) - async def generate_simplified_json(payload: dict) -> dict: - return {} - - async with httpx_cache.AsyncClient(headers=self.headers, http2=True) as client: - content = await self._get_patches_json(client) - - if simplified: - return await generate_simplified_json(content) - - return content + return patches async def _get_contributors(self, client: httpx_cache.AsyncClient, repository: str) -> list: # Get contributors from a given repository. @@ -119,11 +125,16 @@ class Releases: contributors: Dict[str, List] = {} contributors['repositories'] = [] - async with httpx_cache.AsyncClient(headers=self.headers, http2=True) as client: - for repository in repositories: - if 'revanced' in repository: - repo_contributors = await self._get_contributors(client, repository) - data = { 'name': repository, 'contributors': repo_contributors } - contributors['repositories'].append(data) + try: + cached_contributors = await self.InternalCache.get("contributors") + return cached_contributors + except: + async with httpx_cache.AsyncClient(headers=self.headers, http2=True) as client: + for repository in repositories: + if 'revanced' in repository: + repo_contributors = await self._get_contributors(client, repository) + data = { 'name': repository, 'contributors': repo_contributors } + contributors['repositories'].append(data) + await self.InternalCache.store('contributors', contributors) return contributors \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index bb160f0..2a121ad 100644 --- a/poetry.lock +++ b/poetry.lock @@ -371,11 +371,11 @@ six = ">=1.5" [[package]] name = "python-dotenv" -version = "0.20.0" +version = "0.21.0" description = "Read key-value pairs from a .env file and set them as environment variables" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7" [package.extras] cli = ["click (>=5.0)"] @@ -448,11 +448,11 @@ limits = ">=1.5,<2.0" [[package]] name = "sniffio" -version = "1.2.0" +version = "1.3.0" description = "Sniff out which async library your code is running under" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.7" [[package]] name = "starlette" @@ -557,7 +557,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "46ef39f7a5f340eba36c7df892a63409e8796d1472a665849f437d9d650d6086" +content-hash = "66ede5e2aeb5b3b25bd8e15710520a376af8f979d45a6bb17274260853741c66" [metadata.files] aioredis = [ @@ -881,8 +881,8 @@ python-dateutil = [ {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] python-dotenv = [ - {file = "python-dotenv-0.20.0.tar.gz", hash = "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f"}, - {file = "python_dotenv-0.20.0-py3-none-any.whl", hash = "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938"}, + {file = "python-dotenv-0.21.0.tar.gz", hash = "sha256:b77d08274639e3d34145dfa6c7008e66df0f04b7be7a75fd0d5292c191d79045"}, + {file = "python_dotenv-0.21.0-py3-none-any.whl", hash = "sha256:1684eb44636dd462b66c3ee016599815514527ad99965de77f43e0944634a7e5"}, ] pytzdata = [ {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, @@ -940,8 +940,8 @@ slowapi = [ {file = "slowapi-0.1.6.tar.gz", hash = "sha256:bb20650efb860422d7e1e740bc3d6b781aa9f5a947613979bfc15d3c58082244"}, ] sniffio = [ - {file = "sniffio-1.2.0-py3-none-any.whl", hash = "sha256:471b71698eac1c2112a40ce2752bb2f4a4814c22a54a3eed3676bc0f5ca9f663"}, - {file = "sniffio-1.2.0.tar.gz", hash = "sha256:c4666eecec1d3f50960c6bdf61ab7bc350648da6c126e3cf6898d8cd4ddcd3de"}, + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] starlette = [ {file = "starlette-0.19.1-py3-none-any.whl", hash = "sha256:5a60c5c2d051f3a8eb546136aa0c9399773a689595e099e0877704d5888279bf"}, diff --git a/pyproject.toml b/pyproject.toml index 8f1d72d..da0b35c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ orjson = ">=3.8.0" fastapi-cache2 = ">=0.1.9" aioredis = ">=2.0.1" redis = ">=4.3.4" +msgpack = ">=1.0.4" [tool.poetry.dev-dependencies] mypy = ">=0.971" @@ -30,6 +31,7 @@ orjson = ">=3.8.0" fastapi-cache2 = ">=0.1.9" aioredis = ">=2.0.1" redis = ">=4.3.4" +msgpack = ">=1.0.4" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/requirements.txt b/requirements.txt index 7bad909..500bb13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,21 +20,21 @@ httpx==0.23.0; python_version >= "3.7" hyperframe==6.0.1; python_version >= "3.7" and python_full_version >= "3.6.1" and python_version < "4.0" idna==3.3 limits==1.6; python_version >= "3.7" and python_version < "4.0" -msgpack==1.0.4; python_version >= "3.7" and python_version < "4.0" +msgpack==1.0.4 orjson==3.8.0; python_version >= "3.7" packaging==21.3; python_version >= "3.6" pendulum==2.1.2; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.5.0" pydantic==1.10.1; python_version >= "3.7" and python_full_version >= "3.6.1" and python_version < "4.0" pyparsing==3.0.9; python_full_version >= "3.6.8" and python_version >= "3.6" python-dateutil==2.8.2; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.5.0" -python-dotenv==0.20.0; python_version >= "3.7" and python_version < "4.0" +python-dotenv==0.21.0; python_version >= "3.7" and python_version < "4.0" pytzdata==2020.1; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.5.0" pyyaml==6.0; python_version >= "3.7" and python_version < "4.0" redis==4.3.4; python_version >= "3.6" rfc3986==1.5.0; python_version >= "3.7" and python_version < "4.0" six==1.16.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.5.0" slowapi==0.1.6; python_version >= "3.7" and python_version < "4.0" -sniffio==1.2.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2" +sniffio==1.3.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2" starlette==0.19.1; python_version >= "3.7" and python_full_version >= "3.6.1" and python_version < "4.0" toml==0.10.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") typing-extensions==4.3.0; python_version >= "3.7" and python_full_version >= "3.6.1" and python_version < "4.0"