diff --git a/Dockerfile b/Dockerfile index bbfc37e..14b7373 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,25 @@ -FROM python:3.10-slim +#FROM python:3.10-slim +FROM ubuntu:22.04 ARG GITHUB_TOKEN ENV GITHUB_TOKEN $GITHUB_TOKEN +ARG UVICORN_HOST +ENV UVICORN_HOST $UVICORN_HOST + +ARG UVICORN_PORT +ENV UVICORN_PORT $UVICORN_PORT + +ARG UVICORN_LOG_LEVEL +ENV UVICORN_LOG_LEVEL $UVICORN_LOG_LEVEL + WORKDIR /usr/src/app COPY . . RUN apt update && \ - apt-get install build-essential libffi-dev -y && \ + apt-get install build-essential libffi-dev \ + python3 python3-dev python3-pip -y && \ pip install --no-cache-dir -r requirements.txt -CMD [ "python3", "./main.py" ] \ No newline at end of file +CMD [ "/bin/bash", "./run.sh" ] \ No newline at end of file diff --git a/config.toml b/config.toml index 735d286..4cf8861 100644 --- a/config.toml +++ b/config.toml @@ -54,4 +54,10 @@ database = 1 [app] -repositories = ["TeamVanced/VancedMicroG", "revanced/revanced-cli", "revanced/revanced-patches", "revanced/revanced-integrations", "revanced/revanced-manager", "revanced/revanced-patcher"] \ No newline at end of file +repositories = ["TeamVanced/VancedMicroG", "revanced/revanced-cli", "revanced/revanced-patches", "revanced/revanced-integrations", "revanced/revanced-manager"] + +[logging] + +level = "INFO" +json_logs = false +redis_database = 2 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 123a9a3..88b9954 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,9 +16,12 @@ services: container_name: revanced-releases-api image: alexandreteles/revanced-releases-api:latest environment: + - GITHUB_TOKEN=YOUR_GITHUB_TOKEN - REDIS_URL=revanced-releases-api-redis - REDIS_PORT=6379 - - GITHUB_TOKEN=YOUR_GITHUB_TOKEN + - UVICORN_HOST=0.0.0.0 + - UVICORN_PORT=8000 + - UVICORN_LOG_LEVEL=debug ports: - 127.0.0.1:7934:8000 networks: diff --git a/main.py b/main.py index 9b83e93..0998e93 100755 --- a/main.py +++ b/main.py @@ -19,6 +19,8 @@ from fastapi_cache.backends.redis import RedisBackend from modules.Releases import Releases import modules.models.ResponseModels as ResponseModels +import modules.utils.Logger as Logger + """Get latest ReVanced releases from GitHub API.""" # Load config @@ -122,6 +124,10 @@ async def startup() -> None: return None +# setup right before running to make sure no other library overwrites it + +Logger.setup_logging(LOG_LEVEL=config["logging"]["level"], JSON_LOGS=config["logging"]["json_logs"]) + # Run app if __name__ == '__main__': uvicorn.run(app, host=config['uvicorn']['host'], port=config['uvicorn']['port']) \ No newline at end of file diff --git a/modules/InternalCache.py b/modules/InternalCache.py deleted file mode 100644 index a1994a9..0000000 --- a/modules/InternalCache.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -import toml -import msgpack -import aioredis - -# Load config - -config: dict = toml.load("config.toml") - -# Redis connection parameters - -redis_config: dict[ str, str | int ] = { - "url": f"redis://{os.environ['REDIS_URL']}", - "port": os.environ['REDIS_PORT'], - "database": config['internal-cache']['database'], -} - -class InternalCache: - """Implements an internal cache for ReVanced Releases API.""" - - redis_url = f"{redis_config['url']}:{redis_config['port']}/{redis_config['database']}" - redis = aioredis.from_url(redis_url, encoding="utf-8", decode_responses=True) - - async def store(self, key: str, value: dict) -> None: - await self.redis.set(key, msgpack.packb(value), ex=config['internal-cache']['expire']) - - async def delete(self, key: str) -> None: - await self.redis.delete(key) - - async def update(self, key: str, value: dict) -> None: - await self.redis.set(key, msgpack.packb(value), ex=config['internal-cache']['expire']) - - async def get(self, key: str) -> dict: - return msgpack.unpackb(await self.redis.get(key)) - - - - \ No newline at end of file diff --git a/modules/Releases.py b/modules/Releases.py index 5c71ff4..4a22363 100644 --- a/modules/Releases.py +++ b/modules/Releases.py @@ -2,7 +2,9 @@ import os import orjson import httpx_cache from base64 import b64decode -from modules.InternalCache import InternalCache +from modules.utils.InternalCache import InternalCache +import modules.utils.Logger as Logger + class Releases: @@ -12,7 +14,17 @@ class Releases: 'Authorization': "token " + os.environ['GITHUB_TOKEN'] } - httpx_client = httpx_cache.AsyncClient(headers=headers, http2=True) + httpx_logger = Logger.HTTPXLogger() + + httpx_client = httpx_cache.AsyncClient( + headers=headers, + http2=True, + event_hooks={ + 'request': [httpx_logger.log_request], + 'response': [httpx_logger.log_response] + } + ) + InternalCache = InternalCache() async def _get_release(self, repository: str) -> list: @@ -50,13 +62,14 @@ class Releases: dict: A dictionary containing assets from each repository """ - releases: dict[str, list] = {} - releases['tools'] = [] + releases: dict[str, list] - try: - cached_releases = await self.InternalCache.get("releases") - return cached_releases - except: + if await self.InternalCache.exists('releases'): + releases = await self.InternalCache.get('releases') + else: + releases = {} + releases['tools'] = [] + for repository in repositories: files = await self._get_release(repository) if files: @@ -84,10 +97,9 @@ class Releases: Returns: dict: Patches available for a given app """ - try: - cached_patches = await self.InternalCache.get("patches") - return cached_patches - except: + if await self.InternalCache.exists('patches'): + patches = await self.InternalCache.get('patches') + else: patches = await self._get_patches_json() await self.InternalCache.store('patches', patches) @@ -116,13 +128,13 @@ class Releases: dict: A dictionary containing the contributors from each repository """ - contributors: dict[str, list] = {} - contributors['repositories'] = [] + contributors: dict[str, list] - try: - cached_contributors = await self.InternalCache.get("contributors") - return cached_contributors - except: + if await self.InternalCache.exists('contributors'): + contributors = await self.InternalCache.get('contributors') + else: + contributors = {} + contributors['repositories'] = [] for repository in repositories: if 'revanced' in repository: repo_contributors = await self._get_contributors(repository) diff --git a/modules/utils/InternalCache.py b/modules/utils/InternalCache.py new file mode 100644 index 0000000..942a8d9 --- /dev/null +++ b/modules/utils/InternalCache.py @@ -0,0 +1,72 @@ +import os +import toml +import orjson +import msgpack +import aioredis + +import modules.utils.Logger as Logger + +# Load config + +config: dict = toml.load("config.toml") + +# Redis connection parameters + +redis_config: dict[ str, str | int ] = { + "url": f"redis://{os.environ['REDIS_URL']}", + "port": os.environ['REDIS_PORT'], + "database": config['internal-cache']['database'], +} + +class InternalCache: + """Implements an internal cache for ReVanced Releases API.""" + + redis_url = f"{redis_config['url']}:{redis_config['port']}/{redis_config['database']}" + redis = aioredis.from_url(redis_url, encoding="utf-8", decode_responses=True) + + InternalCacheLogger = Logger.InternalCacheLogger() + + async def store(self, key: str, value: dict) -> None: + try: + await self.redis.set(key, orjson.dumps(value), ex=config['internal-cache']['expire']) + await self.InternalCacheLogger.log("SET", None, key) + except aioredis.RedisError as e: + await self.InternalCacheLogger.log("SET", e) + + async def delete(self, key: str) -> None: + try: + await self.redis.delete(key) + await self.InternalCacheLogger.log("DEL", None, key) + except aioredis.RedisError as e: + await self.InternalCacheLogger.log("DEL", e) + + async def update(self, key: str, value: dict) -> None: + try: + await self.redis.set(key, orjson.dumps(value), ex=config['internal-cache']['expire']) + await self.InternalCacheLogger.log("SET", None, key) + except aioredis.RedisError as e: + await self.InternalCacheLogger.log("SET", e) + + async def get(self, key: str) -> dict: + try: + payload = orjson.loads(await self.redis.get(key)) + await self.InternalCacheLogger.log("GET", None, key) + return payload + except aioredis.RedisError as e: + await self.InternalCacheLogger.log("GET", e) + return {} + + async def exists(self, key: str) -> bool: + try: + if await self.redis.exists(key): + await self.InternalCacheLogger.log("EXISTS", None, key) + return True + else: + await self.InternalCacheLogger.log("EXISTS", None, key) + return False + except aioredis.RedisError as e: + await self.InternalCacheLogger.log("EXISTS", e) + return False + + + \ No newline at end of file diff --git a/modules/utils/Logger.py b/modules/utils/Logger.py new file mode 100644 index 0000000..8d7ea3b --- /dev/null +++ b/modules/utils/Logger.py @@ -0,0 +1,92 @@ +import sys +import logging +from loguru import logger +from typing import Optional +from types import FrameType +from redis import RedisError + +class InterceptHandler(logging.Handler): + """Setups a loging handler for uvicorn and FastAPI. + + Args: + logging (logging.Handler) + """ + + def emit(self, record: logging.LogRecord) -> None: + """Emit a log record. + + Args: + record (LogRecord): Logging record + """ + + level: str | int + frame: Optional[FrameType] + depth: int + + # Get corresponding Loguru level if it exists + try: + level = logger.level(record.levelname).name + except ValueError: + level = record.levelno + + frame = logging.currentframe() + depth = 2 + + # Find caller from where originated the logged message + while frame.f_code.co_filename == logging.__file__: + frame = frame.f_back + depth += 1 + + logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage()) + +class HTTPXLogger(): + """Logger adapter for HTTPX.""" + + async def log_request(self, request) -> None: + """Logs HTTPX requests + + Returns: + None + """ + + logger.info(f"[HTTPX] Request: {request.method} {request.url} - Waiting for response") + + async def log_response(self, response) -> None: + """Logs HTTPX responses + + Returns: + None + """ + request = response.request + + logger.info(f"[HTTPX] Response: {request.method} {request.url} - Status: {response.status_code} {response.reason_phrase}") + +class InternalCacheLogger: + async def log(self, operation: str, result: RedisError | None = None, key: str = "",) -> None: + """Logs internal cache operations + + Args: + operation (str): Operation name + key (str): Key used in the operation + """ + if type(result) is RedisError: + logger.error(f"[InternalCache] REDIS {operation} - Failed with error: {result}") + else: + logger.info(f"[InternalCache] REDIS {operation} {key} - OK") + +def setup_logging(LOG_LEVEL: str, JSON_LOGS: bool) -> None: + + """Setup logging for uvicorn and FastAPI.""" + + # intercept everything at the root logger + logging.root.handlers = [InterceptHandler()] + logging.root.setLevel(LOG_LEVEL) + + # remove every other logger's handlers + # and propagate to root logger + for name in logging.root.manager.loggerDict.keys(): + logging.getLogger(name).handlers = [] + logging.getLogger(name).propagate = True + + # configure loguru + logger.configure(handlers=[{"sink": sys.stdout, "serialize": JSON_LOGS}]) \ No newline at end of file diff --git a/modules/utils/__init__.py b/modules/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mypy.ini b/mypy.ini index 0e98c2a..7529c41 100644 --- a/mypy.ini +++ b/mypy.ini @@ -52,3 +52,7 @@ ignore_missing_imports = True [mypy-httpx_cache.*] # No stubs available ignore_missing_imports = True + +[mypy-redis.*] +# No stubs available +ignore_missing_imports = True \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 2c76ceb..1b97975 100644 --- a/poetry.lock +++ b/poetry.lock @@ -63,7 +63,7 @@ tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy" [[package]] name = "certifi" -version = "2022.6.15" +version = "2022.6.15.1" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -276,6 +276,21 @@ python-versions = "*" [package.dependencies] six = ">=1.4.1" +[[package]] +name = "loguru" +version = "0.6.0" +description = "Python logging made (stupidly) simple" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} +win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} + +[package.extras] +dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] + [[package]] name = "msgpack" version = "1.0.4" @@ -563,6 +578,17 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + [[package]] name = "wrapt" version = "1.14.1" @@ -574,7 +600,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "b6bc93886458708baeea4c07ad92f0613d6aeac113875d9ff2e9a3afb71d000d" +content-hash = "a2d02dd2c5903e3654434982ce86d728c2b9c63d6c6520a4bd295c362ac88bb0" [metadata.files] aioredis = [ @@ -598,8 +624,8 @@ attrs = [ {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] certifi = [ - {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, - {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, + {file = "certifi-2022.6.15.1-py3-none-any.whl", hash = "sha256:43dadad18a7f168740e66944e4fa82c6611848ff9056ad910f8f7a3e46ab89e0"}, + {file = "certifi-2022.6.15.1.tar.gz", hash = "sha256:cffdcd380919da6137f76633531a5817e3a9f268575c128249fb637e4f9e73fb"}, ] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, @@ -740,6 +766,10 @@ limits = [ {file = "limits-1.6-py3-none-any.whl", hash = "sha256:12ae4449cf7daadee43edf4096acd9cb9f4bfdec3a995aa9fbd0f72b0b9af762"}, {file = "limits-1.6.tar.gz", hash = "sha256:6c0a57b42647f1141f5a7a0a8479b49e4367c24937a01bd9d4063a595c2dd48a"}, ] +loguru = [ + {file = "loguru-0.6.0-py3-none-any.whl", hash = "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"}, + {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, +] msgpack = [ {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4ab251d229d10498e9a2f3b1e68ef64cb393394ec477e3370c457f9430ce9250"}, {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:112b0f93202d7c0fef0b7810d465fde23c746a2d482e1e2de2aafd2ce1492c88"}, @@ -1115,6 +1145,10 @@ websockets = [ {file = "websockets-10.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55"}, {file = "websockets-10.3.tar.gz", hash = "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"}, ] +win32-setctime = [ + {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, + {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, +] wrapt = [ {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"}, {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"}, diff --git a/portainer-stack.yml b/portainer-stack.yml index 27d5472..2a15864 100644 --- a/portainer-stack.yml +++ b/portainer-stack.yml @@ -18,9 +18,12 @@ services: container_name: revanced-releases-api image: alexandreteles/revanced-releases-api:latest environment: + - GITHUB_TOKEN=YOUR_GITHUB_TOKEN - REDIS_URL=revanced-releases-api-redis - REDIS_PORT=6379 - - GITHUB_TOKEN=YOUR_GITHUB_TOKEN + - UVICORN_HOST=0.0.0.0 + - UVICORN_PORT=8000 + - UVICORN_LOG_LEVEL=debug ports: - 127.0.0.1:7934:8000 networks: diff --git a/pyproject.toml b/pyproject.toml index aeeb2c8..083527a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ fastapi-cache2 = ">=0.1.9" aioredis = {version = ">=2.0.1", extras = ["hiredis"]} redis = ">=4.3.4" msgpack = ">=1.0.4" +loguru = ">=0.6.0" [tool.poetry.dev-dependencies] fastapi = ">=0.82.0" @@ -33,6 +34,7 @@ redis = ">=4.3.4" msgpack = ">=1.0.4" mypy = ">=0.971" types-toml = ">=0.10.8" +loguru = ">=0.6.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/requirements.txt b/requirements.txt index 31cf0ed..72d6824 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aiorwlock==1.3.0; python_version >= "3.7" and python_version < "4.0" and python_ anyio==3.6.1; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2" async-timeout==4.0.2; python_version >= "3.6" attrs==21.4.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.5.0" -certifi==2022.6.15; python_version >= "3.7" and python_version < "4.0" +certifi==2022.6.15.1; python_version >= "3.7" and python_version < "4.0" click==8.1.3; python_version >= "3.7" and python_version < "4.0" colorama==0.4.5; python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" and python_version < "4.0" and platform_system == "Windows" or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.5.0" and python_version < "4.0" and platform_system == "Windows" deprecated==1.2.13; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" @@ -21,6 +21,7 @@ httpx==0.23.0; python_version >= "3.7" hyperframe==6.0.1; python_version >= "3.7" and python_full_version >= "3.6.1" and python_version < "4.0" idna==3.3 limits==1.6; python_version >= "3.7" and python_version < "4.0" +loguru==0.6.0; python_version >= "3.5" msgpack==1.0.4 orjson==3.8.0; python_version >= "3.7" packaging==21.3; python_version >= "3.6" @@ -43,4 +44,5 @@ uvicorn==0.18.3; python_version >= "3.7" uvloop==0.16.0; sys_platform != "win32" and sys_platform != "cygwin" and platform_python_implementation != "PyPy" and python_version >= "3.7" and python_version < "4.0" watchfiles==0.16.1; python_version >= "3.7" and python_version < "4.0" websockets==10.3; python_version >= "3.7" and python_version < "4.0" +win32-setctime==1.1.0; sys_platform == "win32" and python_version >= "3.5" wrapt==1.14.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..0e983be --- /dev/null +++ b/run.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# This script is used to run the application +# It is used by the Dockerfile + +# get number of cores +CORES=$(grep -c ^processor /proc/cpuinfo) + +# Start the application +uvicorn main:app --host="$UVICORN_HOST" --port="$UVICORN_PORT" \ +--workers="$CORES" --log-level="$UVICORN_LOG_LEVEL" --server-header \ +--proxy-headers --forwarded-allow-ips="*" \ No newline at end of file