diff --git a/api/src/config.js b/api/src/config.js index e7e29652..a5ebf3d6 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -1,18 +1,34 @@ import { getVersion } from "@imput/version-info"; -import { loadEnvs, validateEnvs } from "./core/env.js"; +import { loadEnvs, validateEnvs, setupEnvWatcher } from "./core/env.js"; +import * as cluster from "./misc/cluster.js"; const version = await getVersion(); -let env = loadEnvs(); +const env = loadEnvs(); const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`; +export const canonicalEnv = Object.freeze(structuredClone(process.env)); export const setTunnelPort = (port) => env.tunnelPort = port; export const isCluster = env.instanceCount > 1; +export const updateEnv = (newEnv) => { + // tunnelPort is special and needs to get carried over here + newEnv.tunnelPort = env.tunnelPort; + + for (const key in env) { + env[key] = newEnv[key]; + } + + cluster.broadcast({ env_update: newEnv }); +} await validateEnvs(env); +if (env.envFile) { + setupEnvWatcher(); +} + export { env, genericUserAgent, diff --git a/api/src/core/api.js b/api/src/core/api.js index d8ed8ccd..7c0e15b9 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -48,19 +48,23 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { const startTime = new Date(); const startTimestamp = startTime.getTime(); - const serverInfo = JSON.stringify({ - cobalt: { - version: version, - url: env.apiURL, - startTime: `${startTimestamp}`, - durationLimit: env.durationLimit, - turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined, - services: [...env.enabledServices].map(e => { - return friendlyServiceName(e); - }), - }, - git, - }) + const getServerInfo = () => { + return JSON.stringify({ + cobalt: { + version: version, + url: env.apiURL, + startTime: `${startTimestamp}`, + durationLimit: env.durationLimit, + turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined, + services: [...env.enabledServices].map(e => { + return friendlyServiceName(e); + }), + }, + git, + }); + } + + const serverInfo = getServerInfo(); const handleRateExceeded = (_, res) => { const { body } = createResponse("error", { @@ -311,7 +315,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { app.get('/', (_, res) => { res.type('json'); - res.status(200).send(serverInfo); + res.status(200).send(env.envFile ? getServerInfo() : serverInfo); }) app.get('/favicon.ico', (req, res) => { @@ -331,10 +335,6 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes if (env.externalProxy) { - if (env.freebindCIDR) { - throw new Error('Freebind is not available when external proxy is enabled') - } - setGlobalDispatcher(new ProxyAgent(env.externalProxy)) } diff --git a/api/src/core/env.js b/api/src/core/env.js index 34a0d6ef..b50afd1c 100644 --- a/api/src/core/env.js +++ b/api/src/core/env.js @@ -1,6 +1,11 @@ import { Constants } from "youtubei.js"; -import { supportsReusePort } from "../misc/cluster.js"; import { services } from "../processing/service-config.js"; +import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js"; + +import { FileWatcher } from "../misc/file-watcher.js"; +import { isURL } from "../misc/utils.js"; +import * as cluster from "../misc/cluster.js"; +import { Yellow } from "../misc/console-text.js"; const forceLocalProcessingOptions = ["never", "session", "always"]; @@ -68,6 +73,9 @@ export const loadEnvs = (env = process.env) => { // "never" | "session" | "always" forceLocalProcessing: env.FORCE_LOCAL_PROCESSING ?? "never", + + envFile: env.API_ENV_FILE, + envRemoteReloadInterval: 300, }; } @@ -78,7 +86,7 @@ export const validateEnvs = async (env) => { if (env.instanceCount > 1 && !env.redisURL) { throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2"); - } else if (env.instanceCount > 1 && !await supportsReusePort()) { + } else if (env.instanceCount > 1 && !await cluster.supportsReusePort()) { console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js'); console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux'); console.error('(or other OS that supports it). for more info, see `reusePort` option on'); @@ -97,4 +105,81 @@ export const validateEnvs = async (env) => { console.error(`Supported options are are: ${forceLocalProcessingOptions.join(', ')}\n`); throw new Error("Invalid FORCE_LOCAL_PROCESSING"); } + + if (env.externalProxy && env.freebindCIDR) { + throw new Error('freebind is not available when external proxy is enabled') + } +} + +const reloadEnvs = async (contents) => { + const newEnvs = {}; + + for (let line of (await contents).split('\n')) { + line = line.trim(); + if (line === '') { + continue; + } + + const [ key, value ] = line.split(/=(.+)?/); + if (key) { + newEnvs[key] = value || ''; + } + } + + const candidate = { + ...canonicalEnv, + ...newEnvs, + }; + + const parsed = loadEnvs(candidate); + await validateEnvs(parsed); + updateEnv(parsed); +} + +const wrapReload = (contents) => { + reloadEnvs(contents) + .catch((e) => { + console.error(`${Yellow('[!]')} Failed reloading environment variables at ${new Date().toISOString()}.`); + console.error('Error:', e); + }); +} + +let watcher; +const setupWatcherFromFile = (path) => { + const load = () => wrapReload(watcher.read()); + + if (isURL(path)) { + watcher = FileWatcher.fromFileProtocol(path); + } else { + watcher = new FileWatcher({ path }); + } + + watcher.on('file-updated', load); + load(); +} + +const setupWatcherFromFetch = (url) => { + const load = () => wrapReload(fetch(url).then(r => r.text())); + setInterval(load, currentEnv.envRemoteReloadInterval); + load(); +} + +export const setupEnvWatcher = () => { + if (cluster.isPrimary) { + const envFile = currentEnv.envFile; + const isFile = !isURL(envFile) + || new URL(envFile).protocol === 'file:'; + + if (isFile) { + setupWatcherFromFile(envFile); + } else { + setupWatcherFromFetch(envFile); + } + } else if (cluster.isWorker) { + process.on('message', (message) => { + if ('env_update' in message) { + updateEnv(message.env_update); + } + }); + } } diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index 62bf6351..1cde3cdc 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -52,3 +52,12 @@ export function splitFilenameExtension(filename) { export function zip(a, b) { return a.map((value, i) => [ value, b[i] ]); } + +export function isURL(input) { + try { + new URL(input); + return true; + } catch { + return false; + } +} diff --git a/docs/api-env-variables.md b/docs/api-env-variables.md index d44e1cab..92e7a6cf 100644 --- a/docs/api-env-variables.md +++ b/docs/api-env-variables.md @@ -13,6 +13,7 @@ this document is not final and will expand over time. feel free to improve it! | API_REDIS_URL | | `redis://localhost:6379` | | DISABLED_SERVICES | | `bilibili,youtube` | | FORCE_LOCAL_PROCESSING | `never` | `always` | +| API_ENV_FILE | | `/.env` | [*view details*](#general) @@ -111,6 +112,9 @@ when set to `session`, only requests from session (Bearer token) clients will be when set to `always`, all requests will be forced to use on-device processing, no matter the preference. +### API_ENV_FILE +the URL or local path to a `key=value`-style environment variable file. this is used for dynamically reloading environment variables. **not all environment variables are able to be updated by this.** (e.g. the ratelimiters are instantiated when starting cobalt, and cannot be changed) + ## networking [*jump to the table*](#networking-vars)