diff --git a/.github/workflows/docker-develop.yml b/.github/workflows/docker-develop.yml index e89eeae0..43bfc8d1 100644 --- a/.github/workflows/docker-develop.yml +++ b/.github/workflows/docker-develop.yml @@ -1,4 +1,4 @@ -name: Build Docker development image +name: Build development Docker image on: workflow_dispatch: diff --git a/.github/workflows/docker-staging.yml b/.github/workflows/docker-staging.yml new file mode 100644 index 00000000..572a9855 --- /dev/null +++ b/.github/workflows/docker-staging.yml @@ -0,0 +1,55 @@ +name: Build staging Docker image + +on: + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get release metadata + id: release-meta + run: | + version=$(cat package.json | jq -r .version) + echo "commit_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "version=$version" >> $GITHUB_OUTPUT + echo "major_version=$(echo "$version" | cut -d. -f1)" >> $GITHUB_OUTPUT + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + tags: type=raw,value=staging + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e25378b3..914edf2f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: Build Docker image +name: Build release Docker image on: workflow_dispatch: diff --git a/README.md b/README.md index 9a5a05e7..795eb7e3 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,10 @@ this monorepo includes source code for api, frontend, and related packages: - [packages tree](/packages/) it also includes documentation in the [docs tree](/docs/): -- [cobalt api documentation](/docs/api.md) - [how to run a cobalt instance](/docs/run-an-instance.md) -- [how to protect a cobalt instance](/docs/protect-an-instance.md) (recommended if you host a public instance) - -### thank you -cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support! +- [how to protect a cobalt instance](/docs/protect-an-instance.md) +- [cobalt api instance environment variables](/docs/api-env-variables.md) +- [cobalt api documentation](/docs/api.md) ### ethics cobalt is a tool that makes downloading public content easier. it takes **zero liability**. @@ -55,6 +53,9 @@ same content can be downloaded via dev tools of any modern web browser. ### contributing if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away. +### thank you +cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support! + ### licenses for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs. unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE). diff --git a/api/package.json b/api/package.json index faf1c0a2..2d568c21 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.8.3", + "version": "10.9.1", "author": "imput", "exports": "./src/cobalt.js", "type": "module", diff --git a/api/src/config.js b/api/src/config.js index 98da6fe7..bb4994c0 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -28,6 +28,9 @@ const env = { rateLimitWindow: (process.env.RATELIMIT_WINDOW && parseInt(process.env.RATELIMIT_WINDOW)) || 60, rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20, + sessionRateLimitWindow: (process.env.SESSION_RATELIMIT_WINDOW && parseInt(process.env.SESSION_RATELIMIT_WINDOW)) || 60, + sessionRateLimit: (process.env.SESSION_RATELIMIT && parseInt(process.env.SESSION_RATELIMIT)) || 10, + durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800, streamLifespan: (process.env.TUNNEL_LIFESPAN && parseInt(process.env.TUNNEL_LIFESPAN)) || 90, diff --git a/api/src/core/api.js b/api/src/core/api.js index 8e18c929..3231767f 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -75,8 +75,8 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url'); const sessionLimiter = rateLimit({ - windowMs: 60000, - limit: 10, + windowMs: env.sessionRateLimitWindow * 1000, + limit: env.sessionRateLimit, standardHeaders: 'draft-6', legacyHeaders: false, keyGenerator, @@ -92,7 +92,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { keyGenerator: req => req.rateLimitKey || keyGenerator(req), store: await createStore('api'), handler: handleRateExceeded - }) + }); const apiTunnelLimiter = rateLimit({ windowMs: env.rateLimitWindow * 1000, @@ -104,7 +104,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { handler: (_, res) => { return res.sendStatus(429) } - }) + }); app.set('trust proxy', ['loopback', 'uniquelocal']); @@ -176,7 +176,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { return fail(res, "error.api.auth.jwt.invalid"); } - if (!jwt.verify(token)) { + if (!jwt.verify(token, getIP(req, 32))) { return fail(res, "error.api.auth.jwt.invalid"); } @@ -222,7 +222,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { } try { - res.json(jwt.generate()); + res.json(jwt.generate(getIP(req, 32))); } catch { return fail(res, "error.api.generic"); } diff --git a/api/src/misc/utils.js b/api/src/misc/utils.js index 76d7a3eb..62bf6351 100644 --- a/api/src/misc/utils.js +++ b/api/src/misc/utils.js @@ -1,17 +1,26 @@ import { request } from 'undici'; const redirectStatuses = new Set([301, 302, 303, 307, 308]); -export async function getRedirectingURL(url, dispatcher, userAgent) { - const location = await request(url, { +export async function getRedirectingURL(url, dispatcher, headers) { + const params = { dispatcher, method: 'HEAD', - headers: { 'user-agent': userAgent } - }).then(r => { + headers, + redirect: 'manual' + }; + + let location = await request(url, params).then(r => { if (redirectStatuses.has(r.statusCode) && r.headers['location']) { return r.headers['location']; } }).catch(() => null); + location ??= await fetch(url, params).then(r => { + if (redirectStatuses.has(r.status) && r.headers.has('location')) { + return r.headers.get('location'); + } + }).catch(() => null); + return location; } diff --git a/api/src/processing/helpers/youtube-session.js b/api/src/processing/helpers/youtube-session.js index 5235c42a..85f1a6e1 100644 --- a/api/src/processing/helpers/youtube-session.js +++ b/api/src/processing/helpers/youtube-session.js @@ -1,8 +1,11 @@ import * as cluster from "../../misc/cluster.js"; +import { Agent } from "undici"; import { env } from "../../config.js"; import { Green, Yellow } from "../../misc/console-text.js"; +const defaultAgent = new Agent(); + let session; const validateSession = (sessionResponse) => { @@ -32,7 +35,11 @@ const loadSession = async () => { const sessionServerUrl = new URL(env.ytSessionServer); sessionServerUrl.pathname = "/token"; - const newSession = await fetch(sessionServerUrl).then(a => a.json()); + const newSession = await fetch( + sessionServerUrl, + { dispatcher: defaultAgent } + ).then(a => a.json()); + validateSession(newSession); if (!session || session.updated < newSession?.updated) { diff --git a/api/src/processing/request.js b/api/src/processing/request.js index 901e8e96..697e67fc 100644 --- a/api/src/processing/request.js +++ b/api/src/processing/request.js @@ -118,14 +118,13 @@ export function normalizeRequest(request) { )); } -export function getIP(req) { +export function getIP(req, prefix = 56) { const strippedIP = req.ip.replace(/^::ffff:/, ''); const ip = ipaddr.parse(strippedIP); if (ip.kind() === 'ipv4') { return strippedIP; } - const prefix = 56; const v6Bytes = ip.toByteArray(); v6Bytes.fill(0, prefix / 8); diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 00fa4ebf..87a71c38 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -90,7 +90,9 @@ export const services = { "r/u_:user/comments/:id/:title", "r/u_:user/comments/:id/comment/:commentId", - "r/:sub/s/:shareId" + "r/:sub/s/:shareId", + + "video/:shortId", ], subdomains: "*", }, diff --git a/api/src/processing/service-patterns.js b/api/src/processing/service-patterns.js index 8735f123..2412fd46 100644 --- a/api/src/processing/service-patterns.js +++ b/api/src/processing/service-patterns.js @@ -23,7 +23,8 @@ export const testers = { pattern.id?.length <= 16 && !pattern.sub && !pattern.user || (pattern.sub?.length <= 22 && pattern.id?.length <= 16) || (pattern.user?.length <= 22 && pattern.id?.length <= 16) - || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16), + || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16) + || (pattern.shortId?.length <= 16), "rutube": pattern => (pattern.id?.length === 32 && pattern.key?.length <= 32) || diff --git a/api/src/processing/services/instagram.js b/api/src/processing/services/instagram.js index 9cc7dbdf..0fa25527 100644 --- a/api/src/processing/services/instagram.js +++ b/api/src/processing/services/instagram.js @@ -527,7 +527,7 @@ export default function instagram(obj) { // for some reason instagram decides to return HTML // instead of a redirect when requesting with a normal // browser user-agent - 'curl/7.88.1' + {'User-Agent': 'curl/7.88.1'} ).then(match => instagram({ ...obj, ...match, shareId: undefined diff --git a/api/src/processing/services/reddit.js b/api/src/processing/services/reddit.js index 50c78d35..0f506eea 100644 --- a/api/src/processing/services/reddit.js +++ b/api/src/processing/services/reddit.js @@ -50,12 +50,24 @@ async function getAccessToken() { export default async function(obj) { let params = obj; + const accessToken = await getAccessToken(); + const headers = { + 'user-agent': genericUserAgent, + authorization: accessToken && `Bearer ${accessToken}`, + accept: 'application/json' + }; + + if (params.shortId) { + params = await resolveRedirectingURL( + `https://www.reddit.com/video/${params.shortId}`, + obj.dispatcher, headers + ); + } if (!params.id && params.shareId) { params = await resolveRedirectingURL( `https://www.reddit.com/r/${params.sub}/s/${params.shareId}`, - obj.dispatcher, - genericUserAgent + obj.dispatcher, headers ); } @@ -63,17 +75,10 @@ export default async function(obj) { const url = new URL(`https://www.reddit.com/comments/${params.id}.json`); - const accessToken = await getAccessToken(); if (accessToken) url.hostname = 'oauth.reddit.com'; let data = await fetch( - url, { - headers: { - 'User-Agent': genericUserAgent, - accept: 'application/json', - authorization: accessToken && `Bearer ${accessToken}` - } - } + url, { headers } ).then(r => r.json()).catch(() => {}); if (!data || !Array.isArray(data)) { diff --git a/api/src/processing/url.js b/api/src/processing/url.js index 82299999..86c333f6 100644 --- a/api/src/processing/url.js +++ b/api/src/processing/url.js @@ -106,6 +106,14 @@ function aliasURL(url) { url.pathname = `/share/${idPart.slice(-32)}`; } break; + + case "redd": + /* reddit short video links can be treated by changing https://v.redd.it/ + to https://reddit.com/video/.*/ + if (url.hostname === "v.redd.it" && parts.length === 2) { + url = new URL(`https://www.reddit.com/video/${parts[1]}`); + } + break; } return url; @@ -231,11 +239,11 @@ export function extract(url) { return { host, patternMatch }; } -export async function resolveRedirectingURL(url, dispatcher, userAgent) { +export async function resolveRedirectingURL(url, dispatcher, headers) { const originalService = getHostIfValid(normalizeURL(url)); if (!originalService) return; - const canonicalURL = await getRedirectingURL(url, dispatcher, userAgent); + const canonicalURL = await getRedirectingURL(url, dispatcher, headers); if (!canonicalURL) return; const { host, patternMatch } = extract(normalizeURL(canonicalURL)); diff --git a/api/src/security/jwt.js b/api/src/security/jwt.js index 91d6cf9e..557f0b68 100644 --- a/api/src/security/jwt.js +++ b/api/src/security/jwt.js @@ -6,12 +6,19 @@ import { env } from "../config.js"; const toBase64URL = (b) => Buffer.from(b).toString("base64url"); const fromBase64URL = (b) => Buffer.from(b, "base64url").toString(); -const makeHmac = (header, payload) => - createHmac("sha256", env.jwtSecret) - .update(`${header}.${payload}`) - .digest("base64url"); +const makeHmac = (data) => { + return createHmac("sha256", env.jwtSecret) + .update(data) + .digest("base64url"); +} -const generate = () => { +const sign = (header, payload) => + makeHmac(`${header}.${payload}`); + +const getIPHash = (ip) => + makeHmac(ip).slice(0, 8); + +const generate = (ip) => { const exp = Math.floor(new Date().getTime() / 1000) + env.jwtLifetime; const header = toBase64URL(JSON.stringify({ @@ -21,10 +28,11 @@ const generate = () => { const payload = toBase64URL(JSON.stringify({ jti: nanoid(8), + sub: getIPHash(ip), exp, })); - const signature = makeHmac(header, payload); + const signature = sign(header, payload); return { token: `${header}.${payload}.${signature}`, @@ -32,7 +40,7 @@ const generate = () => { }; } -const verify = (jwt) => { +const verify = (jwt, ip) => { const [header, payload, signature] = jwt.split(".", 3); const timestamp = Math.floor(new Date().getTime() / 1000); @@ -40,17 +48,16 @@ const verify = (jwt) => { return false; } - const verifySignature = makeHmac(header, payload); + const verifySignature = sign(header, payload); if (verifySignature !== signature) { return false; } - if (timestamp >= JSON.parse(fromBase64URL(payload)).exp) { - return false; - } + const data = JSON.parse(fromBase64URL(payload)); - return true; + return getIPHash(ip) === data.sub + && timestamp <= data.exp; } export default { diff --git a/api/src/util/tests/instagram.json b/api/src/util/tests/instagram.json index 1df87b9a..4adcf6f8 100644 --- a/api/src/util/tests/instagram.json +++ b/api/src/util/tests/instagram.json @@ -123,6 +123,7 @@ { "name": "private instagram post", "url": "https://www.instagram.com/p/C5_A1TQNPrYw4c2g9KAUTPUl8RVHqiAdAcOOSY0", + "canFail": true, "params": {}, "expected": { "code": 400, diff --git a/api/src/util/tests/reddit.json b/api/src/util/tests/reddit.json index 3afc6126..1dd10ee5 100644 --- a/api/src/util/tests/reddit.json +++ b/api/src/util/tests/reddit.json @@ -56,5 +56,23 @@ "code": 200, "status": "tunnel" } + }, + { + "name": "shortened video link", + "url": "https://v.redd.it/ifg2emt5ck0e1", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, + { + "name": "shortened video link (alternative)", + "url": "https://reddit.com/video/ifg2emt5ck0e1", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } } ] \ No newline at end of file diff --git a/docs/api-env-variables.md b/docs/api-env-variables.md new file mode 100644 index 00000000..34de4b0a --- /dev/null +++ b/docs/api-env-variables.md @@ -0,0 +1,228 @@ +# cobalt api instance environment variables +you can customize your processing instance's behavior using these environment variables. all of them but `API_URL` are optional. +this document is not final and will expand over time. feel free to improve it! + +### general vars +| name | default | value example | +|:--------------------|:----------|:--------------------------------------| +| API_URL | | `https://api.url.example/` | +| API_PORT | `9000` | `1337` | +| COOKIE_PATH | | `/cookies.json` | +| PROCESSING_PRIORITY | | `10` | +| API_INSTANCE_COUNT | | `6` | +| API_REDIS_URL | | `redis://localhost:6379` | +| DISABLED_SERVICES | | `bilibili,youtube` | + +[*view details*](#general) + +### networking vars +| name | default | value example | +|:--------------------|:----------|:--------------------------------------| +| API_LISTEN_ADDRESS | `0.0.0.0` | `127.0.0.1` | +| API_EXTERNAL_PROXY | | `http://user:password@127.0.0.1:8080` | +| FREEBIND_CIDR | | `2001:db8::/32` | + +[*view details*](#networking) + +### limit vars +| name | default | value example | +|:-------------------------|:--------|:--------------| +| DURATION_LIMIT | `10800` | `18000` | +| TUNNEL_LIFESPAN | `90` | `120` | +| RATELIMIT_WINDOW | `60` | `120` | +| RATELIMIT_MAX | `20` | `30` | +| SESSION_RATELIMIT_WINDOW | `60` | `60` | +| SESSION_RATELIMIT | `10` | `10` | + +[*view details*](#limits) + +### security vars +| name | default | value example | +|:------------------|:--------|:--------------------------------------| +| CORS_WILDCARD | `1` | `0` | +| CORS_URL | | `https://web.url.example` | +| TURNSTILE_SITEKEY | | `1x00000000000000000000BB` | +| TURNSTILE_SECRET | | `1x0000000000000000000000000000000AA` | +| JWT_SECRET | | see [details](#security) | +| JWT_EXPIRY | `120` | `240` | +| API_KEY_URL | | `file://keys.json` | +| API_AUTH_REQUIRED | | `1` | + +[*view details*](#security) + +### service-specific vars +| name | value example | +|:---------------------------------|:-------------------------| +| CUSTOM_INNERTUBE_CLIENT | `IOS` | +| YOUTUBE_SESSION_SERVER | `http://localhost:8080/` | +| YOUTUBE_SESSION_INNERTUBE_CLIENT | `WEB_EMBEDDED` | + +[*view details*](#service-specific) + +## general +[*jump to the table*](#general-vars) + +### API_URL +> [!NOTE] +> API_URL is required to run the API instance. + +the URL from which your instance will be accessible. can be external or internal, but it must be a valid URL or else tunnels will not work. + +the value is a URL. + +### API_PORT +port from which the API server will be accessible. + +the value is a number from 1024 to 65535. + +### COOKIE_PATH +path to the `cookies.json` file relative to the current working directory of your cobalt instance (usually the main (src/api) folder). + +### PROCESSING_PRIORITY +`nice` value for ffmpeg subprocesses. available only on unix systems. + +note: the higher the nice value, the lower the priority. you can [read more about nice here](https://en.wikipedia.org/wiki/Nice_(Unix)). + +the value is a number. + +### API_INSTANCE_COUNT +supported only on linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. `API_REDIS_URL` is required to use this option. + +the value is a number. + +### API_REDIS_URL +when configured, cobalt will use this redis instance for tunnel cache. required when `API_INSTANCE_COUNT` is more than 1, because else sub-instance wouldn't be able to share cache. + +the value is a URL. + +### DISABLED_SERVICES +comma-separated list which disables certain services from being used. + +the value is a string of cobalt-supported services. + +## networking +[*jump to the table*](#networking-vars) + +### API_LISTEN_ADDRESS +defines the local address for the api instance. if you are using a docker container, you usually don't need to configure this. + +the value is a local IP address. + +### API_EXTERNAL_PROXY +URL of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only. + +if some feature breaks when using a proxy, please make a new issue about it! + +the value is a URL. + +### FREEBIND_CIDR +IPv6 prefix used for randomly assigning addresses to cobalt requests. available only on linux systems. + +setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all requests it makes for that particular download. + +to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. + +if you want to use this option and run cobalt in a docker container, you also need to set the `API_LISTEN_ADDRESS` env variable to `127.0.0.1` and set `network_mode` for the container to `host`. + +the value is an IPv6 range. + +## limits +[*jump to the table*](#limit-vars) + +### DURATION_LIMIT +media duration limit, in **seconds** + +the value is a number. + +### TUNNEL_LIFESPAN +the duration for which tunnel info is stored in ram, **in seconds**. + +it's recommended to keep this value either default or as low as possible to preserve efficiency and user privacy. + +the value is a number. + +### RATELIMIT_WINDOW +rate limit time window for api requests, but not session requests, in **seconds**. + +the value is a number. + +### RATELIMIT_MAX +amount of api requests to be allowed within the time window of `RATELIMIT_WINDOW`. + +the value is a number. + +### SESSION_RATELIMIT_WINDOW +rate limit time window for session creation requests, in **seconds**. + +the value is a number. + +### SESSION_RATELIMIT +amount of session requests to be allowed within the time window of `SESSION_RATELIMIT_WINDOW`. + +the value is a number. + +## security +[*jump to the table*](#security-vars) + +> [!NOTE] +> in order to enable turnstile bot protection, `TURNSTILE_SITEKEY`, `TURNSTILE_SECRET`, and `JWT_SECRET` must be set. all three at once. + +### CORS_WILDCARD +defines whether cross-origin resource sharing is enabled. when enabled, your instance will be accessible from foreign web pages. + +the value is a number. 0: disabled. 1: enabled. + +### CORS_URL +configures the [cross-origin resource sharing origin](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Access-Control-Allow-Origin). your instance will be available only from this URL if `CORS_WILDCARD` is set to `0`. + +the value is a URL. + +### TURNSTILE_SITEKEY +[cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) sitekey used by the web client to request & solve a challenge to prove that the user is not a bot. + +the value is a specific key. + +### TURNSTILE_SECRET +[cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) secret used by the processing instance to verify that the client solved the challenge successfully. + +the value is a specific key. + +### JWT_SECRET +the secret used for issuing JWT tokens for request authentication. the value must be a random, secure, and long string (over 16 characters). + +the value is a specific key. + +### JWT_EXPIRY +the duration of how long a cobalt-issued JWT token will remain valid, in seconds. + +the value is a number. + +### API_KEY_URL +the URL to the the external or local key database. for local files you have to specify a local path using the `file://` protocol. + +see [the api key section](/docs/protect-an-instance.md#api-key-file-format) in the "how to protect your cobalt instance" document for more details. + +the value is a URL. + +### API_AUTH_REQUIRED +when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled). + +the value is a number. + +## service-specific +[*jump to the table*](#service-specific-vars) + +### CUSTOM_INNERTUBE_CLIENT +innertube client that will be used instead of the default one. + +the value is a string. + +### YOUTUBE_SESSION_SERVER +URL to an instance of [yt-session-generator](https://github.com/imputnet/yt-session-generator). used for automatically pulling `poToken` & `visitor_data` for youtube. can be local or remote. + +the value is a URL. + +### YOUTUBE_SESSION_INNERTUBE_CLIENT +innertube client that's compatible with botguard's (web) `poToken` and `visitor_data`. + +the value is a string. diff --git a/docs/examples/docker-compose.example.yml b/docs/examples/docker-compose.example.yml index ed217ccc..b2ad73c1 100644 --- a/docs/examples/docker-compose.example.yml +++ b/docs/examples/docker-compose.example.yml @@ -49,6 +49,5 @@ services: # init: true # restart: unless-stopped # container_name: yt-session-generator - - # ports: - # - 127.0.0.1:1280:8080 + # labels: + # - com.centurylinklabs.watchtower.scope=cobalt diff --git a/docs/protect-an-instance.md b/docs/protect-an-instance.md index 9b4131c1..30584102 100644 --- a/docs/protect-an-instance.md +++ b/docs/protect-an-instance.md @@ -114,7 +114,7 @@ if you want to use your instance outside of web interface, you'll need an api ke > > if api keys leak, you'll have to update/remove all UUIDs to revoke them. -1. create a `keys.json` file following [the schema and example here](/docs//run-an-instance.md#api-key-file-format). +1. create a `keys.json` file following [the schema and example down below](#api-key-file-format). 2. expose the `keys.json` to the docker container: ```yml @@ -148,3 +148,55 @@ environment: ### why not make keys exclusive by default? keys may be useful for going around rate limiting, while keeping the rest of api rate limited, with no turnstile in place. + +## api key file format +the file is a JSON-serialized object with the following structure: +```typescript + +type KeyFileContents = Record< + UUIDv4String, + { + name?: string, + limit?: number | "unlimited", + ips?: (CIDRString | IPString)[], + userAgents?: string[] + } +>; +``` + +where *`UUIDv4String`* is a stringified version of a UUIDv4 identifier. +- **name** is a field for your own reference, it is not used by cobalt anywhere. + +- **`limit`** specifies how many requests the API key can make during the window specified in the `RATELIMIT_WINDOW` env. + - when omitted, the limit specified in `RATELIMIT_MAX` will be used. + - it can be also set to `"unlimited"`, in which case the API key bypasses all rate limits. + +- **`ips`** contains an array of allowlisted IP ranges, which can be specified both as individual ips or CIDR ranges (e.g. *`["192.168.42.69", "2001:db8::48", "10.0.0.0/8", "fe80::/10"]`*). + - when specified, only requests from these ip ranges can use the specified api key. + - when omitted, any IP can be used to make requests with that API key. + +- **`userAgents`** contains an array of allowed user agents, with support for wildcards (e.g. *`["cobaltbot/1.0", "Mozilla/5.0 * Chrome/*"]`*). + - when specified, requests with a `user-agent` that does not appear in this array will be rejected. + - when omitted, any user agent can be specified to make requests with that API key. + +- if both `ips` and `userAgents` are set, the tokens will be limited by both parameters. +- if cobalt detects any problem with your key file, it will be ignored and a warning will be printed to the console. + +an example key file could look like this: +```json +{ + "b5c7160a-b655-4c7a-b500-de839f094550": { + "limit": 10, + "ips": ["10.0.0.0/8", "192.168.42.42"], + "userAgents": ["*Chrome*"] + }, + "b00b1234-a3e5-99b1-c6d1-dba4512ae190": { + "limit": "unlimited", + "ips": ["192.168.1.2"], + "userAgents": ["cobaltbot/1.0"] + } +} +``` + +if you are configuring a key file, **do not use the UUID from the example** but instead generate your own. you can do this by running the following command if you have node.js installed: +`node -e "console.log(crypto.randomUUID())"` diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index b79e3fe8..9aa7c909 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -1,4 +1,6 @@ # how to run a cobalt instance +this tutorial will help you run your own cobalt processing instance. if your instance is public-facing, we highly recommend that you also [protect it from abuse](/docs/protect-an-instance.md) using turnstile or api keys or both. + ## using docker compose and package from github (recommended) to run the cobalt docker package, you need to have `docker` and `docker-compose` installed and configured. @@ -54,93 +56,5 @@ sudo apt install nscd sudo service nscd start ``` -## list of environment variables for api -| variable name | default | example | description | -|:----------------------|:----------|:------------------------|:------------| -| `API_PORT` | `9000` | `9000` | changes port from which api server is accessible. | -| `API_LISTEN_ADDRESS` | `0.0.0.0` | `127.0.0.1` | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** | -| `API_URL` | ➖ | `https://api.cobalt.tools/` | changes url from which api server is accessible.
***REQUIRED TO RUN THE API***. | -| `API_NAME` | `unknown` | `ams-1` | api server name that is shown in `/api/serverInfo`. | -| `API_EXTERNAL_PROXY` | ➖ | `http://user:password@127.0.0.1:8080`| url of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only. | -| `CORS_WILDCARD` | `1` | `0` | toggles cross-origin resource sharing.
`0`: disabled. `1`: enabled. | -| `CORS_URL` | not used | `https://cobalt.tools` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. | -| `COOKIE_PATH` | not used | `/cookies.json` | path for cookie file relative to main folder. | -| `PROCESSING_PRIORITY` | not used | `10` | changes `nice` value* for ffmpeg subprocess. available only on unix systems. | -| `FREEBIND_CIDR` | ➖ | `2001:db8::/32` | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. see below for more info. | -| `RATELIMIT_WINDOW` | `60` | `120` | rate limit time window in **seconds**. | -| `RATELIMIT_MAX` | `20` | `30` | max requests per time window. requests above this amount will be blocked for the rate limit window duration. | -| `DURATION_LIMIT` | `10800` | `18000` | max allowed video duration in **seconds**. | -| `TUNNEL_LIFESPAN` | `90` | `120` | the duration for which tunnel info is stored in ram, **in seconds**. | -| `TURNSTILE_SITEKEY` | ➖ | `1x00000000000000000000BB` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) sitekey used by browser clients to request a challenge.\*\* | -| `TURNSTILE_SECRET` | ➖ | `1x0000000000000000000000000000000AA` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) secret used by cobalt to verify the client successfully solved the challenge.\*\* | -| `JWT_SECRET` | ➖ | ➖ | the secret used for issuing JWT tokens for request authentication. to choose a value, generate a random, secure, long string (ideally >=16 characters).\*\* | -| `JWT_EXPIRY` | `120` | `240` | the duration of how long a cobalt-issued JWT token will remain valid, in seconds. | -| `API_KEY_URL` | ➖ | `file://keys.json` | the location of the api key database. for loading API keys, cobalt supports HTTP(S) urls, or local files by specifying a local path using the `file://` protocol. see the "api key file format" below for more details. | -| `API_AUTH_REQUIRED` | ➖ | `1` | when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled). | -| `API_REDIS_URL` | ➖ | `redis://localhost:6379` | when set, cobalt uses redis instead of internal memory for the tunnel cache. | -| `API_INSTANCE_COUNT` | ➖ | `2` | supported only on Linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. | -| `DISABLED_SERVICES` | ➖ | `bilibili,youtube` | comma-separated list which disables certain services from being used. | -| `CUSTOM_INNERTUBE_CLIENT` | ➖ | `IOS` | innertube client that will be used instead of the default one. | -| `YOUTUBE_SESSION_SERVER` | ➖ | `http://localhost:8080/` | url to an instance of [invidious' youtube-trusted-session-generator](https://github.com/iv-org/youtube-trusted-session-generator) or its fork/counterpart. used for automatically pulling poToken & visitor_data for youtube. can be local or remote. | - -\* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)). - -\*\* in order to enable turnstile bot protection, all three **`TURNSTILE_SITEKEY`, `TURNSTILE_SECRET` and `JWT_SECRET`** need to be set. - -#### FREEBIND_CIDR -setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all -requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt -in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set -`network_mode` for the container to `host`. - -## api key file format -the file is a JSON-serialized object with the following structure: -```typescript - -type KeyFileContents = Record< - UUIDv4String, - { - name?: string, - limit?: number | "unlimited", - ips?: (CIDRString | IPString)[], - userAgents?: string[] - } ->; -``` - -where *`UUIDv4String`* is a stringified version of a UUIDv4 identifier. -- **name** is a field for your own reference, it is not used by cobalt anywhere. - -- **`limit`** specifies how many requests the API key can make during the window specified in the `RATELIMIT_WINDOW` env. - - when omitted, the limit specified in `RATELIMIT_MAX` will be used. - - it can be also set to `"unlimited"`, in which case the API key bypasses all rate limits. - -- **`ips`** contains an array of allowlisted IP ranges, which can be specified both as individual ips or CIDR ranges (e.g. *`["192.168.42.69", "2001:db8::48", "10.0.0.0/8", "fe80::/10"]`*). - - when specified, only requests from these ip ranges can use the specified api key. - - when omitted, any IP can be used to make requests with that API key. - -- **`userAgents`** contains an array of allowed user agents, with support for wildcards (e.g. *`["cobaltbot/1.0", "Mozilla/5.0 * Chrome/*"]`*). - - when specified, requests with a `user-agent` that does not appear in this array will be rejected. - - when omitted, any user agent can be specified to make requests with that API key. - -- if both `ips` and `userAgents` are set, the tokens will be limited by both parameters. -- if cobalt detects any problem with your key file, it will be ignored and a warning will be printed to the console. - -an example key file could look like this: -```json -{ - "b5c7160a-b655-4c7a-b500-de839f094550": { - "limit": 10, - "ips": ["10.0.0.0/8", "192.168.42.42"], - "userAgents": ["*Chrome*"] - }, - "b00b1234-a3e5-99b1-c6d1-dba4512ae190": { - "limit": "unlimited", - "ips": ["192.168.1.2"], - "userAgents": ["cobaltbot/1.0"] - } -} -``` - -if you are configuring a key file, **do not use the UUID from the example** but instead generate your own. you can do this by running the following command if you have node.js installed: -`node -e "console.log(crypto.randomUUID())"` +## list of environment variables +[this section has moved](/docs/api-env-variables.md) to a dedicated document that is way easier to understand and maintain. go check it out! diff --git a/web/package.json b/web/package.json index 827b6f7b..0e084fb9 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "@imput/cobalt-web", - "version": "10.7.5", + "version": "10.9", "type": "module", "private": true, "scripts": { diff --git a/web/src/lib/api/api.ts b/web/src/lib/api/api.ts index e199f3e9..1eab6e24 100644 --- a/web/src/lib/api/api.ts +++ b/web/src/lib/api/api.ts @@ -2,7 +2,7 @@ import { get } from "svelte/store"; import settings from "$lib/state/settings"; -import { getSession } from "$lib/api/session"; +import { getSession, resetSession } from "$lib/api/session"; import { currentApiURL } from "$lib/api/api-url"; import { turnstileEnabled, turnstileSolved } from "$lib/state/turnstile"; import cachedInfo from "$lib/state/server-info"; @@ -42,7 +42,7 @@ const getAuthorization = async () => { } } -const request = async (request: CobaltSaveRequestBody) => { +const request = async (requestBody: CobaltSaveRequestBody, justRetried = false) => { await getServerInfo(); const getCachedInfo = get(cachedInfo); @@ -75,7 +75,7 @@ const request = async (request: CobaltSaveRequestBody) => { method: "POST", redirect: "manual", signal: AbortSignal.timeout(20000), - body: JSON.stringify(request), + body: JSON.stringify(requestBody), headers: { "Accept": "application/json", "Content-Type": "application/json", @@ -94,9 +94,31 @@ const request = async (request: CobaltSaveRequestBody) => { } }); + if ( + response?.status === 'error' + && response?.error.code === 'error.api.auth.jwt.invalid' + && !justRetried + ) { + resetSession(); + await waitForTurnstile().catch(() => {}); + return request(requestBody, true); + } + return response; } +const waitForTurnstile = async () => { + await getAuthorization(); + return new Promise(resolve => { + const unsub = turnstileSolved.subscribe(solved => { + if (solved) { + unsub(); + resolve(); + } + }); + }); +} + const probeCobaltTunnel = async (url: string) => { const request = await fetch(`${url}&p=1`).catch(() => {}); if (request?.status === 200) { diff --git a/web/src/lib/api/session.ts b/web/src/lib/api/session.ts index 5b3e542b..40304672 100644 --- a/web/src/lib/api/session.ts +++ b/web/src/lib/api/session.ts @@ -62,3 +62,5 @@ export const getSession = async () => { } return newSession; } + +export const resetSession = () => cache = undefined;