merge: 10.9 from main

.github/workflows/docker-develop.yml

@@ -1,4 +1,4 @@
-name: Build Docker development image
+name: Build development Docker image
 
 on:
   workflow_dispatch:

.github/workflows/docker-staging.yml

@@ -0,0 +1,55 @@
+name: Build staging Docker image
+
+on:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get release metadata
+        id: release-meta
+        run: |
+            version=$(cat package.json | jq -r .version)
+            echo "commit_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+            echo "version=$version" >> $GITHUB_OUTPUT
+            echo "major_version=$(echo "$version" | cut -d. -f1)" >> $GITHUB_OUTPUT
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          tags: type=raw,value=staging
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/docker.yml

@@ -1,4 +1,4 @@
-name: Build Docker image
+name: Build release Docker image
 
 on:
   workflow_dispatch:

README.md

@@ -36,12 +36,10 @@ this monorepo includes source code for api, frontend, and related packages:
 - [packages tree](/packages/)
 
 it also includes documentation in the [docs tree](/docs/):
-- [cobalt api documentation](/docs/api.md)
 - [how to run a cobalt instance](/docs/run-an-instance.md)
-- [how to protect a cobalt instance](/docs/protect-an-instance.md) (recommended if you host a public instance)
+- [how to protect a cobalt instance](/docs/protect-an-instance.md)
-
+- [cobalt api instance environment variables](/docs/api-env-variables.md)
-### thank you
+- [cobalt api documentation](/docs/api.md)
-cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
 
 ### ethics
 cobalt is a tool that makes downloading public content easier. it takes **zero liability**.
@@ -55,6 +53,9 @@ same content can be downloaded via dev tools of any modern web browser.
 ### contributing
 if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away.
 
+### thank you
+cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
+
 ### licenses
 for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs.
 unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE).

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "10.8.3",
+    "version": "10.9.1",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",

api/src/config.js

@@ -28,6 +28,9 @@ const env = {
     rateLimitWindow: (process.env.RATELIMIT_WINDOW && parseInt(process.env.RATELIMIT_WINDOW)) || 60,
     rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20,
 
+    sessionRateLimitWindow: (process.env.SESSION_RATELIMIT_WINDOW && parseInt(process.env.SESSION_RATELIMIT_WINDOW)) || 60,
+    sessionRateLimit: (process.env.SESSION_RATELIMIT && parseInt(process.env.SESSION_RATELIMIT)) || 10,
+
     durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800,
     streamLifespan: (process.env.TUNNEL_LIFESPAN && parseInt(process.env.TUNNEL_LIFESPAN)) || 90,
 

api/src/core/api.js

@@ -75,8 +75,8 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
     const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url');
 
     const sessionLimiter = rateLimit({
-        windowMs: 60000,
+        windowMs: env.sessionRateLimitWindow * 1000,
-        limit: 10,
+        limit: env.sessionRateLimit,
         standardHeaders: 'draft-6',
         legacyHeaders: false,
         keyGenerator,
@@ -92,7 +92,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
         keyGenerator: req => req.rateLimitKey || keyGenerator(req),
         store: await createStore('api'),
         handler: handleRateExceeded
-    })
+    });
 
     const apiTunnelLimiter = rateLimit({
         windowMs: env.rateLimitWindow * 1000,
@@ -104,7 +104,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
         handler: (_, res) => {
             return res.sendStatus(429)
         }
-    })
+    });
 
     app.set('trust proxy', ['loopback', 'uniquelocal']);
 
@@ -176,7 +176,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
                 return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            if (!jwt.verify(token)) {
+            if (!jwt.verify(token, getIP(req, 32))) {
                 return fail(res, "error.api.auth.jwt.invalid");
             }
 
@@ -222,7 +222,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
         }
 
         try {
-            res.json(jwt.generate());
+            res.json(jwt.generate(getIP(req, 32)));
         } catch {
             return fail(res, "error.api.generic");
         }

api/src/misc/utils.js

@@ -1,17 +1,26 @@
 import { request } from 'undici';
 const redirectStatuses = new Set([301, 302, 303, 307, 308]);
 
-export async function getRedirectingURL(url, dispatcher, userAgent) {
+export async function getRedirectingURL(url, dispatcher, headers) {
-    const location = await request(url, {
+    const params = {
         dispatcher,
         method: 'HEAD',
-        headers: { 'user-agent': userAgent }
+        headers,
-    }).then(r => {
+        redirect: 'manual'
+    };
+
+    let location = await request(url, params).then(r => {
         if (redirectStatuses.has(r.statusCode) && r.headers['location']) {
             return r.headers['location'];
         }
     }).catch(() => null);
 
+    location ??= await fetch(url, params).then(r => {
+        if (redirectStatuses.has(r.status) && r.headers.has('location')) {
+            return r.headers.get('location');
+        }
+    }).catch(() => null);
+
     return location;
 }
 

api/src/processing/helpers/youtube-session.js

@@ -1,8 +1,11 @@
 import * as cluster from "../../misc/cluster.js";
 
+import { Agent } from "undici";
 import { env } from "../../config.js";
 import { Green, Yellow } from "../../misc/console-text.js";
 
+const defaultAgent = new Agent();
+
 let session;
 
 const validateSession = (sessionResponse) => {
@@ -32,7 +35,11 @@ const loadSession = async () => {
     const sessionServerUrl = new URL(env.ytSessionServer);
     sessionServerUrl.pathname = "/token";
 
-    const newSession = await fetch(sessionServerUrl).then(a => a.json());
+    const newSession = await fetch(
+        sessionServerUrl,
+        { dispatcher: defaultAgent }
+    ).then(a => a.json());
+
     validateSession(newSession);
 
     if (!session || session.updated < newSession?.updated) {

api/src/processing/request.js

@@ -118,14 +118,13 @@ export function normalizeRequest(request) {
     ));
 }
 
-export function getIP(req) {
+export function getIP(req, prefix = 56) {
     const strippedIP = req.ip.replace(/^::ffff:/, '');
     const ip = ipaddr.parse(strippedIP);
     if (ip.kind() === 'ipv4') {
         return strippedIP;
     }
 
-    const prefix = 56;
     const v6Bytes = ip.toByteArray();
           v6Bytes.fill(0, prefix / 8);
 

api/src/processing/service-config.js

@@ -90,7 +90,9 @@ export const services = {
             "r/u_:user/comments/:id/:title",
             "r/u_:user/comments/:id/comment/:commentId",
 
-            "r/:sub/s/:shareId"
+            "r/:sub/s/:shareId",
+
+            "video/:shortId",
         ],
         subdomains: "*",
     },

api/src/processing/service-patterns.js

@@ -23,7 +23,8 @@ export const testers = {
         pattern.id?.length <= 16 && !pattern.sub && !pattern.user
         || (pattern.sub?.length <= 22 && pattern.id?.length <= 16)
         || (pattern.user?.length <= 22 && pattern.id?.length <= 16)
-        || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16),
+        || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16)
+        || (pattern.shortId?.length <= 16),
 
     "rutube": pattern =>
         (pattern.id?.length === 32 && pattern.key?.length <= 32) ||

api/src/processing/services/instagram.js

@@ -527,7 +527,7 @@ export default function instagram(obj) {
             // for some reason instagram decides to return HTML
             // instead of a redirect when requesting with a normal
             // browser user-agent
-            'curl/7.88.1'
+            {'User-Agent': 'curl/7.88.1'}
         ).then(match => instagram({
             ...obj, ...match,
             shareId: undefined

api/src/processing/services/reddit.js

@@ -50,12 +50,24 @@ async function getAccessToken() {
 
 export default async function(obj) {
     let params = obj;
+    const accessToken = await getAccessToken();
+    const headers = {
+        'user-agent': genericUserAgent,
+        authorization: accessToken && `Bearer ${accessToken}`,
+        accept: 'application/json'
+    };
+
+    if (params.shortId) {
+        params = await resolveRedirectingURL(
+            `https://www.reddit.com/video/${params.shortId}`,
+            obj.dispatcher, headers
+        );
+    }
 
     if (!params.id && params.shareId) {
         params = await resolveRedirectingURL(
             `https://www.reddit.com/r/${params.sub}/s/${params.shareId}`,
-            obj.dispatcher,
+            obj.dispatcher, headers
-            genericUserAgent
         );
     }
 
@@ -63,17 +75,10 @@ export default async function(obj) {
 
     const url = new URL(`https://www.reddit.com/comments/${params.id}.json`);
 
-    const accessToken = await getAccessToken();
     if (accessToken) url.hostname = 'oauth.reddit.com';
 
     let data = await fetch(
-        url, {
+        url, { headers }
-            headers: {
-                'User-Agent': genericUserAgent,
-                accept: 'application/json',
-                authorization: accessToken && `Bearer ${accessToken}`
-            }
-        }
     ).then(r => r.json()).catch(() => {});
 
     if (!data || !Array.isArray(data)) {

api/src/processing/url.js

@@ -106,6 +106,14 @@ function aliasURL(url) {
                 url.pathname = `/share/${idPart.slice(-32)}`;
             }
             break;
+            
+        case "redd":
+            /* reddit short video links can be treated by changing https://v.redd.it/<id>
+            to https://reddit.com/video/<id>.*/
+            if (url.hostname === "v.redd.it" && parts.length === 2) {
+                url = new URL(`https://www.reddit.com/video/${parts[1]}`);
+            }
+            break;
     }
 
     return url;
@@ -231,11 +239,11 @@ export function extract(url) {
     return { host, patternMatch };
 }
 
-export async function resolveRedirectingURL(url, dispatcher, userAgent) {
+export async function resolveRedirectingURL(url, dispatcher, headers) {
     const originalService = getHostIfValid(normalizeURL(url));
     if (!originalService) return;
 
-    const canonicalURL = await getRedirectingURL(url, dispatcher, userAgent);
+    const canonicalURL = await getRedirectingURL(url, dispatcher, headers);
     if (!canonicalURL) return;
 
     const { host, patternMatch } = extract(normalizeURL(canonicalURL));

api/src/security/jwt.js

@@ -6,12 +6,19 @@ import { env } from "../config.js";
 const toBase64URL = (b) => Buffer.from(b).toString("base64url");
 const fromBase64URL = (b) => Buffer.from(b, "base64url").toString();
 
-const makeHmac = (header, payload) =>
+const makeHmac = (data) => {
-    createHmac("sha256", env.jwtSecret)
+    return createHmac("sha256", env.jwtSecret)
-        .update(`${header}.${payload}`)
+            .update(data)
-        .digest("base64url");
+            .digest("base64url");
+}
 
-const generate = () => {
+const sign = (header, payload) =>
+        makeHmac(`${header}.${payload}`);
+
+const getIPHash = (ip) =>
+        makeHmac(ip).slice(0, 8);
+
+const generate = (ip) => {
     const exp = Math.floor(new Date().getTime() / 1000) + env.jwtLifetime;
 
     const header = toBase64URL(JSON.stringify({
@@ -21,10 +28,11 @@ const generate = () => {
 
     const payload = toBase64URL(JSON.stringify({
         jti: nanoid(8),
+        sub: getIPHash(ip),
         exp,
     }));
 
-    const signature = makeHmac(header, payload);
+    const signature = sign(header, payload);
 
     return {
         token: `${header}.${payload}.${signature}`,
@@ -32,7 +40,7 @@ const generate = () => {
     };
 }
 
-const verify = (jwt) => {
+const verify = (jwt, ip) => {
     const [header, payload, signature] = jwt.split(".", 3);
     const timestamp = Math.floor(new Date().getTime() / 1000);
 
@@ -40,17 +48,16 @@ const verify = (jwt) => {
         return false;
     }
 
-    const verifySignature = makeHmac(header, payload);
+    const verifySignature = sign(header, payload);
 
     if (verifySignature !== signature) {
         return false;
     }
 
-    if (timestamp >= JSON.parse(fromBase64URL(payload)).exp) {
+    const data = JSON.parse(fromBase64URL(payload));
-        return false;
-    }
 
-    return true;
+    return getIPHash(ip) === data.sub
+            && timestamp <= data.exp;
 }
 
 export default {

api/src/util/tests/instagram.json

@@ -123,6 +123,7 @@
     {
         "name": "private instagram post",
         "url": "https://www.instagram.com/p/C5_A1TQNPrYw4c2g9KAUTPUl8RVHqiAdAcOOSY0",
+        "canFail": true,
         "params": {},
         "expected": {
             "code": 400,

api/src/util/tests/reddit.json

@@ -56,5 +56,23 @@
             "code": 200,
             "status": "tunnel"
         }
+    },
+    {
+        "name": "shortened video link",
+        "url": "https://v.redd.it/ifg2emt5ck0e1",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
+    },
+    {
+        "name": "shortened video link (alternative)",
+        "url": "https://reddit.com/video/ifg2emt5ck0e1",
+        "params": {},
+        "expected": {
+            "code": 200,
+            "status": "tunnel"
+        }
     }
 ]

docs/api-env-variables.md

@@ -0,0 +1,228 @@
+# cobalt api instance environment variables
+you can customize your processing instance's behavior using these environment variables. all of them but `API_URL` are optional.
+this document is not final and will expand over time. feel free to improve it!
+
+### general vars
+| name                | default   | value example                         |
+|:--------------------|:----------|:--------------------------------------|
+| API_URL             |           | `https://api.url.example/`            |
+| API_PORT            | `9000`    | `1337`                                |
+| COOKIE_PATH         |           | `/cookies.json`                       |
+| PROCESSING_PRIORITY |           | `10`                                  |
+| API_INSTANCE_COUNT  |           | `6`                                   |
+| API_REDIS_URL       |           | `redis://localhost:6379`              |
+| DISABLED_SERVICES   |           | `bilibili,youtube`                    |
+
+[*view details*](#general)
+
+### networking vars
+| name                | default   | value example                         |
+|:--------------------|:----------|:--------------------------------------|
+| API_LISTEN_ADDRESS  | `0.0.0.0` | `127.0.0.1`                           |
+| API_EXTERNAL_PROXY  |           | `http://user:password@127.0.0.1:8080` |
+| FREEBIND_CIDR       |           | `2001:db8::/32`                       |
+
+[*view details*](#networking)
+
+### limit vars
+| name                     | default | value example |
+|:-------------------------|:--------|:--------------|
+| DURATION_LIMIT           | `10800` | `18000`       |
+| TUNNEL_LIFESPAN          | `90`    | `120`         |
+| RATELIMIT_WINDOW         | `60`    | `120`         |
+| RATELIMIT_MAX            | `20`    | `30`          |
+| SESSION_RATELIMIT_WINDOW | `60`    | `60`          |
+| SESSION_RATELIMIT        | `10`    | `10`          |
+
+[*view details*](#limits)
+
+### security vars
+| name              | default | value example                         |
+|:------------------|:--------|:--------------------------------------|
+| CORS_WILDCARD     | `1`     | `0`                                   |
+| CORS_URL          |         | `https://web.url.example`             |
+| TURNSTILE_SITEKEY |         | `1x00000000000000000000BB`            |
+| TURNSTILE_SECRET  |         | `1x0000000000000000000000000000000AA` |
+| JWT_SECRET        |         | see [details](#security)              |
+| JWT_EXPIRY        | `120`   | `240`                                 |
+| API_KEY_URL       |         | `file://keys.json`                    |
+| API_AUTH_REQUIRED |         | `1`                                   |
+
+[*view details*](#security)
+
+### service-specific vars
+| name                             | value example            |
+|:---------------------------------|:-------------------------|
+| CUSTOM_INNERTUBE_CLIENT          | `IOS`                    |
+| YOUTUBE_SESSION_SERVER           | `http://localhost:8080/` |
+| YOUTUBE_SESSION_INNERTUBE_CLIENT | `WEB_EMBEDDED`           |
+
+[*view details*](#service-specific)
+
+## general
+[*jump to the table*](#general-vars)
+
+### API_URL
+> [!NOTE]
+> API_URL is required to run the API instance.
+
+the URL from which your instance will be accessible. can be external or internal, but it must be a valid URL or else tunnels will not work.
+
+the value is a URL.
+
+### API_PORT
+port from which the API server will be accessible.
+
+the value is a number from 1024 to 65535.
+
+### COOKIE_PATH
+path to the `cookies.json` file relative to the current working directory of your cobalt instance (usually the main (src/api) folder).
+
+### PROCESSING_PRIORITY
+`nice` value for ffmpeg subprocesses. available only on unix systems.
+
+note: the higher the nice value, the lower the priority. you can [read more about nice here](https://en.wikipedia.org/wiki/Nice_(Unix)).
+
+the value is a number.
+
+### API_INSTANCE_COUNT
+supported only on linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. `API_REDIS_URL` is required to use this option.
+
+the value is a number.
+
+### API_REDIS_URL
+when configured, cobalt will use this redis instance for tunnel cache. required when `API_INSTANCE_COUNT` is more than 1, because else sub-instance wouldn't be able to share cache.
+
+the value is a URL.
+
+### DISABLED_SERVICES
+comma-separated list which disables certain services from being used.
+
+the value is a string of cobalt-supported services.
+
+## networking
+[*jump to the table*](#networking-vars)
+
+### API_LISTEN_ADDRESS
+defines the local address for the api instance. if you are using a docker container, you usually don't need to configure this.
+
+the value is a local IP address.
+
+### API_EXTERNAL_PROXY
+URL of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only.
+
+if some feature breaks when using a proxy, please make a new issue about it!
+
+the value is a URL.
+
+### FREEBIND_CIDR
+IPv6 prefix used for randomly assigning addresses to cobalt requests. available only on linux systems.
+
+setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all requests it makes for that particular download.
+
+to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first.
+
+if you want to use this option and run cobalt in a docker container, you also need to set the `API_LISTEN_ADDRESS` env variable to `127.0.0.1` and set `network_mode` for the container to `host`.
+
+the value is an IPv6 range.
+
+## limits
+[*jump to the table*](#limit-vars)
+
+### DURATION_LIMIT
+media duration limit, in **seconds**
+
+the value is a number.
+
+### TUNNEL_LIFESPAN
+the duration for which tunnel info is stored in ram, **in seconds**.
+
+it's recommended to keep this value either default or as low as possible to preserve efficiency and user privacy.
+
+the value is a number.
+
+### RATELIMIT_WINDOW
+rate limit time window for api requests, but not session requests, in **seconds**.
+
+the value is a number.
+
+### RATELIMIT_MAX
+amount of api requests to be allowed within the time window of `RATELIMIT_WINDOW`.
+
+the value is a number.
+
+### SESSION_RATELIMIT_WINDOW
+rate limit time window for session creation requests, in **seconds**.
+
+the value is a number.
+
+### SESSION_RATELIMIT
+amount of session requests to be allowed within the time window of `SESSION_RATELIMIT_WINDOW`.
+
+the value is a number.
+
+## security
+[*jump to the table*](#security-vars)
+
+> [!NOTE]
+> in order to enable turnstile bot protection, `TURNSTILE_SITEKEY`, `TURNSTILE_SECRET`, and `JWT_SECRET` must be set. all three at once.
+
+### CORS_WILDCARD
+defines whether cross-origin resource sharing is enabled. when enabled, your instance will be accessible from foreign web pages.
+
+the value is a number. 0: disabled. 1: enabled.
+
+### CORS_URL
+configures the [cross-origin resource sharing origin](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Access-Control-Allow-Origin). your instance will be available only from this URL if `CORS_WILDCARD` is set to `0`.
+
+the value is a URL.
+
+### TURNSTILE_SITEKEY
+[cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) sitekey used by the web client to request & solve a challenge to prove that the user is not a bot.
+
+the value is a specific key.
+
+### TURNSTILE_SECRET
+[cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) secret used by the processing instance to verify that the client solved the challenge successfully.
+
+the value is a specific key.
+
+### JWT_SECRET
+the secret used for issuing JWT tokens for request authentication. the value must be a random, secure, and long string (over 16 characters).
+
+the value is a specific key.
+
+### JWT_EXPIRY
+the duration of how long a cobalt-issued JWT token will remain valid, in seconds.
+
+the value is a number.
+
+### API_KEY_URL
+the URL to the the external or local key database. for local files you have to specify a local path using the `file://` protocol.
+
+see [the api key section](/docs/protect-an-instance.md#api-key-file-format) in the "how to protect your cobalt instance" document for more details.
+
+the value is a URL.
+
+### API_AUTH_REQUIRED
+when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled).
+
+the value is a number.
+
+## service-specific
+[*jump to the table*](#service-specific-vars)
+
+### CUSTOM_INNERTUBE_CLIENT
+innertube client that will be used instead of the default one.
+
+the value is a string.
+
+### YOUTUBE_SESSION_SERVER
+URL to an instance of [yt-session-generator](https://github.com/imputnet/yt-session-generator). used for automatically pulling `poToken` & `visitor_data` for youtube. can be local or remote.
+
+the value is a URL.
+
+### YOUTUBE_SESSION_INNERTUBE_CLIENT
+innertube client that's compatible with botguard's (web) `poToken` and `visitor_data`.
+
+the value is a string.

docs/examples/docker-compose.example.yml

@@ -49,6 +49,5 @@ services:
     #     init: true
     #     restart: unless-stopped
     #     container_name: yt-session-generator
-
+    #     labels:
-    #     ports:
+    #       - com.centurylinklabs.watchtower.scope=cobalt
-    #         - 127.0.0.1:1280:8080

docs/protect-an-instance.md

@@ -114,7 +114,7 @@ if you want to use your instance outside of web interface, you'll need an api ke
 >
 > if api keys leak, you'll have to update/remove all UUIDs to revoke them.
 
-1. create a `keys.json` file following [the schema and example here](/docs//run-an-instance.md#api-key-file-format).
+1. create a `keys.json` file following [the schema and example down below](#api-key-file-format).
 
 2. expose the `keys.json` to the docker container:
 ```yml
@@ -148,3 +148,55 @@ environment:
 ### why not make keys exclusive by default?
 keys may be useful for going around rate limiting,
 while keeping the rest of api rate limited, with no turnstile in place.
+
+## api key file format
+the file is a JSON-serialized object with the following structure:
+```typescript
+
+type KeyFileContents = Record<
+    UUIDv4String,
+    {
+        name?: string,
+        limit?: number | "unlimited",
+        ips?: (CIDRString | IPString)[],
+        userAgents?: string[]
+    }
+>;
+```
+
+where *`UUIDv4String`* is a stringified version of a UUIDv4 identifier.
+- **name** is a field for your own reference, it is not used by cobalt anywhere.
+
+- **`limit`** specifies how many requests the API key can make during the window specified in the `RATELIMIT_WINDOW` env.
+    - when omitted, the limit specified in `RATELIMIT_MAX` will be used.
+    - it can be also set to `"unlimited"`, in which case the API key bypasses all rate limits.
+
+- **`ips`** contains an array of allowlisted IP ranges, which can be specified both as individual ips or CIDR ranges (e.g. *`["192.168.42.69", "2001:db8::48", "10.0.0.0/8", "fe80::/10"]`*).
+    - when specified, only requests from these ip ranges can use the specified api key.
+    - when omitted, any IP can be used to make requests with that API key.
+
+- **`userAgents`** contains an array of allowed user agents, with support for wildcards (e.g. *`["cobaltbot/1.0", "Mozilla/5.0 * Chrome/*"]`*).
+    - when specified, requests with a `user-agent` that does not appear in this array will be rejected.
+    - when omitted, any user agent can be specified to make requests with that API key.
+
+- if both `ips` and `userAgents` are set, the tokens will be limited by both parameters.
+- if cobalt detects any problem with your key file, it will be ignored and a warning will be printed to the console.
+
+an example key file could look like this:
+```json
+{
+    "b5c7160a-b655-4c7a-b500-de839f094550": {
+        "limit": 10,
+        "ips": ["10.0.0.0/8", "192.168.42.42"],
+        "userAgents": ["*Chrome*"]
+    },
+    "b00b1234-a3e5-99b1-c6d1-dba4512ae190": {
+        "limit": "unlimited",
+        "ips": ["192.168.1.2"],
+        "userAgents": ["cobaltbot/1.0"]
+    }
+}
+```
+
+if you are configuring a key file, **do not use the UUID from the example** but instead generate your own. you can do this by running the following command if you have node.js installed:
+`node -e "console.log(crypto.randomUUID())"`

docs/run-an-instance.md

@@ -1,4 +1,6 @@
 # how to run a cobalt instance
+this tutorial will help you run your own cobalt processing instance. if your instance is public-facing, we highly recommend that you also [protect it from abuse](/docs/protect-an-instance.md) using turnstile or api keys or both.
+
 ## using docker compose and package from github (recommended)
 to run the cobalt docker package, you need to have `docker` and `docker-compose` installed and configured.
 
@@ -54,93 +56,5 @@ sudo apt install nscd
 sudo service nscd start
 ```
 
-## list of environment variables for api
+## list of environment variables
-| variable name         | default   | example                 | description |
+[this section has moved](/docs/api-env-variables.md) to a dedicated document that is way easier to understand and maintain. go check it out!
-|:----------------------|:----------|:------------------------|:------------|
-| `API_PORT`            | `9000`    | `9000`                  | changes port from which api server is accessible. |
-| `API_LISTEN_ADDRESS`  | `0.0.0.0` | `127.0.0.1`             | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** |
-| `API_URL`             | ➖        | `https://api.cobalt.tools/` | changes url from which api server is accessible. <br> ***REQUIRED TO RUN THE API***. |
-| `API_NAME`            | `unknown` | `ams-1`                 | api server name that is shown in `/api/serverInfo`. |
-| `API_EXTERNAL_PROXY`  | ➖        | `http://user:password@127.0.0.1:8080`| url of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only. |
-| `CORS_WILDCARD`       | `1`       | `0`                     | toggles cross-origin resource sharing. <br> `0`: disabled. `1`: enabled. |
-| `CORS_URL`            | not used  | `https://cobalt.tools`  | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. |
-| `COOKIE_PATH`         | not used  | `/cookies.json`         | path for cookie file relative to main folder. |
-| `PROCESSING_PRIORITY` | not used  | `10`                    | changes `nice` value* for ffmpeg subprocess. available only on unix systems. |
-| `FREEBIND_CIDR`       | ➖        | `2001:db8::/32`         | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. see below for more info. |
-| `RATELIMIT_WINDOW`    | `60`      | `120`                   | rate limit time window in **seconds**. |
-| `RATELIMIT_MAX`       | `20`      | `30`                    | max requests per time window. requests above this amount will be blocked for the rate limit window duration. |
-| `DURATION_LIMIT`      | `10800`   | `18000`                 | max allowed video duration in **seconds**. |
-| `TUNNEL_LIFESPAN`     | `90`      | `120`                   | the duration for which tunnel info is stored in ram, **in seconds**. |
-| `TURNSTILE_SITEKEY`   | ➖        | `1x00000000000000000000BB` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) sitekey used by browser clients to request a challenge.\*\* |
-| `TURNSTILE_SECRET`    | ➖        | `1x0000000000000000000000000000000AA` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) secret used by cobalt to verify the client successfully solved the challenge.\*\* |
-| `JWT_SECRET`          | ➖        | ➖                      | the secret used for issuing JWT tokens for request authentication. to choose a value, generate a random, secure, long string (ideally >=16 characters).\*\* |
-| `JWT_EXPIRY`          | `120`     | `240`                  | the duration of how long a cobalt-issued JWT token will remain valid, in seconds. |
-| `API_KEY_URL`         | ➖        | `file://keys.json`      | the location of the api key database. for loading API keys, cobalt supports HTTP(S) urls, or local files by specifying a local path using the `file://` protocol. see the "api key file format" below for more details.  |
-| `API_AUTH_REQUIRED`   | ➖        | `1`                     | when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled). |
-| `API_REDIS_URL`       | ➖        | `redis://localhost:6379` | when set, cobalt uses redis instead of internal memory for the tunnel cache. |
-| `API_INSTANCE_COUNT`  | ➖        | `2`                     | supported only on Linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. |
-| `DISABLED_SERVICES`   | ➖        | `bilibili,youtube`       | comma-separated list which disables certain services from being used. |
-| `CUSTOM_INNERTUBE_CLIENT` | ➖    | `IOS`                    | innertube client that will be used instead of the default one. |
-| `YOUTUBE_SESSION_SERVER` | ➖    | `http://localhost:8080/`  | url to an instance of [invidious' youtube-trusted-session-generator](https://github.com/iv-org/youtube-trusted-session-generator) or its fork/counterpart. used for automatically pulling poToken & visitor_data for youtube. can be local or remote. |
-
-\* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
-
-\*\* in order to enable turnstile bot protection, all three **`TURNSTILE_SITEKEY`, `TURNSTILE_SECRET` and `JWT_SECRET`** need to be set.
-
-#### FREEBIND_CIDR
-setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all
-requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt
-in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set
-`network_mode` for the container to `host`.
-
-## api key file format
-the file is a JSON-serialized object with the following structure:
-```typescript
-
-type KeyFileContents = Record<
-    UUIDv4String,
-    {
-        name?: string,
-        limit?: number | "unlimited",
-        ips?: (CIDRString | IPString)[],
-        userAgents?: string[]
-    }
->;
-```
-
-where *`UUIDv4String`* is a stringified version of a UUIDv4 identifier.
-- **name** is a field for your own reference, it is not used by cobalt anywhere.
-
-- **`limit`** specifies how many requests the API key can make during the window specified in the `RATELIMIT_WINDOW` env.
-    - when omitted, the limit specified in `RATELIMIT_MAX` will be used.
-    - it can be also set to `"unlimited"`, in which case the API key bypasses all rate limits.
-
-- **`ips`** contains an array of allowlisted IP ranges, which can be specified both as individual ips or CIDR ranges (e.g. *`["192.168.42.69", "2001:db8::48", "10.0.0.0/8", "fe80::/10"]`*).
-    - when specified, only requests from these ip ranges can use the specified api key.
-    - when omitted, any IP can be used to make requests with that API key.
-
-- **`userAgents`** contains an array of allowed user agents, with support for wildcards (e.g. *`["cobaltbot/1.0", "Mozilla/5.0 * Chrome/*"]`*).
-    - when specified, requests with a `user-agent` that does not appear in this array will be rejected.
-    - when omitted, any user agent can be specified to make requests with that API key.
-
-- if both `ips` and `userAgents` are set, the tokens will be limited by both parameters.
-- if cobalt detects any problem with your key file, it will be ignored and a warning will be printed to the console.
-
-an example key file could look like this:
-```json
-{
-    "b5c7160a-b655-4c7a-b500-de839f094550": {
-        "limit": 10,
-        "ips": ["10.0.0.0/8", "192.168.42.42"],
-        "userAgents": ["*Chrome*"]
-    },
-    "b00b1234-a3e5-99b1-c6d1-dba4512ae190": {
-        "limit": "unlimited",
-        "ips": ["192.168.1.2"],
-        "userAgents": ["cobaltbot/1.0"]
-    }
-}
-```
-
-if you are configuring a key file, **do not use the UUID from the example** but instead generate your own. you can do this by running the following command if you have node.js installed:
-`node -e "console.log(crypto.randomUUID())"`

web/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@imput/cobalt-web",
-    "version": "10.7.5",
+    "version": "10.9",
     "type": "module",
     "private": true,
     "scripts": {

web/src/lib/api/api.ts

@@ -2,7 +2,7 @@ import { get } from "svelte/store";
 
 import settings from "$lib/state/settings";
 
-import { getSession } from "$lib/api/session";
+import { getSession, resetSession } from "$lib/api/session";
 import { currentApiURL } from "$lib/api/api-url";
 import { turnstileEnabled, turnstileSolved } from "$lib/state/turnstile";
 import cachedInfo from "$lib/state/server-info";
@@ -42,7 +42,7 @@ const getAuthorization = async () => {
     }
 }
 
-const request = async (request: CobaltSaveRequestBody) => {
+const request = async (requestBody: CobaltSaveRequestBody, justRetried = false) => {
     await getServerInfo();
 
     const getCachedInfo = get(cachedInfo);
@@ -75,7 +75,7 @@ const request = async (request: CobaltSaveRequestBody) => {
         method: "POST",
         redirect: "manual",
         signal: AbortSignal.timeout(20000),
-        body: JSON.stringify(request),
+        body: JSON.stringify(requestBody),
         headers: {
             "Accept": "application/json",
             "Content-Type": "application/json",
@@ -94,9 +94,31 @@ const request = async (request: CobaltSaveRequestBody) => {
         }
     });
 
+    if (
+        response?.status === 'error'
+            && response?.error.code === 'error.api.auth.jwt.invalid'
+            && !justRetried
+    ) {
+        resetSession();
+        await waitForTurnstile().catch(() => {});
+        return request(requestBody, true);
+    }
+
     return response;
 }
 
+const waitForTurnstile = async () => {
+    await getAuthorization();
+    return new Promise<void>(resolve => {
+        const unsub = turnstileSolved.subscribe(solved => {
+            if (solved) {
+                unsub();
+                resolve();
+            }
+        });
+    });
+}
+
 const probeCobaltTunnel = async (url: string) => {
     const request = await fetch(`${url}&p=1`).catch(() => {});
     if (request?.status === 200) {

web/src/lib/api/session.ts

@@ -62,3 +62,5 @@ export const getSession = async () => {
     }
     return newSession;
 }
+
+export const resetSession = () => cache = undefined;