Mirrors/cobalt
1
0
Fork 0
mirror of https://github.com/wukko/cobalt.git synced 2025-04-29 14:04:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

merge: 10.9 from main

Browse Source
This commit is contained in:
wukko 2025-04-02 21:47:49 +06:00
commit 06bc51db54
No known key found for this signature in database
GPG Key ID: 3E30B3F26C7B4AA2
25 changed files with 479 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/docker-develop.yml vendored
View File

@ -1,4 +1,4 @@
name: Build Docker development image
name: Build development Docker image
on:
workflow_dispatch:

55
.github/workflows/docker-staging.yml vendored Normal file
View File

@ -0,0 +1,55 @@
name: Build staging Docker image
on:
workflow_dispatch:
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
build-and-push-image:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to the Container registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get release metadata
id: release-meta
run: |
version=$(cat package.json | jq -r .version)
echo "commit_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
echo "version=$version" >> $GITHUB_OUTPUT
echo "major_version=$(echo "$version" | cut -d. -f1)" >> $GITHUB_OUTPUT
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v5
with:
tags: type=raw,value=staging
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Build and push Docker image
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max

2
.github/workflows/docker.yml vendored
View File

@ -1,4 +1,4 @@
name: Build Docker image
name: Build release Docker image
on:
workflow_dispatch:

11
README.md
View File

@ -36,12 +36,10 @@ this monorepo includes source code for api, frontend, and related packages:
- [packages tree](/packages/)
it also includes documentation in the [docs tree](/docs/):
- [cobalt api documentation](/docs/api.md)
- [how to run a cobalt instance](/docs/run-an-instance.md)
- [how to protect a cobalt instance](/docs/protect-an-instance.md) (recommended if you host a public instance)
### thank you
cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
- [how to protect a cobalt instance](/docs/protect-an-instance.md)
- [cobalt api instance environment variables](/docs/api-env-variables.md)
- [cobalt api documentation](/docs/api.md)
### ethics
cobalt is a tool that makes downloading public content easier. it takes **zero liability**.
@ -55,6 +53,9 @@ same content can be downloaded via dev tools of any modern web browser.
### contributing
if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away.
### thank you
cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
### licenses
for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs.
unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE).

2
api/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@imput/cobalt-api",
"description": "save what you love",
"version": "10.8.3",
"version": "10.9.1",
"author": "imput",
"exports": "./src/cobalt.js",
"type": "module",

3
api/src/config.js
View File

@ -28,6 +28,9 @@ const env = {
rateLimitWindow: (process.env.RATELIMIT_WINDOW && parseInt(process.env.RATELIMIT_WINDOW)) || 60,
rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20,
sessionRateLimitWindow: (process.env.SESSION_RATELIMIT_WINDOW && parseInt(process.env.SESSION_RATELIMIT_WINDOW)) || 60,
sessionRateLimit: (process.env.SESSION_RATELIMIT && parseInt(process.env.SESSION_RATELIMIT)) || 10,
durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800,
streamLifespan: (process.env.TUNNEL_LIFESPAN && parseInt(process.env.TUNNEL_LIFESPAN)) || 90,

12
api/src/core/api.js
View File

@ -75,8 +75,8 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url');
const sessionLimiter = rateLimit({
windowMs: 60000,
limit: 10,
windowMs: env.sessionRateLimitWindow * 1000,
limit: env.sessionRateLimit,
standardHeaders: 'draft-6',
legacyHeaders: false,
keyGenerator,
@ -92,7 +92,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
keyGenerator: req => req.rateLimitKey || keyGenerator(req),
store: await createStore('api'),
handler: handleRateExceeded
})
});
const apiTunnelLimiter = rateLimit({
windowMs: env.rateLimitWindow * 1000,
@ -104,7 +104,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
handler: (_, res) => {
return res.sendStatus(429)
}
})
});
app.set('trust proxy', ['loopback', 'uniquelocal']);
@ -176,7 +176,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
return fail(res, "error.api.auth.jwt.invalid");
}
if (!jwt.verify(token)) {
if (!jwt.verify(token, getIP(req, 32))) {
return fail(res, "error.api.auth.jwt.invalid");
}
@ -222,7 +222,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
}
try {
res.json(jwt.generate());
res.json(jwt.generate(getIP(req, 32)));
} catch {
return fail(res, "error.api.generic");
}

17
api/src/misc/utils.js
View File

@ -1,17 +1,26 @@
import { request } from 'undici';
const redirectStatuses = new Set([301, 302, 303, 307, 308]);
export async function getRedirectingURL(url, dispatcher, userAgent) {
const location = await request(url, {
export async function getRedirectingURL(url, dispatcher, headers) {
const params = {
dispatcher,
method: 'HEAD',
headers: { 'user-agent': userAgent }
}).then(r => {
headers,
redirect: 'manual'
};
let location = await request(url, params).then(r => {
if (redirectStatuses.has(r.statusCode) && r.headers['location']) {
return r.headers['location'];
}
}).catch(() => null);
location ??= await fetch(url, params).then(r => {
if (redirectStatuses.has(r.status) && r.headers.has('location')) {
return r.headers.get('location');
}
}).catch(() => null);
return location;
}

9
api/src/processing/helpers/youtube-session.js
View File

@ -1,8 +1,11 @@
import * as cluster from "../../misc/cluster.js";
import { Agent } from "undici";
import { env } from "../../config.js";
import { Green, Yellow } from "../../misc/console-text.js";
const defaultAgent = new Agent();
let session;
const validateSession = (sessionResponse) => {
@ -32,7 +35,11 @@ const loadSession = async () => {
const sessionServerUrl = new URL(env.ytSessionServer);
sessionServerUrl.pathname = "/token";
const newSession = await fetch(sessionServerUrl).then(a => a.json());
const newSession = await fetch(
sessionServerUrl,
{ dispatcher: defaultAgent }
).then(a => a.json());
validateSession(newSession);
if (!session || session.updated < newSession?.updated) {

3
api/src/processing/request.js
View File

@ -118,14 +118,13 @@ export function normalizeRequest(request) {
));
}
export function getIP(req) {
export function getIP(req, prefix = 56) {
const strippedIP = req.ip.replace(/^::ffff:/, '');
const ip = ipaddr.parse(strippedIP);
if (ip.kind() === 'ipv4') {
return strippedIP;
}
const prefix = 56;
const v6Bytes = ip.toByteArray();
v6Bytes.fill(0, prefix / 8);

4
api/src/processing/service-config.js
View File

@ -90,7 +90,9 @@ export const services = {
"r/u_:user/comments/:id/:title",
"r/u_:user/comments/:id/comment/:commentId",
"r/:sub/s/:shareId"
"r/:sub/s/:shareId",
"video/:shortId",
],
subdomains: "*",
},

3
api/src/processing/service-patterns.js
View File

@ -23,7 +23,8 @@ export const testers = {
pattern.id?.length <= 16 && !pattern.sub && !pattern.user
|| (pattern.sub?.length <= 22 && pattern.id?.length <= 16)
|| (pattern.user?.length <= 22 && pattern.id?.length <= 16)
|| (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16),
|| (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16)
|| (pattern.shortId?.length <= 16),
"rutube": pattern =>
(pattern.id?.length === 32 && pattern.key?.length <= 32) ||

2
api/src/processing/services/instagram.js
View File

@ -527,7 +527,7 @@ export default function instagram(obj) {
// for some reason instagram decides to return HTML
// instead of a redirect when requesting with a normal
// browser user-agent
'curl/7.88.1'
{'User-Agent': 'curl/7.88.1'}
).then(match => instagram({
...obj, ...match,
shareId: undefined

25
api/src/processing/services/reddit.js
View File

@ -50,12 +50,24 @@ async function getAccessToken() {
export default async function(obj) {
let params = obj;
const accessToken = await getAccessToken();
const headers = {
'user-agent': genericUserAgent,
authorization: accessToken && `Bearer ${accessToken}`,
accept: 'application/json'
};
if (params.shortId) {
params = await resolveRedirectingURL(
`https://www.reddit.com/video/${params.shortId}`,
obj.dispatcher, headers
);
}
if (!params.id && params.shareId) {
params = await resolveRedirectingURL(
`https://www.reddit.com/r/${params.sub}/s/${params.shareId}`,
obj.dispatcher,
genericUserAgent
obj.dispatcher, headers
);
}
@ -63,17 +75,10 @@ export default async function(obj) {
const url = new URL(`https://www.reddit.com/comments/${params.id}.json`);
const accessToken = await getAccessToken();
if (accessToken) url.hostname = 'oauth.reddit.com';
let data = await fetch(
url, {
headers: {
'User-Agent': genericUserAgent,
accept: 'application/json',
authorization: accessToken && `Bearer ${accessToken}`
}
}
url, { headers }
).then(r => r.json()).catch(() => {});
if (!data || !Array.isArray(data)) {

12
api/src/processing/url.js
View File

@ -106,6 +106,14 @@ function aliasURL(url) {
url.pathname = `/share/${idPart.slice(-32)}`;
}
break;
case "redd":
/* reddit short video links can be treated by changing https://v.redd.it/<id>
to https://reddit.com/video/<id>.*/
if (url.hostname === "v.redd.it" && parts.length === 2) {
url = new URL(`https://www.reddit.com/video/${parts[1]}`);
}
break;
}
return url;
@ -231,11 +239,11 @@ export function extract(url) {
return { host, patternMatch };
}
export async function resolveRedirectingURL(url, dispatcher, userAgent) {
export async function resolveRedirectingURL(url, dispatcher, headers) {
const originalService = getHostIfValid(normalizeURL(url));
if (!originalService) return;
const canonicalURL = await getRedirectingURL(url, dispatcher, userAgent);
const canonicalURL = await getRedirectingURL(url, dispatcher, headers);
if (!canonicalURL) return;
const { host, patternMatch } = extract(normalizeURL(canonicalURL));

31
api/src/security/jwt.js
View File

@ -6,12 +6,19 @@ import { env } from "../config.js";
const toBase64URL = (b) => Buffer.from(b).toString("base64url");
const fromBase64URL = (b) => Buffer.from(b, "base64url").toString();
const makeHmac = (header, payload) =>
createHmac("sha256", env.jwtSecret)
.update(`${header}.${payload}`)
.digest("base64url");
const makeHmac = (data) => {
return createHmac("sha256", env.jwtSecret)
.update(data)
.digest("base64url");
}
const generate = () => {
const sign = (header, payload) =>
makeHmac(`${header}.${payload}`);
const getIPHash = (ip) =>
makeHmac(ip).slice(0, 8);
const generate = (ip) => {
const exp = Math.floor(new Date().getTime() / 1000) + env.jwtLifetime;
const header = toBase64URL(JSON.stringify({
@ -21,10 +28,11 @@ const generate = () => {
const payload = toBase64URL(JSON.stringify({
jti: nanoid(8),
sub: getIPHash(ip),
exp,
}));
const signature = makeHmac(header, payload);
const signature = sign(header, payload);
return {
token: `${header}.${payload}.${signature}`,
@ -32,7 +40,7 @@ const generate = () => {
};
}
const verify = (jwt) => {
const verify = (jwt, ip) => {
const [header, payload, signature] = jwt.split(".", 3);
const timestamp = Math.floor(new Date().getTime() / 1000);
@ -40,17 +48,16 @@ const verify = (jwt) => {
return false;
}
const verifySignature = makeHmac(header, payload);
const verifySignature = sign(header, payload);
if (verifySignature !== signature) {
return false;
}
if (timestamp >= JSON.parse(fromBase64URL(payload)).exp) {
return false;
}
const data = JSON.parse(fromBase64URL(payload));
return true;
return getIPHash(ip) === data.sub
&& timestamp <= data.exp;
}
export default {

1
api/src/util/tests/instagram.json
View File

@ -123,6 +123,7 @@
{
"name": "private instagram post",
"url": "https://www.instagram.com/p/C5_A1TQNPrYw4c2g9KAUTPUl8RVHqiAdAcOOSY0",
"canFail": true,
"params": {},
"expected": {
"code": 400,

18
api/src/util/tests/reddit.json
View File

@ -56,5 +56,23 @@
"code": 200,
"status": "tunnel"
}
},
{
"name": "shortened video link",
"url": "https://v.redd.it/ifg2emt5ck0e1",
"params": {},
"expected": {
"code": 200,
"status": "tunnel"
}
},
{
"name": "shortened video link (alternative)",
"url": "https://reddit.com/video/ifg2emt5ck0e1",
"params": {},
"expected": {
"code": 200,
"status": "tunnel"
}
}
]

228
docs/api-env-variables.md Normal file
View File

@ -0,0 +1,228 @@
# cobalt api instance environment variables
you can customize your processing instance's behavior using these environment variables. all of them but `API_URL` are optional.
this document is not final and will expand over time. feel free to improve it!
### general vars
| name | default | value example |
|:--------------------|:----------|:--------------------------------------|
| API_URL | | `https://api.url.example/` |
| API_PORT | `9000` | `1337` |
| COOKIE_PATH | | `/cookies.json` |
| PROCESSING_PRIORITY | | `10` |
| API_INSTANCE_COUNT | | `6` |
| API_REDIS_URL | | `redis://localhost:6379` |
| DISABLED_SERVICES | | `bilibili,youtube` |
[*view details*](#general)
### networking vars
| name | default | value example |
|:--------------------|:----------|:--------------------------------------|
| API_LISTEN_ADDRESS | `0.0.0.0` | `127.0.0.1` |
| API_EXTERNAL_PROXY | | `http://user:password@127.0.0.1:8080` |
| FREEBIND_CIDR | | `2001:db8::/32` |
[*view details*](#networking)
### limit vars
| name | default | value example |
|:-------------------------|:--------|:--------------|
| DURATION_LIMIT | `10800` | `18000` |
| TUNNEL_LIFESPAN | `90` | `120` |
| RATELIMIT_WINDOW | `60` | `120` |
| RATELIMIT_MAX | `20` | `30` |
| SESSION_RATELIMIT_WINDOW | `60` | `60` |
| SESSION_RATELIMIT | `10` | `10` |
[*view details*](#limits)
### security vars
| name | default | value example |
|:------------------|:--------|:--------------------------------------|
| CORS_WILDCARD | `1` | `0` |
| CORS_URL | | `https://web.url.example` |
| TURNSTILE_SITEKEY | | `1x00000000000000000000BB` |
| TURNSTILE_SECRET | | `1x0000000000000000000000000000000AA` |
| JWT_SECRET | | see [details](#security) |
| JWT_EXPIRY | `120` | `240` |
| API_KEY_URL | | `file://keys.json` |
| API_AUTH_REQUIRED | | `1` |
[*view details*](#security)
### service-specific vars
| name | value example |
|:---------------------------------|:-------------------------|
| CUSTOM_INNERTUBE_CLIENT | `IOS` |
| YOUTUBE_SESSION_SERVER | `http://localhost:8080/` |
| YOUTUBE_SESSION_INNERTUBE_CLIENT | `WEB_EMBEDDED` |
[*view details*](#service-specific)
## general
[*jump to the table*](#general-vars)
### API_URL
> [!NOTE]
> API_URL is required to run the API instance.
the URL from which your instance will be accessible. can be external or internal, but it must be a valid URL or else tunnels will not work.
the value is a URL.
### API_PORT
port from which the API server will be accessible.
the value is a number from 1024 to 65535.
### COOKIE_PATH
path to the `cookies.json` file relative to the current working directory of your cobalt instance (usually the main (src/api) folder).
### PROCESSING_PRIORITY
`nice` value for ffmpeg subprocesses. available only on unix systems.
note: the higher the nice value, the lower the priority. you can [read more about nice here](https://en.wikipedia.org/wiki/Nice_(Unix)).
the value is a number.
### API_INSTANCE_COUNT
supported only on linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. `API_REDIS_URL` is required to use this option.
the value is a number.
### API_REDIS_URL
when configured, cobalt will use this redis instance for tunnel cache. required when `API_INSTANCE_COUNT` is more than 1, because else sub-instance wouldn't be able to share cache.
the value is a URL.
### DISABLED_SERVICES
comma-separated list which disables certain services from being used.
the value is a string of cobalt-supported services.
## networking
[*jump to the table*](#networking-vars)
### API_LISTEN_ADDRESS
defines the local address for the api instance. if you are using a docker container, you usually don't need to configure this.
the value is a local IP address.
### API_EXTERNAL_PROXY
URL of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only.
if some feature breaks when using a proxy, please make a new issue about it!
the value is a URL.
### FREEBIND_CIDR
IPv6 prefix used for randomly assigning addresses to cobalt requests. available only on linux systems.
setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all requests it makes for that particular download.
to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first.
if you want to use this option and run cobalt in a docker container, you also need to set the `API_LISTEN_ADDRESS` env variable to `127.0.0.1` and set `network_mode` for the container to `host`.
the value is an IPv6 range.
## limits
[*jump to the table*](#limit-vars)
### DURATION_LIMIT
media duration limit, in **seconds**
the value is a number.
### TUNNEL_LIFESPAN
the duration for which tunnel info is stored in ram, **in seconds**.
it's recommended to keep this value either default or as low as possible to preserve efficiency and user privacy.
the value is a number.
### RATELIMIT_WINDOW
rate limit time window for api requests, but not session requests, in **seconds**.
the value is a number.
### RATELIMIT_MAX
amount of api requests to be allowed within the time window of `RATELIMIT_WINDOW`.
the value is a number.
### SESSION_RATELIMIT_WINDOW
rate limit time window for session creation requests, in **seconds**.
the value is a number.
### SESSION_RATELIMIT
amount of session requests to be allowed within the time window of `SESSION_RATELIMIT_WINDOW`.
the value is a number.
## security
[*jump to the table*](#security-vars)
> [!NOTE]
> in order to enable turnstile bot protection, `TURNSTILE_SITEKEY`, `TURNSTILE_SECRET`, and `JWT_SECRET` must be set. all three at once.
### CORS_WILDCARD
defines whether cross-origin resource sharing is enabled. when enabled, your instance will be accessible from foreign web pages.
the value is a number. 0: disabled. 1: enabled.
### CORS_URL
configures the [cross-origin resource sharing origin](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Access-Control-Allow-Origin). your instance will be available only from this URL if `CORS_WILDCARD` is set to `0`.
the value is a URL.
### TURNSTILE_SITEKEY
[cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) sitekey used by the web client to request & solve a challenge to prove that the user is not a bot.
the value is a specific key.
### TURNSTILE_SECRET
[cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) secret used by the processing instance to verify that the client solved the challenge successfully.
the value is a specific key.
### JWT_SECRET
the secret used for issuing JWT tokens for request authentication. the value must be a random, secure, and long string (over 16 characters).
the value is a specific key.
### JWT_EXPIRY
the duration of how long a cobalt-issued JWT token will remain valid, in seconds.
the value is a number.
### API_KEY_URL
the URL to the the external or local key database. for local files you have to specify a local path using the `file://` protocol.
see [the api key section](/docs/protect-an-instance.md#api-key-file-format) in the "how to protect your cobalt instance" document for more details.
the value is a URL.
### API_AUTH_REQUIRED
when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled).
the value is a number.
## service-specific
[*jump to the table*](#service-specific-vars)
### CUSTOM_INNERTUBE_CLIENT
innertube client that will be used instead of the default one.
the value is a string.
### YOUTUBE_SESSION_SERVER
URL to an instance of [yt-session-generator](https://github.com/imputnet/yt-session-generator). used for automatically pulling `poToken` & `visitor_data` for youtube. can be local or remote.
the value is a URL.
### YOUTUBE_SESSION_INNERTUBE_CLIENT
innertube client that's compatible with botguard's (web) `poToken` and `visitor_data`.
the value is a string.

5
docs/examples/docker-compose.example.yml
View File

@ -49,6 +49,5 @@ services:
# init: true
# restart: unless-stopped
# container_name: yt-session-generator
# ports:
# - 127.0.0.1:1280:8080
# labels:
# - com.centurylinklabs.watchtower.scope=cobalt

54
docs/protect-an-instance.md
View File

@ -114,7 +114,7 @@ if you want to use your instance outside of web interface, you'll need an api ke
>
> if api keys leak, you'll have to update/remove all UUIDs to revoke them.
1. create a `keys.json` file following [the schema and example here](/docs//run-an-instance.md#api-key-file-format).
1. create a `keys.json` file following [the schema and example down below](#api-key-file-format).
2. expose the `keys.json` to the docker container:
```yml
@ -148,3 +148,55 @@ environment:
### why not make keys exclusive by default?
keys may be useful for going around rate limiting,
while keeping the rest of api rate limited, with no turnstile in place.
## api key file format
the file is a JSON-serialized object with the following structure:
```typescript
type KeyFileContents = Record<
UUIDv4String,
{
name?: string,
limit?: number | "unlimited",
ips?: (CIDRString | IPString)[],
userAgents?: string[]
}
>;
```
where *`UUIDv4String`* is a stringified version of a UUIDv4 identifier.
- **name** is a field for your own reference, it is not used by cobalt anywhere.
- **`limit`** specifies how many requests the API key can make during the window specified in the `RATELIMIT_WINDOW` env.
- when omitted, the limit specified in `RATELIMIT_MAX` will be used.
- it can be also set to `"unlimited"`, in which case the API key bypasses all rate limits.
- **`ips`** contains an array of allowlisted IP ranges, which can be specified both as individual ips or CIDR ranges (e.g. *`["192.168.42.69", "2001:db8::48", "10.0.0.0/8", "fe80::/10"]`*).
- when specified, only requests from these ip ranges can use the specified api key.
- when omitted, any IP can be used to make requests with that API key.
- **`userAgents`** contains an array of allowed user agents, with support for wildcards (e.g. *`["cobaltbot/1.0", "Mozilla/5.0 * Chrome/*"]`*).
- when specified, requests with a `user-agent` that does not appear in this array will be rejected.
- when omitted, any user agent can be specified to make requests with that API key.
- if both `ips` and `userAgents` are set, the tokens will be limited by both parameters.
- if cobalt detects any problem with your key file, it will be ignored and a warning will be printed to the console.
an example key file could look like this:
```json
{
"b5c7160a-b655-4c7a-b500-de839f094550": {
"limit": 10,
"ips": ["10.0.0.0/8", "192.168.42.42"],
"userAgents": ["*Chrome*"]
},
"b00b1234-a3e5-99b1-c6d1-dba4512ae190": {
"limit": "unlimited",
"ips": ["192.168.1.2"],
"userAgents": ["cobaltbot/1.0"]
}
}
```
if you are configuring a key file, **do not use the UUID from the example** but instead generate your own. you can do this by running the following command if you have node.js installed:
`node -e "console.log(crypto.randomUUID())"`

94
docs/run-an-instance.md
View File

@ -1,4 +1,6 @@
# how to run a cobalt instance
this tutorial will help you run your own cobalt processing instance. if your instance is public-facing, we highly recommend that you also [protect it from abuse](/docs/protect-an-instance.md) using turnstile or api keys or both.
## using docker compose and package from github (recommended)
to run the cobalt docker package, you need to have `docker` and `docker-compose` installed and configured.
@ -54,93 +56,5 @@ sudo apt install nscd
sudo service nscd start
```
## list of environment variables for api
| variable name | default | example | description |
|:----------------------|:----------|:------------------------|:------------|
| `API_PORT` | `9000` | `9000` | changes port from which api server is accessible. |
| `API_LISTEN_ADDRESS` | `0.0.0.0` | `127.0.0.1` | changes address from which api server is accessible. **if you are using docker, you usually don't need to configure this.** |
| `API_URL` | ➖ | `https://api.cobalt.tools/` | changes url from which api server is accessible. <br> ***REQUIRED TO RUN THE API***. |
| `API_NAME` | `unknown` | `ams-1` | api server name that is shown in `/api/serverInfo`. |
| `API_EXTERNAL_PROXY` | ➖ | `http://user:password@127.0.0.1:8080`| url of the proxy that will be passed to [`ProxyAgent`](https://undici.nodejs.org/#/docs/api/ProxyAgent) and used for all external requests. HTTP(S) only. |
| `CORS_WILDCARD` | `1` | `0` | toggles cross-origin resource sharing. <br> `0`: disabled. `1`: enabled. |
| `CORS_URL` | not used | `https://cobalt.tools` | cross-origin resource sharing url. api will be available only from this url if `CORS_WILDCARD` is set to `0`. |
| `COOKIE_PATH` | not used | `/cookies.json` | path for cookie file relative to main folder. |
| `PROCESSING_PRIORITY` | not used | `10` | changes `nice` value* for ffmpeg subprocess. available only on unix systems. |
| `FREEBIND_CIDR` | ➖ | `2001:db8::/32` | IPv6 prefix used for randomly assigning addresses to cobalt requests. only supported on linux systems. see below for more info. |
| `RATELIMIT_WINDOW` | `60` | `120` | rate limit time window in **seconds**. |
| `RATELIMIT_MAX` | `20` | `30` | max requests per time window. requests above this amount will be blocked for the rate limit window duration. |
| `DURATION_LIMIT` | `10800` | `18000` | max allowed video duration in **seconds**. |
| `TUNNEL_LIFESPAN` | `90` | `120` | the duration for which tunnel info is stored in ram, **in seconds**. |
| `TURNSTILE_SITEKEY` | ➖ | `1x00000000000000000000BB` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) sitekey used by browser clients to request a challenge.\*\* |
| `TURNSTILE_SECRET` | ➖ | `1x0000000000000000000000000000000AA` | [cloudflare turnstile](https://www.cloudflare.com/products/turnstile/) secret used by cobalt to verify the client successfully solved the challenge.\*\* |
| `JWT_SECRET` | ➖ | ➖ | the secret used for issuing JWT tokens for request authentication. to choose a value, generate a random, secure, long string (ideally >=16 characters).\*\* |
| `JWT_EXPIRY` | `120` | `240` | the duration of how long a cobalt-issued JWT token will remain valid, in seconds. |
| `API_KEY_URL` | ➖ | `file://keys.json` | the location of the api key database. for loading API keys, cobalt supports HTTP(S) urls, or local files by specifying a local path using the `file://` protocol. see the "api key file format" below for more details. |
| `API_AUTH_REQUIRED` | ➖ | `1` | when set to `1`, the user always needs to be authenticated in some way before they can access the API (either via an api key or via turnstile, if enabled). |
| `API_REDIS_URL` | ➖ | `redis://localhost:6379` | when set, cobalt uses redis instead of internal memory for the tunnel cache. |
| `API_INSTANCE_COUNT` | ➖ | `2` | supported only on Linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. |
| `DISABLED_SERVICES` | ➖ | `bilibili,youtube` | comma-separated list which disables certain services from being used. |
| `CUSTOM_INNERTUBE_CLIENT` | ➖ | `IOS` | innertube client that will be used instead of the default one. |
| `YOUTUBE_SESSION_SERVER` | ➖ | `http://localhost:8080/` | url to an instance of [invidious' youtube-trusted-session-generator](https://github.com/iv-org/youtube-trusted-session-generator) or its fork/counterpart. used for automatically pulling poToken & visitor_data for youtube. can be local or remote. |
\* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)).
\*\* in order to enable turnstile bot protection, all three **`TURNSTILE_SITEKEY`, `TURNSTILE_SECRET` and `JWT_SECRET`** need to be set.
#### FREEBIND_CIDR
setting a `FREEBIND_CIDR` allows cobalt to pick a random IP for every download and use it for all
requests it makes for that particular download. to use freebind in cobalt, you need to follow its [setup instructions](https://github.com/imputnet/freebind.js?tab=readme-ov-file#setup) first. if you configure this option while running cobalt
in a docker container, you also need to set the `API_LISTEN_ADDRESS` env to `127.0.0.1`, and set
`network_mode` for the container to `host`.
## api key file format
the file is a JSON-serialized object with the following structure:
```typescript
type KeyFileContents = Record<
UUIDv4String,
{
name?: string,
limit?: number | "unlimited",
ips?: (CIDRString | IPString)[],
userAgents?: string[]
}
>;
```
where *`UUIDv4String`* is a stringified version of a UUIDv4 identifier.
- **name** is a field for your own reference, it is not used by cobalt anywhere.
- **`limit`** specifies how many requests the API key can make during the window specified in the `RATELIMIT_WINDOW` env.
- when omitted, the limit specified in `RATELIMIT_MAX` will be used.
- it can be also set to `"unlimited"`, in which case the API key bypasses all rate limits.
- **`ips`** contains an array of allowlisted IP ranges, which can be specified both as individual ips or CIDR ranges (e.g. *`["192.168.42.69", "2001:db8::48", "10.0.0.0/8", "fe80::/10"]`*).
- when specified, only requests from these ip ranges can use the specified api key.
- when omitted, any IP can be used to make requests with that API key.
- **`userAgents`** contains an array of allowed user agents, with support for wildcards (e.g. *`["cobaltbot/1.0", "Mozilla/5.0 * Chrome/*"]`*).
- when specified, requests with a `user-agent` that does not appear in this array will be rejected.
- when omitted, any user agent can be specified to make requests with that API key.
- if both `ips` and `userAgents` are set, the tokens will be limited by both parameters.
- if cobalt detects any problem with your key file, it will be ignored and a warning will be printed to the console.
an example key file could look like this:
```json
{
"b5c7160a-b655-4c7a-b500-de839f094550": {
"limit": 10,
"ips": ["10.0.0.0/8", "192.168.42.42"],
"userAgents": ["*Chrome*"]
},
"b00b1234-a3e5-99b1-c6d1-dba4512ae190": {
"limit": "unlimited",
"ips": ["192.168.1.2"],
"userAgents": ["cobaltbot/1.0"]
}
}
```
if you are configuring a key file, **do not use the UUID from the example** but instead generate your own. you can do this by running the following command if you have node.js installed:
`node -e "console.log(crypto.randomUUID())"`
## list of environment variables
[this section has moved](/docs/api-env-variables.md) to a dedicated document that is way easier to understand and maintain. go check it out!

2
web/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@imput/cobalt-web",
"version": "10.7.5",
"version": "10.9",
"type": "module",
"private": true,
"scripts": {

28
web/src/lib/api/api.ts
View File

@ -2,7 +2,7 @@ import { get } from "svelte/store";
import settings from "$lib/state/settings";
import { getSession } from "$lib/api/session";
import { getSession, resetSession } from "$lib/api/session";
import { currentApiURL } from "$lib/api/api-url";
import { turnstileEnabled, turnstileSolved } from "$lib/state/turnstile";
import cachedInfo from "$lib/state/server-info";
@ -42,7 +42,7 @@ const getAuthorization = async () => {
}
}
const request = async (request: CobaltSaveRequestBody) => {
const request = async (requestBody: CobaltSaveRequestBody, justRetried = false) => {
await getServerInfo();
const getCachedInfo = get(cachedInfo);
@ -75,7 +75,7 @@ const request = async (request: CobaltSaveRequestBody) => {
method: "POST",
redirect: "manual",
signal: AbortSignal.timeout(20000),
body: JSON.stringify(request),
body: JSON.stringify(requestBody),
headers: {
"Accept": "application/json",
"Content-Type": "application/json",
@ -94,9 +94,31 @@ const request = async (request: CobaltSaveRequestBody) => {
}
});
if (
response?.status === 'error'
&& response?.error.code === 'error.api.auth.jwt.invalid'
&& !justRetried
) {
resetSession();
await waitForTurnstile().catch(() => {});
return request(requestBody, true);
}
return response;
}
const waitForTurnstile = async () => {
await getAuthorization();
return new Promise<void>(resolve => {
const unsub = turnstileSolved.subscribe(solved => {
if (solved) {
unsub();
resolve();
}
});
});
}
const probeCobaltTunnel = async (url: string) => {
const request = await fetch(`${url}&p=1`).catch(() => {});
if (request?.status === 200) {

2
web/src/lib/api/session.ts
View File

@ -62,3 +62,5 @@ export const getSession = async () => {
}
return newSession;
}
export const resetSession = () => cache = undefined;