From b6cd0ad727878d86a025cbfdef15b368f4fd2f15 Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 19 Mar 2025 18:51:26 +0600 Subject: [PATCH 01/22] api: automatically pull youtube session tokens from a session server if provided, cobalt will pull poToken & visitor_data from an instance of invidious' youtube-trusted-session-generator or its counterpart --- api/src/config.js | 3 + api/src/core/api.js | 6 ++ api/src/processing/helpers/youtube-session.js | 74 +++++++++++++++++++ api/src/processing/services/youtube.js | 19 +++-- 4 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 api/src/processing/helpers/youtube-session.js diff --git a/api/src/config.js b/api/src/config.js index bff1eda3..54b1214e 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -53,7 +53,10 @@ const env = { keyReloadInterval: 900, enabledServices, + customInnertubeClient: process.env.CUSTOM_INNERTUBE_CLIENT, + ytSessionServer: process.env.YOUTUBE_SESSION_SERVER, + ytSessionReloadInterval: 300, } const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; diff --git a/api/src/core/api.js b/api/src/core/api.js index e4d3dfcf..3f9128d8 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -18,8 +18,10 @@ import { verifyTurnstileToken } from "../security/turnstile.js"; import { friendlyServiceName } from "../processing/service-alias.js"; import { verifyStream, getInternalStream } from "../stream/manage.js"; import { createResponse, normalizeRequest, getIP } from "../processing/request.js"; + import * as APIKeys from "../security/api-keys.js"; import * as Cookies from "../processing/cookie/manager.js"; +import * as YouTubeSession from "../processing/helpers/youtube-session.js"; const git = { branch: await getBranch(), @@ -376,6 +378,10 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { if (env.cookiePath) { Cookies.setup(env.cookiePath); } + + if (env.ytSessionServer) { + YouTubeSession.setup(); + } }); if (isCluster) { diff --git a/api/src/processing/helpers/youtube-session.js b/api/src/processing/helpers/youtube-session.js new file mode 100644 index 00000000..5235c42a --- /dev/null +++ b/api/src/processing/helpers/youtube-session.js @@ -0,0 +1,74 @@ +import * as cluster from "../../misc/cluster.js"; + +import { env } from "../../config.js"; +import { Green, Yellow } from "../../misc/console-text.js"; + +let session; + +const validateSession = (sessionResponse) => { + if (!sessionResponse.potoken) { + throw "no poToken in session response"; + } + + if (!sessionResponse.visitor_data) { + throw "no visitor_data in session response"; + } + + if (!sessionResponse.updated) { + throw "no last update timestamp in session response"; + } + + // https://github.com/iv-org/youtube-trusted-session-generator/blob/c2dfe3f/potoken_generator/main.py#L25 + if (sessionResponse.potoken.length < 160) { + console.error(`${Yellow('[!]')} poToken is too short and might not work (${new Date().toISOString()})`); + } +} + +const updateSession = (newSession) => { + session = newSession; +} + +const loadSession = async () => { + const sessionServerUrl = new URL(env.ytSessionServer); + sessionServerUrl.pathname = "/token"; + + const newSession = await fetch(sessionServerUrl).then(a => a.json()); + validateSession(newSession); + + if (!session || session.updated < newSession?.updated) { + cluster.broadcast({ youtube_session: newSession }); + updateSession(newSession); + } +} + +const wrapLoad = (initial = false) => { + loadSession() + .then(() => { + if (initial) { + console.log(`${Green('[✓]')} poToken & visitor_data loaded successfully!`); + } + }) + .catch((e) => { + console.error(`${Yellow('[!]')} Failed loading poToken & visitor_data at ${new Date().toISOString()}.`); + console.error('Error:', e); + }) +} + +export const getYouTubeSession = () => { + return session; +} + +export const setup = () => { + if (cluster.isPrimary) { + wrapLoad(true); + if (env.ytSessionReloadInterval > 0) { + setInterval(wrapLoad, env.ytSessionReloadInterval * 1000); + } + } else if (cluster.isWorker) { + process.on('message', (message) => { + if ('youtube_session' in message) { + updateSession(message.youtube_session); + } + }); + } +} diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index fc712e24..a10a8b08 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -4,6 +4,7 @@ import { fetch } from "undici"; import { Innertube, Session } from "youtubei.js"; import { env } from "../../config.js"; +import { getYouTubeSession } from "../helpers/youtube-session.js"; import { getCookie, updateCookieValues } from "../cookie/manager.js"; const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms @@ -70,16 +71,22 @@ const cloneInnertube = async (customFetch) => { const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date(); const rawCookie = getCookie('youtube'); - const rawCookieValues = rawCookie?.values(); const cookie = rawCookie?.toString(); + const sessionTokens = getYouTubeSession(); + const retrieve_player = Boolean(sessionTokens || cookie); + + if (env.ytSessionServer && !sessionTokens?.potoken) { + throw "no_session_tokens"; + } + if (!innertube || shouldRefreshPlayer) { innertube = await Innertube.create({ fetch: customFetch, - retrieve_player: !!cookie, + retrieve_player, cookie, - po_token: rawCookieValues?.po_token, - visitor_data: rawCookieValues?.visitor_data, + po_token: sessionTokens?.potoken, + visitor_data: sessionTokens?.visitor_data, }); lastRefreshedAt = +new Date(); } @@ -135,7 +142,9 @@ export default async function (o) { }) ); } catch (e) { - if (e.message?.endsWith("decipher algorithm")) { + if (e === "no_session_tokens") { + return { error: "youtube.no_session_tokens" }; + } else if (e.message?.endsWith("decipher algorithm")) { return { error: "youtube.decipher" } } else if (e.message?.includes("refresh access token")) { return { error: "youtube.token_expired" } From 4e6d1c40518fe80ccdf5c0c55c308502f15ffd5b Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 19 Mar 2025 20:32:44 +0600 Subject: [PATCH 02/22] api/tests/youtube: allow HLS tests to fail --- api/src/util/tests/youtube.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/util/tests/youtube.json b/api/src/util/tests/youtube.json index 0655e683..cb4964be 100644 --- a/api/src/util/tests/youtube.json +++ b/api/src/util/tests/youtube.json @@ -189,6 +189,7 @@ { "name": "hls video (h264, 1440p)", "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, "params": { "youtubeVideoCodec": "h264", "videoQuality": "1440", @@ -202,6 +203,7 @@ { "name": "hls video (vp9, 360p)", "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, "params": { "youtubeVideoCodec": "vp9", "videoQuality": "360", @@ -215,6 +217,7 @@ { "name": "hls video (audio mode)", "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, "params": { "downloadMode": "audio", "youtubeHLS": true @@ -227,6 +230,7 @@ { "name": "hls video (audio mode, best format)", "url": "https://www.youtube.com/watch?v=vPwaXytZcgI", + "canFail": true, "params": { "downloadMode": "audio", "youtubeHLS": true, From d1b5983e492d72f406b28cdb6cfa2441e5b4bf5d Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 19 Mar 2025 20:34:56 +0600 Subject: [PATCH 03/22] api/youtube: disable HLS if a session server is used --- api/src/processing/services/youtube.js | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index a10a8b08..ca46d22f 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -151,22 +151,16 @@ export default async function (o) { } else throw e; } - const cookie = getCookie('youtube')?.toString(); - let useHLS = o.youtubeHLS; - // HLS playlists don't contain the av1 video format, at least with the iOS client - if (useHLS && o.format === "av1") { + // HLS playlists don't contain the av1 video format. + // if the session server is used, then iOS client will not work, at least currently. + if (useHLS && (o.format === "av1" || env.ytSessionServer)) { useHLS = false; } let innertubeClient = o.innertubeClient || env.customInnertubeClient || "ANDROID"; - if (cookie) { - useHLS = false; - innertubeClient = "WEB"; - } - if (useHLS) { innertubeClient = "IOS"; } From 073b169a939bc41e53c31e91a9d9ed1dcfa86a1f Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 19 Mar 2025 20:43:31 +0600 Subject: [PATCH 04/22] api: remove code & docs related to youtube oauth it hasn't been functional for a while, unfortunately --- api/package.json | 1 - api/src/processing/cookie/manager.js | 1 - api/src/processing/services/youtube.js | 49 +------------------------ api/src/util/generate-youtube-tokens.js | 38 ------------------- docs/configure-for-youtube.md | 33 ----------------- docs/examples/cookies.example.json | 3 -- 6 files changed, 1 insertion(+), 124 deletions(-) delete mode 100644 api/src/util/generate-youtube-tokens.js delete mode 100644 docs/configure-for-youtube.md diff --git a/api/package.json b/api/package.json index 4f2b21dc..ed78546c 100644 --- a/api/package.json +++ b/api/package.json @@ -11,7 +11,6 @@ "scripts": { "start": "node src/cobalt", "test": "node src/util/test", - "token:youtube": "node src/util/generate-youtube-tokens", "token:jwt": "node src/util/generate-jwt-secret" }, "repository": { diff --git a/api/src/processing/cookie/manager.js b/api/src/processing/cookie/manager.js index 25f41c2c..9e23374b 100644 --- a/api/src/processing/cookie/manager.js +++ b/api/src/processing/cookie/manager.js @@ -13,7 +13,6 @@ const VALID_SERVICES = new Set([ 'reddit', 'twitter', 'youtube', - 'youtube_oauth' ]); const invalidCookies = {}; diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index ca46d22f..49de7758 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -4,8 +4,8 @@ import { fetch } from "undici"; import { Innertube, Session } from "youtubei.js"; import { env } from "../../config.js"; +import { getCookie } from "../cookie/manager.js"; import { getYouTubeSession } from "../helpers/youtube-session.js"; -import { getCookie, updateCookieValues } from "../cookie/manager.js"; const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms @@ -46,27 +46,6 @@ const clientsWithNoCipher = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDR const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320]; -const transformSessionData = (cookie) => { - if (!cookie) - return; - - const values = { ...cookie.values() }; - const REQUIRED_VALUES = ['access_token', 'refresh_token']; - - if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) { - return; - } - - if (values.expires) { - values.expiry_date = values.expires; - delete values.expires; - } else if (!values.expiry_date) { - return; - } - - return values; -} - const cloneInnertube = async (customFetch) => { const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date(); @@ -102,32 +81,6 @@ const cloneInnertube = async (customFetch) => { innertube.session.cache ); - const oauthCookie = getCookie('youtube_oauth'); - const oauthData = transformSessionData(oauthCookie); - - if (!session.logged_in && oauthData) { - await session.oauth.init(oauthData); - session.logged_in = true; - } - - if (session.logged_in && oauthData) { - if (session.oauth.shouldRefreshToken()) { - await session.oauth.refreshAccessToken(); - } - - const cookieValues = oauthCookie.values(); - const oldExpiry = new Date(cookieValues.expiry_date); - const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date); - - if (oldExpiry.getTime() !== newExpiry.getTime()) { - updateCookieValues(oauthCookie, { - ...session.oauth.client_id, - ...session.oauth.oauth2_tokens, - expiry_date: newExpiry.toISOString() - }); - } - } - const yt = new Innertube(session); return yt; } diff --git a/api/src/util/generate-youtube-tokens.js b/api/src/util/generate-youtube-tokens.js deleted file mode 100644 index 5585705a..00000000 --- a/api/src/util/generate-youtube-tokens.js +++ /dev/null @@ -1,38 +0,0 @@ -import { Innertube } from 'youtubei.js'; -import { Red } from '../misc/console-text.js' - -const bail = (...msg) => { - console.error(...msg); - throw new Error(msg); -}; - -const tube = await Innertube.create(); - -tube.session.once( - 'auth-pending', - ({ verification_url, user_code }) => { - console.log(`${Red('[!]')} The token generated by this script is sensitive and you should not share it with anyone!`); - console.log(` By using this token, you are risking your Google account getting terminated.`); - console.log(` You should ${Red('NOT')} use your personal account!`); - console.log(); - console.log(`Open ${verification_url} in a browser and enter ${user_code} when asked for the code.`); - } -); - -tube.session.once('auth-error', (err) => bail('An error occurred:', err)); -tube.session.once('auth', ({ credentials }) => { - if (!credentials.access_token) { - bail('something went wrong'); - } - - console.log( - 'add this cookie to the youtube_oauth array in your cookies file:', - JSON.stringify( - Object.entries(credentials) - .map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`) - .join('; ') - ) - ); -}); - -await tube.session.signIn(); \ No newline at end of file diff --git a/docs/configure-for-youtube.md b/docs/configure-for-youtube.md deleted file mode 100644 index fe286d86..00000000 --- a/docs/configure-for-youtube.md +++ /dev/null @@ -1,33 +0,0 @@ -# how to configure a cobalt instance for youtube -if you get various errors when attempting to download videos that are: -publicly available, not region locked, and not age-restricted; -then your instance's ip address may have bad reputation. - -in this case you have to use disposable google accounts. -there's no other known workaround as of time of writing this document. - -> [!CAUTION] -> **NEVER** use your personal google account for downloading videos via any means. -> you can use any google accounts that you're willing to sacrifice, -> but be prepared to have them **permanently suspended**. -> -> we recommend that you use accounts that don't link back to your personal google account or identity, just in case. -> -> use incognito mode when signing in. -> we also recommend using vpn/proxy services (such as [mullvad](https://mullvad.net/)). - -1. if you haven't done it already, clone the cobalt repo, go to the cloned directory, and run `pnpm install` - -2. run `pnpm -C api token:youtube` - -3. follow instructions, use incognito mode in your browser when signing in. -i cannot stress this enough, but again, **DO NOT USE YOUR PERSONAL GOOGLE ACCOUNT**. - -4. once you have the oauth token, add it to `youtube_oauth` in your cookies file. -you can see an [example here](/docs/examples/cookies.example.json). -you can have several account tokens in this file, if you like. - -5. all done! enjoy freedom. - -### liability -you're responsible for any damage done to any of your google accounts or any other damages. you do this by yourself and at your own risk. diff --git a/docs/examples/cookies.example.json b/docs/examples/cookies.example.json index 7996adeb..73f3378d 100644 --- a/docs/examples/cookies.example.json +++ b/docs/examples/cookies.example.json @@ -10,8 +10,5 @@ ], "twitter": [ "auth_token=; ct0=" - ], - "youtube_oauth": [ - "" ] } From b7fb8d26adb28daae0036058de551875ff817c3c Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 19 Mar 2025 20:49:52 +0600 Subject: [PATCH 05/22] docs/run-an-instance: add info about YOUTUBE_SESSION_SERVER --- docs/run-an-instance.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/run-an-instance.md b/docs/run-an-instance.md index fe8316f5..b79e3fe8 100644 --- a/docs/run-an-instance.md +++ b/docs/run-an-instance.md @@ -81,6 +81,7 @@ sudo service nscd start | `API_INSTANCE_COUNT` | ➖ | `2` | supported only on Linux and node.js `>=23.1.0`. when configured, cobalt will spawn multiple sub-instances amongst which requests will be balanced. | | `DISABLED_SERVICES` | ➖ | `bilibili,youtube` | comma-separated list which disables certain services from being used. | | `CUSTOM_INNERTUBE_CLIENT` | ➖ | `IOS` | innertube client that will be used instead of the default one. | +| `YOUTUBE_SESSION_SERVER` | ➖ | `http://localhost:8080/` | url to an instance of [invidious' youtube-trusted-session-generator](https://github.com/iv-org/youtube-trusted-session-generator) or its fork/counterpart. used for automatically pulling poToken & visitor_data for youtube. can be local or remote. | \* the higher the nice value, the lower the priority. [read more here](https://en.wikipedia.org/wiki/Nice_(Unix)). From f18d28dcfc4c8b2d13cac70c84456b5ebeca3edf Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 20 Mar 2025 00:09:46 +0600 Subject: [PATCH 06/22] web/i18n/error: add api.youtube.no_session_tokens --- web/i18n/en/error.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/i18n/en/error.json b/web/i18n/en/error.json index fe276aa0..2c347951 100644 --- a/web/i18n/en/error.json +++ b/web/i18n/en/error.json @@ -68,5 +68,6 @@ "api.youtube.no_hls_streams": "couldn't find any matching HLS streams for this video. try downloading it without HLS!", "api.youtube.api_error": "youtube updated something about its api and i couldn't get any info about this video. try again in a few seconds, but if this issue sticks, please report it!", "api.youtube.temporary_disabled": "youtube downloading is temporarily disabled due to restrictions from youtube's side. we're already looking for ways to go around them.\n\nwe apologize for the inconvenience and are doing our best to restore this functionality. check cobalt's socials or github for timely updates!", - "api.youtube.drm": "this youtube video is protected by widevine DRM, so i can't download it. try a different link!" + "api.youtube.drm": "this youtube video is protected by widevine DRM, so i can't download it. try a different link!", + "api.youtube.no_session_tokens": "couldn't get required session tokens for youtube. this may be caused by a restriction on youtube's side. try again in a few seconds, but if this issue sticks, please report it!" } From da040f1a092d596ac80cb2d5aba752af469d757f Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 20 Mar 2025 00:11:24 +0600 Subject: [PATCH 07/22] docs/examples/docker: add yt-session-generator example --- docs/examples/docker-compose.example.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/examples/docker-compose.example.yml b/docs/examples/docker-compose.example.yml index e56c0a21..ed217ccc 100644 --- a/docs/examples/docker-compose.example.yml +++ b/docs/examples/docker-compose.example.yml @@ -41,3 +41,14 @@ services: command: --cleanup --scope cobalt --interval 900 --include-restarting volumes: - /var/run/docker.sock:/var/run/docker.sock + + # if needed, use this image for automatically generating poToken & visitor_data + # yt-session-generator: + # image: ghcr.io/imputnet/yt-session-generator:webserver + + # init: true + # restart: unless-stopped + # container_name: yt-session-generator + + # ports: + # - 127.0.0.1:1280:8080 From f8ee005b06a338da20524a6536701c844cd8ccf4 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 20 Mar 2025 00:18:31 +0600 Subject: [PATCH 08/22] api/package: bump version to 10.8 --- api/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/package.json b/api/package.json index ed78546c..a2045151 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.7.10", + "version": "10.8", "author": "imput", "exports": "./src/cobalt.js", "type": "module", From e779506d9edfdf0e556db70bd8c3148b52172b6a Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 20 Mar 2025 10:15:51 +0600 Subject: [PATCH 09/22] api/package: update youtube.js it contains a fix that's necessary for youtube to work rn --- api/package.json | 2 +- pnpm-lock.yaml | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/package.json b/api/package.json index a2045151..eec2296a 100644 --- a/api/package.json +++ b/api/package.json @@ -38,7 +38,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^13.1.0", + "youtubei.js": "^13.2.0", "zod": "^3.23.8" }, "optionalDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 10b56f70..f30791b7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,8 +56,8 @@ importers: specifier: 1.0.3 version: 1.0.3 youtubei.js: - specifier: ^13.1.0 - version: 13.1.0 + specifier: ^13.2.0 + version: 13.2.0 zod: specifier: ^3.23.8 version: 3.23.8 @@ -188,8 +188,8 @@ packages: resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} - '@bufbuild/protobuf@2.1.0': - resolution: {integrity: sha512-+2Mx67Y3skJ4NCD/qNSdBJNWtu6x6Qr53jeNg+QcwiL6mt0wK+3jwHH2x1p7xaYH6Ve2JKOVn0OxU35WsmqI9A==} + '@bufbuild/protobuf@2.2.5': + resolution: {integrity: sha512-/g5EzJifw5GF8aren8wZ/G5oMuPoGeS6MQD3ca8ddcvdXR5UELUfdTZITCGNhNXynY/AYl3Z4plmxdj/tRl/hQ==} '@datastructures-js/heap@4.3.3': resolution: {integrity: sha512-UcUu/DLh/aM4W3C8zZfwxxm6/6FIZUlm3mcAXuNOCa6Aj4iizNvNXQyb8DjZQH2jKSQbMRyNlngP6TPimuGjpQ==} @@ -2286,8 +2286,8 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - youtubei.js@13.1.0: - resolution: {integrity: sha512-uL4TyojAYET0c5NGFD7+ScCod/k8Pc/B+D5tLrunFcz1GaBjRMOGRPcNGaRmnhwisegU7ibtw0iUxCN+BZ0ang==} + youtubei.js@13.2.0: + resolution: {integrity: sha512-esbSvWS12Dz/cVlHhnL/PSE84a/mVpQdzwPDIkRQu/NHJVxv0isBUcm3hJnYB1jg1LYvomV0YeOrYv5qWwJREA==} zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -2299,7 +2299,7 @@ snapshots: '@jridgewell/gen-mapping': 0.3.5 '@jridgewell/trace-mapping': 0.3.25 - '@bufbuild/protobuf@2.1.0': {} + '@bufbuild/protobuf@2.2.5': {} '@datastructures-js/heap@4.3.3': {} @@ -4242,9 +4242,9 @@ snapshots: yocto-queue@0.1.0: {} - youtubei.js@13.1.0: + youtubei.js@13.2.0: dependencies: - '@bufbuild/protobuf': 2.1.0 + '@bufbuild/protobuf': 2.2.5 jintr: 3.2.1 tslib: 2.6.3 undici: 5.28.4 From 24ce19d09fd2040fc7d66fe336ec3df18593ddeb Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 20 Mar 2025 10:58:15 +0600 Subject: [PATCH 10/22] api/youtube: use both ios & web_embedded client depending on request this ensures better reliability & reduces rate limiting of either clients --- api/src/config.js | 1 + api/src/processing/services/youtube.js | 59 +++++++++++++++++--------- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/api/src/config.js b/api/src/config.js index 54b1214e..98da6fe7 100644 --- a/api/src/config.js +++ b/api/src/config.js @@ -57,6 +57,7 @@ const env = { customInnertubeClient: process.env.CUSTOM_INNERTUBE_CLIENT, ytSessionServer: process.env.YOUTUBE_SESSION_SERVER, ytSessionReloadInterval: 300, + ytSessionInnertubeClient: process.env.YOUTUBE_SESSION_INNERTUBE_CLIENT, } const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 49de7758..219209ad 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -46,7 +46,7 @@ const clientsWithNoCipher = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDR const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320]; -const cloneInnertube = async (customFetch) => { +const cloneInnertube = async (customFetch, useSession) => { const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date(); const rawCookie = getCookie('youtube'); @@ -55,7 +55,7 @@ const cloneInnertube = async (customFetch) => { const sessionTokens = getYouTubeSession(); const retrieve_player = Boolean(sessionTokens || cookie); - if (env.ytSessionServer && !sessionTokens?.potoken) { + if (useSession && env.ytSessionServer && !sessionTokens?.potoken) { throw "no_session_tokens"; } @@ -64,8 +64,8 @@ const cloneInnertube = async (customFetch) => { fetch: customFetch, retrieve_player, cookie, - po_token: sessionTokens?.potoken, - visitor_data: sessionTokens?.visitor_data, + po_token: useSession ? sessionTokens?.potoken : undefined, + visitor_data: useSession ? sessionTokens?.visitor_data : undefined, }); lastRefreshedAt = +new Date(); } @@ -86,13 +86,46 @@ const cloneInnertube = async (customFetch) => { } export default async function (o) { + const quality = o.quality === "max" ? 9000 : Number(o.quality); + + let useHLS = o.youtubeHLS; + let innertubeClient = o.innertubeClient || env.customInnertubeClient || "IOS"; + + // HLS playlists from the iOS client don't contain the av1 video format. + if (useHLS && o.format === "av1") { + useHLS = false; + } + + if (useHLS) { + innertubeClient = "IOS"; + } + + // iOS client doesn't have adaptive formats of resolution >1080p, + // so we use the WEB_EMBEDDED client instead for those cases + const useSession = + env.ytSessionServer && ( + ( + !useHLS + && innertubeClient === "IOS" + && ( + (quality > 1080 && o.format !== "h264") + || o.format === "vp9" + ) + ) + ); + + if (useSession) { + innertubeClient = env.ytSessionInnertubeClient || "WEB_EMBEDDED"; + } + let yt; try { yt = await cloneInnertube( (input, init) => fetch(input, { ...init, dispatcher: o.dispatcher - }) + }), + useSession ); } catch (e) { if (e === "no_session_tokens") { @@ -104,20 +137,6 @@ export default async function (o) { } else throw e; } - let useHLS = o.youtubeHLS; - - // HLS playlists don't contain the av1 video format. - // if the session server is used, then iOS client will not work, at least currently. - if (useHLS && (o.format === "av1" || env.ytSessionServer)) { - useHLS = false; - } - - let innertubeClient = o.innertubeClient || env.customInnertubeClient || "ANDROID"; - - if (useHLS) { - innertubeClient = "IOS"; - } - let info; try { info = await yt.getBasicInfo(o.id, innertubeClient); @@ -196,8 +215,6 @@ export default async function (o) { } } - const quality = o.quality === "max" ? 9000 : Number(o.quality); - const normalizeQuality = res => { const shortestSide = Math.min(res.height, res.width); return videoQualities.find(qual => qual >= shortestSide); From ee94513580359d4dfffe7d35acf316dea09fc097 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 20 Mar 2025 18:11:04 +0600 Subject: [PATCH 11/22] api/package: bump version to 10.8.1 --- api/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/package.json b/api/package.json index eec2296a..525bc061 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.8", + "version": "10.8.1", "author": "imput", "exports": "./src/cobalt.js", "type": "module", From cf17f53405c03362859d11d29e7bf72f50a15075 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 21 Mar 2025 21:29:25 +0600 Subject: [PATCH 12/22] api/youtube: use the iOS client for <=1080p vp9 videos --- api/src/processing/services/youtube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index 219209ad..e1cbf018 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -109,7 +109,7 @@ export default async function (o) { && innertubeClient === "IOS" && ( (quality > 1080 && o.format !== "h264") - || o.format === "vp9" + || (quality > 1080 && o.format !== "vp9") ) ) ); From b93099620f09a8d91300a26c97191eb06c55d87a Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 21 Mar 2025 21:30:47 +0600 Subject: [PATCH 13/22] api/match/youtube: use 1080 dummy quality for audio-only downloads --- api/src/processing/match.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/match.js b/api/src/processing/match.js index e2d6aa07..ee4fdc1a 100644 --- a/api/src/processing/match.js +++ b/api/src/processing/match.js @@ -109,7 +109,7 @@ export default async function({ host, patternMatch, params }) { } if (url.hostname === "music.youtube.com" || isAudioOnly) { - fetchInfo.quality = "max"; + fetchInfo.quality = "1080"; fetchInfo.format = "vp9"; fetchInfo.isAudioOnly = true; fetchInfo.isAudioMuted = false; From c7c20c2157989a032b330979d18a0f369a5ee625 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 21 Mar 2025 21:52:21 +0600 Subject: [PATCH 14/22] api/tests/xiaohongshu: update the live photo picker link --- api/src/util/tests/xiaohongshu.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json index 0cca9393..e3feedb3 100644 --- a/api/src/util/tests/xiaohongshu.json +++ b/api/src/util/tests/xiaohongshu.json @@ -10,7 +10,7 @@ }, { "name": "picker with multiple live photos", - "url": "https://www.xiaohongshu.com/explore/67847fa1000000000203e6ed?xsec_token=CBzyP7Y44PPpsM20lgxqrIIJMHqOLemusDsRcmsX0cTpk", + "url": "https://www.xiaohongshu.com/explore/67c691b4000000000d0159cc?xsec_token=CB8p1eyB5DiFkwlUpy1BTeVsI9oOve6ppNjuDzo8V8p5w=", "params": {}, "expected": { "code": 200, From 1be9a8674547d12d2a802205b6585d059b2f512c Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 21 Mar 2025 22:16:49 +0600 Subject: [PATCH 15/22] api/tests/xiaohongshu: update the video link & allow to fail all links expire apparently --- api/src/util/tests/xiaohongshu.json | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json index e3feedb3..a169cc23 100644 --- a/api/src/util/tests/xiaohongshu.json +++ b/api/src/util/tests/xiaohongshu.json @@ -1,7 +1,8 @@ [ { - "name": "long link video", - "url": "https://www.xiaohongshu.com/discovery/item/6789065900000000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share", + "name": "video (might have expired)", + "url": "https://www.xiaohongshu.com/explore/67cc17a3000000000e00726a?xsec_token=CBSFRtbF57so920elY1kbIX4fE1nhrwlpGZs9m6pIFpwo=", + "canFail": true, "params": {}, "expected": { "code": 200, @@ -9,8 +10,9 @@ } }, { - "name": "picker with multiple live photos", + "name": "picker with multiple live photos (might have expired)", "url": "https://www.xiaohongshu.com/explore/67c691b4000000000d0159cc?xsec_token=CB8p1eyB5DiFkwlUpy1BTeVsI9oOve6ppNjuDzo8V8p5w=", + "canFail": true, "params": {}, "expected": { "code": 200, @@ -18,8 +20,9 @@ } }, { - "name": "one photo", + "name": "one photo (might have expired)", "url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA", + "canFail": true, "params": {}, "expected": { "code": 200, @@ -27,7 +30,7 @@ } }, { - "name": "short link, might expire eventually", + "name": "short link (might have expired)", "url": "https://xhslink.com/a/czn4z6c1tic4", "canFail": true, "params": {}, From 36516598f985da2772c8a90eceb4cf9f04940ab8 Mon Sep 17 00:00:00 2001 From: wukko Date: Fri, 21 Mar 2025 22:34:03 +0600 Subject: [PATCH 16/22] api/package: bump version to 10.8.2 --- api/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/package.json b/api/package.json index 525bc061..48f75d1e 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.8.1", + "version": "10.8.2", "author": "imput", "exports": "./src/cobalt.js", "type": "module", From 0a7cf7580cdedd8494686f19641fb24fc3e80326 Mon Sep 17 00:00:00 2001 From: lostdusty <47502554+lostdusty@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:43:53 -0300 Subject: [PATCH 17/22] api/core: remove non-printable unicode character in boot message (#1182) --- api/src/core/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/core/api.js b/api/src/core/api.js index 3f9128d8..82449c30 100644 --- a/api/src/core/api.js +++ b/api/src/core/api.js @@ -356,7 +356,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => { }, () => { if (isPrimary) { console.log(`\n` + - Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" + + Bright(Cyan("cobalt ")) + Bright("API ^ω^") + "\n" + "~~~~~~\n" + Bright("version: ") + version + "\n" + From d13b97c86251c769055a04884894cf26833db0c6 Mon Sep 17 00:00:00 2001 From: jj Date: Sun, 23 Mar 2025 17:59:17 +0100 Subject: [PATCH 18/22] api/cookies.example.json: add youtube example --- docs/examples/cookies.example.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/examples/cookies.example.json b/docs/examples/cookies.example.json index 73f3378d..d788b2dd 100644 --- a/docs/examples/cookies.example.json +++ b/docs/examples/cookies.example.json @@ -10,5 +10,8 @@ ], "twitter": [ "auth_token=; ct0=" + ], + "youtube": [ + "cookie=; b=" ] } From 2f38260e23968eda68e985468ff53cccfab2de10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Vuong=20=F0=9F=8D=82?= Date: Tue, 25 Mar 2025 18:11:49 +0700 Subject: [PATCH 19/22] api/service-config: add tiktok lite url pattern --- api/src/processing/service-config.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index 1dc8bf30..b228b3de 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -136,12 +136,13 @@ export const services = { tiktok: { patterns: [ ":user/video/:postId", + "i18n/share/video/:postId/", ":shortLink", "t/:shortLink", ":user/photo/:postId", "v/:postId.html" ], - subdomains: ["vt", "vm", "m"], + subdomains: ["vt", "vm", "m", "t"], }, tumblr: { patterns: [ From ab13f783268ce45778297a83f4e1ffcef393d6b9 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 25 Mar 2025 18:31:12 +0600 Subject: [PATCH 20/22] api/tiktok: normalize short link URL & catch empty patternMatch --- api/src/processing/services/tiktok.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/processing/services/tiktok.js b/api/src/processing/services/tiktok.js index 6fec01d8..93e07c50 100644 --- a/api/src/processing/services/tiktok.js +++ b/api/src/processing/services/tiktok.js @@ -1,6 +1,6 @@ import Cookie from "../cookie/cookie.js"; -import { extract } from "../url.js"; +import { extract, normalizeURL } from "../url.js"; import { genericUserAgent } from "../../config.js"; import { updateCookie } from "../cookie/manager.js"; import { createStream } from "../../stream/manage.js"; @@ -23,8 +23,8 @@ export default async function(obj) { if (html.startsWith(' Date: Tue, 25 Mar 2025 18:32:05 +0600 Subject: [PATCH 21/22] api/service-config/tiktok: remove trailing forward slash from a pattern --- api/src/processing/service-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/processing/service-config.js b/api/src/processing/service-config.js index b228b3de..00fa4ebf 100644 --- a/api/src/processing/service-config.js +++ b/api/src/processing/service-config.js @@ -136,7 +136,7 @@ export const services = { tiktok: { patterns: [ ":user/video/:postId", - "i18n/share/video/:postId/", + "i18n/share/video/:postId", ":shortLink", "t/:shortLink", ":user/photo/:postId", From 2d38d6300375b41c584b45f97a01a753c545b8f7 Mon Sep 17 00:00:00 2001 From: wukko Date: Tue, 25 Mar 2025 19:11:19 +0600 Subject: [PATCH 22/22] api/package: update youtubei.js to 13.3.0 --- api/package.json | 4 ++-- pnpm-lock.yaml | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/package.json b/api/package.json index 48f75d1e..86091c9d 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.8.2", + "version": "10.8.3", "author": "imput", "exports": "./src/cobalt.js", "type": "module", @@ -38,7 +38,7 @@ "set-cookie-parser": "2.6.0", "undici": "^5.19.1", "url-pattern": "1.0.3", - "youtubei.js": "^13.2.0", + "youtubei.js": "^13.3.0", "zod": "^3.23.8" }, "optionalDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f30791b7..76584fb1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,8 +56,8 @@ importers: specifier: 1.0.3 version: 1.0.3 youtubei.js: - specifier: ^13.2.0 - version: 13.2.0 + specifier: ^13.3.0 + version: 13.3.0 zod: specifier: ^3.23.8 version: 3.23.8 @@ -1468,8 +1468,8 @@ packages: resolution: {integrity: sha512-bZsjR/iRjl1Nk1UkjGpAzLNfQtzuijhn2g+pbZb98HQ1Gk8vM9hfbxeMBP+M2/UUdwj0RqGG3mlvk2MsAqwvEw==} engines: {node: 20 || >=22} - jintr@3.2.1: - resolution: {integrity: sha512-yjKUBuwTTg4nc4izMysxuIk0BKh45hnbc1KnXE6LxagIGZn5od+I2elpuRY9IIm3EiKiUZxhxV89a0iX+xoEZg==} + jintr@3.3.0: + resolution: {integrity: sha512-ZsaajJ4Hr5XR0tSPhOZOTjFhxA0qscKNSOs41NRjx7ZOGwpfdp8NKIBEUtvUPbA37JXyv1sJlgeOOZHjr3h76Q==} joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} @@ -2286,8 +2286,8 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - youtubei.js@13.2.0: - resolution: {integrity: sha512-esbSvWS12Dz/cVlHhnL/PSE84a/mVpQdzwPDIkRQu/NHJVxv0isBUcm3hJnYB1jg1LYvomV0YeOrYv5qWwJREA==} + youtubei.js@13.3.0: + resolution: {integrity: sha512-tbl7rxltpgKoSsmfGUe9JqWUAzv6HFLqrOn0N85EbTn5DLt24EXrjClnXdxyr3PBARMJ3LC4vbll100a0ABsYw==} zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} @@ -3519,7 +3519,7 @@ snapshots: dependencies: '@isaacs/cliui': 8.0.2 - jintr@3.2.1: + jintr@3.3.0: dependencies: acorn: 8.14.0 @@ -4242,10 +4242,10 @@ snapshots: yocto-queue@0.1.0: {} - youtubei.js@13.2.0: + youtubei.js@13.3.0: dependencies: '@bufbuild/protobuf': 2.2.5 - jintr: 3.2.1 + jintr: 3.3.0 tslib: 2.6.3 undici: 5.28.4