Mirrors/cobalt
1
0
Fork 0
mirror of https://github.com/wukko/cobalt.git synced 2025-06-13 05:37:44 +02:00
Code Issues Packages Projects Releases Wiki Activity

merge: changes from main

Browse Source
This commit is contained in:
wukko
2025-03-27 20:01:46 +06:00
parent 6a13ca347d 2d38d63003
commit 0b29121c53
18 changed files with 186 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
api/src/config.js
View File

@ -53,7 +53,11 @@ const env = {
keyReloadInterval: 900,
enabledServices,
customInnertubeClient: process.env.CUSTOM_INNERTUBE_CLIENT,
ytSessionServer: process.env.YOUTUBE_SESSION_SERVER,
ytSessionReloadInterval: 300,
ytSessionInnertubeClient: process.env.YOUTUBE_SESSION_INNERTUBE_CLIENT,
}
const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";

10
api/src/core/api.js
View File

@ -18,9 +18,11 @@ import { verifyTurnstileToken } from "../security/turnstile.js";
import { friendlyServiceName } from "../processing/service-alias.js";
import { verifyStream } from "../stream/manage.js";
import { createResponse, normalizeRequest, getIP } from "../processing/request.js";
import { setupTunnelHandler } from "./itunnel.js";
import * as APIKeys from "../security/api-keys.js";
import * as Cookies from "../processing/cookie/manager.js";
import { setupTunnelHandler } from "./itunnel.js";
import * as YouTubeSession from "../processing/helpers/youtube-session.js";
const git = {
branch: await getBranch(),
@ -340,7 +342,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
}, () => {
if (isPrimary) {
console.log(`\n` +
Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" +
Bright(Cyan("cobalt ")) + Bright("API ^ω^") + "\n" +
"~~~~~~\n" +
Bright("version: ") + version + "\n" +
@ -362,6 +364,10 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
if (env.cookiePath) {
Cookies.setup(env.cookiePath);
}
if (env.ytSessionServer) {
YouTubeSession.setup();
}
});
setupTunnelHandler();

1
api/src/processing/cookie/manager.js
View File

@ -13,7 +13,6 @@ const VALID_SERVICES = new Set([
'reddit',
'twitter',
'youtube',
'youtube_oauth'
]);
const invalidCookies = {};

74
api/src/processing/helpers/youtube-session.js Normal file
View File

@ -0,0 +1,74 @@
import * as cluster from "../../misc/cluster.js";
import { env } from "../../config.js";
import { Green, Yellow } from "../../misc/console-text.js";
let session;
const validateSession = (sessionResponse) => {
if (!sessionResponse.potoken) {
throw "no poToken in session response";
}
if (!sessionResponse.visitor_data) {
throw "no visitor_data in session response";
}
if (!sessionResponse.updated) {
throw "no last update timestamp in session response";
}
// https://github.com/iv-org/youtube-trusted-session-generator/blob/c2dfe3f/potoken_generator/main.py#L25
if (sessionResponse.potoken.length < 160) {
console.error(`${Yellow('[!]')} poToken is too short and might not work (${new Date().toISOString()})`);
}
}
const updateSession = (newSession) => {
session = newSession;
}
const loadSession = async () => {
const sessionServerUrl = new URL(env.ytSessionServer);
sessionServerUrl.pathname = "/token";
const newSession = await fetch(sessionServerUrl).then(a => a.json());
validateSession(newSession);
if (!session || session.updated < newSession?.updated) {
cluster.broadcast({ youtube_session: newSession });
updateSession(newSession);
}
}
const wrapLoad = (initial = false) => {
loadSession()
.then(() => {
if (initial) {
console.log(`${Green('[✓]')} poToken & visitor_data loaded successfully!`);
}
})
.catch((e) => {
console.error(`${Yellow('[!]')} Failed loading poToken & visitor_data at ${new Date().toISOString()}.`);
console.error('Error:', e);
})
}
export const getYouTubeSession = () => {
return session;
}
export const setup = () => {
if (cluster.isPrimary) {
wrapLoad(true);
if (env.ytSessionReloadInterval > 0) {
setInterval(wrapLoad, env.ytSessionReloadInterval * 1000);
}
} else if (cluster.isWorker) {
process.on('message', (message) => {
if ('youtube_session' in message) {
updateSession(message.youtube_session);
}
});
}
}

2
api/src/processing/match.js
View File

@ -109,7 +109,7 @@ export default async function({ host, patternMatch, params }) {
}
if (url.hostname === "music.youtube.com" || isAudioOnly) {
fetchInfo.quality = "max";
fetchInfo.quality = "1080";
fetchInfo.format = "vp9";
fetchInfo.isAudioOnly = true;
fetchInfo.isAudioMuted = false;

3
api/src/processing/service-config.js
View File

@ -136,12 +136,13 @@ export const services = {
tiktok: {
patterns: [
":user/video/:postId",
"i18n/share/video/:postId",
":shortLink",
"t/:shortLink",
":user/photo/:postId",
"v/:postId.html"
],
subdomains: ["vt", "vm", "m"],
subdomains: ["vt", "vm", "m", "t"],
},
tumblr: {
patterns: [

6
api/src/processing/services/tiktok.js
View File

@ -1,6 +1,6 @@
import Cookie from "../cookie/cookie.js";
import { extract } from "../url.js";
import { extract, normalizeURL } from "../url.js";
import { genericUserAgent } from "../../config.js";
import { updateCookie } from "../cookie/manager.js";
import { createStream } from "../../stream/manage.js";
@ -23,8 +23,8 @@ export default async function(obj) {
if (html.startsWith('<a href="https://')) {
const extractedURL = html.split('<a href="')[1].split('?')[0];
const { patternMatch } = extract(extractedURL);
postId = patternMatch.postId;
const { patternMatch } = extract(normalizeURL(extractedURL));
postId = patternMatch?.postId;
}
}
if (!postId) return { error: "fetch.short_link" };

127
api/src/processing/services/youtube.js
View File

@ -4,7 +4,8 @@ import { fetch } from "undici";
import { Innertube, Session } from "youtubei.js";
import { env } from "../../config.js";
import { getCookie, updateCookieValues } from "../cookie/manager.js";
import { getCookie } from "../cookie/manager.js";
import { getYouTubeSession } from "../helpers/youtube-session.js";
const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms
@ -45,41 +46,26 @@ const clientsWithNoCipher = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDR
const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320];
const transformSessionData = (cookie) => {
if (!cookie)
return;
const values = { ...cookie.values() };
const REQUIRED_VALUES = ['access_token', 'refresh_token'];
if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
return;
}
if (values.expires) {
values.expiry_date = values.expires;
delete values.expires;
} else if (!values.expiry_date) {
return;
}
return values;
}
const cloneInnertube = async (customFetch) => {
const cloneInnertube = async (customFetch, useSession) => {
const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date();
const rawCookie = getCookie('youtube');
const rawCookieValues = rawCookie?.values();
const cookie = rawCookie?.toString();
const sessionTokens = getYouTubeSession();
const retrieve_player = Boolean(sessionTokens || cookie);
if (useSession && env.ytSessionServer && !sessionTokens?.potoken) {
throw "no_session_tokens";
}
if (!innertube || shouldRefreshPlayer) {
innertube = await Innertube.create({
fetch: customFetch,
retrieve_player: !!cookie,
retrieve_player,
cookie,
po_token: rawCookieValues?.po_token,
visitor_data: rawCookieValues?.visitor_data,
po_token: useSession ? sessionTokens?.potoken : undefined,
visitor_data: useSession ? sessionTokens?.visitor_data : undefined,
});
lastRefreshedAt = +new Date();
}
@ -95,73 +81,62 @@ const cloneInnertube = async (customFetch) => {
innertube.session.cache
);
const oauthCookie = getCookie('youtube_oauth');
const oauthData = transformSessionData(oauthCookie);
if (!session.logged_in && oauthData) {
await session.oauth.init(oauthData);
session.logged_in = true;
}
if (session.logged_in && oauthData) {
if (session.oauth.shouldRefreshToken()) {
await session.oauth.refreshAccessToken();
}
const cookieValues = oauthCookie.values();
const oldExpiry = new Date(cookieValues.expiry_date);
const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);
if (oldExpiry.getTime() !== newExpiry.getTime()) {
updateCookieValues(oauthCookie, {
...session.oauth.client_id,
...session.oauth.oauth2_tokens,
expiry_date: newExpiry.toISOString()
});
}
}
const yt = new Innertube(session);
return yt;
}
export default async function (o) {
const quality = o.quality === "max" ? 9000 : Number(o.quality);
let useHLS = o.youtubeHLS;
let innertubeClient = o.innertubeClient || env.customInnertubeClient || "IOS";
// HLS playlists from the iOS client don't contain the av1 video format.
if (useHLS && o.format === "av1") {
useHLS = false;
}
if (useHLS) {
innertubeClient = "IOS";
}
// iOS client doesn't have adaptive formats of resolution >1080p,
// so we use the WEB_EMBEDDED client instead for those cases
const useSession =
env.ytSessionServer && (
(
!useHLS
&& innertubeClient === "IOS"
&& (
(quality > 1080 && o.format !== "h264")
|| (quality > 1080 && o.format !== "vp9")
)
)
);
if (useSession) {
innertubeClient = env.ytSessionInnertubeClient || "WEB_EMBEDDED";
}
let yt;
try {
yt = await cloneInnertube(
(input, init) => fetch(input, {
...init,
dispatcher: o.dispatcher
})
}),
useSession
);
} catch (e) {
if (e.message?.endsWith("decipher algorithm")) {
if (e === "no_session_tokens") {
return { error: "youtube.no_session_tokens" };
} else if (e.message?.endsWith("decipher algorithm")) {
return { error: "youtube.decipher" }
} else if (e.message?.includes("refresh access token")) {
return { error: "youtube.token_expired" }
} else throw e;
}
const cookie = getCookie('youtube')?.toString();
let useHLS = o.youtubeHLS;
// HLS playlists don't contain the av1 video format, at least with the iOS client
if (useHLS && o.format === "av1") {
useHLS = false;
}
let innertubeClient = o.innertubeClient || env.customInnertubeClient || "ANDROID";
if (cookie) {
useHLS = false;
innertubeClient = "WEB";
}
if (useHLS) {
innertubeClient = "IOS";
}
let info;
try {
info = await yt.getBasicInfo(o.id, innertubeClient);
@ -240,8 +215,6 @@ export default async function (o) {
}
}
const quality = o.quality === "max" ? 9000 : Number(o.quality);
const normalizeQuality = res => {
const shortestSide = Math.min(res.height, res.width);
return videoQualities.find(qual => qual >= shortestSide);

38
api/src/util/generate-youtube-tokens.js
View File

@ -1,38 +0,0 @@
import { Innertube } from 'youtubei.js';
import { Red } from '../misc/console-text.js'
const bail = (...msg) => {
console.error(...msg);
throw new Error(msg);
};
const tube = await Innertube.create();
tube.session.once(
'auth-pending',
({ verification_url, user_code }) => {
console.log(`${Red('[!]')} The token generated by this script is sensitive and you should not share it with anyone!`);
console.log(` By using this token, you are risking your Google account getting terminated.`);
console.log(` You should ${Red('NOT')} use your personal account!`);
console.log();
console.log(`Open ${verification_url} in a browser and enter ${user_code} when asked for the code.`);
}
);
tube.session.once('auth-error', (err) => bail('An error occurred:', err));
tube.session.once('auth', ({ credentials }) => {
if (!credentials.access_token) {
bail('something went wrong');
}
console.log(
'add this cookie to the youtube_oauth array in your cookies file:',
JSON.stringify(
Object.entries(credentials)
.map(([k, v]) => `${k}=${v instanceof Date ? v.toISOString() : v}`)
.join('; ')
)
);
});
await tube.session.signIn();

15
api/src/util/tests/xiaohongshu.json
View File

@ -1,7 +1,8 @@
[
{
"name": "long link video",
"url": "https://www.xiaohongshu.com/discovery/item/6789065900000000210035fc?source=webshare&xhsshare=pc_web&xsec_token=CBustnz_Twf1BSybpe5-D-BzUb-Bx28DPLb418TN9S9Kk&xsec_source=pc_share",
"name": "video (might have expired)",
"url": "https://www.xiaohongshu.com/explore/67cc17a3000000000e00726a?xsec_token=CBSFRtbF57so920elY1kbIX4fE1nhrwlpGZs9m6pIFpwo=",
"canFail": true,
"params": {},
"expected": {
"code": 200,
@ -9,8 +10,9 @@
}
},
{
"name": "picker with multiple live photos",
"url": "https://www.xiaohongshu.com/explore/67847fa1000000000203e6ed?xsec_token=CBzyP7Y44PPpsM20lgxqrIIJMHqOLemusDsRcmsX0cTpk",
"name": "picker with multiple live photos (might have expired)",
"url": "https://www.xiaohongshu.com/explore/67c691b4000000000d0159cc?xsec_token=CB8p1eyB5DiFkwlUpy1BTeVsI9oOve6ppNjuDzo8V8p5w=",
"canFail": true,
"params": {},
"expected": {
"code": 200,
@ -18,8 +20,9 @@
}
},
{
"name": "one photo",
"name": "one photo (might have expired)",
"url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA",
"canFail": true,
"params": {},
"expected": {
"code": 200,
@ -27,7 +30,7 @@
}
},
{
"name": "short link, might expire eventually",
"name": "short link (might have expired)",
"url": "https://xhslink.com/a/czn4z6c1tic4",
"canFail": true,
"params": {},

4
api/src/util/tests/youtube.json
View File

@ -189,6 +189,7 @@
{
"name": "hls video (h264, 1440p)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"canFail": true,
"params": {
"youtubeVideoCodec": "h264",
"videoQuality": "1440",
@ -202,6 +203,7 @@
{
"name": "hls video (vp9, 360p)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"canFail": true,
"params": {
"youtubeVideoCodec": "vp9",
"videoQuality": "360",
@ -215,6 +217,7 @@
{
"name": "hls video (audio mode)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"canFail": true,
"params": {
"downloadMode": "audio",
"youtubeHLS": true
@ -227,6 +230,7 @@
{
"name": "hls video (audio mode, best format)",
"url": "https://www.youtube.com/watch?v=vPwaXytZcgI",
"canFail": true,
"params": {
"downloadMode": "audio",
"youtubeHLS": true,