Mirrors/cobalt
1
0
Fork 0
mirror of https://github.com/wukko/cobalt.git synced 2025-06-12 13:17:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

5.0-dev1

Browse Source 
- rewrote and/or optimized all service modules
- rewrote matching and processing modules to optimize readability and performance
- added support for reddit gifs
- fixed various issues with twitter error explanations
- code optimizations and enhancements (such as finally getting rid of ==, prettier and more readable formatting, etc)
- added branch information
- all functions in currentCommit submodule run only once and cache received data
- added a test script. only twitter and soundcloud are 100% covered and tested atm, will add tests (and probably fixes) for the rest of services in next commits
- changed some localization strings for russian
- added more clarity to rate limit message
- moved services folder into processing folder
This commit is contained in:
wukko
2023-02-12 13:40:49 +06:00
parent 3432c91482
commit dacaaf5b27
39 changed files with 1139 additions and 825 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
src/modules/processing/match.js
View File

@ -5,23 +5,24 @@ import loc from "../../localization/manager.js";
import { testers } from "./servicesPatternTesters.js";
import bilibili from "../services/bilibili.js";
import reddit from "../services/reddit.js";
import twitter from "../services/twitter.js";
import youtube from "../services/youtube.js";
import vk from "../services/vk.js";
import tiktok from "../services/tiktok.js";
import tumblr from "../services/tumblr.js";
import bilibili from "./services/bilibili.js";
import reddit from "./services/reddit.js";
import twitter from "./services/twitter.js";
import youtube from "./services/youtube.js";
import vk from "./services/vk.js";
import tiktok from "./services/tiktok.js";
import tumblr from "./services/tumblr.js";
import matchActionDecider from "./matchActionDecider.js";
import vimeo from "../services/vimeo.js";
import soundcloud from "../services/soundcloud.js";
import vimeo from "./services/vimeo.js";
import soundcloud from "./services/soundcloud.js";
export default async function (host, patternMatch, url, lang, obj) {
try {
if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
if (!(testers[host](patternMatch))) throw Error();
let r, isAudioOnly = !!obj.isAudioOnly;
if (!testers[host]) return apiJSON(0, { t: errorUnsupported(lang) });
if (!(testers[host](patternMatch))) return apiJSON(0, { t: brokenLink(lang) });
let r;
switch (host) {
case "twitter":
r = await twitter({
@ -29,14 +30,14 @@ export default async function (host, patternMatch, url, lang, obj) {
spaceId: patternMatch["spaceId"] ? patternMatch["spaceId"] : false,
lang: lang
});
if (r.isAudioOnly) obj.isAudioOnly = true
break;
case "vk":
r = await vk({
url: url,
userId: patternMatch["userId"],
videoId: patternMatch["videoId"],
lang: lang, quality: obj.vQuality
lang: lang,
quality: obj.vQuality
});
break;
case "bilibili":
@ -48,10 +49,11 @@ export default async function (host, patternMatch, url, lang, obj) {
case "youtube":
let fetchInfo = {
id: patternMatch["id"].slice(0, 11),
lang: lang, quality: obj.vQuality,
lang: lang,
quality: obj.vQuality,
format: "webm"
};
if (url.match('music.youtube.com') || obj.isAudioOnly == true) obj.vFormat = "audio";
if (url.match('music.youtube.com') || isAudioOnly === true) obj.vFormat = "audio";
switch (obj.vFormat) {
case "mp4":
fetchInfo["format"] = "mp4";
@ -60,7 +62,7 @@ export default async function (host, patternMatch, url, lang, obj) {
fetchInfo["format"] = "webm";
fetchInfo["isAudioOnly"] = true;
fetchInfo["quality"] = "max";
obj.isAudioOnly = true;
isAudioOnly = true;
break;
}
r = await youtube(fetchInfo);
@ -69,7 +71,8 @@ export default async function (host, patternMatch, url, lang, obj) {
r = await reddit({
sub: patternMatch["sub"],
id: patternMatch["id"],
title: patternMatch["title"], lang: lang,
title: patternMatch["title"],
lang: lang,
});
break;
case "douyin":
@ -77,28 +80,33 @@ export default async function (host, patternMatch, url, lang, obj) {
r = await tiktok({
host: host,
postId: patternMatch["postId"],
id: patternMatch["id"], lang: lang,
noWatermark: obj.isNoTTWatermark, fullAudio: obj.isTTFullAudio,
isAudioOnly: obj.isAudioOnly
id: patternMatch["id"],
lang: lang,
noWatermark: obj.isNoTTWatermark,
fullAudio: obj.isTTFullAudio,
isAudioOnly: isAudioOnly
});
if (r.isAudioOnly) obj.isAudioOnly = true;
break;
case "tumblr":
r = await tumblr({
id: patternMatch["id"], url: url, user: patternMatch["user"] ? patternMatch["user"] : false,
id: patternMatch["id"],
url: url,
user: patternMatch["user"] ? patternMatch["user"] : false,
lang: lang
});
break;
case "vimeo":
r = await vimeo({
id: patternMatch["id"].slice(0, 11), quality: obj.vQuality,
id: patternMatch["id"].slice(0, 11),
quality: obj.vQuality,
lang: lang
});
break;
case "soundcloud":
obj.isAudioOnly = true;
isAudioOnly = true;
r = await soundcloud({
author: patternMatch["author"], song: patternMatch["song"], url: url,
author: patternMatch["author"],
song: patternMatch["song"], url: url,
shortLink: patternMatch["shortLink"] ? patternMatch["shortLink"] : false,
accessKey: patternMatch["accessKey"] ? patternMatch["accessKey"] : false,
format: obj.aFormat,
@ -108,10 +116,15 @@ export default async function (host, patternMatch, url, lang, obj) {
default:
return apiJSON(0, { t: errorUnsupported(lang) });
}
return !r.error ? matchActionDecider(r, host, obj.ip, obj.aFormat, obj.isAudioOnly, lang, obj.isAudioMuted) : apiJSON(0, {
t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error)
});
if (r.isAudioOnly) isAudioOnly = true;
let isAudioMuted = isAudioOnly ? false : obj.isAudioMuted;
if (r.error) return apiJSON(0, { t: Array.isArray(r.error) ? loc(lang, r.error[0], r.error[1]) : loc(lang, r.error) });
return matchActionDecider(r, host, obj.ip, obj.aFormat, isAudioOnly, lang, isAudioMuted);
} catch (e) {
console.log(e)
return apiJSON(0, { t: genericError(lang, host) })
}
}

242
src/modules/processing/matchActionDecider.js
View File

@ -1,121 +1,137 @@
import { audioIgnore, services, supportedAudio } from "../config.js"
import { apiJSON } from "../sub/utils.js"
import { audioIgnore, services, supportedAudio } from "../config.js";
import { apiJSON } from "../sub/utils.js";
import loc from "../../localization/manager.js";
export default function(r, host, ip, audioFormat, isAudioOnly, lang, isAudioMuted) {
if (!isAudioOnly && !r.picker && !isAudioMuted) {
switch (host) {
case "twitter":
return apiJSON(1, { u: r.urls });
case "vk":
return apiJSON(2, {
type: "bridge", u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "bilibili":
return apiJSON(2, {
type: "render", u: r.urls, service: host, ip: ip,
filename: r.filename,
time: r.time
});
case "youtube":
return apiJSON(2, {
type: r.type, u: r.urls, service: host, ip: ip,
filename: r.filename,
time: r.time,
});
case "reddit":
return apiJSON(r.typeId, {
type: r.type, u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "tiktok":
return apiJSON(2, {
type: "bridge", u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "douyin":
return apiJSON(2, {
type: "bridge", u: r.urls, service: host, ip: ip,
filename: r.filename,
});
case "tumblr":
return apiJSON(1, { u: r.urls });
case "vimeo":
if (Array.isArray(r.urls)) {
return apiJSON(2, {
type: "render", u: r.urls, service: host, ip: ip,
filename: r.filename
});
} else {
return apiJSON(1, { u: r.urls });
}
}
} else if (isAudioMuted && !isAudioOnly) {
let isSplit = Array.isArray(r.urls);
return apiJSON(2, {
type: isSplit ? "bridge" : "mute",
u: isSplit ? r.urls[0] : r.urls,
let action,
responseType = 2,
defaultParams = {
u: r.urls,
service: host,
ip: ip,
filename: r.filename,
mute: true,
});
} else if (r.picker) {
switch (host) {
case "douyin":
case "tiktok":
let type = "render";
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3"
type = "bridge"
}
return apiJSON(5, {
type: type,
picker: r.picker,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip,
filename: r.audioFilename, isAudioOnly: true, audioFormat: audioFormat, copy: audioFormat === "best" ? true : false,
})
case "twitter":
return apiJSON(5, {
picker: r.picker, service: host
})
}
} else if (isAudioOnly) {
if ((host === "reddit" && r.typeId === 1) || (host === "vimeo" && !r.filename) || audioIgnore.includes(host)) return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
let type = "render";
let copy = false;
if (!supportedAudio.includes(audioFormat)) audioFormat = "best";
if ((host == "tiktok" || host == "douyin") && services.tiktok.audioFormats.includes(audioFormat)) {
if (r.isMp3) {
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3"
type = "bridge"
}
} else if (audioFormat === "best") {
audioFormat = "m4a"
type = "bridge"
}
}
if ((audioFormat === "best" && services[host]["bestAudio"]) || services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) {
audioFormat = services[host]["bestAudio"]
type = "bridge"
} else if (audioFormat === "best") {
audioFormat = "m4a"
copy = true
if (r.audioFilename.includes("twitterspaces")) {
audioFormat = "mp3"
copy = false
}
}
return apiJSON(2, {
type: type,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls, service: host, ip: ip,
filename: r.audioFilename, isAudioOnly: true,
audioFormat: audioFormat, copy: copy, fileMetadata: r.fileMetadata ? r.fileMetadata : false
})
} else {
return apiJSON(0, { t: loc(lang, 'ErrorSomethingWentWrong') });
},
params = {}
if (isAudioMuted) action = "muteVideo";
if (!isAudioOnly && !r.picker && !isAudioMuted) action = "video";
if (r.picker) action = "picker";
if (isAudioOnly) action = "audio";
if (action === "picker" || action === "audio") {
defaultParams.filename = r.audioFilename;
defaultParams.isAudioOnly = true;
defaultParams.audioFormat = audioFormat;
}
switch (action) {
case "video":
switch (host) {
case "bilibili":
params = { type: "render", time: r.time };
break;
case "youtube":
params = { type: r.type, time: r.time };
break;
case "reddit":
responseType = r.typeId;
params = { type: r.type };
break;
case "vimeo":
if (Array.isArray(r.urls)) {
params = { type: "render" }
} else {
responseType = 1;
}
break;
case "vk":
case "douyin":
case "tiktok":
params = { type: "bridge" };
break;
case "tumblr":
case "twitter":
responseType = 1;
break;
}
break;
case "muteVideo":
params = {
type: Array.isArray(r.urls) ? "bridge" : "mute",
u: Array.isArray(r.urls) ? r.urls[0] : r.urls,
mute: true
}
break;
case "picker":
responseType = 5;
switch (host) {
case "twitter":
params = { picker: r.picker };
break;
case "douyin":
case "tiktok":
let pickerType = "render";
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3";
pickerType = "bridge"
}
params = {
type: pickerType,
picker: r.picker,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
copy: audioFormat === "best" ? true : false
}
}
break;
case "audio":
if ((host === "reddit" && r.typeId === 1) || (host === "vimeo" && !r.filename) || audioIgnore.includes(host)) return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
let processType = "render";
let copy = false;
if (!supportedAudio.includes(audioFormat)) audioFormat = "best";
if ((host === "tiktok" || host === "douyin") && services.tiktok.audioFormats.includes(audioFormat)) {
if (r.isMp3) {
if (audioFormat === "mp3" || audioFormat === "best") {
audioFormat = "mp3";
processType = "bridge"
}
} else if (audioFormat === "best") {
audioFormat = "m4a";
processType = "bridge"
}
}
if ((audioFormat === "best" && services[host]["bestAudio"])
|| services[host]["bestAudio"] && (audioFormat === services[host]["bestAudio"])) {
audioFormat = services[host]["bestAudio"];
processType = "bridge"
} else if (audioFormat === "best") {
audioFormat = "m4a";
copy = true;
if (r.audioFilename.includes("twitterspaces")) {
audioFormat = "mp3"
copy = false
}
}
params = {
type: processType,
u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
audioFormat: audioFormat,
copy: copy,
fileMetadata: r.fileMetadata ? r.fileMetadata : false
}
break;
default:
return apiJSON(0, { t: loc(lang, 'ErrorEmptyDownload') });
}
return apiJSON(responseType, {...defaultParams, ...params})
}

28
src/modules/processing/services/bilibili.js Normal file
View File

@ -0,0 +1,28 @@
import { genericUserAgent, maxVideoDuration } from "../../config.js";
// TO-DO: quality picking & bilibili.tv support
export default async function(obj) {
let html = await fetch(`https://bilibili.com/video/${obj.id}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!(html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"'))) return { error: 'ErrorEmptyDownload' };
let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
if (streamData.data.timelength > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let video = streamData["data"]["dash"]["video"].filter((v) => {
if (!v["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
let audio = streamData["data"]["dash"]["audio"].filter((a) => {
if (!a["baseUrl"].includes("https://upos-sz-mirrorcosov.bilivideo.com/")) return true;
}).sort((a, b) => Number(b.bandwidth) - Number(a.bandwidth));
return {
urls: [video[0]["baseUrl"], audio[0]["baseUrl"]],
time: streamData.data.timelength,
audioFilename: `bilibili_${obj.id}_audio`,
filename: `bilibili_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.mp4`
};
}

28
src/modules/processing/services/reddit.js Normal file
View File

@ -0,0 +1,28 @@
import { maxVideoDuration } from "../../config.js";
export default async function(obj) {
let data = await fetch(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}/${obj.name}.json`).then((r) => { return r.json() }).catch(() => { return false });
if (!data) return { error: 'ErrorCouldntFetch' };
data = data[0]["data"]["children"][0]["data"];
if (data.url.endsWith('.gif')) return { typeId: 1, urls: data.url };
if (!"reddit_video" in data["secure_media"]) return { error: 'ErrorEmptyDownload' };
if (data["secure_media"]["reddit_video"]["duration"] * 1000 > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let video = data["secure_media"]["reddit_video"]["fallback_url"].split('?')[0],
audio = video.match('.mp4') ? `${video.split('_')[0]}_audio.mp4` : `${data["secure_media"]["reddit_video"]["fallback_url"].split('DASH')[0]}audio`;
await fetch(audio, { method: "HEAD" }).then((r) => {if (r.status != 200) audio = ''}).catch(() => {audio = ''});
let id = data["secure_media"]["reddit_video"]["fallback_url"].split('/')[3];
if (!audio.length > 0) return { typeId: 1, urls: video };
return {
typeId: 2,
type: "render",
urls: [video, audio],
audioFilename: `reddit_${id}_audio`,
filename: `reddit_${id}.mp4`
};
}

74
src/modules/processing/services/soundcloud.js Normal file
View File

@ -0,0 +1,74 @@
import { maxAudioDuration } from "../../config.js";
let cachedID = {};
async function findClientID() {
try {
let sc = await fetch('https://soundcloud.com/').then((r) => { return r.text() }).catch(() => { return false });
let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
if (cachedID.version === scVersion) return cachedID.id;
let scripts = sc.matchAll(/<script.+src="(.+)">/g);
let clientid;
for (let script of scripts) {
let url = script[1];
if (url && !url.startsWith('https://a-v2.sndcdn.com')) return;
let scrf = await fetch(url).then((r) => {return r.text()}).catch(() => { return false });
let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
if (id && typeof id[0] === 'string') {
clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
break;
}
}
cachedID.version = scVersion;
cachedID.id = clientid;
return clientid;
} catch (e) {
return false;
}
}
export default async function(obj) {
let html;
if (!obj.author && !obj.song && obj.shortLink) {
html = await fetch(`https://soundcloud.app.goo.gl/${obj.shortLink}/`).then((r) => { return r.text() }).catch(() => { return false });
}
if (obj.author && obj.song) {
html = await fetch(`https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`).then((r) => { return r.text() }).catch(() => { return false });
}
if (!html) return { error: 'ErrorCouldntFetch'};
if (!(html.includes('<script>window.__sc_hydration = ')
&& html.includes('"format":{"protocol":"progressive","mime_type":"audio/mpeg"},')
&& html.includes('{"hydratable":"sound","data":'))) {
return { error: ['ErrorBrokenLink', 'soundcloud'] }
}
let json = JSON.parse(html.split('{"hydratable":"sound","data":')[1].split('}];</script>')[0])
if (!json["media"]["transcodings"]) return { error: 'ErrorEmptyDownload' };
let clientId = await findClientID();
if (!clientId) return { error: 'ErrorSoundCloudNoClientId' };
let fileUrlBase = json.media.transcodings[0]["url"].replace("/hls", "/progressive")
let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
if (!fileUrl.substring(0, 54) === "https://api-v2.soundcloud.com/media/soundcloud:tracks:") return { error: 'ErrorEmptyDownload' };
if (json.duration > maxAudioDuration) return { error: ['ErrorLengthAudioConvert', maxAudioDuration / 60000] };
let file = await fetch(fileUrl).then(async (r) => { return (await r.json()).url }).catch(() => { return false });
if (!file) return { error: 'ErrorCouldntFetch' };
return {
urls: file,
audioFilename: `soundcloud_${json.id}`,
fileMetadata: {
title: json.title,
artist: json.user.username,
}
}
}

112
src/modules/processing/services/tiktok.js Normal file
View File

@ -0,0 +1,112 @@
import { genericUserAgent } from "../../config.js";
let userAgent = genericUserAgent.split(' Chrome/1')[0],
config = {
tiktok: {
short: "https://vt.tiktok.com/",
api: "https://api2.musical.ly/aweme/v1/feed/?aweme_id={postId}&version_code=262&app_name=musical_ly&channel=App&device_id=null&os_version=14.4.2&device_platform=iphone&device_type=iPhone9&region=US&carrier_region=US",
},
douyin: {
short: "https://v.douyin.com/",
api: "https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={postId}",
}
}
function selector(j, h, id) {
if (!j) return false;
let t;
switch (h) {
case "tiktok":
t = j["aweme_list"].filter((v) => { if (v["aweme_id"] === id) return true });
break;
case "douyin":
t = j['item_list'].filter((v) => { if (v["aweme_id"] === id) return true });
break;
}
if (!t.length > 0) return false;
return t[0];
}
export default async function(obj) {
if (!obj.postId) {
let html = await fetch(`${config[obj.host]["short"]}${obj.id}`, {
redirect: "manual",
headers: { "user-agent": userAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (html.slice(0, 17) === '<a href="https://' && html.includes('/video/')) {
obj.postId = html.split('/video/')[1].split('?')[0].replace("/", '')
} else if (html.slice(0, 32) === '<a href="https://m.tiktok.com/v/' && html.includes('/v/')) {
obj.postId = html.split('/v/')[1].split('.html')[0].replace("/", '')
}
}
if (!obj.postId) return { error: 'ErrorCantGetID' };
let detail;
detail = await fetch(config[obj.host]["api"].replace("{postId}", obj.postId), {
headers: {"user-agent": "TikTok 26.2.0 rv:262018 (iPhone; iOS 14.4.2; en_US) Cronet"}
}).then((r) => { return r.json() }).catch(() => { return false });
detail = selector(detail, obj.host, obj.postId);
if (!detail) return { error: 'ErrorCouldntFetch' };
let video, videoFilename, audioFilename, isMp3, audio, images, filenameBase = `${obj.host}_${obj.postId}`;
if (obj.host === "tiktok") {
images = detail["image_post_info"] ? detail["image_post_info"]["images"] : false
} else {
images = detail["images"] ? detail["images"] : false
}
if (!obj.isAudioOnly && !images) {
video = obj.host === "tiktok" ? detail["video"]["download_addr"]["url_list"][0] : detail['video']['play_addr']['url_list'][0];
videoFilename = `${filenameBase}_video.mp4`;
if (obj.noWatermark) {
video = obj.host === "tiktok" ? detail["video"]["play_addr"]["url_list"][0] : detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play");
videoFilename = `${filenameBase}_video_nw.mp4` // nw - no watermark
}
} else {
let fallback = obj.host === "douyin" ? detail["video"]["play_addr"]["url_list"][0].replace("playwm", "play") : detail["video"]["play_addr"]["url_list"][0];
audio = fallback;
audioFilename = `${filenameBase}_audio_fv`; // fv - from video
if (obj.fullAudio || fallback.includes("music")) {
audio = detail["music"]["play_url"]["url_list"][0]
audioFilename = `${filenameBase}_audio`
}
if (audio.slice(-4) === ".mp3") isMp3 = true;
}
if (video) return {
urls: video,
filename: videoFilename
}
if (images && obj.isAudioOnly) {
return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}
if (images) {
let imageLinks = [];
for (let i in images) {
let sel = obj.host === "tiktok" ? images[i]["display_image"]["url_list"] : images[i]["url_list"];
sel = sel.filter((p) => { if (p.includes(".jpeg?")) return true; })
imageLinks.push({url: sel[0]})
}
return {
picker: imageLinks,
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}
if (audio) return {
urls: audio,
audioFilename: audioFilename,
isAudioOnly: true,
isMp3: isMp3,
}
}

14
src/modules/processing/services/tumblr.js Normal file
View File

@ -0,0 +1,14 @@
import { genericUserAgent } from "../../config.js";
export default async function(obj) {
let html = await fetch(`https://${
obj.user ? obj.user : obj.url.split('.')[0].replace('https://', '')
}.tumblr.com/post/${obj.id}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes('property="og:video" content="https://va.media.tumblr.com/')) return { error: 'ErrorEmptyDownload' };
return { urls: `https://va.media.tumblr.com/${html.split('property="og:video" content="https://va.media.tumblr.com/')[1].split('"')[0]}`, audioFilename: `tumblr_${obj.id}_audio` }
}

101
src/modules/processing/services/twitter.js Normal file
View File

@ -0,0 +1,101 @@
import { genericUserAgent } from "../../config.js";
function bestQuality(arr) {
return arr.filter((v) => { if (v["content_type"] === "video/mp4") return true }).sort((a, b) => Number(b.bitrate) - Number(a.bitrate))[0]["url"].split("?")[0]
}
const apiURL = "https://api.twitter.com/1.1"
// TO-DO: move from 1.1 api to graphql
export default async function(obj) {
let _headers = {
"user-agent": genericUserAgent,
"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
// ^ no explicit content, but with multi media support
"host": "api.twitter.com"
};
let req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
let showURL = `${apiURL}/statuses/show/${obj.id}.json?tweet_mode=extended&include_user_entities=0&trim_user=1&include_entities=0&cards_platform=Web-12&include_cards=1`;
if (!obj.spaceId) {
let req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch((e) => { return false });
if (!req_status) {
_headers.authorization = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw";
// ^ explicit content, but no multi media support
delete _headers["x-guest-token"]
req_act = await fetch(`${apiURL}/guest/activate.json`, {
method: "POST",
headers: _headers
}).then((r) => { return r.status === 200 ? r.json() : false}).catch(() => { return false });
if (!req_act) return { error: 'ErrorCouldntFetch' };
_headers["x-guest-token"] = req_act["guest_token"];
req_status = await fetch(showURL, { headers: _headers }).then((r) => { return r.status === 200 ? r.json() : false }).catch(() => { return false });
}
if (!req_status) return { error: 'ErrorCouldntFetch' };
if (!req_status["extended_entities"] || !req_status["extended_entities"]["media"]) return { error: 'ErrorNoVideosInTweet' };
let single, multiple = [], media = req_status["extended_entities"]["media"];
media = media.filter((i) => { if (i["type"] === "video" || i["type"] === "animated_gif") return true })
if (media.length > 1) {
for (let i in media) { multiple.push({type: "video", thumb: media[i]["media_url_https"], url: bestQuality(media[i]["video_info"]["variants"])}) }
} else if (media.length === 1) {
single = bestQuality(media[0]["video_info"]["variants"])
} else {
return { error: 'ErrorNoVideosInTweet' }
}
if (single) {
return { urls: single, filename: `twitter_${obj.id}.mp4`, audioFilename: `twitter_${obj.id}_audio` }
} else if (multiple) {
return { picker: multiple }
} else {
return { error: 'ErrorNoVideosInTweet' }
}
} else {
_headers["host"] = "twitter.com"
_headers["content-type"] = "application/json"
let query = {
variables: {"id": obj.spaceId,"isMetatagsQuery":true,"withSuperFollowsUserFields":true,"withDownvotePerspective":false,"withReactionsMetadata":false,"withReactionsPerspective":false,"withSuperFollowsTweetFields":true,"withReplays":true},
features: {"spaces_2022_h2_clipping":true,"spaces_2022_h2_spaces_communities":true,"verified_phone_label_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_uc_gql_enabled":true,"vibe_api_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":false,"interactive_text_enabled":true,"responsive_web_text_conversations_enabled":false,"responsive_web_enhance_cards_enabled":true}
}
query.variables = new URLSearchParams(JSON.stringify(query.variables)).toString().slice(0, -1);
query.features = new URLSearchParams(JSON.stringify(query.features)).toString().slice(0, -1);
query = `https://twitter.com/i/api/graphql/wJ5g4zf7v8qPHSQbaozYuw/AudioSpaceById?variables=${query.variables}&features=${query.features}`
let AudioSpaceById = await fetch(query, { headers: _headers }).then((r) => {return r.status === 200 ? r.json() : false}).catch((e) => { return false });
if (!AudioSpaceById) return { error: 'ErrorEmptyDownload' };
if (!AudioSpaceById.data.audioSpace.metadata) return { error: 'ErrorEmptyDownload' };
if (!AudioSpaceById.data.audioSpace.metadata.is_space_available_for_replay === true) return { error: 'TwitterSpaceWasntRecorded' };
let streamStatus = await fetch(
`https://twitter.com/i/api/1.1/live_video_stream/status/${AudioSpaceById.data.audioSpace.metadata.media_key}`, { headers: _headers }
).then((r) =>{ return r.status === 200 ? r.json() : false }).catch(() => { return false });
if (!streamStatus) return { error: 'ErrorCouldntFetch' };
let participants = AudioSpaceById.data.audioSpace.participants.speakers;
let listOfParticipants = `Twitter Space speakers: `;
for (let i in participants) { listOfParticipants += `@${participants[i]["twitter_screen_name"]}, ` }
listOfParticipants = listOfParticipants.slice(0, -2);
return {
urls: streamStatus.source.noRedirectPlaybackUrl,
audioFilename: `twitterspaces_${obj.spaceId}`,
isAudioOnly: true,
fileMetadata: {
title: AudioSpaceById.data.audioSpace.metadata.title,
artist: `Twitter Space by @${AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.screen_name}`,
comment: listOfParticipants,
// cover: AudioSpaceById.data.audioSpace.metadata.creator_results.result.legacy.profile_image_url_https.replace("_normal", "")
}
}
}
}

77
src/modules/processing/services/vimeo.js Normal file
View File

@ -0,0 +1,77 @@
import { maxVideoDuration, quality, services } from "../../config.js";
export default async function(obj) {
let api = await fetch(`https://player.vimeo.com/video/${obj.id}/config`).then((r) => { return r.json() }).catch(() => { return false });
if (!api) return { error: 'ErrorCouldntFetch' };
let downloadType = "dash";
if (JSON.stringify(api).includes('"progressive":[{')) downloadType = "progressive";
switch(downloadType) {
case "progressive":
let all = api["request"]["files"]["progressive"].sort((a, b) => Number(b.width) - Number(a.width));
let best = all[0];
try {
if (obj.quality != "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in all) {
let currQuality = parseInt(all[i]["quality"].replace('p', ''), 10)
if (currQuality === pref) {
best = all[i];
break
}
if (currQuality < pref) {
best = all[i-1];
break
}
}
}
} catch (e) {
best = all[0]
}
return { urls: best["url"], filename: `tumblr_${obj.id}.mp4` };
case "dash":
if (api.video.duration > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let masterJSONURL = api["request"]["files"]["dash"]["cdns"]["akfire_interconnect_quic"]["url"];
let masterJSON = await fetch(masterJSONURL).then((r) => { return r.json() }).catch(() => { return false });
if (!masterJSON) return { error: 'ErrorCouldntFetch' };
if (!masterJSON.video) return { error: 'ErrorEmptyDownload' };
let type = "parcel";
if (masterJSON.base_url === "../") type = "chop";
let masterJSON_Video = masterJSON.video.sort((a, b) => Number(b.width) - Number(a.width));
let masterJSON_Audio = masterJSON.audio.sort((a, b) => Number(b.bitrate) - Number(a.bitrate)).filter((a)=> {if (a['mime_type'] === "audio/mp4") return true;});
let bestVideo = masterJSON_Video[0], bestAudio = masterJSON_Audio[0];
switch (type) {
case "parcel":
if (obj.quality != "max") {
let pref = parseInt(quality[obj.quality], 10)
for (let i in masterJSON_Video) {
let currQuality = parseInt(services.vimeo.resolutionMatch[masterJSON_Video[i]["width"]], 10)
if (currQuality < pref) {
break;
} else if (String(currQuality) === String(pref)) {
bestVideo = masterJSON_Video[i]
}
}
}
let baseUrl = masterJSONURL.split("/sep/")[0];
let videoUrl = `${baseUrl}/parcel/video/${bestVideo.index_segment.split('?')[0]}`,
audioUrl = `${baseUrl}/parcel/audio/${bestAudio.index_segment.split('?')[0]}`;
return { urls: [videoUrl, audioUrl], audioFilename: `vimeo_${obj.id}_audio`, filename: `vimeo_${obj.id}_${bestVideo["width"]}x${bestVideo["height"]}.mp4` }
case "chop": // TO-DO: support for chop stream type
default:
return { error: 'ErrorEmptyDownload' }
}
default:
return { error: 'ErrorEmptyDownload' }
}
}

45
src/modules/processing/services/vk.js Normal file
View File

@ -0,0 +1,45 @@
import { xml2json } from "xml-js";
import { genericUserAgent, maxVideoDuration, services } from "../../config.js";
import selectQuality from "../../stream/selectQuality.js";
export default async function(obj) {
let html;
html = await fetch(`https://vk.com/video-${obj.userId}_${obj.videoId}`, {
headers: { "user-agent": genericUserAgent }
}).then((r) => { return r.text() }).catch(() => { return false });
if (!html) return { error: 'ErrorCouldntFetch' };
if (!html.includes(`{"lang":`)) return { error: 'ErrorEmptyDownload' };
let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
if (!Number(js["mvData"]["is_active_live"]) === 0) return { error: 'ErrorLiveVideo' };
if (js["mvData"]["duration"] > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
let mpd = JSON.parse(xml2json(js["player"]["params"][0]["manifest"], { compact: true, spaces: 4 }));
let repr = mpd["MPD"]["Period"]["AdaptationSet"]["Representation"];
if (!mpd["MPD"]["Period"]["AdaptationSet"]["Representation"]) repr = mpd["MPD"]["Period"]["AdaptationSet"][0]["Representation"];
let selectedQuality,
attr = repr[repr.length - 1]["_attributes"],
qualities = Object.keys(services.vk.quality_match);
for (let i in qualities) {
if (qualities[i] === attr["height"]) {
selectedQuality = `url${attr["height"]}`;
break
}
if (qualities[i] === attr["width"]) {
selectedQuality = `url${attr["width"]}`;
break
}
}
let maxQuality = js["player"]["params"][0][selectedQuality].split('type=')[1].slice(0, 1);
let userQuality = selectQuality('vk', obj.quality, Object.entries(services.vk.quality_match).reduce((r, [k, v]) => { r[v] = k; return r; })[maxQuality]);
let userRepr = repr[services.vk.representation_match[userQuality]]["_attributes"];
if (!selectedQuality in js["player"]["params"][0]) return { error: 'ErrorEmptyDownload' };
return {
urls: js["player"]["params"][0][`url${userQuality}`],
filename: `vk_${obj.userId}_${obj.videoId}_${userRepr["width"]}x${userRepr['height']}.mp4`
}
}

88
src/modules/processing/services/youtube.js Normal file
View File

@ -0,0 +1,88 @@
import ytdl from "better-ytdl-core";
import { maxVideoDuration, quality as mq } from "../../config.js";
import selectQuality from "../../stream/selectQuality.js";
export default async function(obj) {
let isAudioOnly = !!obj.isAudioOnly,
infoInitial = await ytdl.getInfo(obj.id);
if (!infoInitial) return { error: 'ErrorCantConnectToServiceAPI' };
let info = infoInitial.formats;
if (info[0]["isLive"]) return { error: 'ErrorLiveVideo' };
let videoMatch = [], fullVideoMatch = [], video = [],
audio = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasAudio"] && !a["hasVideo"] && a["container"] == obj.format) return true
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (audio.length === 0) return { error: 'ErrorBadFetch' };
if (audio[0]["approxDurationMs"] > maxVideoDuration) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] };
if (!isAudioOnly) {
video = info.filter((a) => {
if (!a["isHLS"] && !a["isDashMPD"] && a["hasVideo"] && a["container"] == obj.format) {
if (obj.quality != "max") {
if (a["hasAudio"] && mq[obj.quality] == a["height"]) {
fullVideoMatch.push(a)
} else if (!a["hasAudio"] && mq[obj.quality] == a["height"]) {
videoMatch.push(a)
}
}
return true
}
}).sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
if (obj.quality != "max") {
if (videoMatch.length === 0) {
let ss = selectQuality("youtube", obj.quality, video[0]["qualityLabel"].slice(0, 5).replace('p', '').trim());
videoMatch = video.filter((a) => {
if (a["qualityLabel"].slice(0, 5).replace('p', '').trim() == ss) return true
})
} else if (fullVideoMatch.length > 0) {
videoMatch = [fullVideoMatch[0]]
}
} else videoMatch = [video[0]];
if (obj.quality === "los") videoMatch = [video[video.length - 1]];
}
if (video.length === 0) isAudioOnly = true;
if (isAudioOnly) {
let r = {
type: "render",
isAudioOnly: true,
urls: audio[0]["url"],
audioFilename: `youtube_${obj.id}_audio`,
fileMetadata: {
title: infoInitial.videoDetails.title,
artist: infoInitial.videoDetails.ownerChannelName.replace("- Topic", "").trim(),
}
}
if (infoInitial.videoDetails.description) {
let isAutoGenAudio = infoInitial.videoDetails.description.startsWith("Provided to YouTube by");
if (isAutoGenAudio) {
let descItems = infoInitial.videoDetails.description.split("\n\n")
r.fileMetadata.album = descItems[2]
r.fileMetadata.copyright = descItems[3]
if (descItems[4].startsWith("Released on:")) r.fileMetadata.date = descItems[4].replace("Released on: ", '').trim();
}
}
return r
}
let singleTest;
if (videoMatch.length > 0) {
singleTest = videoMatch[0]["hasVideo"] && videoMatch[0]["hasAudio"];
return {
type: singleTest ? "bridge" : "render",
urls: singleTest ? videoMatch[0]["url"] : [videoMatch[0]["url"], audio[0]["url"]],
time: videoMatch[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${videoMatch[0]["width"]}x${videoMatch[0]["height"]}.${obj.format}`
}
}
singleTest = video[0]["hasVideo"] && video[0]["hasAudio"];
return {
type: singleTest ? "bridge" : "render",
urls: singleTest ? video[0]["url"] : [video[0]["url"], audio[0]["url"]],
time: video[0]["approxDurationMs"],
filename: `youtube_${obj.id}_${video[0]["width"]}x${video[0]["height"]}.${obj.format}`
}
}

1
src/modules/processing/servicesConfig.json
View File

@ -8,6 +8,7 @@
"enabled": true
},
"reddit": {
"alias": "reddit videos & gifs",
"patterns": ["r/:sub/comments/:id/:title"],
"enabled": true
},

29
src/modules/processing/servicesPatternTesters.js
View File

@ -1,27 +1,28 @@
export const testers = {
"twitter": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length < 20) || (patternMatch["spaceId"] && patternMatch["spaceId"].length === 13),
"twitter": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length < 20)
|| (patternMatch["spaceId"] && patternMatch["spaceId"].length === 13),
"vk": (patternMatch) => (patternMatch["userId"] && patternMatch["videoId"] &&
patternMatch["userId"].length <= 10 && patternMatch["videoId"].length === 9),
"vk": (patternMatch) => (patternMatch["userId"] && patternMatch["videoId"]
&& patternMatch["userId"].length <= 10 && patternMatch["videoId"].length === 9),
"bilibili": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length >= 12),
"youtube": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length >= 11),
"reddit": (patternMatch) => (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"] &&
patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96),
"reddit": (patternMatch) => (patternMatch["sub"] && patternMatch["id"] && patternMatch["title"]
&& patternMatch["sub"].length <= 22 && patternMatch["id"].length <= 10 && patternMatch["title"].length <= 96),
"tiktok": (patternMatch) => ((patternMatch["user"] && patternMatch["postId"] && patternMatch["postId"].length <= 21) ||
(patternMatch["id"] && patternMatch["id"].length <= 13)),
"tiktok": (patternMatch) => ((patternMatch["user"] && patternMatch["postId"] && patternMatch["postId"].length <= 21)
|| (patternMatch["id"] && patternMatch["id"].length <= 13)),
"douyin": (patternMatch) => ((patternMatch["postId"] && patternMatch["postId"].length <= 21) ||
(patternMatch["id"] && patternMatch["id"].length <= 13)),
"douyin": (patternMatch) => ((patternMatch["postId"] && patternMatch["postId"].length <= 21)
|| (patternMatch["id"] && patternMatch["id"].length <= 13)),
"tumblr": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length < 21) ||
(patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)),
"tumblr": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length < 21)
|| (patternMatch["id"] && patternMatch["id"].length < 21 && patternMatch["user"] && patternMatch["user"].length <= 32)),
"vimeo": (patternMatch) => ((patternMatch["id"] && patternMatch["id"].length <= 11)),
"soundcloud": (patternMatch) => ((patternMatch["author"] && patternMatch["song"] && (patternMatch["author"].length + patternMatch["song"].length) <= 96) ||
(patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32))
};
"soundcloud": (patternMatch) => ((patternMatch["author"] && patternMatch["song"]
&& (patternMatch["author"].length + patternMatch["song"].length) <= 96) || (patternMatch["shortLink"] && patternMatch["shortLink"].length <= 32))
}