From fa7af1bf444e3a473118ca5243380f32863d24cb Mon Sep 17 00:00:00 2001 From: Snazzah Date: Sat, 29 Apr 2023 14:33:36 -0500 Subject: [PATCH] feat: add twitch vod/clip support --- src/modules/processing/match.js | 10 + src/modules/processing/services/twitch.js | 223 ++++++++++++++++++ src/modules/processing/servicesConfig.json | 6 + .../processing/servicesPatternTesters.js | 4 +- src/modules/stream/types.js | 2 +- src/modules/sub/utils.js | 3 + src/test/tests.json | 57 +++++ 7 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 src/modules/processing/services/twitch.js diff --git a/src/modules/processing/match.js b/src/modules/processing/match.js index 3328a32c..aef1c175 100644 --- a/src/modules/processing/match.js +++ b/src/modules/processing/match.js @@ -17,6 +17,7 @@ import vimeo from "./services/vimeo.js"; import soundcloud from "./services/soundcloud.js"; import instagram from "./services/instagram.js"; import vine from "./services/vine.js"; +import twitch from "./services/twitch.js"; export default async function (host, patternMatch, url, lang, obj) { try { @@ -110,6 +111,15 @@ export default async function (host, patternMatch, url, lang, obj) { case "vine": r = await vine({ id: patternMatch["id"] }); break; + case "twitch": + r = await twitch({ + vodId: patternMatch["video"] ? patternMatch["video"] : false, + clipId: patternMatch["clip"] ? patternMatch["clip"] : false, + lang: lang, quality: obj.vQuality, + isAudioOnly: obj.isAudioOnly, + format: obj.vFormat + }); + break; default: return apiJSON(0, { t: errorUnsupported(lang) }); } diff --git a/src/modules/processing/services/twitch.js b/src/modules/processing/services/twitch.js new file mode 100644 index 00000000..bd8a8690 --- /dev/null +++ b/src/modules/processing/services/twitch.js @@ -0,0 +1,223 @@ +import { maxVideoDuration } from "../../config.js"; + +const gqlURL = "https://gql.twitch.tv/gql"; +const m3u8URL = "https://usher.ttvnw.net"; + +function parseM3U8Line(line) { + const result = {}; + + let str = '', inQuotes = false, keyName = null, escaping = false; + for (let i = 0; i < line.length; i++) { + const char = line[i]; + if (char === '"' && !escaping) { + inQuotes = !inQuotes; + continue; + } else if (char === ',' && !escaping && !inQuotes) { + if (!keyName) break; + result[keyName] = str; + keyName = null; + str = ''; + continue; + } else if (char === '\\' && !escaping) { + escaping = true; + continue; + } else if (char === '=' && !escaping && !inQuotes) { + keyName = str; + str = ''; + continue; + } + + str += char; + escaping = false; + } + + if (keyName) result[keyName] = str; + return result; +} + +function getM3U8Formats(m3u8body) { + let formats = []; + const formatLines = m3u8body.split('\n').slice(2); + + for (let i = 0; i < formatLines.length; i += 3) { + const mediaLine = parseM3U8Line(formatLines[i].split(':')[1]); + const streamLine = parseM3U8Line(formatLines[i + 1].split(':')[1]); + formats.push({ + id: mediaLine['GROUP-ID'], + name: mediaLine.NAME, + resolution: streamLine.RESOLUTION ? streamLine.RESOLUTION.split('x') : null, + url: formatLines[i + 2] + }); + } + return formats; +}; + +export default async function(obj) { + try { + let _headers = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" }; + + if (!obj.clipId && !obj.vodId) return { error: 'ErrorCantGetID' }; + + if (obj.vodId) { + const req_metadata = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify([ + { + "operationName": "VideoMetadata", + "variables": { + "channelLogin": "", + "videoID": obj.vodId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687" + } + } + } + ]) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; + const vodMetadata = req_metadata[0].data.video; + + if (vodMetadata.previewThumbnailURL.endsWith('/404_processing_{width}x{height}.png')) return { error: 'ErrorLiveVideo' }; + if (vodMetadata.lengthSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!vodMetadata.owner) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... + + const req_token = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify({ + query: `{ + videoPlaybackAccessToken( + id: "${obj.vodId}", + params: { + platform: "web", + playerBackend: "mediaplayer", + playerType: "site" + } + ) + { + value + signature + } + }` + }) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_token) return { error: 'ErrorCouldntFetch' }; + + const access_token = req_token.data.videoPlaybackAccessToken; + const req_m3u8 = await fetch(`${m3u8URL}/vod/${obj.vodId}.m3u8?${new URLSearchParams({ + allow_source: 'true', + allow_audio_only: 'true', + allow_spectre: 'true', + player: 'twitchweb', + playlist_include_framerate: 'true', + nauth: access_token.value, + nauthsig: access_token.signature + })}`, { + headers: _headers + }).then((r) => { return r.status == 200 ? r.text() : false;}).catch(() => {return false}); + if (!req_m3u8) return { error: 'ErrorCouldntFetch' }; + + const formats = getM3U8Formats(req_m3u8); + const generalMeta = { + title: vodMetadata.title, + artist: `Twitch Broadcast by @${vodMetadata.owner.login}`, + } + + if (!obj.isAudioOnly) { + const format = formats.find(f => f.resolution && f.resolution[1] == obj.quality) || formats[0]; + + return { + urls: format.url, + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + filename: `twitchvod_${obj.vodId}_${format.resolution[0]}x${format.resolution[1]}.mp4` + }; + } else { + return { + type: "render", + isM3U8: true, + time: vodMetadata.lengthSeconds * 1000, + urls: formats.find(f => f.id === 'audio_only').url, + audioFilename: `twitchvod_${obj.vodId}_audio`, + fileMetadata: generalMeta + } + } + } else if (obj.clipId) { + const req_metadata = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify({ + query: `{ + clip(slug: "${obj.clipId}") { + broadcaster { + login + } + createdAt + curator { + login + } + durationSeconds + id + medium: thumbnailURL(width: 480, height: 272) + title + videoQualities { + quality + sourceURL + } + } + }` + }) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_metadata) return { error: 'ErrorCouldntFetch' }; + const clipMetadata = req_metadata.data.clip; + if (clipMetadata.durationSeconds > maxVideoDuration / 1000) return { error: ['ErrorLengthLimit', maxVideoDuration / 60000] }; + if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) return { error: 'ErrorEmptyDownload' }; // Streamer was banned... + + const req_token = await fetch(gqlURL, { + method: "POST", + headers: _headers, + body: JSON.stringify([ + { + "operationName": "VideoAccessToken_Clip", + "variables": { + "slug": obj.clipId + }, + "extensions": { + "persistedQuery": { + "version": 1, + "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11" + } + } + } + ]) + }).then((r) => { return r.status == 200 ? r.json() : false;}).catch(() => {return false}); + if (!req_token) return { error: 'ErrorCouldntFetch' }; + + const generalMeta = { + title: clipMetadata.title, + artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`, + } + + const access_token = req_token[0].data.clip.playbackAccessToken; + const formats = clipMetadata.videoQualities; + const format = formats.find(f => f.quality == obj.quality) || formats[0]; + + return { + type: "bridge", + urls: `${format.sourceURL}?${new URLSearchParams({ + sig: access_token.signature, + token: access_token.value + })}`, + filename: `twitchclip_${clipMetadata.id}_${format.quality}.mp4`, + audioFilename: `twitchclip_${clipMetadata.id}_audio`, + fileMetadata: generalMeta + }; + } + } catch (err) { + return { error: 'ErrorBadFetch' }; + } +} \ No newline at end of file diff --git a/src/modules/processing/servicesConfig.json b/src/modules/processing/servicesConfig.json index 2bb7f8fe..0318d4e2 100644 --- a/src/modules/processing/servicesConfig.json +++ b/src/modules/processing/servicesConfig.json @@ -62,6 +62,12 @@ "tld": "co", "patterns": ["v/:id"], "enabled": true + }, + "twitch": { + "alias": "twitch vods & videos & clips", + "tld": "tv", + "patterns": ["videos/:video", ":channel/clip/:clip"], + "enabled": true } } } diff --git a/src/modules/processing/servicesPatternTesters.js b/src/modules/processing/servicesPatternTesters.js index 8f70613c..35eaa902 100644 --- a/src/modules/processing/servicesPatternTesters.js +++ b/src/modules/processing/servicesPatternTesters.js @@ -28,5 +28,7 @@ export const testers = { "instagram": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), - "vine": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12) + "vine": (patternMatch) => (patternMatch["id"] && patternMatch["id"].length <= 12), + + "twitch": (patternMatch) => ((patternMatch["channel"] && patternMatch["clip"] && patternMatch["clip"].length <= 100 || patternMatch["video"] && patternMatch["video"].length <= 10)), } diff --git a/src/modules/stream/types.js b/src/modules/stream/types.js index a4eb233f..3aa259f6 100644 --- a/src/modules/stream/types.js +++ b/src/modules/stream/types.js @@ -145,7 +145,7 @@ export function streamVideoOnly(streamInfo, res) { '-c', 'copy' ] if (streamInfo.mute) args.push('-an'); - if (streamInfo.service === "vimeo") args.push('-bsf:a', 'aac_adtstoasc'); + if (streamInfo.service === "vimeo" || streamInfo.service === "twitch") args.push('-bsf:a', 'aac_adtstoasc'); if (format === "mp4") args.push('-movflags', 'faststart+frag_keyframe+empty_moov'); args.push('-f', format, 'pipe:3'); const ffmpegProcess = spawn(ffmpeg, args, { diff --git a/src/modules/sub/utils.js b/src/modules/sub/utils.js index 27a17b82..4bcfc6d9 100644 --- a/src/modules/sub/utils.js +++ b/src/modules/sub/utils.js @@ -84,6 +84,9 @@ export function cleanURL(url, host) { if (url.includes('youtube.com/shorts/')) { url = url.split('?')[0].replace('shorts/', 'watch?v='); } + if (url.includes('clips.twitch.tv')) { + url = url.split('?')[0].replace('clips.twitch.tv/', 'twitch.tv/_/clip/'); + } return url.slice(0, 128) } export function verifyLanguageCode(code) { diff --git a/src/test/tests.json b/src/test/tests.json index 158a0ea5..57fbefdd 100644 --- a/src/test/tests.json +++ b/src/test/tests.json @@ -869,5 +869,62 @@ "code": 200, "status": "stream" } + }], + "twitch": [{ + "name": "clip", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "clip (isAudioOnly)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "clip (isAudioMuted)", + "url": "https://twitch.tv/rtgame/clip/TubularInventiveSardineCorgiDerp-PM47mJQQ2vsL5B5G", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "video", + "url": "https://twitch.tv/videos/1315890970", + "params": {}, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "video (isAudioOnly)", + "url": "https://twitch.tv/videos/1315890970", + "params": { + "isAudioOnly": true + }, + "expected": { + "code": 200, + "status": "stream" + } + }, { + "name": "video (isAudioMuted)", + "url": "https://twitch.tv/videos/1315890970", + "params": { + "isAudioMuted": true + }, + "expected": { + "code": 200, + "status": "stream" + } }] } \ No newline at end of file