diff --git a/api/package.json b/api/package.json index fc5304ed..1e8f506a 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@imput/cobalt-api", "description": "save what you love", - "version": "10.7.7", + "version": "10.7.9", "author": "imput", "exports": "./src/cobalt.js", "type": "module", diff --git a/api/src/processing/services/pinterest.js b/api/src/processing/services/pinterest.js index ea4275cb..15566cc4 100644 --- a/api/src/processing/services/pinterest.js +++ b/api/src/processing/services/pinterest.js @@ -23,7 +23,7 @@ export default async function(o) { const videoLink = [...html.matchAll(videoRegex)] .map(([, link]) => link) - .find(a => a.endsWith('.mp4') && a.includes('720p')); + .find(a => a.endsWith('.mp4')); if (videoLink) return { urls: videoLink, diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.js index b4a1d557..a4f4505e 100644 --- a/api/src/processing/services/twitter.js +++ b/api/src/processing/services/twitter.js @@ -24,6 +24,11 @@ const badContainerEnd = new Date(1702605600000); function needsFixing(media) { const representativeId = media.source_status_id_str ?? media.id_str; + + // syndication api doesn't have media ids in its response, + // so we just assume it's all good + if (!representativeId) return false; + const mediaTimestamp = new Date( Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH) ); @@ -53,6 +58,25 @@ const getGuestToken = async (dispatcher, forceReload = false) => { } } +const requestSyndication = async(dispatcher, tweetId) => { + // thank you + // https://github.com/yt-dlp/yt-dlp/blob/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9/yt_dlp/extractor/twitter.py#L1334 + const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, ''); + const syndicationUrl = new URL("https://cdn.syndication.twimg.com/tweet-result"); + + syndicationUrl.searchParams.set("id", tweetId); + syndicationUrl.searchParams.set("token", token(tweetId)); + + const result = await fetch(syndicationUrl, { + headers: { + "user-agent": genericUserAgent + }, + dispatcher + }); + + return result; +} + const requestTweet = async(dispatcher, tweetId, token, cookie) => { const graphqlTweetURL = new URL(graphqlURL); @@ -87,36 +111,24 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => { let result = await fetch(graphqlTweetURL, { headers, dispatcher }); updateCookie(cookie, result.headers); - // we might have been missing the `ct0` cookie, retry + // we might have been missing the ct0 cookie, retry if (result.status === 403 && result.headers.get('set-cookie')) { - result = await fetch(graphqlTweetURL, { - headers: { - ...headers, - 'x-csrf-token': cookie.values().ct0 - }, - dispatcher - }); + const cookieValues = cookie?.values(); + if (cookieValues?.ct0) { + result = await fetch(graphqlTweetURL, { + headers: { + ...headers, + 'x-csrf-token': cookieValues.ct0 + }, + dispatcher + }); + } } return result } -export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { - const cookie = await getCookie('twitter'); - - let guestToken = await getGuestToken(dispatcher); - if (!guestToken) return { error: "fetch.fail" }; - - let tweet = await requestTweet(dispatcher, id, guestToken); - - // get new token & retry if old one expired - if ([403, 429].includes(tweet.status)) { - guestToken = await getGuestToken(dispatcher, true); - tweet = await requestTweet(dispatcher, id, guestToken) - } - - tweet = await tweet.json(); - +const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => { let tweetTypename = tweet?.data?.tweetResult?.result?.__typename; if (!tweetTypename) { @@ -127,13 +139,13 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { const reason = tweet?.data?.tweetResult?.result?.reason; switch(reason) { case "Protected": - return { error: "content.post.private" } + return { error: "content.post.private" }; case "NsfwLoggedOut": if (cookie) { tweet = await requestTweet(dispatcher, id, guestToken, cookie); tweet = await tweet.json(); tweetTypename = tweet?.data?.tweetResult?.result?.__typename; - } else return { error: "content.post.age" } + } else return { error: "content.post.age" }; } } @@ -150,7 +162,69 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities; } - let media = (repostedTweet?.media || baseTweet?.extended_entities?.media); + return (repostedTweet?.media || baseTweet?.extended_entities?.media); +} + +const testResponse = (result) => { + const contentLength = result.headers.get("content-length"); + + if (!contentLength || contentLength === '0') { + return false; + } + + if (!result.headers.get("content-type").startsWith("application/json")) { + return false; + } + + return true; +} + +export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { + const cookie = await getCookie('twitter'); + + let syndication = false; + + let guestToken = await getGuestToken(dispatcher); + if (!guestToken) return { error: "fetch.fail" }; + + // for now we assume that graphql api will come back after some time, + // so we try it first + + let tweet = await requestTweet(dispatcher, id, guestToken); + + // get new token & retry if old one expired + if ([403, 429].includes(tweet.status)) { + guestToken = await getGuestToken(dispatcher, true); + if (cookie) { + tweet = await requestTweet(dispatcher, id, guestToken, cookie); + } else { + tweet = await requestTweet(dispatcher, id, guestToken); + } + } + + const testGraphql = testResponse(tweet); + + // if graphql requests fail, then resort to tweet embed api + if (!testGraphql) { + syndication = true; + tweet = await requestSyndication(dispatcher, id); + + const testSyndication = testResponse(tweet); + + // if even syndication request failed, then cry out loud + if (!testSyndication) { + return { error: "fetch.fail" }; + } + } + + tweet = await tweet.json(); + + let media = + syndication + ? tweet.mediaDetails + : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie); + + if (!media) return { error: "fetch.empty" }; // check if there's a video at given index (/video/) if (index >= 0 && index < media?.length) { @@ -163,7 +237,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) { service: "twitter", type: "proxy", url, filename, - }) + }); switch (media?.length) { case undefined: diff --git a/api/src/processing/services/youtube.js b/api/src/processing/services/youtube.js index a4c54a82..b12d2cee 100644 --- a/api/src/processing/services/youtube.js +++ b/api/src/processing/services/youtube.js @@ -428,6 +428,10 @@ export default async function (o) { } } + if (video?.drm_families || audio?.drm_families) { + return { error: "youtube.drm" }; + } + const fileMetadata = { title: basicInfo.title.trim(), artist: basicInfo.author.replace("- Topic", "").trim() diff --git a/api/src/util/tests/twitter.json b/api/src/util/tests/twitter.json index 4fc5900f..4139e39d 100644 --- a/api/src/util/tests/twitter.json +++ b/api/src/util/tests/twitter.json @@ -169,6 +169,15 @@ "status": "tunnel" } }, + { + "name": "gif", + "url": "https://x.com/thelastromances/status/1897839691212202479", + "params": {}, + "expected": { + "code": 200, + "status": "tunnel" + } + }, { "name": "inexistent post", "url": "https://twitter.com/test/status/9487653", diff --git a/web/i18n/en/error.json b/web/i18n/en/error.json index eed9eec5..70574af2 100644 --- a/web/i18n/en/error.json +++ b/web/i18n/en/error.json @@ -67,5 +67,6 @@ "api.youtube.token_expired": "couldn't get this video because the youtube token expired and i couldn't refresh it. try again in a few seconds, but if it still doesn't work, tell the instance owner about this error!", "api.youtube.no_hls_streams": "couldn't find any matching HLS streams for this video. try downloading it without HLS!", "api.youtube.api_error": "youtube updated something about its api and i couldn't get any info about this video. try again in a few seconds, but if this issue sticks, please report it!", - "api.youtube.temporary_disabled": "youtube downloading is temporarily disabled due to restrictions from youtube's side. we're already looking for ways to go around them.\n\nwe apologize for the inconvenience and are doing our best to restore this functionality. check cobalt's socials or github for timely updates!" + "api.youtube.temporary_disabled": "youtube downloading is temporarily disabled due to restrictions from youtube's side. we're already looking for ways to go around them.\n\nwe apologize for the inconvenience and are doing our best to restore this functionality. check cobalt's socials or github for timely updates!", + "api.youtube.drm": "this youtube video is protected by widevine DRM, so i can't download it. try a different link!" }