From 99265d594bc171b2881312782cb9551241f3882b Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 22 Jan 2025 14:41:44 +0600 Subject: [PATCH 1/6] api/readme: update list of supported services & list of dependencies --- api/README.md | 53 ++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/api/README.md b/api/README.md index 84c534ea..70d85de6 100644 --- a/api/README.md +++ b/api/README.md @@ -11,12 +11,9 @@ we recommend [deploying your own instance](/docs/run-an-instance.md) if you wish you can read [the api documentation here](/docs/api.md). -> [!WARNING] -> the v7 public api (/api/json) will be shut down on **november 11th, 2024**. -> you can access documentation for it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md). - ## supported services -this list is not final and keeps expanding over time. if support for a service you want is missing, create an issue (or a pull request 👀). +this list is not final and keeps expanding over time! +if the desired service isn't supported yet, feel free to create an appropriate issue (or a pull request 👀). | service | video + audio | only audio | only video | metadata | rich file names | | :-------- | :-----------: | :--------: | :--------: | :------: | :-------------: | @@ -39,12 +36,13 @@ this list is not final and keeps expanding over time. if support for a service y | twitter/x | ✅ | ✅ | ✅ | ➖ | ➖ | | vimeo | ✅ | ✅ | ✅ | ✅ | ✅ | | vk videos & clips | ✅ | ❌ | ✅ | ✅ | ✅ | +| xiaohongshu | ✅ | ✅ | ✅ | ➖ | ➖ | | youtube | ✅ | ✅ | ✅ | ✅ | ✅ | | emoji | meaning | | :-----: | :---------------------- | | ✅ | supported | -| ➖ | impossible/unreasonable | +| ➖ | unreasonable/impossible | | ❌ | not supported | ### additional notes or features (per service) @@ -71,36 +69,35 @@ as long as you: - provide a link to the license and indicate if changes to the code were made, and - release the code under the **same license** -## acknowledgements +## open source acknowledgements ### ffmpeg -cobalt heavily relies on ffmpeg for converting and merging media files. it's an absolutely amazing piece of software offered for anyone for free, yet doesn't receive as much credit as it should. +cobalt relies on ffmpeg for muxing and encoding media files. ffmpeg is absolutely spectacular and we're privileged to have an ability to use it for free, just like anyone else. we believe it should be way more recognized. you can [support ffmpeg here](https://ffmpeg.org/donations.html)! -#### ffmpeg-static -we use [ffmpeg-static](https://github.com/eugeneware/ffmpeg-static) to get binaries for ffmpeg depending on the platform. - -you can support the developer via various methods listed on their github page! (linked above) - ### youtube.js -cobalt relies on [youtube.js](https://github.com/LuanRT/YouTube.js) for interacting with the innertube api, it wouldn't have been possible without it. +cobalt relies on **[youtube.js](https://github.com/LuanRT/YouTube.js)** for interacting with youtube's innertube api, it wouldn't have been possible without this package. -you can support the developer via various methods listed on their github page! (linked above) +you can support the developer via various methods listed on their github page! +(linked above) ### many others -cobalt also depends on: +cobalt-api also depends on: -- [content-disposition-header](https://www.npmjs.com/package/content-disposition-header) to simplify the provision of `content-disposition` headers. -- [cors](https://www.npmjs.com/package/cors) to manage cross-origin resource sharing within expressjs. -- [dotenv](https://www.npmjs.com/package/dotenv) to load environment variables from the `.env` file. -- [express](https://www.npmjs.com/package/express) as the backbone of cobalt servers. -- [express-rate-limit](https://www.npmjs.com/package/express-rate-limit) to rate limit api endpoints. -- [hls-parser](https://www.npmjs.com/package/hls-parser) to parse `m3u8` playlists for certain services. -- [ipaddr.js](https://www.npmjs.com/package/ipaddr.js) to parse ip addresses (for rate limiting). -- [nanoid](https://www.npmjs.com/package/nanoid) to generate unique (temporary) identifiers for each requested stream. -- [psl](https://www.npmjs.com/package/psl) as the domain name parser. -- [set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser) to parse cookies that cobalt receives from certain services. -- [undici](https://www.npmjs.com/package/undici) for making http requests. -- [url-pattern](https://www.npmjs.com/package/url-pattern) to match provided links with supported patterns. +- **[content-disposition-header](https://www.npmjs.com/package/content-disposition-header)** to simplify the provision of `content-disposition` headers. +- **[cors](https://www.npmjs.com/package/cors)** to manage cross-origin resource sharing within expressjs. +- **[dotenv](https://www.npmjs.com/package/dotenv)** to load environment variables from the `.env` file. +- **[express](https://www.npmjs.com/package/express)** as the backbone of cobalt servers. +- **[express-rate-limit](https://www.npmjs.com/package/express-rate-limit)** to rate limit api endpoints. +- **[ffmpeg-static](https://www.npmjs.com/package/ffmpeg-static)** to get binaries for ffmpeg depending on the platform. +- **[hls-parser](https://www.npmjs.com/package/hls-parser)** to parse HLS playlists according to spec (very impressive stuff). +- **[ipaddr.js](https://www.npmjs.com/package/ipaddr.js)** to parse ip addresses (used for rate limiting). +- **[nanoid](https://www.npmjs.com/package/nanoid)** to generate unique identifiers for each requested tunnel. +- **[set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser)** to parse cookies that cobalt receives from certain services. +- **[undici](https://www.npmjs.com/package/undici)** for making http requests. +- **[url-pattern](https://www.npmjs.com/package/url-pattern)** to match provided links with supported patterns. +- **[zod](https://www.npmjs.com/package/zod)** to lock down the api request schema. +- **[@datastructures-js/priority-queue](https://www.npmjs.com/package/@datastructures-js/priority-queue)** for sorting stream caches for future clean up (without redis). +- **[@imput/psl](https://www.npmjs.com/package/@imput/psl)** as the domain name parser, our fork of [psl](https://www.npmjs.com/package/psl). ...and many other packages that these packages rely on. From 3be98a14b369289b4b50b2ec9147d76287c3f2e3 Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 22 Jan 2025 14:46:09 +0600 Subject: [PATCH 2/6] readme: update some phrasing & add a link to bluesky --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8168a4df..9a5a05e7 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,13 @@ 💬 community discord server +
🐦 twitter + + 🦋 bluesky +


@@ -34,11 +38,10 @@ this monorepo includes source code for api, frontend, and related packages: it also includes documentation in the [docs tree](/docs/): - [cobalt api documentation](/docs/api.md) - [how to run a cobalt instance](/docs/run-an-instance.md) -- [how to protect a cobalt instance](/docs/protect-an-instance.md) -- [how to configure a cobalt instance for youtube](/docs/configure-for-youtube.md) +- [how to protect a cobalt instance](/docs/protect-an-instance.md) (recommended if you host a public instance) ### thank you -cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt) and the main processing servers are hosted on their network. we really appreciate their kindness and support! +cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support! ### ethics cobalt is a tool that makes downloading public content easier. it takes **zero liability**. @@ -50,7 +53,7 @@ it can only download free & publicly accessible content. same content can be downloaded via dev tools of any modern web browser. ### contributing -thank you for considering making a contribution to cobalt! please check the [contributing guidelines here](/CONTRIBUTING.md) before making a pull request. +if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away. ### licenses for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs. From 899d1efdea2d1b23cb88d9d33ca50245ff094a69 Mon Sep 17 00:00:00 2001 From: wukko Date: Wed, 22 Jan 2025 14:46:30 +0600 Subject: [PATCH 3/6] web/about/general: update infra partner phrasing --- web/i18n/en/about/general.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/i18n/en/about/general.md b/web/i18n/en/about/general.md index 2334ab8e..1f5e53fd 100644 --- a/web/i18n/en/about/general.md +++ b/web/i18n/en/about/general.md @@ -25,7 +25,7 @@ no ads, trackers, paywalls, or other nonsense. just a convenient web app that wo cobalt was created for public benefit, to protect people from ads and malware pushed by its alternatives. we believe that the best software is safe, open, and accessible. -it's possible to keep the main instances up thanks to our long-standing infrastructure partner, [royalehosting.net]({partners.royalehosting})! +a part of our infrastructure is provided by our long-standing partner, [royalehosting.net]({partners.royalehosting})!
From 31f6ff9b87ee7cf23dce7bf9217f8901a6e59e08 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 23 Jan 2025 21:51:06 +0600 Subject: [PATCH 4/6] api/tests/loom: update test links the old video is unavailable for an unknown reason. it's unplayable in a regular browser and also loom's own landing page. --- api/src/util/tests/loom.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/util/tests/loom.json b/api/src/util/tests/loom.json index 1849a000..cc4273d3 100644 --- a/api/src/util/tests/loom.json +++ b/api/src/util/tests/loom.json @@ -1,7 +1,7 @@ [ { "name": "1080p video", - "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885", "params": {}, "expected": { "code": 200, @@ -10,7 +10,7 @@ }, { "name": "1080p video (muted)", - "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885", "params": { "downloadMode": "mute" }, @@ -21,7 +21,7 @@ }, { "name": "1080p video (audio only)", - "url": "https://www.loom.com/share/313bf71d20ca47b2a35b6634cefdb761", + "url": "https://www.loom.com/share/d165fd054a294d8a8587807bcc50c885", "params": { "downloadMode": "audio" }, From 7b31817fdb7501233d3eddc375db4311cbe4fc3c Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 23 Jan 2025 21:58:41 +0600 Subject: [PATCH 5/6] api/tests/xiaohongshu: update photo test link --- api/src/util/tests/xiaohongshu.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/util/tests/xiaohongshu.json b/api/src/util/tests/xiaohongshu.json index 425a6a2e..de632a77 100644 --- a/api/src/util/tests/xiaohongshu.json +++ b/api/src/util/tests/xiaohongshu.json @@ -19,7 +19,7 @@ }, { "name": "one photo", - "url": "https://www.xiaohongshu.com/explore/6788b56200000000210008c8?xsec_token=CBSDiWU4N-DgirHrOVbIWrlKfUNFHKwm-Wsjqz7dIMc_k", + "url": "https://www.xiaohongshu.com/explore/676e132d000000000b016f68?xsec_token=ABRv6LKzizOFeSaf2HnnBkdBqniB5Ak1fI8tMAHzO31jA", "params": {}, "expected": { "code": 200, From 906d9293337a8916a4c7da8ca1ab792f925233c8 Mon Sep 17 00:00:00 2001 From: wukko Date: Thu, 23 Jan 2025 22:00:02 +0600 Subject: [PATCH 6/6] api/tests/pinterest: update the gif link because the id changed??? --- api/src/util/tests/pinterest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/util/tests/pinterest.json b/api/src/util/tests/pinterest.json index 6308adb4..4760dd36 100644 --- a/api/src/util/tests/pinterest.json +++ b/api/src/util/tests/pinterest.json @@ -68,7 +68,7 @@ }, { "name": "regular gif", - "url": "https://www.pinterest.com/pin/814447913881127862/", + "url": "https://www.pinterest.com/pin/643170390530326178/", "params": {}, "expected": { "code": 200, @@ -77,7 +77,7 @@ }, { "name": "regular gif (.ca TLD)", - "url": "https://www.pinterest.ca/pin/814447913881127862/", + "url": "https://www.pinterest.ca/pin/643170390530326178/", "params": {}, "expected": { "code": 200,