From f72b7dbbbb0e31c4d6a44ec35b4ab96b1aecdc30 Mon Sep 17 00:00:00 2001 From: Stefan <17972991+stefancruz@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:59:54 +0100 Subject: [PATCH 1/3] feat: add bichute and dailymotion to embedded sources --- .gitmodules | 12 ++++++++++++ app/src/stable/AndroidManifest.xml | 10 ++++++++++ app/src/stable/assets/sources/bitchute | 1 + app/src/stable/assets/sources/dailymotion | 1 + app/src/stable/res/raw/plugin_config.json | 4 +++- app/src/unstable/AndroidManifest.xml | 10 ++++++++++ app/src/unstable/assets/sources/bitchute | 1 + app/src/unstable/assets/sources/dailymotion | 1 + app/src/unstable/res/raw/plugin_config.json | 4 +++- 9 files changed, 42 insertions(+), 2 deletions(-) create mode 160000 app/src/stable/assets/sources/bitchute create mode 160000 app/src/stable/assets/sources/dailymotion create mode 160000 app/src/unstable/assets/sources/bitchute create mode 160000 app/src/unstable/assets/sources/dailymotion diff --git a/.gitmodules b/.gitmodules index cfc1f2fa..388021f4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -70,3 +70,15 @@ [submodule "app/src/unstable/assets/sources/spotify"] path = app/src/unstable/assets/sources/spotify url = ../plugins/spotify.git +[submodule "app/src/stable/assets/sources/bitchute"] + path = app/src/stable/assets/sources/bitchute + url = ../plugins/bitchute.git +[submodule "app/src/unstable/assets/sources/bitchute"] + path = app/src/unstable/assets/sources/bitchute + url = ../plugins/bitchute.git +[submodule "app/src/unstable/assets/sources/dailymotion"] + path = app/src/unstable/assets/sources/dailymotion + url = ../plugins/dailymotion.git +[submodule "app/src/stable/assets/sources/dailymotion"] + path = app/src/stable/assets/sources/dailymotion + url = ../plugins/dailymotion.git diff --git a/app/src/stable/AndroidManifest.xml b/app/src/stable/AndroidManifest.xml index a5fdd260..a30bb7af 100644 --- a/app/src/stable/AndroidManifest.xml +++ b/app/src/stable/AndroidManifest.xml @@ -33,6 +33,11 @@ + + + + + @@ -57,6 +62,11 @@ + + + + + diff --git a/app/src/stable/assets/sources/bitchute b/app/src/stable/assets/sources/bitchute new file mode 160000 index 00000000..41fe8f79 --- /dev/null +++ b/app/src/stable/assets/sources/bitchute @@ -0,0 +1 @@ +Subproject commit 41fe8f79735176cd8a0ae8c971936cafed00fa16 diff --git a/app/src/stable/assets/sources/dailymotion b/app/src/stable/assets/sources/dailymotion new file mode 160000 index 00000000..069aa3d3 --- /dev/null +++ b/app/src/stable/assets/sources/dailymotion @@ -0,0 +1 @@ +Subproject commit 069aa3d31a35559e45c1fe1ea1eb2a94d3b5d120 diff --git a/app/src/stable/res/raw/plugin_config.json b/app/src/stable/res/raw/plugin_config.json index a1da4004..3b7dacec 100644 --- a/app/src/stable/res/raw/plugin_config.json +++ b/app/src/stable/res/raw/plugin_config.json @@ -10,7 +10,9 @@ "aac9e9f0-24b5-11ee-be56-0242ac120002": "sources/patreon/PatreonConfig.json", "9d703ff5-c556-4962-a990-4f000829cb87": "sources/nebula/NebulaConfig.json", "cf8ea74d-ad9b-489e-a083-539b6aa8648c": "sources/bilibili/build/BiliBiliConfig.json", - "4e365633-6d3f-4267-8941-fdc36631d813": "sources/spotify/build/SpotifyConfig.json" + "4e365633-6d3f-4267-8941-fdc36631d813": "sources/spotify/build/SpotifyConfig.json", + "9c87e8db-e75d-48f4-afe5-2d203d4b95c5": "sources/dailymotion/build/DailymotionConfig.json", + "e8b1ad5f-0c6d-497d-a5fa-0a785a16d902": "sources/bitchute/BitchuteConfig.json" }, "SOURCES_EMBEDDED_DEFAULT": [ "35ae969a-a7db-11ed-afa1-0242ac120002" diff --git a/app/src/unstable/AndroidManifest.xml b/app/src/unstable/AndroidManifest.xml index 9affd8ab..1304222e 100644 --- a/app/src/unstable/AndroidManifest.xml +++ b/app/src/unstable/AndroidManifest.xml @@ -34,6 +34,11 @@ + + + + + @@ -58,6 +63,11 @@ + + + + + diff --git a/app/src/unstable/assets/sources/bitchute b/app/src/unstable/assets/sources/bitchute new file mode 160000 index 00000000..41fe8f79 --- /dev/null +++ b/app/src/unstable/assets/sources/bitchute @@ -0,0 +1 @@ +Subproject commit 41fe8f79735176cd8a0ae8c971936cafed00fa16 diff --git a/app/src/unstable/assets/sources/dailymotion b/app/src/unstable/assets/sources/dailymotion new file mode 160000 index 00000000..069aa3d3 --- /dev/null +++ b/app/src/unstable/assets/sources/dailymotion @@ -0,0 +1 @@ +Subproject commit 069aa3d31a35559e45c1fe1ea1eb2a94d3b5d120 diff --git a/app/src/unstable/res/raw/plugin_config.json b/app/src/unstable/res/raw/plugin_config.json index 551c5470..4e4cc1dc 100644 --- a/app/src/unstable/res/raw/plugin_config.json +++ b/app/src/unstable/res/raw/plugin_config.json @@ -10,7 +10,9 @@ "aac9e9f0-24b5-11ee-be56-0242ac120002": "sources/patreon/PatreonConfig.json", "9d703ff5-c556-4962-a990-4f000829cb87": "sources/nebula/NebulaConfig.json", "cf8ea74d-ad9b-489e-a083-539b6aa8648c": "sources/bilibili/build/BiliBiliConfig.json", - "4e365633-6d3f-4267-8941-fdc36631d813": "sources/spotify/build/SpotifyConfig.json" + "4e365633-6d3f-4267-8941-fdc36631d813": "sources/spotify/build/SpotifyConfig.json", + "9c87e8db-e75d-48f4-afe5-2d203d4b95c5": "sources/dailymotion/build/DailymotionConfig.json", + "e8b1ad5f-0c6d-497d-a5fa-0a785a16d902": "sources/bitchute/BitchuteConfig.json" }, "SOURCES_EMBEDDED_DEFAULT": [ "35ae969a-a7db-11ed-afa1-0242ac120002" From bf685a607fc62a7107e6be78f2e73270697204ac Mon Sep 17 00:00:00 2001 From: Koen Date: Thu, 29 Aug 2024 13:42:18 +0000 Subject: [PATCH 2/3] Download fixes. --- .../platformplayer/downloads/VideoDownload.kt | 118 ++++++++++++++++-- .../services/DownloadService.kt | 12 +- 2 files changed, 118 insertions(+), 12 deletions(-) diff --git a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt index 574a2875..b5e96b11 100644 --- a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt +++ b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt @@ -58,6 +58,7 @@ import kotlinx.serialization.Transient import java.io.File import java.io.FileOutputStream import java.io.IOException +import java.lang.Thread.sleep import java.time.OffsetDateTime import java.util.UUID import java.util.concurrent.Executors @@ -156,6 +157,7 @@ class VideoDownload { this.targetBitrate = targetBitrate; this.hasVideoRequestExecutor = video is JSSource && video.hasRequestExecutor; this.requiresLiveVideoSource = false; + this.requiresLiveAudioSource = false; this.targetVideoName = videoSource?.name; } constructor(video: IPlatformVideoDetails, videoSource: IVideoSource?, audioSource: IAudioSource?, subtitleSource: SubtitleRawSource?) { @@ -667,7 +669,7 @@ class VideoDownload { if(Settings.instance.downloads.byteRangeDownload && head?.containsKey("accept-ranges") == true && head.containsKey("content-length")) { val concurrency = Settings.instance.downloads.getByteRangeThreadCount(); - Logger.i(TAG, "Download $name ByteRange Parallel (${concurrency})"); + Logger.i(TAG, "Download $name ByteRange Parallel (${concurrency}): " + videoUrl); sourceLength = head["content-length"]!!.toLong(); onProgress(sourceLength, 0, 0); downloadSource_Ranges(name, client, fileStream, videoUrl, sourceLength, 1024*512, concurrency, onProgress); @@ -751,6 +753,76 @@ class VideoDownload { onProgress(sourceLength, totalRead, 0); return sourceLength; } + /*private fun downloadSource_Sequential(client: ManagedHttpClient, fileStream: FileOutputStream, url: String, onProgress: (Long, Long, Long) -> Unit): Long { + val progressRate: Int = 4096 * 25 + var lastProgressCount: Int = 0 + val speedRate: Int = 4096 * 25 + var readSinceLastSpeedTest: Long = 0 + var timeSinceLastSpeedTest: Long = System.currentTimeMillis() + + var lastSpeed: Long = 0 + + var totalRead: Long = 0 + var sourceLength: Long + val buffer = ByteArray(4096) + + var isPartialDownload = false + var result: ManagedHttpClient.Response? = null + do { + result = client.get(url, if (isPartialDownload) hashMapOf("Range" to "bytes=$totalRead-") else hashMapOf()) + if (isPartialDownload) { + if (result.code != 206) + throw IllegalStateException("Failed to download source, byte range fallback failed. Web[${result.code}] Error") + } else { + if (!result.isOk) + throw IllegalStateException("Failed to download source. Web[${result.code}] Error") + } + if (result.body == null) + throw IllegalStateException("Failed to download source. Web[${result.code}] No response") + + isPartialDownload = true + sourceLength = result.body!!.contentLength() + val sourceStream = result.body!!.byteStream() + + try { + while (true) { + val read = sourceStream.read(buffer) + if (read <= 0) { + break + } + + fileStream.write(buffer, 0, read) + + totalRead += read + readSinceLastSpeedTest += read + + if (totalRead / progressRate > lastProgressCount) { + onProgress(sourceLength, totalRead, lastSpeed) + lastProgressCount++ + } + if (readSinceLastSpeedTest > speedRate) { + val lastSpeedTime = timeSinceLastSpeedTest + timeSinceLastSpeedTest = System.currentTimeMillis() + val timeSince = timeSinceLastSpeedTest - lastSpeedTime + if (timeSince > 0) + lastSpeed = (readSinceLastSpeedTest / (timeSince / 1000.0)).toLong() + readSinceLastSpeedTest = 0 + } + + if (isCancelled) + throw CancellationException("Cancelled") + } + } catch (e: Throwable) { + Logger.w(TAG, "Sequential download was interrupted, trying to fallback to byte ranges", e) + } finally { + sourceStream.close() + result.body?.close() + } + } while (totalRead < sourceLength) + + onProgress(sourceLength, totalRead, 0) + return sourceLength + }*/ private fun downloadSource_Ranges(name: String, client: ManagedHttpClient, fileStream: FileOutputStream, url: String, sourceLength: Long, rangeSize: Int, concurrency: Int = 1, onProgress: (Long, Long, Long) -> Unit) { val progressRate: Int = 4096 * 5; var lastProgressCount: Int = 0; @@ -823,18 +895,42 @@ class VideoDownload { return tasks.map { it.get() }; } private fun requestByteRange(client: ManagedHttpClient, url: String, rangeStart: Long, rangeEnd: Long): Triple { - val toRead = rangeEnd - rangeStart; - val req = client.get(url, mutableMapOf(Pair("Range", "bytes=${rangeStart}-${rangeEnd}"))); - if(!req.isOk) - throw IllegalStateException("Range request failed Code [${req.code}] due to: ${req.message}"); - if(req.body == null) - throw IllegalStateException("Range request failed, No body"); - val read = req.body.contentLength(); + var retryCount = 0 + var lastException: Throwable? = null - if(read < toRead) - throw IllegalStateException("Byte-Range request attempted to provide less (${read} < ${toRead})"); + while (retryCount <= 3) { + try { + val toRead = rangeEnd - rangeStart; + val req = client.get(url, mutableMapOf(Pair("Range", "bytes=${rangeStart}-${rangeEnd}"))); + if (!req.isOk) { + val bodyString = req.body?.string() + req.body?.close() + throw IllegalStateException("Range request failed Code [${req.code}] due to: ${req.message}"); + } + if (req.body == null) + throw IllegalStateException("Range request failed, No body"); + val read = req.body.contentLength(); - return Triple(req.body.bytes(), rangeStart, rangeEnd); + if (read < toRead) + throw IllegalStateException("Byte-Range request attempted to provide less (${read} < ${toRead})"); + + return Triple(req.body.bytes(), rangeStart, rangeEnd); + } catch (e: Throwable) { + Logger.w(TAG, "Failed to download range (url=${url} bytes=${rangeStart}-${rangeEnd})", e) + + retryCount++ + lastException = e + + sleep(when (retryCount) { + 1 -> 1000 + ((Math.random() * 300.0).toLong() - 150) + 2 -> 2000 + ((Math.random() * 300.0).toLong() - 150) + 3 -> 4000 + ((Math.random() * 300.0).toLong() - 150) + else -> 1000 + ((Math.random() * 300.0).toLong() - 150) + }) + } + } + + throw lastException!! } fun validate() { diff --git a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt index 6955cbe9..86f8a07b 100644 --- a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt +++ b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt @@ -28,7 +28,12 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.cancel import kotlinx.coroutines.launch +import okhttp3.OkHttpClient +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.Proxy import java.net.SocketException +import java.time.Duration import java.time.OffsetDateTime class DownloadService : Service() { @@ -44,7 +49,12 @@ class DownloadService : Service() { private var _notificationManager: NotificationManager? = null; private var _notificationChannel: NotificationChannel? = null; - private val _client = ManagedHttpClient(); + private val _client = ManagedHttpClient(OkHttpClient.Builder() + //.proxy(Proxy(Proxy.Type.HTTP, InetSocketAddress(InetAddress.getByName("192.168.1.175"), 8081))) + .readTimeout(Duration.ofSeconds(30)) + .writeTimeout(Duration.ofSeconds(30)) + .connectTimeout(Duration.ofSeconds(30)) + .callTimeout(Duration.ofMinutes(30))) private var _started = false; From f41a971cd8852f31f1b0b851362c0cc2adc5b694 Mon Sep 17 00:00:00 2001 From: Kelvin Date: Thu, 29 Aug 2024 15:43:29 +0200 Subject: [PATCH 3/3] Fix download states --- .../platforms/js/models/sources/JSSource.kt | 2 +- .../platformplayer/downloads/VideoDownload.kt | 42 ++++++++++++++++--- .../platformplayer/helpers/VideoHelper.kt | 4 +- .../services/DownloadService.kt | 6 +-- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/app/src/main/java/com/futo/platformplayer/api/media/platforms/js/models/sources/JSSource.kt b/app/src/main/java/com/futo/platformplayer/api/media/platforms/js/models/sources/JSSource.kt index a658f5cb..45ace168 100644 --- a/app/src/main/java/com/futo/platformplayer/api/media/platforms/js/models/sources/JSSource.kt +++ b/app/src/main/java/com/futo/platformplayer/api/media/platforms/js/models/sources/JSSource.kt @@ -50,7 +50,7 @@ abstract class JSSource { _requestExecutor = obj.getOrDefault(_config, "requestExecutor", "JSSource.requestExecutor", null)?.let { JSRequest(plugin, it, null, null, true); } - hasRequestExecutor = _requestModifier != null || obj.has("getRequestExecutor"); + hasRequestExecutor = _requestExecutor != null || obj.has("getRequestExecutor"); } fun getRequestModifier(): IRequestModifier? { diff --git a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt index 574a2875..a1a11d91 100644 --- a/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt +++ b/app/src/main/java/com/futo/platformplayer/downloads/VideoDownload.kt @@ -93,6 +93,18 @@ class VideoDownload { @Transient val audioSourceToUse: IAudioSource? get () = if(requiresLiveAudioSource) audioSourceLive as IAudioSource? else audioSource as IAudioSource?; + var requireVideoSource: Boolean = false; + var requireAudioSource: Boolean = false; + + @Contextual + @Transient + val isVideoDownloadReady: Boolean get() = !requireVideoSource || + ((requiresLiveVideoSource && isLiveVideoSourceValid) || (!requiresLiveVideoSource && videoSource != null)); + @Contextual + @Transient + val isAudioDownloadReady: Boolean get() = !requireAudioSource || + ((requiresLiveAudioSource && isLiveAudioSourceValid) || (!requiresLiveAudioSource && audioSource != null)); + var subtitleSource: SubtitleRawSource?; @kotlinx.serialization.Serializable(with = OffsetDateTimeNullableSerializer::class) @@ -157,6 +169,8 @@ class VideoDownload { this.hasVideoRequestExecutor = video is JSSource && video.hasRequestExecutor; this.requiresLiveVideoSource = false; this.targetVideoName = videoSource?.name; + this.requireVideoSource = targetPixelCount != null + this.requireAudioSource = targetBitrate != null; //TODO: May not be a valid check.. can only be determined after live fetch? } constructor(video: IPlatformVideoDetails, videoSource: IVideoSource?, audioSource: IAudioSource?, subtitleSource: SubtitleRawSource?) { this.video = SerializedPlatformVideo.fromVideo(video); @@ -175,6 +189,8 @@ class VideoDownload { this.targetAudioName = audioSource?.name; this.targetPixelCount = if(videoSource != null) (videoSource.width * videoSource.height).toLong() else null; this.targetBitrate = if(audioSource != null) audioSource.bitrate.toLong() else null; + this.requireVideoSource = videoSource != null; + this.requireAudioSource = audioSource != null; } fun withGroup(groupType: String, groupID: String): VideoDownload { @@ -320,8 +336,10 @@ class VideoDownload { throw DownloadException("Audio source is not supported for downloading (yet)", false); } - if(((!requiresLiveVideoSource && videoSource == null) || (requiresLiveVideoSource && !isLiveVideoSourceValid)) || ((!requiresLiveAudioSource && audioSource == null) || (requiresLiveAudioSource && !isLiveAudioSourceValid))) - throw DownloadException("No valid sources found for video/audio"); + if(!isVideoDownloadReady) + throw DownloadException("No valid sources found for video"); + if(!isAudioDownloadReady) + throw DownloadException("No valid sources found for audio"); } } @@ -367,6 +385,7 @@ class VideoDownload { sourcesToDownload.add(async { Logger.i(TAG, "Started downloading video"); + var lastEmit = 0L; val progressCallback = { length: Long, totalRead: Long, speed: Long -> synchronized(progressLock) { lastVideoLength = length; @@ -379,9 +398,14 @@ class VideoDownload { val total = lastVideoRead + lastAudioRead; if(totalLength > 0) { val percentage = (total / totalLength.toDouble()); - onProgress?.invoke(percentage); progress = percentage; - onProgressChanged.emit(percentage); + + val now = System.currentTimeMillis(); + if(now - lastEmit > 200) { + lastEmit = System.currentTimeMillis(); + onProgress?.invoke(percentage); + onProgressChanged.emit(percentage); + } } } } @@ -401,6 +425,7 @@ class VideoDownload { sourcesToDownload.add(async { Logger.i(TAG, "Started downloading audio"); + var lastEmit = 0L; val progressCallback = { length: Long, totalRead: Long, speed: Long -> synchronized(progressLock) { lastAudioLength = length; @@ -413,9 +438,14 @@ class VideoDownload { val total = lastVideoRead + lastAudioRead; if(totalLength > 0) { val percentage = (total / totalLength.toDouble()); - onProgress?.invoke(percentage); progress = percentage; - onProgressChanged.emit(percentage); + + val now = System.currentTimeMillis(); + if(now - lastEmit > 200) { + lastEmit = System.currentTimeMillis(); + onProgress?.invoke(percentage); + onProgressChanged.emit(percentage); + } } } } diff --git a/app/src/main/java/com/futo/platformplayer/helpers/VideoHelper.kt b/app/src/main/java/com/futo/platformplayer/helpers/VideoHelper.kt index 860c2daa..23fa2b73 100644 --- a/app/src/main/java/com/futo/platformplayer/helpers/VideoHelper.kt +++ b/app/src/main/java/com/futo/platformplayer/helpers/VideoHelper.kt @@ -193,7 +193,7 @@ class VideoHelper { fun estimateSourceSize(source: IVideoSource?): Int { if(source == null) return 0; - if(source is IVideoUrlSource) { + if(source is IVideoSource) { if(source.bitrate ?: 0 <= 0 || source.duration.toInt() == 0) return 0; return (source.duration / 8).toInt() * source.bitrate!!; @@ -202,7 +202,7 @@ class VideoHelper { } fun estimateSourceSize(source: IAudioSource?): Int { if(source == null) return 0; - if(source is IAudioUrlSource) { + if(source is IAudioSource) { if(source.bitrate <= 0 || source.duration?.toInt() ?: 0 == 0) return 0; return (source.duration!! / 8).toInt() * source.bitrate; diff --git a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt index 6955cbe9..b9c56420 100644 --- a/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt +++ b/app/src/main/java/com/futo/platformplayer/services/DownloadService.kt @@ -195,9 +195,7 @@ class DownloadService : Service() { download.targetBitrate = download.audioSource!!.bitrate.toLong(); download.audioSource = null; } - if(download.videoDetails == null || - ((download.videoSource == null && download.audioSource == null) && - (download.requiresLiveVideoSource && !download.isLiveVideoSourceValid) && (download.requiresLiveAudioSource && !download.isLiveAudioSourceValid))) + if(download.videoDetails == null || (!download.isVideoDownloadReady || !download.isAudioDownloadReady)) download.changeState(VideoDownload.State.PREPARING); notifyDownload(download); @@ -214,7 +212,7 @@ class DownloadService : Service() { download.progress = progress; val currentTime = System.currentTimeMillis(); - if (currentTime - lastNotifyTime > 500) { + if (currentTime - lastNotifyTime > 800) { notifyDownload(download); lastNotifyTime = currentTime; }