Working DBCache, test plugin

app/src/main/java/com/futo/platformplayer/Settings.kt

@@ -8,7 +8,6 @@ import android.webkit.CookieManager
 import androidx.lifecycle.lifecycleScope
 import com.futo.platformplayer.activities.*
 import com.futo.platformplayer.api.http.ManagedHttpClient
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.constructs.Event0
 import com.futo.platformplayer.fragment.mainactivity.bottombar.MenuBottomBarFragment
 import com.futo.platformplayer.logging.Logger
@@ -276,7 +275,7 @@ class Settings : FragmentedStorageFileJson() {
         @FormField(R.string.clear_channel_cache, FieldForm.BUTTON, R.string.clear_channel_cache_description, 14)
         fun clearChannelCache() {
             UIDialogs.toast(SettingsActivity.getActivity()!!, "Started clearing..");
-            ChannelContentCache.instance.clear();
+            StateCache.instance.clear();
             UIDialogs.toast(SettingsActivity.getActivity()!!, "Finished clearing");
         }
     }

app/src/main/java/com/futo/platformplayer/SettingsDev.kt

@@ -2,6 +2,7 @@ package com.futo.platformplayer
 
 import android.content.Context
 import android.webkit.CookieManager
+import androidx.lifecycle.lifecycleScope
 import androidx.work.Constraints
 import androidx.work.Data
 import androidx.work.ExistingPeriodicWorkPolicy
@@ -20,12 +21,12 @@ import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
 import com.futo.platformplayer.api.media.platforms.js.SourcePluginDescriptor
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.background.BackgroundWorker
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.engine.V8Plugin
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.serializers.FlexibleBooleanSerializer
 import com.futo.platformplayer.states.StateAnnouncement
 import com.futo.platformplayer.states.StateApp
+import com.futo.platformplayer.states.StateCache
 import com.futo.platformplayer.states.StateDeveloper
 import com.futo.platformplayer.states.StateDownloads
 import com.futo.platformplayer.states.StateSubscriptions
@@ -82,26 +83,74 @@ class SettingsDev : FragmentedStorageFileJson() {
         var backgroundSubscriptionFetching: Boolean = false;
     }
 
+
+    @FormField(R.string.cache, FieldForm.GROUP, -1, 3)
+    val cache: Cache = Cache();
+    @Serializable
+    class Cache {
+
+        @FormField(R.string.subscriptions_cache_5000, FieldForm.BUTTON, -1, 1)
+        fun subscriptionsCache5000() {
+            StateApp.instance.scope.launch(Dispatchers.IO) {
+                try {
+                    val subsCache =
+                        StateSubscriptions.instance.getSubscriptionsFeedWithExceptions(cacheScope = this)?.first;
+
+                    var total = 0;
+                    var page = 0;
+                    var lastToast = System.currentTimeMillis();
+                    while(subsCache!!.hasMorePages() && total < 5000) {
+                        subsCache!!.nextPage();
+                        total += subsCache!!.getResults().size;
+                        page++;
+
+                        if(page % 10 == 0)
+                            withContext(Dispatchers.Main) {
+                                val diff = System.currentTimeMillis() - lastToast;
+                                lastToast = System.currentTimeMillis();
+                                UIDialogs.toast(
+                                    SettingsActivity.getActivity()!!,
+                                    "Page: ${page}, Total: ${total}, Speed: ${diff}ms"
+                                );
+                            }
+                        Thread.sleep(250);
+                    }
+
+                    withContext(Dispatchers.Main) {
+                        UIDialogs.toast(
+                            SettingsActivity.getActivity()!!,
+                            "FINISHED Page: ${page}, Total: ${total}"
+                        );
+                    }
+                }
+                catch(ex: Throwable) {
+                    Logger.e("SettingsDev", ex.message, ex);
+                    Logger.i("SettingsDev", "Failed: ${ex.message}");
+                }
+            }
+        }
+    }
+
     @FormField(R.string.crash_me, FieldForm.BUTTON,
-        R.string.crashes_the_application_on_purpose, 2)
+        R.string.crashes_the_application_on_purpose, 3)
     fun crashMe() {
         throw java.lang.IllegalStateException("This is an uncaught exception triggered on purpose!");
     }
 
     @FormField(R.string.delete_announcements, FieldForm.BUTTON,
-        R.string.delete_all_announcements, 2)
+        R.string.delete_all_announcements, 3)
     fun deleteAnnouncements() {
         StateAnnouncement.instance.deleteAllAnnouncements();
     }
 
     @FormField(R.string.clear_cookies, FieldForm.BUTTON,
-        R.string.clear_all_cookies_from_the_cookieManager, 2)
+        R.string.clear_all_cookies_from_the_cookieManager, 3)
     fun clearCookies() {
         val cookieManager: CookieManager = CookieManager.getInstance()
         cookieManager.removeAllCookies(null);
     }
     @FormField(R.string.test_background_worker, FieldForm.BUTTON,
-        R.string.test_background_worker_description, 3)
+        R.string.test_background_worker_description, 4)
     fun triggerBackgroundUpdate() {
         val act = SettingsActivity.getActivity()!!;
         UIDialogs.toast(SettingsActivity.getActivity()!!, "Starting test background worker");
@@ -113,10 +162,10 @@ class SettingsDev : FragmentedStorageFileJson() {
         wm.enqueue(req);
     }
     @FormField(R.string.clear_channel_cache, FieldForm.BUTTON,
-        R.string.test_background_worker_description, 3)
+        R.string.test_background_worker_description, 4)
     fun clearChannelContentCache() {
         UIDialogs.toast(SettingsActivity.getActivity()!!, "Clearing cache");
-        ChannelContentCache.instance.clearToday();
+        StateCache.instance.clearToday();
         UIDialogs.toast(SettingsActivity.getActivity()!!, "Cleared");
     }
 
@@ -363,6 +412,17 @@ class SettingsDev : FragmentedStorageFileJson() {
         }
     }
 
+
+    @Contextual
+    @Transient
+    @FormField(R.string.info, FieldForm.GROUP, -1, 19)
+    var info = Info();
+    @Serializable
+    class Info {
+        @FormField(R.string.dev_info_channel_cache_size, FieldForm.READONLYTEXT, -1, 1, "channelCacheSize")
+        var channelCacheStartupCount = StateCache.instance.channelCacheStartupCount;
+    }
+
     //region BOILERPLATE
     override fun encode(): String {
         return Json.encodeToString(this);

app/src/main/java/com/futo/platformplayer/cache/ChannelContentCache.kt

@@ -1,213 +0,0 @@
-package com.futo.platformplayer.cache
-
-import com.futo.platformplayer.api.media.models.contents.IPlatformContent
-import com.futo.platformplayer.api.media.models.video.SerializedPlatformContent
-import com.futo.platformplayer.api.media.structures.DedupContentPager
-import com.futo.platformplayer.api.media.structures.IPager
-import com.futo.platformplayer.api.media.structures.PlatformContentPager
-import com.futo.platformplayer.logging.Logger
-import com.futo.platformplayer.polycentric.PolycentricCache
-import com.futo.platformplayer.resolveChannelUrl
-import com.futo.platformplayer.serializers.PlatformContentSerializer
-import com.futo.platformplayer.states.StatePlatform
-import com.futo.platformplayer.states.StateSubscriptions
-import com.futo.platformplayer.stores.FragmentedStorage
-import com.futo.platformplayer.stores.v2.ManagedStore
-import com.futo.platformplayer.toSafeFileName
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.launch
-import java.time.OffsetDateTime
-import kotlin.streams.toList
-import kotlin.system.measureTimeMillis
-
-class ChannelContentCache {
-    private val _targetCacheSize = 3000;
-    val _channelCacheDir = FragmentedStorage.getOrCreateDirectory("channelCache");
-    val _channelContents: HashMap<String, ManagedStore<SerializedPlatformContent>>;
-    init {
-        val allFiles = _channelCacheDir.listFiles() ?: arrayOf();
-        val initializeTime = measureTimeMillis {
-            _channelContents = HashMap(allFiles
-                .filter { it.isDirectory }
-                .parallelStream().map {
-                    Pair(it.name, FragmentedStorage.storeJson(_channelCacheDir, it.name, PlatformContentSerializer())
-                            .withoutBackup()
-                            .load())
-                }.toList().associate { it })
-        }
-        val minDays = OffsetDateTime.now().minusDays(10);
-        val totalItems = _channelContents.map { it.value.count() }.sum();
-        val toTrim = totalItems - _targetCacheSize;
-        val trimmed: Int;
-        if(toTrim > 0) {
-            val redundantContent = _channelContents.flatMap { it.value.getItems().filter { it.datetime != null && it.datetime!!.isBefore(minDays) }.drop(9) }
-                .sortedBy { it.datetime!! }.take(toTrim);
-            for(content in redundantContent)
-                uncacheContent(content);
-            trimmed = redundantContent.size;
-        }
-        else trimmed = 0;
-        Logger.i(TAG, "ChannelContentCache time: ${initializeTime}ms channels: ${allFiles.size}, videos: ${totalItems}, trimmed: ${trimmed}, total: ${totalItems - trimmed}");
-    }
-
-    fun clear() {
-        synchronized(_channelContents) {
-            for(channel in _channelContents)
-                for(content in channel.value.getItems())
-                    uncacheContent(content);
-        }
-    }
-    fun clearToday() {
-        val yesterday = OffsetDateTime.now().minusDays(1);
-        synchronized(_channelContents) {
-            for(channel in _channelContents)
-                for(content in channel.value.getItems().filter { it.datetime?.isAfter(yesterday) == true })
-                    uncacheContent(content);
-        }
-    }
-
-    fun getChannelCachePager(channelUrl: String): PlatformContentPager {
-        val validID = channelUrl.toSafeFileName();
-
-        val validStores = _channelContents
-            .filter { it.key == validID }
-            .map { it.value };
-
-        val items = validStores.flatMap { it.getItems() }
-            .sortedByDescending { it.datetime };
-        return PlatformContentPager(items, Math.min(150, items.size));
-    }
-    fun getSubscriptionCachePager(): DedupContentPager {
-        Logger.i(TAG, "Subscriptions CachePager get subscriptions");
-        val subs = StateSubscriptions.instance.getSubscriptions();
-        Logger.i(TAG, "Subscriptions CachePager polycentric urls");
-        val allUrls = subs.map {
-            val otherUrls =  PolycentricCache.instance.getCachedProfile(it.channel.url)?.profile?.ownedClaims?.mapNotNull { c -> c.claim.resolveChannelUrl() } ?: listOf();
-            if(!otherUrls.contains(it.channel.url))
-                return@map listOf(listOf(it.channel.url), otherUrls).flatten();
-            else
-                return@map otherUrls;
-        }.flatten().distinct();
-        Logger.i(TAG, "Subscriptions CachePager compiling");
-        val validSubIds = allUrls.map { it.toSafeFileName() }.toHashSet();
-
-        val validStores = _channelContents
-            .filter { validSubIds.contains(it.key) }
-            .map { it.value };
-
-        val items = validStores.flatMap { it.getItems() }
-            .sortedByDescending { it.datetime };
-
-        return DedupContentPager(PlatformContentPager(items, Math.min(30, items.size)), StatePlatform.instance.getEnabledClients().map { it.id });
-    }
-
-    fun uncacheContent(content: SerializedPlatformContent) {
-        val store = getContentStore(content);
-        store?.delete(content);
-    }
-    fun cacheContents(contents: List<IPlatformContent>): List<IPlatformContent> {
-        return contents.filter { cacheContent(it) };
-    }
-    fun cacheContent(content: IPlatformContent, doUpdate: Boolean = false): Boolean {
-        if(content.author.url.isEmpty())
-            return false;
-
-        val channelId = content.author.url.toSafeFileName();
-        val store = getContentStore(channelId).let {
-            if(it == null) {
-                Logger.i(TAG, "New Channel Cache for channel ${content.author.name}");
-                val store = FragmentedStorage.storeJson<SerializedPlatformContent>(_channelCacheDir, channelId, PlatformContentSerializer()).load();
-                _channelContents.put(channelId, store);
-                return@let store;
-            }
-            else return@let it;
-        }
-        val serialized = SerializedPlatformContent.fromContent(content);
-        val existing = store.findItems { it.url == content.url };
-
-        if(existing.isEmpty() || doUpdate) {
-            if(existing.isNotEmpty())
-                existing.forEach { store.delete(it) };
-
-            store.save(serialized);
-        }
-
-        return existing.isEmpty();
-    }
-
-    private fun getContentStore(content: IPlatformContent): ManagedStore<SerializedPlatformContent>? {
-        val channelId = content.author.url.toSafeFileName();
-        return getContentStore(channelId);
-    }
-    private fun getContentStore(channelId: String): ManagedStore<SerializedPlatformContent>? {
-        return synchronized(_channelContents) {
-            var channelStore = _channelContents.get(channelId);
-            return@synchronized channelStore;
-        }
-    }
-
-    companion object {
-        private val TAG = "ChannelCache";
-
-        private val _lock = Object();
-        private var _instance: ChannelContentCache? = null;
-        val instance: ChannelContentCache get() {
-            synchronized(_lock) {
-                if(_instance == null) {
-                    _instance = ChannelContentCache();
-                }
-            }
-            return _instance!!;
-        }
-
-        fun cachePagerResults(scope: CoroutineScope, pager: IPager<IPlatformContent>, onNewCacheHit: ((IPlatformContent)->Unit)? = null): IPager<IPlatformContent> {
-            return ChannelVideoCachePager(pager, scope, onNewCacheHit);
-        }
-    }
-
-    class ChannelVideoCachePager(val pager: IPager<IPlatformContent>, private val scope: CoroutineScope, private val onNewCacheItem: ((IPlatformContent)->Unit)? = null): IPager<IPlatformContent> {
-
-        init {
-            val results = pager.getResults();
-
-            Logger.i(TAG, "Caching ${results.size} subscription initial results [${pager.hashCode()}]");
-            scope.launch(Dispatchers.IO) {
-                try {
-                    val newCacheItems = instance.cacheContents(results);
-                    if(onNewCacheItem != null)
-                        newCacheItems.forEach { onNewCacheItem!!(it) }
-                } catch (e: Throwable) {
-                    Logger.e(TAG, "Failed to cache videos.", e);
-                }
-            }
-        }
-
-        override fun hasMorePages(): Boolean {
-            return pager.hasMorePages();
-        }
-
-        override fun nextPage() {
-            pager.nextPage();
-            val results = pager.getResults();
-
-            Logger.i(TAG, "Caching ${results.size} subscription results");
-            scope.launch(Dispatchers.IO) {
-                try {
-                    val newCacheItems = instance.cacheContents(results);
-                    if(onNewCacheItem != null)
-                        newCacheItems.forEach { onNewCacheItem!!(it) }
-                } catch (e: Throwable) {
-                    Logger.e(TAG, "Failed to cache videos.", e);
-                }
-            }
-        }
-
-        override fun getResults(): List<IPlatformContent> {
-            val results = pager.getResults();
-
-            return results;
-        }
-
-    }
-}

app/src/main/java/com/futo/platformplayer/fragment/channel/tab/ChannelContentsFragment.kt

@@ -24,7 +24,6 @@ import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.api.media.structures.IRefreshPager
 import com.futo.platformplayer.api.media.structures.IReplacerPager
 import com.futo.platformplayer.api.media.structures.MultiPager
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.constructs.Event1
 import com.futo.platformplayer.constructs.Event2
 import com.futo.platformplayer.constructs.TaskHandler
@@ -32,6 +31,7 @@ import com.futo.platformplayer.engine.exceptions.PluginException
 import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.fragment.mainactivity.main.FeedView
 import com.futo.platformplayer.fragment.mainactivity.main.PolycentricProfile
+import com.futo.platformplayer.states.StateCache
 import com.futo.platformplayer.states.StatePolycentric
 import com.futo.platformplayer.states.StateSubscriptions
 import com.futo.platformplayer.views.FeedStyle
@@ -78,7 +78,7 @@ class ChannelContentsFragment : Fragment(), IChannelTabFragment {
     private val _taskLoadVideos = TaskHandler<IPlatformChannel, IPager<IPlatformContent>>({lifecycleScope}, {
             val livePager = getContentPager(it);
             return@TaskHandler if(_channel?.let { StateSubscriptions.instance.isSubscribed(it) } == true)
-                ChannelContentCache.cachePagerResults(lifecycleScope, livePager);
+                StateCache.cachePagerResults(lifecycleScope, livePager);
             else livePager;
         }).success { livePager ->
             setLoading(false);

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/FeedView.kt

@@ -352,6 +352,7 @@ abstract class FeedView<TFragment, TResult, TConverted, TPager, TViewHolder> : L
     }
 
     private fun loadPagerInternal(pager: TPager, cache: ItemCache<TResult>? = null) {
+        Logger.i(TAG, "Setting new internal pager on feed");
         _cache = cache;
 
         detachPagerEvents();
@@ -397,6 +398,7 @@ abstract class FeedView<TFragment, TResult, TConverted, TPager, TViewHolder> : L
         }
     }
 
+    var _lastNextPage = false;
     private fun loadNextPage() {
         synchronized(_pager_lock) {
             val pager: TPager = recyclerData.pager ?: return;
@@ -405,9 +407,14 @@ abstract class FeedView<TFragment, TResult, TConverted, TPager, TViewHolder> : L
 
             //loadCachedPage();
             if (pager.hasMorePages()) {
+                _lastNextPage = true;
                 setLoading(true);
                 _nextPageHandler.run(pager);
             }
+            else if(_lastNextPage) {
+                Logger.i(TAG, "End of page reached");
+                _lastNextPage = false;
+            }
         }
     }
 

app/src/main/java/com/futo/platformplayer/fragment/mainactivity/main/SubscriptionsFeedFragment.kt

@@ -15,13 +15,13 @@ import com.futo.platformplayer.api.media.models.video.IPlatformVideo
 import com.futo.platformplayer.api.media.platforms.js.JSClient
 import com.futo.platformplayer.api.media.structures.EmptyPager
 import com.futo.platformplayer.api.media.structures.IPager
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.constructs.TaskHandler
 import com.futo.platformplayer.engine.exceptions.PluginException
 import com.futo.platformplayer.exceptions.ChannelException
 import com.futo.platformplayer.exceptions.RateLimitException
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.states.StateApp
+import com.futo.platformplayer.states.StateCache
 import com.futo.platformplayer.states.StatePlatform
 import com.futo.platformplayer.states.StateSubscriptions
 import com.futo.platformplayer.stores.FragmentedStorage
@@ -132,8 +132,10 @@ class SubscriptionsFeedFragment : MainFragment() {
 
                 if(StateSubscriptions.instance.getOldestUpdateTime().getNowDiffMinutes() > 5 && Settings.instance.subscriptions.fetchOnTabOpen)
                     loadResults(false);
-                else if(recyclerData.results.size == 0)
+                else if(recyclerData.results.size == 0) {
                     loadCache();
+                    setLoading(false);
+                }
             }
 
             val announcementsView = _announcementsView;
@@ -306,12 +308,18 @@ class SubscriptionsFeedFragment : MainFragment() {
 
 
         private fun loadCache() {
-            Logger.i(TAG, "Subscriptions load cache");
+            fragment.lifecycleScope.launch(Dispatchers.IO) {
-            val cachePager = ChannelContentCache.instance.getSubscriptionCachePager();
+                Logger.i(TAG, "Subscriptions retrieving cache");
-            val results = cachePager.getResults();
+                val cachePager = StateCache.instance.getSubscriptionCachePager();
-            Logger.i(TAG, "Subscriptions show cache (${results.size})");
+                Logger.i(TAG, "Subscriptions retrieved cache");
-            setTextCentered(if (results.isEmpty()) context.getString(R.string.no_results_found_swipe_down_to_refresh) else null);
+
-            setPager(cachePager);
+                withContext(Dispatchers.Main) {
+                    val results = cachePager.getResults();
+                    Logger.i(TAG, "Subscriptions show cache (${results.size})");
+                    setTextCentered(if (results.isEmpty()) context.getString(R.string.no_results_found_swipe_down_to_refresh) else null);
+                    setPager(cachePager);
+                }
+            }
         }
         private fun loadResults(withRefetch: Boolean = false) {
             setLoading(true);

app/src/main/java/com/futo/platformplayer/states/StateApp.kt

@@ -26,7 +26,6 @@ import com.futo.platformplayer.api.media.models.video.SerializedPlatformVideo
 import com.futo.platformplayer.api.media.platforms.js.DevJSClient
 import com.futo.platformplayer.api.media.platforms.js.JSClient
 import com.futo.platformplayer.background.BackgroundWorker
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.casting.StateCasting
 import com.futo.platformplayer.constructs.Event0
 import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
@@ -387,7 +386,7 @@ class StateApp {
             try {
                 Logger.i(TAG, "MainApp Started: Initializing [ChannelContentCache]");
                 val time = measureTimeMillis {
-                    ChannelContentCache.instance;
+                    StateCache.instance;
                 }
                 Logger.i(TAG, "ChannelContentCache initialized in ${time}ms");
             } catch (e: Throwable) {

app/src/main/java/com/futo/platformplayer/states/StateCache.kt

@@ -6,7 +6,6 @@ import com.futo.platformplayer.api.media.structures.DedupContentPager
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.api.media.structures.MultiChronoContentPager
 import com.futo.platformplayer.api.media.structures.PlatformContentPager
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.polycentric.PolycentricCache
 import com.futo.platformplayer.resolveChannelUrl
@@ -16,12 +15,18 @@ import com.futo.platformplayer.stores.db.ManagedDBStore
 import com.futo.platformplayer.stores.db.types.DBChannelCache
 import com.futo.platformplayer.stores.db.types.DBHistory
 import com.futo.platformplayer.toSafeFileName
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import java.time.OffsetDateTime
+import kotlin.system.measureTimeMillis
 
 class StateCache {
     private val _channelCache = ManagedDBStore.create("channelCache", DBChannelCache.Descriptor(), PlatformContentSerializer())
         .load();
 
+    val channelCacheStartupCount = _channelCache.count();
+
     fun clear() {
         _channelCache.deleteAll();
     }
@@ -36,6 +41,12 @@ class StateCache {
             it.obj;
         }
     }
+    fun getChannelCachePager(channelUrls: List<String>): IPager<IPlatformContent> {
+        val pagers = MultiChronoContentPager(channelUrls.map { _channelCache.queryPager(DBChannelCache.Index::channelUrl, it, 20) {
+            it.obj;
+        } }, false, 20);
+        return DedupContentPager(pagers, StatePlatform.instance.getEnabledClients().map { it.id });
+    }
     fun getSubscriptionCachePager(): DedupContentPager {
         Logger.i(TAG, "Subscriptions CachePager get subscriptions");
         val subs = StateSubscriptions.instance.getSubscriptions();
@@ -47,10 +58,15 @@ class StateCache {
             else
                 return@map otherUrls;
         }.flatten().distinct();
-        Logger.i(TAG, "Subscriptions CachePager compiling");
 
-        val pagers = MultiChronoContentPager(allUrls.map { getChannelCachePager(it) }, false, 20);
+        Logger.i(TAG, "Subscriptions CachePager get pagers");
-        return DedupContentPager(pagers, StatePlatform.instance.getEnabledClients().map { it.id });
+        val pagers = allUrls.parallelStream().map { getChannelCachePager(it) }.toList();
+
+        Logger.i(TAG, "Subscriptions CachePager compiling");
+        val pager = MultiChronoContentPager(pagers, false, 20);
+        pager.initialize();
+        Logger.i(TAG, "Subscriptions CachePager compiled");
+        return DedupContentPager(pager, StatePlatform.instance.getEnabledClients().map { it.id });
     }
 
 
@@ -63,8 +79,8 @@ class StateCache {
         if(item != null)
             _channelCache.delete(item);
     }
-    fun cacheContents(contents: List<IPlatformContent>): List<IPlatformContent> {
+    fun cacheContents(contents: List<IPlatformContent>, doUpdate: Boolean = false): List<IPlatformContent> {
-        return contents.filter { cacheContent(it) };
+        return contents.filter { cacheContent(it, doUpdate) };
     }
     fun cacheContent(content: IPlatformContent, doUpdate: Boolean = false): Boolean {
         if(content.author.url.isEmpty())
@@ -102,5 +118,58 @@ class StateCache {
                 _instance = null;
             }
         }
+
+
+        fun cachePagerResults(scope: CoroutineScope, pager: IPager<IPlatformContent>, onNewCacheHit: ((IPlatformContent)->Unit)? = null): IPager<IPlatformContent> {
+            return ChannelContentCachePager(pager, scope, onNewCacheHit);
+        }
+    }
+    class ChannelContentCachePager(val pager: IPager<IPlatformContent>, private val scope: CoroutineScope, private val onNewCacheItem: ((IPlatformContent)->Unit)? = null): IPager<IPlatformContent> {
+
+        init {
+            val results = pager.getResults();
+
+            Logger.i(TAG, "Caching ${results.size} subscription initial results [${pager.hashCode()}]");
+            scope.launch(Dispatchers.IO) {
+                try {
+                    val newCacheItems = StateCache.instance.cacheContents(results, true);
+                    if(onNewCacheItem != null)
+                        newCacheItems.forEach { onNewCacheItem!!(it) }
+                } catch (e: Throwable) {
+                    Logger.e(TAG, "Failed to cache videos.", e);
+                }
+            }
+        }
+
+        override fun hasMorePages(): Boolean {
+            return pager.hasMorePages();
+        }
+
+        override fun nextPage() {
+            pager.nextPage();
+            val results = pager.getResults();
+
+            scope.launch(Dispatchers.IO) {
+                try {
+                    val newCacheItemsCount: Int;
+                    val ms = measureTimeMillis {
+                        val newCacheItems = instance.cacheContents(results, true);
+                        newCacheItemsCount = newCacheItems.size;
+                        if(onNewCacheItem != null)
+                            newCacheItems.forEach { onNewCacheItem!!(it) }
+                    }
+                    Logger.i(TAG, "Caching ${results.size} subscription results, updated ${newCacheItemsCount} (${ms}ms)");
+                } catch (e: Throwable) {
+                    Logger.e(TAG, "Failed to cache ${results.size} videos.", e);
+                }
+            }
+        }
+
+        override fun getResults(): List<IPlatformContent> {
+            val results = pager.getResults();
+
+            return results;
+        }
+
     }
 }

app/src/main/java/com/futo/platformplayer/states/StateSubscriptions.kt

@@ -10,7 +10,6 @@ import com.futo.platformplayer.api.media.platforms.js.JSClient
 import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
 import com.futo.platformplayer.api.media.structures.*
 import com.futo.platformplayer.api.media.structures.ReusablePager.Companion.asReusable
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.constructs.Event0
 import com.futo.platformplayer.constructs.Event1
 import com.futo.platformplayer.constructs.Event2

app/src/main/java/com/futo/platformplayer/stores/db/ManagedDBStore.kt

@@ -7,6 +7,7 @@ import androidx.sqlite.db.SimpleSQLiteQuery
 import com.futo.platformplayer.api.media.structures.AdhocPager
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.assume
+import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.states.StateApp
 import com.futo.platformplayer.stores.v2.JsonStoreSerializer
 import com.futo.platformplayer.stores.v2.StoreSerializer
@@ -264,6 +265,7 @@ class ManagedDBStore<I: ManagedDBIndex<T>, T, D: ManagedDBDatabase<T, I, DA>, DA
     fun queryPager(field: KProperty<*>, obj: Any, pageSize: Int): IPager<I> = queryPager(validateFieldName(field), obj, pageSize);
     fun queryPager(field: String, obj: Any, pageSize: Int): IPager<I> {
         return AdhocPager({
+            Logger.i("ManagedDBStore", "Next Page [query: ${obj}](${it}) ${pageSize}");
            queryPage(field, obj, it - 1, pageSize);
         });
     }

app/src/main/java/com/futo/platformplayer/stores/db/types/DBChannelCache.kt

@@ -3,7 +3,9 @@ package com.futo.platformplayer.stores.db.types
 import androidx.room.ColumnInfo
 import androidx.room.Dao
 import androidx.room.Database
+import androidx.room.Entity
 import androidx.room.Ignore
+import androidx.room.Index
 import androidx.room.PrimaryKey
 import com.futo.platformplayer.api.media.models.video.SerializedPlatformContent
 import com.futo.platformplayer.models.HistoryVideo
@@ -25,7 +27,7 @@ class DBChannelCache {
     //These classes solely exist for bounding generics for type erasure
     @Dao
     interface DBDAO: ManagedDBDAOBase<SerializedPlatformContent, Index> {}
-    @Database(entities = [Index::class], version = 2)
+    @Database(entities = [Index::class], version = 4)
     abstract class DB: ManagedDBDatabase<SerializedPlatformContent, Index, DBDAO>() {
         abstract override fun base(): DBDAO;
     }
@@ -37,6 +39,11 @@ class DBChannelCache {
         override fun indexClass(): KClass<Index> = Index::class;
     }
 
+    @Entity(TABLE_NAME, indices = [
+        androidx.room.Index(value = ["url"]),
+        androidx.room.Index(value = ["channelUrl"]),
+        androidx.room.Index(value = ["datetime"], orders = [androidx.room.Index.Order.DESC])
+    ])
     class Index: ManagedDBIndex<SerializedPlatformContent> {
         @ColumnIndex
         @PrimaryKey(true)
@@ -49,7 +56,7 @@ class DBChannelCache {
         var channelUrl: String? = null;
 
         @ColumnIndex
-        @ColumnOrdered(0)
+        @ColumnOrdered(0, true)
         var datetime: Long? = null;
 
 

app/src/main/java/com/futo/platformplayer/subscription/CachedSubscriptionAlgorithm.kt

@@ -5,10 +5,10 @@ import com.futo.platformplayer.api.media.platforms.js.JSClient
 import com.futo.platformplayer.api.media.structures.DedupContentPager
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.api.media.structures.PlatformContentPager
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.models.Subscription
 import com.futo.platformplayer.polycentric.PolycentricCache
 import com.futo.platformplayer.resolveChannelUrl
+import com.futo.platformplayer.states.StateCache
 import com.futo.platformplayer.states.StatePlatform
 import com.futo.platformplayer.states.StateSubscriptions
 import com.futo.platformplayer.toSafeFileName
@@ -27,13 +27,16 @@ class CachedSubscriptionAlgorithm(pageSize: Int = 150, scope: CoroutineScope, al
     override fun getSubscriptions(subs: Map<Subscription, List<String>>): Result {
         val validSubIds = subs.flatMap { it.value } .map { it.toSafeFileName() }.toHashSet();
 
-        val validStores = ChannelContentCache.instance._channelContents
+        /*
+        val validStores = StateCache.instance._channelContents
             .filter { validSubIds.contains(it.key) }
-            .map { it.value };
+            .map { it.value };*/
 
+        /*
         val items = validStores.flatMap { it.getItems() }
             .sortedByDescending { it.datetime };
+        */
 
-        return Result(DedupContentPager(PlatformContentPager(items, Math.min(_pageSize, items.size)), StatePlatform.instance.getEnabledClients().map { it.id }), listOf());
+        return Result(DedupContentPager(StateCache.instance.getChannelCachePager(subs.flatMap { it.value }.distinct()), StatePlatform.instance.getEnabledClients().map { it.id }), listOf());
     }
 }

app/src/main/java/com/futo/platformplayer/subscription/SimpleSubscriptionAlgorithm.kt

@@ -8,7 +8,6 @@ import com.futo.platformplayer.api.media.platforms.js.SourcePluginConfig
 import com.futo.platformplayer.api.media.structures.DedupContentPager
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.api.media.structures.MultiChronoContentPager
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.engine.exceptions.PluginException
 import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.engine.exceptions.ScriptCriticalException
@@ -17,6 +16,7 @@ import com.futo.platformplayer.findNonRuntimeException
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.models.Subscription
 import com.futo.platformplayer.polycentric.PolycentricCache
+import com.futo.platformplayer.states.StateCache
 import com.futo.platformplayer.states.StatePlatform
 import com.futo.platformplayer.states.StatePolycentric
 import com.futo.platformplayer.states.StateSubscriptions
@@ -157,7 +157,7 @@ class SimpleSubscriptionAlgorithm(
                     val time = measureTimeMillis {
                         pager = StatePlatform.instance.getChannelContent(platformClient, url, true, threadPool.poolSize, toIgnore);
 
-                        pager = ChannelContentCache.cachePagerResults(scope, pager!!) {
+                        pager = StateCache.cachePagerResults(scope, pager!!) {
                             onNewCacheHit.emit(sub, it);
                         };
 
@@ -176,7 +176,7 @@ class SimpleSubscriptionAlgorithm(
                         throw channelEx;
                     else {
                         Logger.i(StateSubscriptions.TAG, "Channel ${sub.channel.name} failed, substituting with cache");
-                        pager = ChannelContentCache.instance.getChannelCachePager(sub.channel.url);
+                        pager = StateCache.instance.getChannelCachePager(sub.channel.url);
                     }
                 }
             }

app/src/main/java/com/futo/platformplayer/subscription/SubscriptionsTaskFetchAlgorithm.kt

@@ -11,7 +11,6 @@ import com.futo.platformplayer.api.media.structures.DedupContentPager
 import com.futo.platformplayer.api.media.structures.EmptyPager
 import com.futo.platformplayer.api.media.structures.IPager
 import com.futo.platformplayer.api.media.structures.MultiChronoContentPager
-import com.futo.platformplayer.cache.ChannelContentCache
 import com.futo.platformplayer.engine.exceptions.PluginException
 import com.futo.platformplayer.engine.exceptions.ScriptCaptchaRequiredException
 import com.futo.platformplayer.engine.exceptions.ScriptCriticalException
@@ -21,6 +20,7 @@ import com.futo.platformplayer.fragment.mainactivity.main.SubscriptionsFeedFragm
 import com.futo.platformplayer.logging.Logger
 import com.futo.platformplayer.models.Subscription
 import com.futo.platformplayer.states.StateApp
+import com.futo.platformplayer.states.StateCache
 import com.futo.platformplayer.states.StatePlatform
 import com.futo.platformplayer.states.StateSubscriptions
 import kotlinx.coroutines.CoroutineScope
@@ -108,7 +108,7 @@ abstract class SubscriptionsTaskFetchAlgorithm(
                 val sub = if(!entry.value.isEmpty()) entry.value[0].task.sub else null;
                 val liveTasks = entry.value.filter { !it.task.fromCache };
                 val cachedTasks = entry.value.filter { it.task.fromCache };
-                val livePager = if(!liveTasks.isEmpty()) ChannelContentCache.cachePagerResults(scope, MultiChronoContentPager(liveTasks.map { it.pager!! }, true).apply { this.initialize() }, {
+                val livePager = if(!liveTasks.isEmpty()) StateCache.cachePagerResults(scope, MultiChronoContentPager(liveTasks.map { it.pager!! }, true).apply { this.initialize() }, {
                     onNewCacheHit.emit(sub!!, it);
                 }) else null;
                 val cachedPager = if(!cachedTasks.isEmpty()) MultiChronoContentPager(cachedTasks.map { it.pager!! }, true).apply { this.initialize() } else null;
@@ -142,7 +142,7 @@ abstract class SubscriptionsTaskFetchAlgorithm(
                             return@submit SubscriptionTaskResult(task, null, null);
                         else {
                             cachedChannels.add(task.url);
-                            return@submit SubscriptionTaskResult(task, ChannelContentCache.instance.getChannelCachePager(task.url), null);
+                            return@submit SubscriptionTaskResult(task, StateCache.instance.getChannelCachePager(task.url), null);
                         }
                     }
                 }
@@ -197,7 +197,7 @@ abstract class SubscriptionsTaskFetchAlgorithm(
                         throw channelEx;
                     else {
                         Logger.i(StateSubscriptions.TAG, "Channel ${task.sub.channel.name} failed, substituting with cache");
-                        pager = ChannelContentCache.instance.getChannelCachePager(task.sub.channel.url);
+                        pager = StateCache.instance.getChannelCachePager(task.sub.channel.url);
                         taskEx = ex;
                         return@submit SubscriptionTaskResult(task, pager, taskEx);
                     }

app/src/main/res/values/strings.xml

@@ -399,6 +399,7 @@
     <string name="version_code">Version Code</string>
     <string name="version_name">Version Name</string>
     <string name="version_type">Version Type</string>
+    <string name="dev_info_channel_cache_size">Channel Cache Size (Startup)</string>
     <string name="when_watching_a_video_in_preview_mode_resume_at_the_position_when_opening_the_video_code">When watching a video in preview mode, resume at the position when opening the video code</string>
     <string name="please_enable_logging_to_submit_logs">Please enable logging to submit logs</string>
     <string name="embedded_plugins_reinstalled_a_reboot_is_recommended">Embedded plugins reinstalled, a reboot is recommended</string>
@@ -424,6 +425,7 @@
     <string name="developer_mode">Developer Mode</string>
     <string name="development_server">Development Server</string>
     <string name="experimental">Experimental</string>
+    <string name="cache">Cache</string>
     <string name="fill_storage_till_error">Fill storage till error</string>
     <string name="inject">Inject</string>
     <string name="injects_a_test_source_config_local_into_v8">Injects a test source config (local) into V8</string>
@@ -432,6 +434,7 @@
     <string name="removes_all_subscriptions">Removes all subscriptions</string>
     <string name="settings_related_to_development_server_be_careful_as_it_may_open_your_phone_to_security_vulnerabilities">Settings related to development server, be careful as it may open your phone to security vulnerabilities</string>
     <string name="start_server">Start Server</string>
+    <string name="subscriptions_cache_5000">Subscriptions Cache 5000</string>
     <string name="start_server_on_boot">Start Server on boot</string>
     <string name="starts_a_devServer_on_port_11337_may_expose_vulnerabilities">Starts a DevServer on port 11337, may expose vulnerabilities.</string>
     <string name="test_v8_communication_speed">Test V8 Communication speed</string>

app/src/unstable/assets/sources/test/TestConfig.json

@@ -0,0 +1,24 @@
+{
+	"name": "Testing",
+	"description": "Just for testing.",
+	"author": "FUTO",
+	"authorUrl": "https://futo.org",
+
+	"platformUrl": "https://odysee.com",
+	"sourceUrl": "https://plugins.grayjay.app/Test/TestConfig.json",
+	"repositoryUrl": "https://futo.org",
+	"scriptUrl": "./TestScript.js",
+	"version": 31,
+	
+	"iconUrl": "./odysee.png",
+	"id": "1c05bfc3-08b9-42d0-93d3-6d52e0fd34d8",
+	
+	"scriptSignature": "",
+	"scriptPublicKey": "",
+	"packages": ["Http"],
+	
+	"allowEval": false,
+	"allowUrls": [],
+
+	"supportedClaimTypes": []
+}

app/src/unstable/assets/sources/test/TestScript.js

@@ -0,0 +1,45 @@
+var config = {};
+
+//Source Methods
+source.enable = function(conf){
+	config = conf ?? {};
+	//log(config);
+}
+source.getHome = function() {
+    return new ContentPager([
+        source.getContentDetails("whatever")
+    ]);
+};
+
+//Video
+source.isContentDetailsUrl = function(url) {
+	return REGEX_DETAILS_URL.test(url)
+};
+source.getContentDetails = function(url) {
+	return new PlatformVideoDetails({
+		id: new PlatformID("Test", "Something", config.id),
+		name: "Test Video",
+		thumbnails: new Thumbnails([]),
+		author: new PlatformAuthorLink(new PlatformID("Test", "TestID", config.id),
+			"TestAuthor",
+			"None",
+			""),
+		datetime: parseInt(new Date().getTime() / 1000),
+		duration: 0,
+		viewCount: 0,
+		url: "",
+		isLive: false,
+		description: "",
+		rating: new RatingLikes(0),
+		video: new VideoSourceDescriptor([
+			new HLSSource({
+				name: "HLS",
+				url: "",
+				duration: 0,
+				priority: true
+			})
+		])
+	});
+};
+
+log("LOADED");