fix(YouTube - Open Shorts in regular player): Do not exit app when pressing back button in regular player (#5020)

extensions/youtube/src/main/java/app/revanced/extension/youtube/patches/OpenShortsInRegularPlayerPatch.java

@@ -31,6 +31,8 @@ public class OpenShortsInRegularPlayerPatch {
 
     private static WeakReference<Activity> mainActivityRef = new WeakReference<>(null);
 
+    private static volatile boolean overrideBackPressToExit;
+
     /**
      * Injection point.
      */
@@ -38,6 +40,18 @@ public class OpenShortsInRegularPlayerPatch {
         mainActivityRef = new WeakReference<>(activity);
     }
 
+    /**
+     * Injection point.
+     */
+    public static boolean overrideBackPressToExit(boolean original) {
+        if (overrideBackPressToExit) {
+            Logger.printDebug(() -> "Overriding back press to exit activity");
+            return false;
+        }
+
+        return original;
+    }
+
     /**
      * Injection point.
      */
@@ -45,6 +59,7 @@ public class OpenShortsInRegularPlayerPatch {
         try {
             ShortsPlayerType type = Settings.SHORTS_PLAYER_TYPE.get();
             if (type == ShortsPlayerType.SHORTS_PLAYER) {
+                overrideBackPressToExit = false;
                 return false; // Default unpatched behavior.
             }
 
@@ -61,13 +76,17 @@ public class OpenShortsInRegularPlayerPatch {
                 // set to open in the regular player, so it's ignored as
                 // checking the map makes the patch more complicated.
                 Logger.printDebug(() -> "Ignoring Short with no videoId");
+                overrideBackPressToExit = false;
                 return false;
             }
             
             if (NavigationButton.getSelectedNavigationButton() == NavigationButton.SHORTS) {
+                overrideBackPressToExit = false;
                 return false; // Always use Shorts player for the Shorts nav button.
             }
 
+            overrideBackPressToExit = true;
+
             final boolean forceFullScreen = (type == ShortsPlayerType.REGULAR_PLAYER_FULLSCREEN);
             OpenVideosFullscreenHookPatch.setOpenNextVideoFullscreen(forceFullScreen);
 

extensions/youtube/src/main/java/app/revanced/extension/youtube/patches/OpenVideosFullscreenHookPatch.java

@@ -33,7 +33,7 @@ public class OpenVideosFullscreenHookPatch {
         }
 
         if (!isFullScreenPatchIncluded()) {
-            return false;
+            return original;
         }
 
         return Settings.OPEN_VIDEOS_FULLSCREEN_PORTRAIT.get();

patches/src/main/kotlin/app/revanced/patches/youtube/layout/player/fullscreen/Fingerprints.kt

@@ -15,9 +15,6 @@ internal val openVideosFullscreenPortraitFingerprint = fingerprint {
     }
 }
 
-/**
- * Used to enable opening regular videos fullscreen.
- */
 internal val openVideosFullscreenHookPatchExtensionFingerprint = fingerprint {
     accessFlags(AccessFlags.PRIVATE, AccessFlags.STATIC)
     returns("Z")

patches/src/main/kotlin/app/revanced/patches/youtube/layout/shortsplayer/Fingerprints.kt

@@ -54,3 +54,11 @@ internal val shortsPlaybackIntentFingerprint = fingerprint {
         "reels_fragment_descriptor"
     )
 }
+
+internal val exitVideoPlayerFingerprint = fingerprint {
+    returns("V")
+    parameters()
+    literal {
+        mdx_drawer_layout_id
+    }
+}

patches/src/main/kotlin/app/revanced/patches/youtube/layout/shortsplayer/OpenShortsInRegularPlayerPatch.kt

@@ -1,11 +1,16 @@
 package app.revanced.patches.youtube.layout.shortsplayer
 
 import app.revanced.patcher.extensions.InstructionExtensions.addInstruction
+import app.revanced.patcher.extensions.InstructionExtensions.addInstructions
 import app.revanced.patcher.extensions.InstructionExtensions.addInstructionsWithLabels
 import app.revanced.patcher.extensions.InstructionExtensions.getInstruction
 import app.revanced.patcher.patch.bytecodePatch
+import app.revanced.patcher.patch.resourcePatch
 import app.revanced.patches.all.misc.resources.addResources
 import app.revanced.patches.all.misc.resources.addResourcesPatch
+import app.revanced.patches.shared.misc.mapping.get
+import app.revanced.patches.shared.misc.mapping.resourceMappingPatch
+import app.revanced.patches.shared.misc.mapping.resourceMappings
 import app.revanced.patches.shared.misc.settings.preference.ListPreference
 import app.revanced.patches.youtube.layout.player.fullscreen.openVideosFullscreenHookPatch
 import app.revanced.patches.youtube.misc.extension.sharedExtensionPatch
@@ -19,12 +24,29 @@ import app.revanced.patches.youtube.shared.mainActivityOnCreateFingerprint
 import app.revanced.util.findFreeRegister
 import app.revanced.util.getReference
 import app.revanced.util.indexOfFirstInstructionOrThrow
+import app.revanced.util.indexOfFirstInstructionReversedOrThrow
+import com.android.tools.smali.dexlib2.Opcode
 import com.android.tools.smali.dexlib2.iface.instruction.OneRegisterInstruction
 import com.android.tools.smali.dexlib2.iface.reference.MethodReference
 
 private const val EXTENSION_CLASS_DESCRIPTOR =
     "Lapp/revanced/extension/youtube/patches/OpenShortsInRegularPlayerPatch;"
 
+internal var mdx_drawer_layout_id = -1L
+    private set
+
+private val openShortsInRegularPlayerResourcePatch = resourcePatch {
+    dependsOn(resourceMappingPatch)
+
+    execute {
+        mdx_drawer_layout_id = resourceMappings[
+            "id",
+            "mdx_drawer_layout",
+        ]
+
+    }
+}
+
 @Suppress("unused")
 val openShortsInRegularPlayerPatch = bytecodePatch(
     name = "Open Shorts in regular player",
@@ -36,7 +58,8 @@ val openShortsInRegularPlayerPatch = bytecodePatch(
         addResourcesPatch,
         openVideosFullscreenHookPatch,
         navigationBarHookPatch,
-        versionCheckPatch
+        versionCheckPatch,
+        openShortsInRegularPlayerResourcePatch
     )
 
     compatibleWith(
@@ -127,5 +150,28 @@ val openShortsInRegularPlayerPatch = bytecodePatch(
                 ${extensionInstructions(0, 1)}
             """
         )
+
+        // Fix issue with back button exiting the app instead of minimizing the player.
+        // Without this change this issue can be difficult to reproduce, but seems to occur
+        // most often with 'open video in regular player' and not open in fullscreen player.
+        exitVideoPlayerFingerprint.method.apply {
+            // Method call for Activity.finish()
+            val finishIndex = indexOfFirstInstructionOrThrow {
+                val reference = getReference<MethodReference>()
+                reference?.name == "finish"
+            }
+
+            // Index of PlayerType.isWatchWhileMaximizedOrFullscreen()
+            val index = indexOfFirstInstructionReversedOrThrow(finishIndex, Opcode.MOVE_RESULT)
+            val register = getInstruction<OneRegisterInstruction>(index).registerA
+
+            addInstructions(
+                index + 1,
+                """
+                    invoke-static { v$register }, ${EXTENSION_CLASS_DESCRIPTOR}->overrideBackPressToExit(Z)Z    
+                    move-result v$register
+                """
+            )
+        }
     }
 }