From 6d4e503b161d4225bd746a2f218f0130c0de86a3 Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Tue, 1 Dec 2020 06:21:27 -0500 Subject: [PATCH 1/5] fix: rename duplicate attributes to not start with numeric --- .../src/main/java/brut/androlib/res/data/ResResSpec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResResSpec.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResResSpec.java index 0e9aac4d..37a7b5b5 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResResSpec.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/data/ResResSpec.java @@ -37,7 +37,7 @@ public class ResResSpec { ResResSpec resResSpec = type.getResSpecUnsafe(name); if (resResSpec != null) { - cleanName = name + "_APKTOOL_DUPLICATENAME_" + id.toString(); + cleanName = String.format("APKTOOL_DUPLICATE_%s_%s", type.toString(), id.toString()); } else { cleanName = ((name == null || name.isEmpty()) ? ("APKTOOL_DUMMYVAL_" + id.toString()) : name); } From 201b5976bb10a29a989d6a625d7c63c9882433f2 Mon Sep 17 00:00:00 2001 From: Comnir Date: Thu, 10 Dec 2020 12:25:16 +0200 Subject: [PATCH 2/5] Add tests for StringBlock#decodeString with failing tests for code points above 0x10FFFF (issue 2299). --- .../androlib/res/decoder/StringBlock.java | 11 ++++- ...tringBlockWithSurrogatePairInUtf8Test.java | 49 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 brut.apktool/apktool-lib/src/test/java/brut/androlib/res/decoder/StringBlockWithSurrogatePairInUtf8Test.java diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/decoder/StringBlock.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/decoder/StringBlock.java index a388ba19..625cdde2 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/decoder/StringBlock.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/decoder/StringBlock.java @@ -18,6 +18,8 @@ package brut.androlib.res.decoder; import brut.androlib.res.xml.ResXmlEncoders; import brut.util.ExtDataInput; +import com.google.common.annotations.VisibleForTesting; + import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.*; @@ -254,6 +256,12 @@ public class StringBlock { private StringBlock() { } + @VisibleForTesting + StringBlock(byte[] strings, boolean isUTF8) { + m_strings = strings; + m_isUTF8 = isUTF8; + } + /** * Returns style information - array of int triplets, where in each triplet: * * first int is index of tag name ('b','i', etc.) * second int is tag @@ -288,7 +296,8 @@ public class StringBlock { return style; } - private String decodeString(int offset, int length) { + @VisibleForTesting + String decodeString(int offset, int length) { try { return (m_isUTF8 ? UTF8_DECODER : UTF16LE_DECODER).decode( ByteBuffer.wrap(m_strings, offset, length)).toString(); diff --git a/brut.apktool/apktool-lib/src/test/java/brut/androlib/res/decoder/StringBlockWithSurrogatePairInUtf8Test.java b/brut.apktool/apktool-lib/src/test/java/brut/androlib/res/decoder/StringBlockWithSurrogatePairInUtf8Test.java new file mode 100644 index 00000000..eadc09dc --- /dev/null +++ b/brut.apktool/apktool-lib/src/test/java/brut/androlib/res/decoder/StringBlockWithSurrogatePairInUtf8Test.java @@ -0,0 +1,49 @@ +package brut.androlib.res.decoder; + +import org.junit.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.Assert.assertEquals; + +public class StringBlockWithSurrogatePairInUtf8Test { + @Test + public void decodeSingleOctet() { + final String actual = new StringBlock("abcDEF123".getBytes(StandardCharsets.UTF_8), true).decodeString(0, 9); + assertEquals("Incorrect decoding", "abcDEF123", actual); + } + + @Test + public void decodeTwoOctets() { + final String actual0 = new StringBlock(new byte[] { (byte) 0xC2, (byte) 0x80}, true).decodeString(0, 2); + assertEquals("Incorrect decoding", "\u0080", actual0); + + final String actual1 = new StringBlock(new byte[] { (byte) 0xDF, (byte) 0xBF}, true).decodeString(0, 2); + assertEquals("Incorrect decoding", "\u07FF", actual1); + } + + @Test + public void decodeThreeOctets() { + final String actual0 = new StringBlock(new byte[] { (byte) 0xE0, (byte) 0xA0, (byte) 0x80}, true).decodeString(0, 3); + assertEquals("Incorrect decoding", "\u0800", actual0); + + final String actual1 = new StringBlock(new byte[] { (byte) 0xEF, (byte) 0xBF, (byte) 0xBF}, true).decodeString(0, 3); + assertEquals("Incorrect decoding", "\uFFFF", actual1); + } + + @Test + public void decodeSurrogatePair_when_givesAsThreeOctetsFromInvalidRangeOfUtf8() { + // See: https://github.com/iBotPeaches/Apktool/issues/2299 + final String actual0 = new StringBlock(new byte[] { (byte) 0xED, (byte) 0xA0, (byte) 0xBD, (byte) 0xED, (byte) 0xB4, (byte) 0x86}, true).decodeString(0, 6); + assertEquals("Incorrect decoding", "\uD83D\uDD06", actual0); + } + + @Test + public void decodeSurrogatePair_when_givesAsThreeOctetsFromTheValidRangeOfUtf8() { + // \u10FFFF is encoded in UTF-8 as "0xDBFF 0xDFFF" (4-byte encoding), + // but when used in Android resources which are encoded in UTF-8, 3-byte encoding is used, + // so each of these is encoded as 3-bytes + final String actual0 = new StringBlock(new byte[] { (byte) 0xED, (byte) 0xAF, (byte) 0xBF, (byte) 0xED, (byte) 0xBF, (byte) 0xBF}, true).decodeString(0, 6); + assertEquals("Incorrect decoding", "\uDBFF\uDFFF", actual0); + } +} From f1321c84378e622154b5a31d1ef7ab77fd4de7cd Mon Sep 17 00:00:00 2001 From: Comnir Date: Thu, 10 Dec 2020 12:33:06 +0200 Subject: [PATCH 3/5] fix issue 2299: Unicode code points higher than 0x10000 decoding fails. - Use CESU8 decoder instead of UTF-8 in StringBlock. - DEX uses Modified UTF-8 which is close to CESU-8 (https://source.android.com/devices/tech/dalvik/dex-format#mutf-8) --- .../brut/androlib/res/decoder/StringBlock.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/decoder/StringBlock.java b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/decoder/StringBlock.java index 625cdde2..2bff1b04 100644 --- a/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/decoder/StringBlock.java +++ b/brut.apktool/apktool-lib/src/main/java/brut/androlib/res/decoder/StringBlock.java @@ -298,10 +298,22 @@ public class StringBlock { @VisibleForTesting String decodeString(int offset, int length) { + final ByteBuffer wrappedBuffer = ByteBuffer.wrap(m_strings, offset, length); try { - return (m_isUTF8 ? UTF8_DECODER : UTF16LE_DECODER).decode( - ByteBuffer.wrap(m_strings, offset, length)).toString(); + return (m_isUTF8 ? UTF8_DECODER : UTF16LE_DECODER).decode(wrappedBuffer).toString(); } catch (CharacterCodingException ex) { + LOGGER.warning("Failed to decode a string at offset " + offset + " of length " + length); + if (!m_isUTF8) { + return null; + } + } + + try { + // in some places, Android uses 3-byte UTF-8 sequences instead of 4-bytes. + // If decoding failed, we try to use CESU-8 decoder, which is closer to what Android actually uses. + return CESU8_DECODER.decode(wrappedBuffer).toString(); + } catch (CharacterCodingException e) { + LOGGER.warning("Failed to decode a string with CESU-8 decoder."); return null; } } @@ -362,6 +374,7 @@ public class StringBlock { private final CharsetDecoder UTF16LE_DECODER = Charset.forName("UTF-16LE").newDecoder(); private final CharsetDecoder UTF8_DECODER = Charset.forName("UTF-8").newDecoder(); + private final CharsetDecoder CESU8_DECODER = Charset.forName("CESU8").newDecoder(); private static final Logger LOGGER = Logger.getLogger(StringBlock.class.getName()); // ResChunk_header = header.type (0x0001) + header.headerSize (0x001C) From 961f1f94f697abcb0726a10653140d4b6311b376 Mon Sep 17 00:00:00 2001 From: Comnir Date: Thu, 10 Dec 2020 14:44:01 +0200 Subject: [PATCH 4/5] Rename variables in tests. --- .../decoder/StringBlockWithSurrogatePairInUtf8Test.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/brut.apktool/apktool-lib/src/test/java/brut/androlib/res/decoder/StringBlockWithSurrogatePairInUtf8Test.java b/brut.apktool/apktool-lib/src/test/java/brut/androlib/res/decoder/StringBlockWithSurrogatePairInUtf8Test.java index eadc09dc..8e864b46 100644 --- a/brut.apktool/apktool-lib/src/test/java/brut/androlib/res/decoder/StringBlockWithSurrogatePairInUtf8Test.java +++ b/brut.apktool/apktool-lib/src/test/java/brut/androlib/res/decoder/StringBlockWithSurrogatePairInUtf8Test.java @@ -34,8 +34,8 @@ public class StringBlockWithSurrogatePairInUtf8Test { @Test public void decodeSurrogatePair_when_givesAsThreeOctetsFromInvalidRangeOfUtf8() { // See: https://github.com/iBotPeaches/Apktool/issues/2299 - final String actual0 = new StringBlock(new byte[] { (byte) 0xED, (byte) 0xA0, (byte) 0xBD, (byte) 0xED, (byte) 0xB4, (byte) 0x86}, true).decodeString(0, 6); - assertEquals("Incorrect decoding", "\uD83D\uDD06", actual0); + final String actual = new StringBlock(new byte[] { (byte) 0xED, (byte) 0xA0, (byte) 0xBD, (byte) 0xED, (byte) 0xB4, (byte) 0x86}, true).decodeString(0, 6); + assertEquals("Incorrect decoding", "\uD83D\uDD06", actual); } @Test @@ -43,7 +43,7 @@ public class StringBlockWithSurrogatePairInUtf8Test { // \u10FFFF is encoded in UTF-8 as "0xDBFF 0xDFFF" (4-byte encoding), // but when used in Android resources which are encoded in UTF-8, 3-byte encoding is used, // so each of these is encoded as 3-bytes - final String actual0 = new StringBlock(new byte[] { (byte) 0xED, (byte) 0xAF, (byte) 0xBF, (byte) 0xED, (byte) 0xBF, (byte) 0xBF}, true).decodeString(0, 6); - assertEquals("Incorrect decoding", "\uDBFF\uDFFF", actual0); + final String actual = new StringBlock(new byte[] { (byte) 0xED, (byte) 0xAF, (byte) 0xBF, (byte) 0xED, (byte) 0xBF, (byte) 0xBF}, true).decodeString(0, 6); + assertEquals("Incorrect decoding", "\uDBFF\uDFFF", actual); } } From 785cb4f89d946496958cffe33fe7bb4963563d0e Mon Sep 17 00:00:00 2001 From: Connor Tumbleson Date: Fri, 11 Dec 2020 07:06:14 -0500 Subject: [PATCH 5/5] test: add High Brightness Symbol into tests for feature test against #2299 --- .../test/resources/aapt1/testapp/res/values-mcc001/strings.xml | 1 + .../src/test/resources/aapt2/testapp/res/values/strings.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/brut.apktool/apktool-lib/src/test/resources/aapt1/testapp/res/values-mcc001/strings.xml b/brut.apktool/apktool-lib/src/test/resources/aapt1/testapp/res/values-mcc001/strings.xml index 65ccbe69..58b57eb1 100644 --- a/brut.apktool/apktool-lib/src/test/resources/aapt1/testapp/res/values-mcc001/strings.xml +++ b/brut.apktool/apktool-lib/src/test/resources/aapt1/testapp/res/values-mcc001/strings.xml @@ -41,4 +41,5 @@ bar" [Ţåþ ţö ţýþé þåššŵöŕð one two three] []Ţåþ ţö ţýþé þåššŵöŕð one two three [Ţåþ ţö ţýþé þåššŵöŕð one two three] + 🔆 diff --git a/brut.apktool/apktool-lib/src/test/resources/aapt2/testapp/res/values/strings.xml b/brut.apktool/apktool-lib/src/test/resources/aapt2/testapp/res/values/strings.xml index 250aa62f..4452f158 100644 --- a/brut.apktool/apktool-lib/src/test/resources/aapt2/testapp/res/values/strings.xml +++ b/brut.apktool/apktool-lib/src/test/resources/aapt2/testapp/res/values/strings.xml @@ -2,4 +2,5 @@ testapp + 🔆