From 06d108d791ea155c7ac7422f05e19c509b59788a Mon Sep 17 00:00:00 2001 From: Ben Gruver Date: Sun, 1 Mar 2015 13:26:25 -0800 Subject: [PATCH] Refactor how the PARAM_LIST_OR_ID thing is parsed This gets rids of the empty placeholder tokens, which cause errors in IDEA's built-in lexer support code --- .../java/org/jf/baksmali/RoundtripTest.java | 69 +++++++++++++++++++ smali/src/main/antlr3/smaliParser.g | 7 +- smali/src/main/jflex/smaliLexer.flex | 8 +-- .../LexerTest/TypeAndIdentifierTest.tokens | 38 +++++----- smalidea/src/main/antlr3/smalideaParser.g | 8 +-- .../java/org/jf/smalidea/SmaliTokens.java | 6 +- 6 files changed, 97 insertions(+), 39 deletions(-) create mode 100644 baksmali/src/test/java/org/jf/baksmali/RoundtripTest.java diff --git a/baksmali/src/test/java/org/jf/baksmali/RoundtripTest.java b/baksmali/src/test/java/org/jf/baksmali/RoundtripTest.java new file mode 100644 index 00000000..266497f9 --- /dev/null +++ b/baksmali/src/test/java/org/jf/baksmali/RoundtripTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jf.baksmali; + +import org.antlr.runtime.RecognitionException; +import org.jf.baksmali.Adaptors.ClassDefinition; +import org.jf.dexlib2.iface.ClassDef; +import org.jf.smali.SmaliTestUtils; +import org.jf.util.IndentingWriter; +import org.jf.util.TextUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringWriter; + +public class RoundtripTest { + @Test + public void testParamListMethodName() throws IOException, RecognitionException { + String text = "" + + ".class Lblah;\n" + + ".super Ljava/lang/Object;\n" + + "# virtual methods\n" + + ".method public abstract II()V\n" + + ".end method\n"; + ClassDef classDef = SmaliTestUtils.compileSmali(text); + + baksmaliOptions options = new baksmaliOptions(); + options.useImplicitReferences = true; + + StringWriter stringWriter = new StringWriter(); + IndentingWriter writer = new IndentingWriter(stringWriter); + ClassDefinition classDefinition = new ClassDefinition(options, classDef); + classDefinition.writeTo(writer); + writer.close(); + + Assert.assertEquals(TextUtils.normalizeWhitespace(text), + TextUtils.normalizeWhitespace(stringWriter.toString())); + } +} diff --git a/smali/src/main/antlr3/smaliParser.g b/smali/src/main/antlr3/smaliParser.g index 0e8c2aa4..bf84af43 100644 --- a/smali/src/main/antlr3/smaliParser.g +++ b/smali/src/main/antlr3/smaliParser.g @@ -131,8 +131,7 @@ tokens { PACKED_SWITCH_DIRECTIVE; PARAM_LIST_END; PARAM_LIST_START; - PARAM_LIST_OR_ID_END; - PARAM_LIST_OR_ID_START; + PARAM_LIST_OR_ID_PRIMITIVE_TYPE; PARAMETER_DIRECTIVE; POSITIVE_INTEGER_LITERAL; PRIMITIVE_TYPE; @@ -529,7 +528,7 @@ registers_directive }; param_list_or_id - : PARAM_LIST_OR_ID_START PRIMITIVE_TYPE+ PARAM_LIST_OR_ID_END; + : PARAM_LIST_OR_ID_PRIMITIVE_TYPE+; /*identifiers are much more general than most languages. Any of the below can either be the indicated type OR an identifier, depending on the context*/ @@ -584,7 +583,7 @@ method_prototype param_list : PARAM_LIST_START nonvoid_type_descriptor* PARAM_LIST_END -> nonvoid_type_descriptor* - | PARAM_LIST_OR_ID_START PRIMITIVE_TYPE* PARAM_LIST_OR_ID_END -> PRIMITIVE_TYPE* + | (PARAM_LIST_OR_ID_PRIMITIVE_TYPE -> PRIMITIVE_TYPE[$PARAM_LIST_OR_ID_PRIMITIVE_TYPE])+ | nonvoid_type_descriptor*; array_descriptor diff --git a/smali/src/main/jflex/smaliLexer.flex b/smali/src/main/jflex/smaliLexer.flex index 2849ab89..9c4a3c3a 100644 --- a/smali/src/main/jflex/smaliLexer.flex +++ b/smali/src/main/jflex/smaliLexer.flex @@ -307,9 +307,9 @@ Type = {PrimitiveType} | {ClassDescriptor} | {ArrayPrefix} ({ClassDescriptor} | } { - {PrimitiveType} { return newToken(PRIMITIVE_TYPE); } - [^] { yypushback(1); yybegin(YYINITIAL); return newToken(PARAM_LIST_OR_ID_END); } - <> { yybegin(YYINITIAL); return newToken(PARAM_LIST_OR_ID_END); } + {PrimitiveType} { return newToken(PARAM_LIST_OR_ID_PRIMITIVE_TYPE); } + [^] { yypushback(1); yybegin(YYINITIAL); } + <> { yybegin(YYINITIAL); } } { @@ -632,9 +632,9 @@ Type = {PrimitiveType} | {ClassDescriptor} | {ArrayPrefix} ({ClassDescriptor} | } {PrimitiveType} {PrimitiveType}+ { + // go back and re-lex it as a PARAM_LIST_OR_ID yypushback(yylength()); yybegin(PARAM_LIST_OR_ID); - return newToken(PARAM_LIST_OR_ID_START); } {Type} {Type}+ { diff --git a/smali/src/test/resources/LexerTest/TypeAndIdentifierTest.tokens b/smali/src/test/resources/LexerTest/TypeAndIdentifierTest.tokens index 03e79f03..0e850f4a 100644 --- a/smali/src/test/resources/LexerTest/TypeAndIdentifierTest.tokens +++ b/smali/src/test/resources/LexerTest/TypeAndIdentifierTest.tokens @@ -38,24 +38,20 @@ CLASS_DESCRIPTOR("LI/I/I;") ARRAY_TYPE_PREFIX("[[") PRIMITIVE_TYPE("I") -PARAM_LIST_OR_ID_START("") -PRIMITIVE_TYPE("I") -PRIMITIVE_TYPE("I") -PRIMITIVE_TYPE("I") -PRIMITIVE_TYPE("I") -PRIMITIVE_TYPE("I") -PARAM_LIST_OR_ID_END("") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I") -PARAM_LIST_OR_ID_START("") -PRIMITIVE_TYPE("Z") -PRIMITIVE_TYPE("B") -PRIMITIVE_TYPE("S") -PRIMITIVE_TYPE("C") -PRIMITIVE_TYPE("I") -PRIMITIVE_TYPE("J") -PRIMITIVE_TYPE("F") -PRIMITIVE_TYPE("D") -PARAM_LIST_OR_ID_END("") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("Z") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("B") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("S") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("C") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("J") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("F") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("D") PARAM_LIST_START("") PRIMITIVE_TYPE("I") @@ -111,11 +107,9 @@ ARRAY_TYPE_PREFIX("[") INVALID_TOKEN(";") MEMBER_NAME("") -PARAM_LIST_OR_ID_START("") -PRIMITIVE_TYPE("I") -PRIMITIVE_TYPE("I") -PRIMITIVE_TYPE("I") -PARAM_LIST_OR_ID_END("") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I") +PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I") ARRAY_TYPE_PREFIX("[") PRIMITIVE_TYPE("I") diff --git a/smalidea/src/main/antlr3/smalideaParser.g b/smalidea/src/main/antlr3/smalideaParser.g index 17e0445f..1d18269d 100644 --- a/smalidea/src/main/antlr3/smalideaParser.g +++ b/smalidea/src/main/antlr3/smalideaParser.g @@ -360,7 +360,7 @@ registers_directive } param_list_or_id - : PARAM_LIST_OR_ID_START primitive_type+ PARAM_LIST_OR_ID_END; + : PARAM_LIST_OR_ID_PRIMITIVE_TYPE+; /*identifiers are much more general than most languages. Any of the below can either be the indicated type OR an identifier, depending on the context*/ @@ -478,7 +478,6 @@ colon param_list_inner : ((PARAM_LIST_START param* PARAM_LIST_END) - | (PARAM_LIST_OR_ID_START param* PARAM_LIST_OR_ID_END) | (param+)); catch [RecognitionException re] { Marker errorMarker = mark(); @@ -511,8 +510,7 @@ param_list_reference Marker marker = mark(); } : ((PARAM_LIST_START nonvoid_type_descriptor* PARAM_LIST_END) - | (PARAM_LIST_OR_ID_START nonvoid_type_descriptor* PARAM_LIST_OR_ID_END) - | (nonvoid_type_descriptor*)) + | (nonvoid_type_descriptor)*) { marker.done(SmaliElementTypes.METHOD_REFERENCE_PARAM_LIST); }; catch [RecognitionException re] { recover(input, re); @@ -521,7 +519,7 @@ param_list_reference primitive_type @init { Marker marker = mark(); } - : PRIMITIVE_TYPE + : (PRIMITIVE_TYPE | PARAM_LIST_OR_ID_PRIMITIVE_TYPE) { finishToken(marker, SmaliElementTypes.PRIMITIVE_TYPE); }; catch [RecognitionException re] { recover(input, re); diff --git a/smalidea/src/main/java/org/jf/smalidea/SmaliTokens.java b/smalidea/src/main/java/org/jf/smalidea/SmaliTokens.java index 6c10955b..18bd35a1 100644 --- a/smalidea/src/main/java/org/jf/smalidea/SmaliTokens.java +++ b/smalidea/src/main/java/org/jf/smalidea/SmaliTokens.java @@ -143,8 +143,7 @@ public class SmaliTokens { @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PACKED_SWITCH_DIRECTIVE; @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_END; @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_START; - @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_OR_ID_END; - @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_OR_ID_START; + @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_OR_ID_PRIMITIVE_TYPE; @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAMETER_DIRECTIVE; @SuppressWarnings({"UnusedDeclaration"}) public static IElementType POSITIVE_INTEGER_LITERAL; @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PRIMITIVE_TYPE; @@ -263,8 +262,7 @@ public class SmaliTokens { tokenColors.put("OPEN_PAREN", SmaliHighlightingColors.PARENS); tokenColors.put("PACKED_SWITCH_DIRECTIVE", SmaliHighlightingColors.DIRECTIVE); tokenColors.put("PARAM_LIST_END", SmaliHighlightingColors.TYPE); - tokenColors.put("PARAM_LIST_OR_ID_END", SmaliHighlightingColors.TYPE); - tokenColors.put("PARAM_LIST_OR_ID_START", SmaliHighlightingColors.TYPE); + tokenColors.put("PARAM_LIST_OR_ID_PRIMITIVE_TYPE", SmaliHighlightingColors.TYPE); tokenColors.put("PARAM_LIST_START", SmaliHighlightingColors.TYPE); tokenColors.put("PARAMETER_DIRECTIVE", SmaliHighlightingColors.DIRECTIVE); tokenColors.put("POSITIVE_INTEGER_LITERAL", SmaliHighlightingColors.NUMBER);