Refactor how the PARAM_LIST_OR_ID thing is parsed

This gets rids of the empty placeholder tokens, which cause errors
in IDEA's built-in lexer support code

baksmali/src/test/java/org/jf/baksmali/RoundtripTest.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.jf.baksmali;
+
+import org.antlr.runtime.RecognitionException;
+import org.jf.baksmali.Adaptors.ClassDefinition;
+import org.jf.dexlib2.iface.ClassDef;
+import org.jf.smali.SmaliTestUtils;
+import org.jf.util.IndentingWriter;
+import org.jf.util.TextUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
+public class RoundtripTest {
+    @Test
+    public void testParamListMethodName() throws IOException, RecognitionException {
+        String text = "" +
+                ".class Lblah;\n" +
+                ".super Ljava/lang/Object;\n" +
+                "# virtual methods\n" +
+                ".method public abstract II()V\n" +
+                ".end method\n";
+        ClassDef classDef = SmaliTestUtils.compileSmali(text);
+
+        baksmaliOptions options = new baksmaliOptions();
+        options.useImplicitReferences = true;
+
+        StringWriter stringWriter = new StringWriter();
+        IndentingWriter writer = new IndentingWriter(stringWriter);
+        ClassDefinition classDefinition = new ClassDefinition(options, classDef);
+        classDefinition.writeTo(writer);
+        writer.close();
+
+        Assert.assertEquals(TextUtils.normalizeWhitespace(text),
+                TextUtils.normalizeWhitespace(stringWriter.toString()));
+    }
+}

smali/src/main/antlr3/smaliParser.g

@@ -131,8 +131,7 @@ tokens {
   PACKED_SWITCH_DIRECTIVE;
   PARAM_LIST_END;
   PARAM_LIST_START;
-  PARAM_LIST_OR_ID_END;
+  PARAM_LIST_OR_ID_PRIMITIVE_TYPE;
-  PARAM_LIST_OR_ID_START;
   PARAMETER_DIRECTIVE;
   POSITIVE_INTEGER_LITERAL;
   PRIMITIVE_TYPE;
@@ -529,7 +528,7 @@ registers_directive
     };
 
 param_list_or_id
-  : PARAM_LIST_OR_ID_START PRIMITIVE_TYPE+ PARAM_LIST_OR_ID_END;
+  : PARAM_LIST_OR_ID_PRIMITIVE_TYPE+;
 
 /*identifiers are much more general than most languages. Any of the below can either be
 the indicated type OR an identifier, depending on the context*/
@@ -584,7 +583,7 @@ method_prototype
 
 param_list
   : PARAM_LIST_START nonvoid_type_descriptor* PARAM_LIST_END -> nonvoid_type_descriptor*
-  | PARAM_LIST_OR_ID_START PRIMITIVE_TYPE* PARAM_LIST_OR_ID_END -> PRIMITIVE_TYPE*
+  | (PARAM_LIST_OR_ID_PRIMITIVE_TYPE -> PRIMITIVE_TYPE[$PARAM_LIST_OR_ID_PRIMITIVE_TYPE])+
   | nonvoid_type_descriptor*;
 
 array_descriptor

smali/src/main/jflex/smaliLexer.flex

@@ -307,9 +307,9 @@ Type = {PrimitiveType} | {ClassDescriptor} | {ArrayPrefix} ({ClassDescriptor} |
 }
 
 <PARAM_LIST_OR_ID> {
-    {PrimitiveType} { return newToken(PRIMITIVE_TYPE); }
+    {PrimitiveType} { return newToken(PARAM_LIST_OR_ID_PRIMITIVE_TYPE); }
-    [^] { yypushback(1); yybegin(YYINITIAL); return newToken(PARAM_LIST_OR_ID_END); }
+    [^] { yypushback(1); yybegin(YYINITIAL); }
-    <<EOF>> { yybegin(YYINITIAL); return newToken(PARAM_LIST_OR_ID_END); }
+    <<EOF>> { yybegin(YYINITIAL); }
 }
 
 <PARAM_LIST> {
@@ -632,9 +632,9 @@ Type = {PrimitiveType} | {ClassDescriptor} | {ArrayPrefix} ({ClassDescriptor} |
     }
 
     {PrimitiveType} {PrimitiveType}+ {
+        // go back and re-lex it as a PARAM_LIST_OR_ID
         yypushback(yylength());
         yybegin(PARAM_LIST_OR_ID);
-        return newToken(PARAM_LIST_OR_ID_START);
     }
 
     {Type} {Type}+ {

smali/src/test/resources/LexerTest/TypeAndIdentifierTest.tokens

@@ -38,24 +38,20 @@ CLASS_DESCRIPTOR("LI/I/I;")
 ARRAY_TYPE_PREFIX("[[")
 PRIMITIVE_TYPE("I")
 
-PARAM_LIST_OR_ID_START("")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
-PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
-PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
-PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
-PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
-PRIMITIVE_TYPE("I")
-PARAM_LIST_OR_ID_END("")
 
-PARAM_LIST_OR_ID_START("")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("Z")
-PRIMITIVE_TYPE("Z")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("B")
-PRIMITIVE_TYPE("B")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("S")
-PRIMITIVE_TYPE("S")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("C")
-PRIMITIVE_TYPE("C")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
-PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("J")
-PRIMITIVE_TYPE("J")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("F")
-PRIMITIVE_TYPE("F")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("D")
-PRIMITIVE_TYPE("D")
-PARAM_LIST_OR_ID_END("")
 
 PARAM_LIST_START("")
 PRIMITIVE_TYPE("I")
@@ -111,11 +107,9 @@ ARRAY_TYPE_PREFIX("[") INVALID_TOKEN(";")
 
 MEMBER_NAME("<linit>")
 
-PARAM_LIST_OR_ID_START("")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
-PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
-PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
-PRIMITIVE_TYPE("I")
-PARAM_LIST_OR_ID_END("")
 
 ARRAY_TYPE_PREFIX("[")
 PRIMITIVE_TYPE("I")

smalidea/src/main/antlr3/smalideaParser.g

@@ -360,7 +360,7 @@ registers_directive
   }
 
 param_list_or_id
-  : PARAM_LIST_OR_ID_START primitive_type+ PARAM_LIST_OR_ID_END;
+  : PARAM_LIST_OR_ID_PRIMITIVE_TYPE+;
 
 /*identifiers are much more general than most languages. Any of the below can either be
 the indicated type OR an identifier, depending on the context*/
@@ -478,7 +478,6 @@ colon
 
 param_list_inner
   : ((PARAM_LIST_START param* PARAM_LIST_END)
-    | (PARAM_LIST_OR_ID_START param* PARAM_LIST_OR_ID_END)
     | (param+));
   catch [RecognitionException re] {
     Marker errorMarker = mark();
@@ -511,8 +510,7 @@ param_list_reference
     Marker marker = mark();
   }
   : ((PARAM_LIST_START nonvoid_type_descriptor* PARAM_LIST_END)
-    | (PARAM_LIST_OR_ID_START nonvoid_type_descriptor* PARAM_LIST_OR_ID_END)
+    | (nonvoid_type_descriptor)*)
-    | (nonvoid_type_descriptor*))
   { marker.done(SmaliElementTypes.METHOD_REFERENCE_PARAM_LIST); };
   catch [RecognitionException re] {
     recover(input, re);
@@ -521,7 +519,7 @@ param_list_reference
 
 primitive_type
   @init { Marker marker = mark(); }
-  : PRIMITIVE_TYPE
+  : (PRIMITIVE_TYPE | PARAM_LIST_OR_ID_PRIMITIVE_TYPE)
   { finishToken(marker, SmaliElementTypes.PRIMITIVE_TYPE); };
   catch [RecognitionException re] {
     recover(input, re);

smalidea/src/main/java/org/jf/smalidea/SmaliTokens.java

@@ -143,8 +143,7 @@ public class SmaliTokens {
     @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PACKED_SWITCH_DIRECTIVE;
     @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_END;
     @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_START;
-    @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_OR_ID_END;
+    @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_OR_ID_PRIMITIVE_TYPE;
-    @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAM_LIST_OR_ID_START;
     @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PARAMETER_DIRECTIVE;
     @SuppressWarnings({"UnusedDeclaration"}) public static IElementType POSITIVE_INTEGER_LITERAL;
     @SuppressWarnings({"UnusedDeclaration"}) public static IElementType PRIMITIVE_TYPE;
@@ -263,8 +262,7 @@ public class SmaliTokens {
         tokenColors.put("OPEN_PAREN", SmaliHighlightingColors.PARENS);
         tokenColors.put("PACKED_SWITCH_DIRECTIVE", SmaliHighlightingColors.DIRECTIVE);
         tokenColors.put("PARAM_LIST_END", SmaliHighlightingColors.TYPE);
-        tokenColors.put("PARAM_LIST_OR_ID_END", SmaliHighlightingColors.TYPE);
+        tokenColors.put("PARAM_LIST_OR_ID_PRIMITIVE_TYPE", SmaliHighlightingColors.TYPE);
-        tokenColors.put("PARAM_LIST_OR_ID_START", SmaliHighlightingColors.TYPE);
         tokenColors.put("PARAM_LIST_START", SmaliHighlightingColors.TYPE);
         tokenColors.put("PARAMETER_DIRECTIVE", SmaliHighlightingColors.DIRECTIVE);
         tokenColors.put("POSITIVE_INTEGER_LITERAL", SmaliHighlightingColors.NUMBER);