Add a lexer test for api 29

This tests that spaces are not supported in simple names and class
descriptors prior to api 30

smali/src/test/java/org/jf/smali/LexerTest.java

@@ -47,6 +47,7 @@ import static org.jf.smali.expectedTokensTestGrammarParser.ExpectedToken;
 
 public class LexerTest {
     private static final HashMap<String, Integer> tokenTypesByName;
+    private static final int MOST_RECENT_API = 10000;
 
     static {
         tokenTypesByName = new HashMap<String, Integer>();
@@ -116,6 +117,11 @@ public class LexerTest {
         runTest("TypeAndIdentifierTest");
     }
 
+    @Test
+    public void TypeAndIdentifierTest_api29() {
+        runTest("TypeAndIdentifierTest_api29", 29);
+    }
+
     @Test
     public void SymbolTest() {
         runTest("SymbolTest", false);
@@ -127,10 +133,18 @@ public class LexerTest {
     }
 
     public void runTest(String test) {
-        runTest(test, true);
+        runTest(test, true, MOST_RECENT_API);
     }
 
     public void runTest(String test, boolean discardHiddenTokens) {
+        runTest(test, discardHiddenTokens, MOST_RECENT_API);
+    }
+
+    public void runTest(String test, int apiLevel) {
+        runTest(test, true, apiLevel);
+    }
+
+    public void runTest(String test, boolean discardHiddenTokens, int apiLevel) {
         String smaliFile = String.format("LexerTest%s%s.smali", File.separatorChar, test);
         String tokensFile = String.format("LexerTest%s%s.tokens", File.separatorChar, test);
 
@@ -159,7 +173,7 @@ public class LexerTest {
             Assert.fail("Could not load " + smaliFile);
         }
         smaliFlexLexer lexer = new smaliFlexLexer(new InputStreamReader(smaliStream),
-            10000 /* most recent API level */);
+            apiLevel);
         lexer.setSourceFile(new File(test + ".smali"));
         lexer.setSuppressErrors(true);
 

smali/src/test/resources/LexerTest/TypeAndIdentifierTest_api29.smali

@@ -0,0 +1,73 @@
+Z
+B
+S
+C
+I
+J
+F
+D
+V
+
+Ljava/lang/String;
+LI;
+LV;
+LI/I/I;
+L`single`;
+L`java`/lang/String;
+L`java`/`lang`/`String`;
+Lspace/test/`20 a0 1680 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 200a 202f 205f 3000　`;
+
+[Z
+[B
+[S
+[C
+[I
+[J
+[F
+[D
+[Ljava/lang/String;
+[L`java`/lang/String;
+[L`spaaaace spaaaace`;
+[L`spaaaace spaaaace`/`spaaaace spaaaace`;
+[LI/I/I;
+[[LI/I/I;
+[[I
+
+IIIII
+ZBSCIJFD
+ILa;[La;[I
+Ljava/lang/String;Ljava/lang/String;
+[I[I[I
+[I[Z
+[I[Ljava/lang/String;
+
+<init>
+<clinit>
+<blah>
+<init->
+
+Ljava/lang/String
+L`java`/lang/String
+L;
+L``;
+L``
+LI
+L[Ljava/lang/String;
+L`[Ljava/lang/String;
+LInvalidCharIn321\`[`;
+
+[
+[V
+[java/lang/String;
+[;
+
+<linit>
+
+III
+
+[I->clone()Ljava/lang/Object;
+
+`this is the quote that never ends
+`simple_name_in_backticks`
+`simple_name_with_spaces_20 a0 1680 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 200a 202f 205f 3000　`
+

smali/src/test/resources/LexerTest/TypeAndIdentifierTest_api29.tokens

@@ -0,0 +1,129 @@
+PRIMITIVE_TYPE("Z")
+PRIMITIVE_TYPE("B")
+PRIMITIVE_TYPE("S")
+PRIMITIVE_TYPE("C")
+PRIMITIVE_TYPE("I")
+PRIMITIVE_TYPE("J")
+PRIMITIVE_TYPE("F")
+PRIMITIVE_TYPE("D")
+VOID_TYPE("V")
+
+CLASS_DESCRIPTOR("Ljava/lang/String;")
+CLASS_DESCRIPTOR("LI;")
+CLASS_DESCRIPTOR("LV;")
+CLASS_DESCRIPTOR("LI/I/I;")
+CLASS_DESCRIPTOR("Lsingle;")
+CLASS_DESCRIPTOR("Ljava/lang/String;")
+CLASS_DESCRIPTOR("Ljava/lang/String;")
+INVALID_TOKEN("Lspace/test/20 a0\u00a01680\u16802000\u20002001\u20012002\u20022003\u20032004\u20042005\u20052006\u20062007\u20072008\u20082009\u2009200a\u200a202f\u202f205f\u205f3000\u3000;")
+
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("Z")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("B")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("S")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("C")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("I")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("J")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("F")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("D")
+ARRAY_TYPE_PREFIX("[")
+CLASS_DESCRIPTOR("Ljava/lang/String;")
+ARRAY_TYPE_PREFIX("[")
+CLASS_DESCRIPTOR("Ljava/lang/String;")
+ARRAY_TYPE_PREFIX("[")
+INVALID_TOKEN("Lspaaaace spaaaace;")
+ARRAY_TYPE_PREFIX("[")
+INVALID_TOKEN("Lspaaaace spaaaace/spaaaace spaaaace;")
+ARRAY_TYPE_PREFIX("[")
+CLASS_DESCRIPTOR("LI/I/I;")
+ARRAY_TYPE_PREFIX("[[")
+CLASS_DESCRIPTOR("LI/I/I;")
+ARRAY_TYPE_PREFIX("[[")
+PRIMITIVE_TYPE("I")
+
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
+
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("Z")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("B")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("S")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("C")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("J")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("F")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("D")
+
+PRIMITIVE_TYPE("I")
+CLASS_DESCRIPTOR("La;")
+ARRAY_TYPE_PREFIX("[")
+CLASS_DESCRIPTOR("La;")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("I")
+
+CLASS_DESCRIPTOR("Ljava/lang/String;")
+CLASS_DESCRIPTOR("Ljava/lang/String;")
+
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("I")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("I")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("I")
+
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("I")
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("Z")
+
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("I")
+ARRAY_TYPE_PREFIX("[")
+CLASS_DESCRIPTOR("Ljava/lang/String;")
+
+MEMBER_NAME("<init>")
+MEMBER_NAME("<clinit>")
+MEMBER_NAME("<blah>")
+MEMBER_NAME("<init->")
+
+SIMPLE_NAME("Ljava") INVALID_TOKEN("/") SIMPLE_NAME("lang") INVALID_TOKEN("/") SIMPLE_NAME("String")
+SIMPLE_NAME("L") SIMPLE_NAME("java") INVALID_TOKEN("/") SIMPLE_NAME("lang") INVALID_TOKEN("/") SIMPLE_NAME("String")
+SIMPLE_NAME("L") INVALID_TOKEN(";")
+SIMPLE_NAME("L") INVALID_TOKEN("`") INVALID_TOKEN("`") INVALID_TOKEN(";")
+SIMPLE_NAME("L") INVALID_TOKEN("`") INVALID_TOKEN("`")
+SIMPLE_NAME("LI")
+SIMPLE_NAME("L") ARRAY_TYPE_PREFIX("[") CLASS_DESCRIPTOR("Ljava/lang/String;")
+SIMPLE_NAME("L") INVALID_TOKEN("`") ARRAY_TYPE_PREFIX("[") CLASS_DESCRIPTOR("Ljava/lang/String;")
+SIMPLE_NAME("LInvalidCharIn321") INVALID_TOKEN("\\") INVALID_TOKEN("`") ARRAY_TYPE_PREFIX("[") INVALID_TOKEN("`") INVALID_TOKEN(";")
+
+ARRAY_TYPE_PREFIX("[")
+ARRAY_TYPE_PREFIX("[") VOID_TYPE("V")
+ARRAY_TYPE_PREFIX("[") SIMPLE_NAME("java") INVALID_TOKEN("/") SIMPLE_NAME("lang") INVALID_TOKEN("/") SIMPLE_NAME("String") INVALID_TOKEN(";")
+ARRAY_TYPE_PREFIX("[") INVALID_TOKEN(";")
+
+MEMBER_NAME("<linit>")
+
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
+PARAM_LIST_OR_ID_PRIMITIVE_TYPE("I")
+
+ARRAY_TYPE_PREFIX("[")
+PRIMITIVE_TYPE("I")
+ARROW("->")
+SIMPLE_NAME("clone")
+OPEN_PAREN("(")
+CLOSE_PAREN(")")
+CLASS_DESCRIPTOR("Ljava/lang/Object;")
+
+INVALID_TOKEN("`") SIMPLE_NAME("this") SIMPLE_NAME("is") SIMPLE_NAME("the") SIMPLE_NAME("quote") SIMPLE_NAME("that") SIMPLE_NAME("never") SIMPLE_NAME("ends")
+SIMPLE_NAME("simple_name_in_backticks")
+SIMPLE_NAME("simple_name_with_spaces_20 a0\u00a01680\u16802000\u20002001\u20012002\u20022003\u20032004\u20042005\u20052006\u20062007\u20072008\u20082009\u2009200a\u200a202f\u202f205f\u205f3000\u3000")