Add a hidden command for printing out the lexical tokens of a smali file

This is useful for creating/updating the .tokens file for the lexer tests.

smali/src/main/java/org/jf/smali/Main.java

@@ -81,6 +81,7 @@ public class Main extends Command {
         List<JCommander> commandHierarchy = main.getCommandHierarchy();
 
         ExtendedCommands.addExtendedCommand(jc, new AssembleCommand(commandHierarchy));
+        ExtendedCommands.addExtendedCommand(jc, new PrintTokensCommand(commandHierarchy));
         ExtendedCommands.addExtendedCommand(jc, new HelpCommand(commandHierarchy));
         ExtendedCommands.addExtendedCommand(jc, new HlepCommand(commandHierarchy));
 

smali/src/main/java/org/jf/smali/PrintTokensCommand.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2019, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.jf.smali;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.jf.util.jcommander.Command;
+import org.jf.util.jcommander.ExtendedParameter;
+import org.jf.util.jcommander.ExtendedParameters;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+import java.util.List;
+
+@Parameters(
+        commandDescription = "Lexes the given smali file(s) and prints the tokens.",
+        hidden = true)
+@ExtendedParameters(
+        commandName = "tokens")
+public class PrintTokensCommand extends Command {
+
+    @Parameter(names = {"-h", "-?", "--help"}, help = true,
+            description = "Show usage information for this command.")
+    private boolean help;
+
+    @Parameter(names = {"-a", "--api"},
+            description = "The numeric api level to use while assembling.")
+    @ExtendedParameter(argumentNames = "api")
+    private int apiLevel = 15;
+
+    @Parameter(description = "Assembles the given files. If a directory is specified, it will be " +
+            "recursively searched for any files with a .smali prefix")
+    @ExtendedParameter(argumentNames = "[<file>|<dir>]+")
+    private List<String> input;
+
+    public PrintTokensCommand(@Nonnull List<JCommander> commandAncestors) {
+        super(commandAncestors);
+    }
+
+    @Override public void run() {
+        if (help || input == null || input.isEmpty()) {
+            usage();
+            return;
+        }
+
+        try {
+            Smali.printTokens(getOptions(), input);
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    protected SmaliOptions getOptions() {
+        SmaliOptions options = new SmaliOptions();
+
+        options.apiLevel = apiLevel;
+
+        return options;
+    }
+}

smali/src/main/java/org/jf/smali/Smali.java

@@ -40,6 +40,7 @@ import org.antlr.runtime.tree.CommonTreeNodeStream;
 import org.jf.dexlib2.Opcodes;
 import org.jf.dexlib2.writer.builder.DexBuilder;
 import org.jf.dexlib2.writer.io.FileDataStore;
+import org.jf.util.StringUtils;
 
 import javax.annotation.Nonnull;
 import java.io.File;
@@ -47,7 +48,6 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Arrays;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
@@ -133,6 +133,47 @@ public class Smali {
         return true;
     }
 
+    /**
+     * Prints the lexical tokens for the given files.
+     *
+     * @param options a SmaliOptions object with the options to use
+     * @param input The files/directories to process
+     * @return true if assembly completed with no errors, or false if errors were encountered
+     */
+    public static boolean printTokens(final SmaliOptions options, List<String> input) throws IOException {
+        TreeSet<File> filesToProcessSet = new TreeSet<File>();
+
+        for (String fileToProcess: input) {
+            File argFile = new File(fileToProcess);
+
+            if (!argFile.exists()) {
+                throw new IllegalArgumentException("Cannot find file or directory \"" + fileToProcess + "\"");
+            }
+
+            if (argFile.isDirectory()) {
+                getSmaliFilesInDir(argFile, filesToProcessSet);
+            } else if (argFile.isFile()) {
+                filesToProcessSet.add(argFile);
+            }
+        }
+
+        boolean errors = false;
+
+        for (final File file: filesToProcessSet) {
+            try {
+                errors |= !printTokensForSingleFile(file, options);
+            } catch (Exception ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        if (errors) {
+            return false;
+        }
+
+        return true;
+    }
+
     private static void getSmaliFilesInDir(@Nonnull File dir, @Nonnull Set<File> smaliFiles) {
         File[] files = dir.listFiles();
         if (files != null) {
@@ -166,7 +207,13 @@ public class Smali {
                         continue;
                     }
 
-                    System.out.println(smaliParser.tokenNames[token.getType()] + ": " + token.getText());
+                    String tokenName;
+                    if (token.getType() == -1) {
+                        tokenName = "EOF";
+                    } else {
+                        tokenName = smaliParser.tokenNames[token.getType()];
+                    }
+                    System.out.println(tokenName + ": " + token.getText());
                 }
 
                 System.out.flush();
@@ -206,4 +253,40 @@ public class Smali {
             }
         }
     }
+
+    private static boolean printTokensForSingleFile(File smaliFile, SmaliOptions options)
+            throws Exception {
+        FileInputStream fis = null;
+        try {
+            fis = new FileInputStream(smaliFile);
+            InputStreamReader reader = new InputStreamReader(fis, "UTF-8");
+
+            LexerErrorInterface lexer = new smaliFlexLexer(reader, options.apiLevel);
+            ((smaliFlexLexer)lexer).setSourceFile(smaliFile);
+            CommonTokenStream tokens = new CommonTokenStream((TokenSource)lexer);
+            tokens.fill();
+
+            for (int i=0; i<tokens.size(); i++) {
+                Token token = tokens.get(i);
+                if (token.getChannel() == smaliParser.HIDDEN) {
+                    continue;
+                }
+
+                String tokenName;
+                if (token.getType() == -1) {
+                    tokenName = "EOF";
+                } else {
+                    tokenName = smaliParser.tokenNames[token.getType()];
+                }
+                System.out.println(tokenName + "(\"" + StringUtils.escapeString(token.getText()) + "\")");
+            }
+            System.out.flush();
+
+            return lexer.getNumberOfSyntaxErrors() == 0;
+        } finally {
+            if (fis != null) {
+                fis.close();
+            }
+        }
+    }
 }