From d762deacc9272a5806ac6c6ac9247c70d4c3c363 Mon Sep 17 00:00:00 2001 From: Ben Gruver Date: Mon, 16 Sep 2019 16:27:27 -0700 Subject: [PATCH] Add a hidden command for printing out the lexical tokens of a smali file This is useful for creating/updating the .tokens file for the lexer tests. --- smali/src/main/java/org/jf/smali/Main.java | 1 + .../java/org/jf/smali/PrintTokensCommand.java | 90 +++++++++++++++++++ smali/src/main/java/org/jf/smali/Smali.java | 87 +++++++++++++++++- 3 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 smali/src/main/java/org/jf/smali/PrintTokensCommand.java diff --git a/smali/src/main/java/org/jf/smali/Main.java b/smali/src/main/java/org/jf/smali/Main.java index 6b56fddb..429adc34 100644 --- a/smali/src/main/java/org/jf/smali/Main.java +++ b/smali/src/main/java/org/jf/smali/Main.java @@ -81,6 +81,7 @@ public class Main extends Command { List commandHierarchy = main.getCommandHierarchy(); ExtendedCommands.addExtendedCommand(jc, new AssembleCommand(commandHierarchy)); + ExtendedCommands.addExtendedCommand(jc, new PrintTokensCommand(commandHierarchy)); ExtendedCommands.addExtendedCommand(jc, new HelpCommand(commandHierarchy)); ExtendedCommands.addExtendedCommand(jc, new HlepCommand(commandHierarchy)); diff --git a/smali/src/main/java/org/jf/smali/PrintTokensCommand.java b/smali/src/main/java/org/jf/smali/PrintTokensCommand.java new file mode 100644 index 00000000..6bdf5dff --- /dev/null +++ b/smali/src/main/java/org/jf/smali/PrintTokensCommand.java @@ -0,0 +1,90 @@ +/* + * Copyright 2019, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.jf.smali; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import org.jf.util.jcommander.Command; +import org.jf.util.jcommander.ExtendedParameter; +import org.jf.util.jcommander.ExtendedParameters; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.List; + +@Parameters( + commandDescription = "Lexes the given smali file(s) and prints the tokens.", + hidden = true) +@ExtendedParameters( + commandName = "tokens") +public class PrintTokensCommand extends Command { + + @Parameter(names = {"-h", "-?", "--help"}, help = true, + description = "Show usage information for this command.") + private boolean help; + + @Parameter(names = {"-a", "--api"}, + description = "The numeric api level to use while assembling.") + @ExtendedParameter(argumentNames = "api") + private int apiLevel = 15; + + @Parameter(description = "Assembles the given files. If a directory is specified, it will be " + + "recursively searched for any files with a .smali prefix") + @ExtendedParameter(argumentNames = "[|]+") + private List input; + + public PrintTokensCommand(@Nonnull List commandAncestors) { + super(commandAncestors); + } + + @Override public void run() { + if (help || input == null || input.isEmpty()) { + usage(); + return; + } + + try { + Smali.printTokens(getOptions(), input); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + + protected SmaliOptions getOptions() { + SmaliOptions options = new SmaliOptions(); + + options.apiLevel = apiLevel; + + return options; + } +} diff --git a/smali/src/main/java/org/jf/smali/Smali.java b/smali/src/main/java/org/jf/smali/Smali.java index 855092ac..fbeefdea 100644 --- a/smali/src/main/java/org/jf/smali/Smali.java +++ b/smali/src/main/java/org/jf/smali/Smali.java @@ -40,6 +40,7 @@ import org.antlr.runtime.tree.CommonTreeNodeStream; import org.jf.dexlib2.Opcodes; import org.jf.dexlib2.writer.builder.DexBuilder; import org.jf.dexlib2.writer.io.FileDataStore; +import org.jf.util.StringUtils; import javax.annotation.Nonnull; import java.io.File; @@ -47,7 +48,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.util.Arrays; -import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -133,6 +133,47 @@ public class Smali { return true; } + /** + * Prints the lexical tokens for the given files. + * + * @param options a SmaliOptions object with the options to use + * @param input The files/directories to process + * @return true if assembly completed with no errors, or false if errors were encountered + */ + public static boolean printTokens(final SmaliOptions options, List input) throws IOException { + TreeSet filesToProcessSet = new TreeSet(); + + for (String fileToProcess: input) { + File argFile = new File(fileToProcess); + + if (!argFile.exists()) { + throw new IllegalArgumentException("Cannot find file or directory \"" + fileToProcess + "\""); + } + + if (argFile.isDirectory()) { + getSmaliFilesInDir(argFile, filesToProcessSet); + } else if (argFile.isFile()) { + filesToProcessSet.add(argFile); + } + } + + boolean errors = false; + + for (final File file: filesToProcessSet) { + try { + errors |= !printTokensForSingleFile(file, options); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + if (errors) { + return false; + } + + return true; + } + private static void getSmaliFilesInDir(@Nonnull File dir, @Nonnull Set smaliFiles) { File[] files = dir.listFiles(); if (files != null) { @@ -166,7 +207,13 @@ public class Smali { continue; } - System.out.println(smaliParser.tokenNames[token.getType()] + ": " + token.getText()); + String tokenName; + if (token.getType() == -1) { + tokenName = "EOF"; + } else { + tokenName = smaliParser.tokenNames[token.getType()]; + } + System.out.println(tokenName + ": " + token.getText()); } System.out.flush(); @@ -206,4 +253,40 @@ public class Smali { } } } + + private static boolean printTokensForSingleFile(File smaliFile, SmaliOptions options) + throws Exception { + FileInputStream fis = null; + try { + fis = new FileInputStream(smaliFile); + InputStreamReader reader = new InputStreamReader(fis, "UTF-8"); + + LexerErrorInterface lexer = new smaliFlexLexer(reader, options.apiLevel); + ((smaliFlexLexer)lexer).setSourceFile(smaliFile); + CommonTokenStream tokens = new CommonTokenStream((TokenSource)lexer); + tokens.fill(); + + for (int i=0; i