jimple2cpg: improvement for large Android apk loading (#2331)

This commit address two issues:

While loading Android apk with soot's src_prec_apk_c_j, it takes more time than src_prec_apk. In my experiment, It takes about 70s to process a modern (large, ~ 120MB, with ~ 100,000 classes) apk in Scene.v().loadNecessaryClasses(), however it only takes about 7s, which is 10x faster, to process the same file with src_prec_apk. So I add an option to accept the path to android.jar to make it work.

When Soot creates JimpleBody for some methods, it may throws exception StmtSwitch: type of throw argument is not a RefType. This problem is discussed in Exception in Jimple Body Creation with original names: IllegalStateException: UnitThrowAnalysis StmtSwitch: type of throw argument is not a RefType! soot-oss/soot#1256, but AFAIK it's not yet fixed, so the workaround is to disable jb.use-original-names when processing apk, since in most cases there are no original names when dealing with bytecode anyway.

joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/Jimple2Cpg.scala

@@ -10,7 +10,7 @@ import io.joern.x2cpg.{SourceFiles, X2CpgFrontend}
 import io.shiftleft.codepropertygraph.Cpg
 import org.slf4j.LoggerFactory
 import soot.options.Options
-import soot.{G, PhaseOptions, Scene}
+import soot.{G, PackManager, Scene}
 
 import java.io.{File => JFile}
 import java.nio.file.Paths
@@ -53,15 +53,21 @@ class Jimple2Cpg extends X2CpgFrontend[Config] {
 
   private val logger = LoggerFactory.getLogger(classOf[Jimple2Cpg])
 
-  def sootLoadApk(input: String, framework: String = null): Unit = {
+  def sootLoadApk(input: String, framework: Option[String] = None): Unit = {
     Options.v().set_process_dir(List(input).asJava)
-    if (framework != null) {
-      Options.v().set_src_prec(Options.src_prec_apk)
-      Options.v().set_force_android_jar(framework)
-    } else {
-      Options.v().set_src_prec(Options.src_prec_apk_c_j)
+    framework match {
+      case Some(value) if value.nonEmpty => {
+        Options.v().set_src_prec(Options.src_prec_apk)
+        Options.v().set_force_android_jar(value)
+      }
+      case _ => {
+        Options.v().set_src_prec(Options.src_prec_apk_c_j)
+      }
     }
     Options.v().set_process_multiple_dex(true)
+    // workaround for Soot's bug while parsing large apk.
+    // see: https://github.com/soot-oss/soot/issues/1256
+    Options.v().setPhaseOption("jb", "use-original-names:false")
   }
 
   def sootLoadClass(inputDir: String): Unit = {
@@ -109,7 +115,7 @@ class Jimple2Cpg extends X2CpgFrontend[Config] {
         val ext = config.inputPath.split("\\.").lastOption.getOrElse("")
         ext match {
           case "jar" | "zip"      => sootLoadClass(config.inputPath)
-          case "apk" | "dex"      => sootLoadApk(config.inputPath)
+          case "apk" | "dex"      => sootLoadApk(config.inputPath, config.android)
           case "jimple" | "class" => sootLoadSource(config.inputPath, ext)
           // case "war" => sootLoadClass(unpackPath/WEB-INF/classes)
           case _ => {
@@ -125,6 +131,7 @@ class Jimple2Cpg extends X2CpgFrontend[Config] {
         new TypeNodePass(astCreator.global.usedTypes.keys().asScala.toList, cpg)
           .createAndApply()
       }
+
       // Clear classes from Soot
       G.reset()
     }
@@ -166,8 +173,8 @@ class Jimple2Cpg extends X2CpgFrontend[Config] {
     Options.v().set_no_bodies_for_excluded(true)
     Options.v().set_allow_phantom_refs(true)
     // keep variable names
-    Options.v.setPhaseOption("jb.sils", "enabled:false")
-    PhaseOptions.v().setPhaseOption("jb", "use-original-names:true")
+    Options.v().setPhaseOption("jb.sils", "enabled:false")
+    Options.v().setPhaseOption("jb", "use-original-names:true")
     // output jimple
     Options.v().set_output_format(Options.output_format_jimple)
     Options.v().set_output_dir(ProgramHandlingUtil.getUnpackingDir.toString)

joern-cli/frontends/jimple2cpg/src/main/scala/io/joern/jimple2cpg/Main.scala

@@ -6,8 +6,11 @@ import scopt.OParser
 
 /** Command line configuration parameters
   */
-final case class Config(inputPath: String = "", outputPath: String = X2CpgConfig.defaultOutputPath)
-    extends X2CpgConfig[Config] {
+final case class Config(
+  inputPath: String = "",
+  outputPath: String = X2CpgConfig.defaultOutputPath,
+  android: Option[String] = None
+) extends X2CpgConfig[Config] {
 
   override def withInputPath(inputPath: String): Config =
     copy(inputPath = inputPath)
@@ -20,8 +23,13 @@ private object Frontend {
 
   val cmdLineParser: OParser[Unit, Config] = {
     val builder = OParser.builder[Config]
-    import builder.programName
-    OParser.sequence(programName("jimple2cpg"))
+    import builder._
+    OParser.sequence(
+      programName("jimple2cpg"),
+      opt[String]("android")
+        .text("Optional path to android.jar while processing apk file.")
+        .action((android, config) => config.copy(android = Option(android)))
+    )
   }
 }