Use new script manager. (#178)

* Update gitignore to ignore preprocessor.

* Update Joern to use the latest script manager.

* Remove unused project.

* Symlink to scripts folder.

* Run scalafmt.

.gitignore

@@ -9,7 +9,7 @@ fuzzyc2cpg
 .idea/
 joern-cli.zip
 joern-server.zip
-fuzzyppcli-dist
+fuzzyppcli/
 cpg.bin.zip
 
 

build.sbt

@@ -4,8 +4,8 @@ name := "joern"
 organization := "io.shiftleft"
 ThisBuild / scalaVersion := "2.12.8"
 
-val cpgVersion = "0.10.126"
-val fuzzyc2cpgVersion = "1.1.12"
+val cpgVersion = "0.10.131"
+val fuzzyc2cpgVersion = "1.1.13"
 
 ThisBuild / resolvers += Resolver.mavenLocal
 

joern-cli/build.sbt

@@ -1,5 +1,3 @@
-import java.io.FilenameFilter
-
 enablePlugins(JavaAppPackaging)
 enablePlugins(UniversalPlugin)
 

scripts/ast-for-funcs-dump/ast-for-funcs-dump.scala → joern-cli/src/main/resources/scripts/ast-for-funcs-dump.sc

@@ -72,19 +72,19 @@
               // ...
  */
 
-import java.io._
-
-import scala.collection.JavaConverters._
-
-import io.circe.syntax._
+import gremlin.scala._
 import io.circe.generic.semiauto._
+import io.circe.syntax._
 import io.circe.{Encoder, Json}
+import org.apache.tinkerpop.gremlin.structure.{Edge, VertexProperty}
 
 import io.shiftleft.codepropertygraph.generated.nodes.AstNode
+import io.shiftleft.joern.console.Console.cpg
+import io.shiftleft.semanticcpg.language._
 
-import gremlin.scala._
-import org.apache.tinkerpop.gremlin.structure.Edge
-import org.apache.tinkerpop.gremlin.structure.VertexProperty
+import java.io._
+
+import scala.collection.JavaConverters._
 
 final case class AstForFuncsFunction(function: String, id: String, AST: List[AstNode])
 

scripts/ast-for-funcs/ast-for-funcs.scala → joern-cli/src/main/resources/scripts/ast-for-funcs.sc

@@ -71,17 +71,17 @@
               // ...
  */
 
-import scala.collection.JavaConverters._
-
-import io.circe.syntax._
+import gremlin.scala._
 import io.circe.generic.semiauto._
+import io.circe.syntax._
 import io.circe.{Encoder, Json}
+import org.apache.tinkerpop.gremlin.structure.{Edge, VertexProperty}
 
 import io.shiftleft.codepropertygraph.generated.nodes.AstNode
+import io.shiftleft.joern.console.Console.cpg
+import io.shiftleft.semanticcpg.language._
 
-import gremlin.scala._
-import org.apache.tinkerpop.gremlin.structure.Edge
-import org.apache.tinkerpop.gremlin.structure.VertexProperty
+import scala.collection.JavaConverters._
 
 final case class AstForFuncsFunction(function: String, id: String, AST: List[AstNode])
 final case class AstForFuncsResult(functions: List[AstForFuncsFunction])

scripts/cfg-for-funcs-dump/cfg-for-funcs-dump.scala → joern-cli/src/main/resources/scripts/cfg-for-funcs-dump.sc

@@ -68,20 +68,20 @@
               // ...
  */
 
-import java.io._
-
-import scala.collection.JavaConverters._
-import io.circe.syntax._
+import gremlin.scala._
 import io.circe.generic.semiauto._
+import io.circe.syntax._
 import io.circe.{Encoder, Json}
+import org.apache.tinkerpop.gremlin.structure.{Edge, VertexProperty}
 
+import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes}
+import io.shiftleft.joern.console.Console.cpg
+import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.language.types.expressions.generalizations.CfgNode
-import io.shiftleft.codepropertygraph.generated.EdgeTypes
-import io.shiftleft.codepropertygraph.generated.nodes
 
-import gremlin.scala._
-import org.apache.tinkerpop.gremlin.structure.Edge
-import org.apache.tinkerpop.gremlin.structure.VertexProperty
+import java.io._
+
+import scala.collection.JavaConverters._
 
 final case class CfgForFuncsFunction(function: String, id: String, CFG: List[nodes.CfgNode])
 

scripts/cfg-for-funcs/cfg-for-funcs.scala → joern-cli/src/main/resources/scripts/cfg-for-funcs.sc

@@ -68,19 +68,18 @@
               // ...
  */
 
-import scala.collection.JavaConverters._
-
-import io.circe.syntax._
+import gremlin.scala._
 import io.circe.generic.semiauto._
+import io.circe.syntax._
 import io.circe.{Encoder, Json}
+import org.apache.tinkerpop.gremlin.structure.{Edge, VertexProperty}
 
+import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes}
+import io.shiftleft.joern.console.Console.cpg
+import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.language.types.expressions.generalizations.CfgNode
-import io.shiftleft.codepropertygraph.generated.EdgeTypes
-import io.shiftleft.codepropertygraph.generated.nodes
 
-import gremlin.scala._
-import org.apache.tinkerpop.gremlin.structure.Edge
-import org.apache.tinkerpop.gremlin.structure.VertexProperty
+import scala.collection.JavaConverters._
 
 final case class CfgForFuncsFunction(function: String, id: String, CFG: List[nodes.CfgNode])
 final case class CfgForFuncsResult(functions: List[CfgForFuncsFunction])

scripts/cfgToDot/cfgToDot.scala → joern-cli/src/main/resources/scripts/cfgToDot.sc

@@ -34,13 +34,14 @@
    }
  */
 
-import java.nio.file.Paths
-
 import gremlin.scala._
-import io.shiftleft.codepropertygraph.generated._
-import io.shiftleft.semanticcpg.utils.ExpandTo
 import org.apache.tinkerpop.gremlin.structure.Direction
+
 import io.shiftleft.Implicits.JavaIteratorDeco
+import io.shiftleft.codepropertygraph.generated._
+import io.shiftleft.joern.console.Console.cpg
+
+import java.nio.file.Paths
 
 /** Some helper functions: adapted from ReachingDefPass.scala in codeproperty graph repo */
 def vertexToStr(vertex: Vertex): String = {

scripts/graph-for-funcs/graph-for-funcs.scala → joern-cli/src/main/resources/scripts/graph-for-funcs.sc

@@ -26,15 +26,16 @@ import io.circe.syntax._
 import io.circe.generic.semiauto._
 import io.circe.{Encoder, Json}
 
-import io.shiftleft.dataflowengine.language._
 import io.shiftleft.semanticcpg.language.types.expressions.generalizations.CfgNode
 import io.shiftleft.codepropertygraph.generated.EdgeTypes
 import io.shiftleft.codepropertygraph.generated.NodeTypes
 import io.shiftleft.codepropertygraph.generated.nodes
+import io.shiftleft.dataflowengine.language._
+import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.language.types.expressions.Call
 import io.shiftleft.semanticcpg.language.types.structure.Local
 import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn
-
+import io.shiftleft.joern.console.Console.cpg
 import gremlin.scala._
 import org.apache.tinkerpop.gremlin.structure.Edge
 import org.apache.tinkerpop.gremlin.structure.VertexProperty

scripts/list-funcs/list-funcs.scala → joern-cli/src/main/resources/scripts/list-funcs.sc

@@ -14,4 +14,7 @@
    List("<operator>.indirectMemberAccess", "<operator>.assignment", "free_list", "free", "<operator>.notEquals")
  */
 
+import io.shiftleft.joern.console.Console.cpg
+import io.shiftleft.semanticcpg.language._
+
 cpg.method.name.l

scripts/pdg-for-funcs-dump/pdg-for-funcs-dump.scala → joern-cli/src/main/resources/scripts/pdg-for-funcs-dump.sc

@@ -64,24 +64,23 @@
               // ...
  */
 
-import java.io._
-
-import scala.collection.JavaConverters._
-import io.circe.syntax._
+import gremlin.scala._
 import io.circe.generic.semiauto._
+import io.circe.syntax._
 import io.circe.{Encoder, Json}
+import org.apache.tinkerpop.gremlin.structure.{Edge, VertexProperty}
 
-import io.shiftleft.dataflowengine.language._
 import io.shiftleft.codepropertygraph.generated.nodes.MethodParameterIn
+import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, nodes}
+import io.shiftleft.dataflowengine.language._
+import io.shiftleft.joern.console.Console.cpg
+import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.language.types.expressions.Call
 import io.shiftleft.semanticcpg.language.types.structure.Local
-import io.shiftleft.codepropertygraph.generated.EdgeTypes
-import io.shiftleft.codepropertygraph.generated.NodeTypes
-import io.shiftleft.codepropertygraph.generated.nodes
 
-import gremlin.scala._
-import org.apache.tinkerpop.gremlin.structure.Edge
-import org.apache.tinkerpop.gremlin.structure.VertexProperty
+import java.io._
+
+import scala.collection.JavaConverters._
 
 implicit val encodeFuncFunction: Encoder[PdgForFuncsFunction] = deriveEncoder
 

joern-cli/src/main/resources/scripts/scripts.json

@@ -0,0 +1,40 @@
+{
+  "scripts": [
+    {
+        "name": "ast-for-funcs",
+        "description": "Returns the corresponding AST for each function as Json object."
+    },
+    {
+      "name": "ast-for-funcs-dump",
+      "description": "Prints the corresponding AST for each function as Json string to a file."
+    },
+    {
+      "name": "cfg-for-funcs",
+      "description": "Returns the corresponding CFG for each function as Json object."
+    },
+    {
+      "name": "cfg-for-funcs-dump",
+      "description": "Prints the corresponding CFG for each function as Json string to a file."
+    },
+    {
+      "name": "cfgToDot",
+      "description": "Generates a dot graph for your cfg."
+    },
+    {
+      "name": "functions-to-dot",
+      "description": "Generates a list of strings, each containing a dot graph for every internal function in the provided CPG."
+    },
+    {
+      "name": "graph-for-funcs",
+      "description": "Returns the corresponding graph (AST+CFG+PDG) for each function as Json object."
+    },
+    {
+      "name": "list-funcs",
+      "description": "Lists all functions."
+    },
+    {
+      "name": "pdg-for-funcs-dump",
+      "description": "Prints the corresponding PDG for each function as Json string to a file."
+    }
+  ]
+}

joern-cli/src/main/scala/io/shiftleft/joern/JoernScriptExecutor.scala

@@ -37,12 +37,12 @@ class JoernScriptExecutor extends CpgQueryExecutor[AnyRef] {
   }
 
   override def executeQuery(cpg: Cpg, query: String): IO[UUID] =
-    ??? // unused within Joern
+    ??? // unused in the Joern CLI
 
   override def executeQuerySync(cpg: Cpg, query: String): IO[CpgOperationResult[AnyRef]] =
     underlying.executeQuerySync(cpg, query)
 
   override def retrieveQueryResult(queryId: UUID): OptionT[IO, CpgOperationResult[AnyRef]] =
-    ??? // unused within Joern
+    ??? // unused in the Joern CLI
 
 }

joern-cli/src/main/scala/io/shiftleft/joern/JoernScriptManager.scala

@@ -1,30 +1,5 @@
 package io.shiftleft.joern
 
-import cats.effect.IO
-import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.console.ScriptManager
-import io.shiftleft.console.query.{CpgOperationFailure, CpgOperationResult, CpgOperationSuccess}
 
-class JoernScriptManager(executor: JoernScriptExecutor = new JoernScriptExecutor()) extends ScriptManager(executor) {
-
-  private def scriptContent(name: String): String =
-    (DEFAULT_SCRIPTS_FOLDER / name / s"$name.scala").lines.mkString(System.lineSeparator())
-
-  private def handleQueryResult(result: IO[CpgOperationResult[AnyRef]]): AnyRef = {
-    val scriptExecutionResult = for {
-      queryResult <- result
-      result <- queryResult match {
-        case CpgOperationSuccess(result) => IO(result)
-        case CpgOperationFailure(ex)     => IO.raiseError[AnyRef](ex)
-      }
-    } yield result
-    scriptExecutionResult.handleErrorWith(IO(_)).unsafeRunSync()
-  }
-
-  def runScript(name: String, cpgFilename: String): AnyRef =
-    handleQueryResult(executor.executeQuerySync(CpgLoader.load(cpgFilename), scriptContent(name)))
-
-  def runScript(name: String, cpg: Cpg): AnyRef =
-    handleQueryResult(executor.executeQuerySync(cpg, scriptContent(name)))
-
-}
+class JoernScriptManager(executor: JoernScriptExecutor = new JoernScriptExecutor()) extends ScriptManager(executor)

joern-server/build.sbt

@@ -2,7 +2,7 @@ enablePlugins(JavaAppPackaging)
 enablePlugins(UniversalPlugin)
 
 organization := "io.shiftleft"
-name := "Joern Server"
+name := "joern-server"
 maintainer := "[email protected]"
 
 resolvers += Classpaths.typesafeReleases

project/Projects.scala

@@ -1,7 +1,6 @@
-
 import sbt._
 
 object Projects {
-       lazy val joerncli = project.in(file("joern-cli"))
-       lazy val joernserver = project.in(file("joern-server"))
+  lazy val joerncli = project.in(file("joern-cli"))
+  lazy val joernserver = project.in(file("joern-server"))
 }

scripts

@@ -0,0 +1 @@
+joern-cli/src/main/resources/scripts