integrate querydb to this repo to avoid cyclic dependency (#649)

* move querydb over here, integrate build.sbt

* adjust path

* maintainer config

* cleanup/refactor createDistribution

* drop log4j dep to avoid multiple logging implementations

* joern/createDistribution: invoke and include querydb/createDistribution

* WIP - old version of querydb/install.sh

* WIP

* update joernscan

* github actions config: run querydb/createDistribution, upload querydb

* readme

* fmt

.github/workflows/release.yml

@@ -67,3 +67,12 @@ jobs:
           asset_path: target/joern-cli.zip.sha512
           asset_name: joern-cli.zip.sha512
           asset_content_type: text/plain
+      - name: Upload querydb.zip
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: querydb/target/querydb.zip
+          asset_name: querydb.zip
+          asset_content_type: application/zip

build.sbt

@@ -1,14 +1,16 @@
 name := "joern"
-organization := "io.joern"
+ThisBuild / organization := "io.joern"
 ThisBuild / scalaVersion := "2.13.5"
 // don't upgrade to 2.13.6 until https://github.com/com-lihaoyi/Ammonite/issues/1182 is resolved
-ThisBuild /Test /fork := true
+
 val cpgVersion = "1.3.388"
 val ghidra2cpgVersion = "0.0.47"
 val js2cpgVersion = "0.2.21"
 val javasrc2cpgVersion = "0.0.14"
 val jimple2cpgVersion = "0.0.7"
 
+ThisBuild /Test /fork := true
+
 ThisBuild / resolvers ++= Seq(
   Resolver.mavenLocal,
   Resolver.jcenterRepo,
@@ -25,12 +27,15 @@ homepage := Some(url("https://joern.io/"))
 licenses := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0"))
 
 lazy val joerncli = project.in(file("joern-cli"))
+lazy val querydb = project.in(file("querydb"))
 
 lazy val createDistribution = taskKey[File]("Create a complete Joern distribution")
 createDistribution := {
   val distributionFile = file("target/joern-cli.zip")
   val zip = (joerncli/Universal/packageBin).value
+
   IO.copyFile(zip, distributionFile)
+  val querydbDistribution = (querydb/createDistribution).value
 
   println(s"created distribution - resulting files: $distributionFile")
   distributionFile

joern-cli/src/main/scala/io/shiftleft/joern/JoernScan.scala

@@ -18,7 +18,7 @@ import scala.jdk.CollectionConverters._
 import scala.reflect.runtime.universe._
 
 object JoernScanConfig {
-  val defaultDbVersion: String = "0.0.108"
+  val defaultDbVersion: String = "latest"
 }
 
 case class JoernScanConfig(src: String = "",
@@ -213,9 +213,9 @@ object JoernScan extends App with BridgeBase {
 
   private def urlForVersion(version: String): String = {
     if (version == "latest") {
-      "https://github.com/joernio/query-database/releases/latest/download/querydb.zip"
+      "https://github.com/joernio/joern/releases/latest/download/querydb.zip"
     } else {
-      s"https://github.com/joernio/query-database/releases/download/v$version/querydb.zip"
+      s"https://github.com/joernio/joern/releases/download/v$version/querydb.zip"
     }
   }
 

querydb-install.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+set -o nounset
+set -eu
+
+if [ "$(uname)" = 'Darwin' ]; then
+  # get script location
+  # https://unix.stackexchange.com/a/96238
+  if [ "${BASH_SOURCE:-x}" != 'x' ]; then
+    this_script=$BASH_SOURCE
+  elif [ "${ZSH_VERSION:-x}" != 'x' ]; then
+    setopt function_argzero
+    this_script=$0
+  elif eval '[[ -n ${.sh.file} ]]' 2>/dev/null; then
+    eval 'this_script=${.sh.file}'
+  else
+    echo 1>&2 "Unsupported shell. Please use bash, ksh93 or zsh."
+    exit 2
+  fi
+  relative_directory=$(dirname "$this_script")
+  SCRIPT_ABS_DIR=$(cd "$relative_directory" && pwd)
+else
+  SCRIPT_ABS_PATH=$(readlink -f "$0")
+  SCRIPT_ABS_DIR=$(dirname "$SCRIPT_ABS_PATH")
+fi
+
+# Check required tools are installed.
+check_installed() {
+  if ! type "$1" > /dev/null; then
+    echo "Please ensure you have $1 installed."
+    exit 1
+  fi
+}
+
+
+# https://stackoverflow.com/a/28085062
+: ${DEFAULT_JOERN_INSTALL_DIR:=$PWD/joern-cli/target/universal/stage}
+echo "where is the joern distribution installed?
+note: you can e.g. create one locally using 'sbt stage'
+[$DEFAULT_JOERN_INSTALL_DIR]:"
+read JOERN_INSTALL_DIR
+if [ -z "$JOERN_INSTALL_DIR" ]; then
+  JOERN_INSTALL_DIR=$DEFAULT_JOERN_INSTALL_DIR
+fi
+
+echo "building the plugin"
+sbt querydb/createDistribution
+readonly QUERYDB_ZIP=$PWD/querydb/target/querydb.zip
+
+echo "Installing plugin"
+pushd $JOERN_INSTALL_DIR
+  ./joern --remove-plugin querydb
+  ./joern --add-plugin $QUERYDB_ZIP
+popd

querydb/README.md

@@ -0,0 +1,155 @@
+# Joern Query Database ("Joern-Scan")
+
+This is the central query database for the open-source code analysis
+platform [Joern](https://github.com/ShiftLeftSecurity/joern). It has
+two purposes:
+
+* It provides the batteries required to turn Joern into a ready-to-run code scanning tool.
+* Its queries serve as examples useful for those looking to write their own queries.
+* [built on JDK11](https://github.com/ShiftLeftSecurity/overflowdb/blob/master/.github/workflows/release.yml) but runs on JRE >= 1.8
+
+The query database is distributed as a standalone library that
+includes Joern as a dependency. This means that it is not necessary to
+install Joern to make use of the queries in the database.
+
+At the same time, the database is a Joern extension, that is, when
+dynamically loaded at startup, its functionality becomes available on
+the interactive Joern shell and in Joern scripts.
+
+You can fork this project to build your own custom queries and
+scanners or kindly send a PR to to this repo to have them considered
+for inclusion in the default distribution.
+
+## Installing and running
+
+The installation script downloads joern and installs it in a sub-directory.
+The query database is installed as an extension.
+
+```
+./install.sh
+```
+
+You can run all queries as follows:
+
+```
+./joern-scan path/to/code
+```
+
+For example,
+
+```
+mkdir foo
+echo "int foo(int a, int b, int c, int d, int e, int f) {}" > foo/foo.c
+./joern-scan foo
+```
+
+runs all queries on the sample code in the directory `foo`, determining that the function `foo`
+has too many parameters.
+
+## Adding your own queries
+
+Please follow the rules below for a tear-free query writing experience:
+
+* Queries in the package `io.joern.scanners` are picked up automatically at runtime,
+  so please put your queries there.
+* Each query must begin with the annotation `@q` and must be placed in a query bundle.
+  A query bundle is simply an `object` that derives from `QueryBundle`
+* Queries can have parameters,but you must provide a default value for each parameter
+* Please add unit tests for queries. These also serve as a spec for what your query does.
+* Please format the code before sending a PR using `sbt scalafmt` and `sbt test:scalafmt`
+
+Take a look at the query bundle `Metrics` at `src/main/scala/io/joern/scanners/c/Metrics.scala`
+as an example:
+
+```
+object Metrics extends QueryBundle {
+
+  @q
+  def tooManyParameters(n: Int = 4): Query =
+    Query.make(
+      name = "too-many-params",
+      author = Crew.fabs,
+      title = s"Number of parameters larger than $n",
+      description = s"This query identifies functions with more than $n formal parameters",
+      score = 1.0,
+      withStrRep({ cpg =>
+        cpg.method.internal.filter(_.parameter.size > n)
+      }),
+      tags = List(QueryTags.metrics)
+    )
+
+  @q
+  def tooHighComplexity(n: Int = 4): Query =
+    Query.make(
+      name = "too-high-complexity",
+      author = Crew.fabs,
+      title = s"Cyclomatic complexity higher than $n",
+      description = s"This query identifies functions with a cyclomatic complexity higher than $n",
+      score = 1.0,
+      withStrRep({ cpg =>
+        cpg.method.internal.filter(_.controlStructure.size > n)
+      }),
+      tags = List(QueryTags.metrics)
+    )
+  ...
+}
+```
+
+Corresponding tests for queries are located in
+`src/test/scala/io/joern/scanners`. For example, tests for the metrics
+queries are located in
+`src/test/scala/io/joern/scanners/c/MetricsTests.scala`:
+
+```
+class MetricsTests extends Suite {
+
+  override val code = """
+    int too_many_params(int a, int b, int c, int d, int e) {
+    }
+	...
+	"""
+
+  "find functions with too many parameters" in {
+    Metrics.tooManyParameters(4)(cpg).map(_.evidence) match {
+      case List(List(method: nodes.Method)) =>
+        method.name shouldBe "too_many_params"
+      case _ => fail
+    }
+  }
+  ...
+}
+```
+
+These tests can be run individually from the IntelliJ IDE during query
+development.
+
+## Building/Testing the database
+
+We use the Scala Build Tool (sbt). Please make sure you have sbt
+installed. The version does not matter as sbt will fetch the required
+version based on the build file (`build.sbt`).
+
+Once `sbt` is installed, you can build and test the database as
+follows:
+
+```
+sbt test
+```
+
+You can test newly developed queries 
+
+If you want to test newly created queries with `joern-scan` as follows:
+
+```
+sbt joerncli/stage
+./querydb-install.sh && ./joern-scan <src>
+```
+
+## Exporting the database to JSON
+
+After running `install.sh`, you can launch
+```
+./joern-scan --dump
+```
+to create a file named `querydb.json` that contains the list of all available queries
+along with its meta information.

querydb/build.sbt

@@ -0,0 +1,80 @@
+name := "querydb"
+
+enablePlugins(JavaAppPackaging)
+
+libraryDependencies ++= Seq(
+  "com.lihaoyi" %% "sourcecode" % "0.1.9",
+  "com.lihaoyi" %% "upickle" % "1.2.2",
+  "com.github.pathikrit" %% "better-files" % "3.8.0",
+  "com.github.scopt" %% "scopt" % "3.7.1",
+  "io.joern" %% "ghidra2cpg" % Versions.ghidra2cpg,
+  "io.shiftleft" %% "semanticcpg" % Versions.cpg,
+  "io.shiftleft" %% "console" % Versions.cpg,
+  "io.shiftleft" %% "dataflowengineoss" % Versions.cpg,
+  "io.shiftleft" %% "fuzzyc2cpg-tests" % Versions.cpg % Test classifier "tests",
+  "io.shiftleft" %% "c2cpg-tests" % Versions.cpg % Test classifier "tests",
+  "io.shiftleft" %% "semanticcpg-tests" % Versions.cpg % Test classifier "tests",
+  "io.shiftleft" %% "fuzzyc2cpg" % Versions.cpg % Test,
+  "io.shiftleft" %% "c2cpg" % Versions.cpg % Test,
+  "org.scalatest" %% "scalatest" % "3.1.1" % Test,
+  "io.joern" %% "ghidra2cpg-tests" % Versions.ghidra2cpg % Test classifier "tests"
+)
+
+Compile/doc/sources := Seq.empty
+Compile/packageDoc/publishArtifact := false
+
+topLevelDirectory := Some(name.value)
+
+lazy val createDistribution = taskKey[File]("Create binary distribution of extension")
+createDistribution := {
+  import better.files._
+  val pkgBin = (Universal/packageBin).value
+  val tmpDstArchive = "/tmp/querydb.zip"
+  val dstArchive = (target.value / "querydb.zip").toScala
+  if (dstArchive.exists) dstArchive.delete()
+  IO.copy(
+    List((pkgBin, file(tmpDstArchive))),
+    CopyOptions(overwrite = true, preserveLastModified = true, preserveExecutable = true)
+  )
+
+  File.usingTemporaryDirectory("querydb") { dir =>
+    File(tmpDstArchive).unzipTo(dir)
+    dir.listRecursively.filter { x =>
+      val name = x.toString
+      name.contains("org.scala") ||
+      name.contains("net.sf.trove4") ||
+      name.contains("com.google.guava") ||
+      name.contains("org.apache.logging") ||
+      name.contains("com.google.protobuf") ||
+      name.contains("com.lihaoyi") ||
+      name.contains("io.shiftleft") ||
+      name.contains("org.typelevel") ||
+      name.contains("io.undertow") ||
+      name.contains("com.chuusai") ||
+      name.contains("io.get-coursier") ||
+      name.contains("io.circe") ||
+      name.contains("net.java.dev") ||
+      name.contains("com.github.javaparser") ||
+      name.contains("org.javassist") ||
+      name.contains("com.lihaoyi.ammonite") ||
+      name.contains("io.joern.ghidra2cpg") ||
+      name.contains("net.oneandone")
+    }.foreach(_.delete())
+    dir.zipTo(dstArchive)
+    File(tmpDstArchive).delete()
+  }
+  println(s"created distribution - resulting files: $dstArchive")
+
+  dstArchive.toJava
+}
+
+Compile/scalacOptions ++= Seq(
+  "-Xfatal-warnings",
+  "-feature",
+  "-deprecation",
+  "-language:implicitConversions",
+)
+
+fork := true
+
+maintainer := "[email protected]"