From 4b5da19825d73283d7ba73a97ff39f5971b93949 Mon Sep 17 00:00:00 2001 From: Xavier Pinho Date: Sun, 19 Jan 2025 12:01:39 +0000 Subject: [PATCH] [c#] refactor local summary creator into its own util (#5236) --- .../joern/csharpsrc2cpg/CSharpSrc2Cpg.scala | 37 +--------------- .../utils/ProgramSummaryCreator.scala | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/ProgramSummaryCreator.scala diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/CSharpSrc2Cpg.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/CSharpSrc2Cpg.scala index 0bb524516293..37929aebb057 100644 --- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/CSharpSrc2Cpg.scala +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/CSharpSrc2Cpg.scala @@ -5,7 +5,7 @@ import io.joern.csharpsrc2cpg.astcreation.AstCreator import io.joern.csharpsrc2cpg.datastructures.CSharpProgramSummary import io.joern.csharpsrc2cpg.parser.DotNetJsonParser import io.joern.csharpsrc2cpg.passes.{AstCreationPass, DependencyPass} -import io.joern.csharpsrc2cpg.utils.{DependencyDownloader, DotNetAstGenRunner} +import io.joern.csharpsrc2cpg.utils.{DependencyDownloader, DotNetAstGenRunner, ProgramSummaryCreator} import io.joern.x2cpg.X2Cpg.withNewEmptyCpg import io.joern.x2cpg.astgen.AstGenRunner.AstGenRunnerResult import io.joern.x2cpg.astgen.ParserResult @@ -35,13 +35,7 @@ class CSharpSrc2Cpg extends X2CpgFrontend[Config] { File.usingTemporaryDirectory("csharpsrc2cpgOut") { tmpDir => val astGenResult = new DotNetAstGenRunner(config).execute(tmpDir) val astCreators = CSharpSrc2Cpg.processAstGenRunnerResults(astGenResult.parsedFiles, config) - // Pre-parse the AST creators for high level structures - val internalProgramSummary = summarizeAstCreators(astCreators) - // Load built-in and/or external summaries for missing imports - val builtinSummary = createBuiltinSummary(config, internalProgramSummary) - val internalAndBuiltinSummary = internalProgramSummary ++= builtinSummary - val externalSummary = createExternalSummary(config, internalAndBuiltinSummary) - val localSummary = internalAndBuiltinSummary ++= externalSummary + val localSummary = ProgramSummaryCreator.from(astCreators, config) val hash = HashUtil.sha256(astCreators.map(_.parserResult).map(x => Paths.get(x.fullPath))) new MetaDataPass(cpg, Languages.CSHARPSRC, config.inputPath, Option(hash)).createAndApply() @@ -61,33 +55,6 @@ class CSharpSrc2Cpg extends X2CpgFrontend[Config] { } } - private def summarizeAstCreators(astCreators: Seq[AstCreator]): CSharpProgramSummary = { - ConcurrentTaskUtil - .runUsingThreadPool(astCreators.map(x => () => x.summarize()).iterator) - .flatMap { - case Failure(exception) => - logger.warn(s"Unable to pre-parse C# file, skipping - ", exception) - None - case Success(summary) => Option(summary) - } - .foldLeft(CSharpProgramSummary(imports = CSharpProgramSummary.initialImports))(_ ++= _) - } - - private def createBuiltinSummary(config: Config, internalSummary: CSharpProgramSummary): CSharpProgramSummary = { - Option - .when(config.useBuiltinSummaries)(CSharpProgramSummary.builtinTypesSummary) - .map(_.filter(namespacePred = (ns, _) => internalSummary.imports.contains(ns))) - .getOrElse(CSharpProgramSummary()) - } - - private def createExternalSummary(config: Config, internalSummary: CSharpProgramSummary): CSharpProgramSummary = { - val jsonFilePaths = config.externalSummaryPaths - Option - .when(jsonFilePaths.nonEmpty)(CSharpProgramSummary.externalTypesSummary(jsonFilePaths)) - .map(_.filter(namespacePred = (ns, _) => internalSummary.imports.contains(ns))) - .getOrElse(CSharpProgramSummary()) - } - private def buildFiles(config: Config): List[String] = { SourceFiles.determine( config.inputPath, diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/ProgramSummaryCreator.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/ProgramSummaryCreator.scala new file mode 100644 index 000000000000..721db6fae599 --- /dev/null +++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/ProgramSummaryCreator.scala @@ -0,0 +1,43 @@ +package io.joern.csharpsrc2cpg.utils + +import io.joern.csharpsrc2cpg.Config +import io.joern.csharpsrc2cpg.astcreation.AstCreator +import io.joern.csharpsrc2cpg.datastructures.CSharpProgramSummary +import io.joern.x2cpg.utils.ConcurrentTaskUtil +import org.slf4j.LoggerFactory + +import scala.util.{Failure, Success} + +/** Builds a `CSharpProgramSummary` by pre-parsing AST creators for high level structures, taking into account related + * frontend options. + */ +object ProgramSummaryCreator { + + private val logger = LoggerFactory.getLogger(getClass) + + def from(astCreators: Seq[AstCreator], config: Config): CSharpProgramSummary = { + val internalSummary = summarizeAstCreators(astCreators) + val externalSummary = buildExternalSummary(config.useBuiltinSummaries, config.externalSummaryPaths) + internalSummary ++= externalSummary.filter(namespacePred = (ns, _) => internalSummary.imports.contains(ns)) + } + + private def summarizeAstCreators(astCreators: Seq[AstCreator]): CSharpProgramSummary = { + ConcurrentTaskUtil + .runUsingThreadPool(astCreators.map(x => () => x.summarize()).iterator) + .flatMap { + case Failure(exception) => + logger.warn(s"Unable to pre-parse C# file, skipping - ", exception) + None + case Success(summary) => Option(summary) + } + .foldLeft(CSharpProgramSummary(imports = CSharpProgramSummary.initialImports))(_ ++= _) + } + + private def buildExternalSummary(withBuiltinTypes: Boolean, withJsonFiles: Set[String]): CSharpProgramSummary = { + val builtin = if withBuiltinTypes then CSharpProgramSummary.builtinTypesSummary else CSharpProgramSummary() + val fromJson = + if withJsonFiles.nonEmpty then CSharpProgramSummary.externalTypesSummary(withJsonFiles) + else CSharpProgramSummary() + builtin ++= fromJson + } +}