diff --git a/VSharp.API/VSharp.cs b/VSharp.API/VSharp.cs index f6b186161..20ea57652 100644 --- a/VSharp.API/VSharp.cs +++ b/VSharp.API/VSharp.cs @@ -191,10 +191,10 @@ private static Statistics StartExploration( stopOnCoverageAchieved: 100, randomSeed: options.RandomSeed, stepsLimit: options.StepsLimit, - aiAgentTrainingOptions: options.AIAgentTrainingOptions == null ? FSharpOption.None : FSharpOption.Some(options.AIAgentTrainingOptions), + aiOptions: options.AIOptions == null ? FSharpOption.None : FSharpOption.Some(options.AIOptions), pathToModel: options.PathToModel == null ? FSharpOption.None : FSharpOption.Some(options.PathToModel), - useGPU: options.UseGPU == null ? FSharpOption.None : FSharpOption.Some(options.UseGPU), - optimize: options.Optimize == null ? FSharpOption.None : FSharpOption.Some(options.Optimize) + useGPU: options.UseGPU, + optimize: options.Optimize ); var fuzzerOptions = diff --git a/VSharp.API/VSharpOptions.cs b/VSharp.API/VSharpOptions.cs index 7d6d7901b..c26eca6ef 100644 --- a/VSharp.API/VSharpOptions.cs +++ b/VSharp.API/VSharpOptions.cs @@ -113,7 +113,7 @@ public readonly record struct VSharpOptions public readonly bool ReleaseBranches = DefaultReleaseBranches; public readonly int RandomSeed = DefaultRandomSeed; public readonly uint StepsLimit = DefaultStepsLimit; - public readonly AIAgentTrainingOptions AIAgentTrainingOptions = null; + public readonly AIOptions? AIOptions = null; public readonly string PathToModel = DefaultPathToModel; public readonly bool UseGPU = false; public readonly bool Optimize = false; @@ -133,7 +133,7 @@ public readonly record struct VSharpOptions /// If true and timeout is specified, a part of allotted time in the end is given to execute remaining states without branching. /// Fixed seed for random operations. Used if greater than or equal to zero. /// Number of symbolic machine steps to stop execution after. Zero value means no limit. - /// Settings for AI searcher training. + /// Settings for AI searcher training. /// Path to ONNX file with model to use in AI searcher. /// Specifies whether the ONNX execution session should use a CUDA-enabled GPU. /// Enabling options like parallel execution and various graph transformations to enhance performance of ONNX. @@ -150,7 +150,7 @@ public VSharpOptions( bool releaseBranches = DefaultReleaseBranches, int randomSeed = DefaultRandomSeed, uint stepsLimit = DefaultStepsLimit, - AIAgentTrainingOptions aiAgentTrainingOptions = null, + AIOptions? aiOptions = null, string pathToModel = DefaultPathToModel, bool useGPU = false, bool optimize = false) @@ -167,7 +167,7 @@ public VSharpOptions( ReleaseBranches = releaseBranches; RandomSeed = randomSeed; StepsLimit = stepsLimit; - AIAgentTrainingOptions = aiAgentTrainingOptions; + AIOptions = aiOptions; PathToModel = pathToModel; UseGPU = useGPU; Optimize = optimize; diff --git a/VSharp.Explorer/AISearcher.fs b/VSharp.Explorer/AISearcher.fs index b02f9a423..3e99b67bb 100644 --- a/VSharp.Explorer/AISearcher.fs +++ b/VSharp.Explorer/AISearcher.fs @@ -2,20 +2,31 @@ namespace VSharp.Explorer open System.Collections.Generic open Microsoft.ML.OnnxRuntime +open System +open System.Text +open System.Text.Json open VSharp open VSharp.IL.Serializer open VSharp.ML.GameServer.Messages -type internal AISearcher(oracle: Oracle, aiAgentTrainingOptions: Option) = +type AIMode = + | Runner + | TrainingSendModel + | TrainingSendEachStep + + +type internal AISearcher(oracle: Oracle, aiAgentTrainingMode: Option) = let stepsToSwitchToAI = - match aiAgentTrainingOptions with + match aiAgentTrainingMode with | None -> 0u - | Some options -> options.stepsToSwitchToAI + | Some(SendModel options) -> options.aiAgentTrainingOptions.stepsToSwitchToAI + | Some(SendEachStep options) -> options.aiAgentTrainingOptions.stepsToSwitchToAI let stepsToPlay = - match aiAgentTrainingOptions with + match aiAgentTrainingMode with | None -> 0u - | Some options -> options.stepsToPlay + | Some(SendModel options) -> options.aiAgentTrainingOptions.stepsToPlay + | Some(SendEachStep options) -> options.aiAgentTrainingOptions.stepsToPlay let mutable lastCollectedStatistics = Statistics() let mutable defaultSearcherSteps = 0u @@ -25,14 +36,17 @@ type internal AISearcher(oracle: Oracle, aiAgentTrainingOptions: Option BFSSearcher() :> IForwardSearcher - | Some options -> - match options.defaultSearchStrategy with + let pickSearcher = + function | BFSMode -> BFSSearcher() :> IForwardSearcher | DFSMode -> DFSSearcher() :> IForwardSearcher | x -> failwithf $"Unexpected default searcher {x}. DFS and BFS supported for now." + match aiAgentTrainingMode with + | None -> BFSSearcher() :> IForwardSearcher + | Some(SendModel options) -> pickSearcher options.aiAgentTrainingOptions.aiBaseOptions.defaultSearchStrategy + | Some(SendEachStep options) -> pickSearcher options.aiAgentTrainingOptions.aiBaseOptions.defaultSearchStrategy + let mutable stepsPlayed = 0u let isInAIMode () = @@ -41,59 +55,6 @@ type internal AISearcher(oracle: Oracle, aiAgentTrainingOptions: Option() let availableStates = HashSet<_>() - let updateGameState (delta: GameState) = - match gameState with - | None -> gameState <- Some delta - | Some s -> - let updatedBasicBlocks = delta.GraphVertices |> Array.map (fun b -> b.Id) |> HashSet - let updatedStates = delta.States |> Array.map (fun s -> s.Id) |> HashSet - - let vertices = - s.GraphVertices - |> Array.filter (fun v -> updatedBasicBlocks.Contains v.Id |> not) - |> ResizeArray<_> - - vertices.AddRange delta.GraphVertices - - let edges = - s.Map - |> Array.filter (fun e -> updatedBasicBlocks.Contains e.VertexFrom |> not) - |> ResizeArray<_> - - edges.AddRange delta.Map - let activeStates = vertices |> Seq.collect (fun v -> v.States) |> HashSet - - let states = - let part1 = - s.States - |> Array.filter (fun s -> activeStates.Contains s.Id && (not <| updatedStates.Contains s.Id)) - |> ResizeArray<_> - - part1.AddRange delta.States - - part1.ToArray() - |> Array.map (fun s -> - State( - s.Id, - s.Position, - s.PathCondition, - s.VisitedAgainVertices, - s.VisitedNotCoveredVerticesInZone, - s.VisitedNotCoveredVerticesOutOfZone, - s.StepWhenMovedLastTime, - s.InstructionsVisitedInCurrentBlock, - s.History, - s.Children |> Array.filter activeStates.Contains - )) - - let pathConditionVertices = - ResizeArray s.PathConditionVertices - - pathConditionVertices.AddRange delta.PathConditionVertices - - gameState <- - Some - <| GameState(vertices.ToArray(), states, pathConditionVertices.ToArray(), edges.ToArray()) let init states = @@ -128,15 +89,18 @@ type internal AISearcher(oracle: Oracle, aiAgentTrainingOptions: Option ignore - let inTrainMode = aiAgentTrainingOptions.IsSome - + let aiMode = + match aiAgentTrainingMode with + | Some(SendEachStep _) -> TrainingSendEachStep + | Some(SendModel _) -> TrainingSendModel + | None -> Runner let pick selector = if useDefaultSearcher then defaultSearcherSteps <- defaultSearcherSteps + 1u if Seq.length availableStates > 0 then let gameStateDelta = collectGameStateDelta () - updateGameState gameStateDelta + gameState <- AISearcher.updateGameState gameStateDelta gameState let statistics = computeStatistics gameState.Value Application.applicationGraphDelta.Clear() lastCollectedStatistics <- statistics @@ -149,7 +113,7 @@ type internal AISearcher(oracle: Oracle, aiAgentTrainingOptions: Option 0u then - gameStateDelta - else - gameState.Value + match aiMode with + | TrainingSendEachStep + | TrainingSendModel -> + if stepsPlayed > 0u then + gameStateDelta + else + gameState.Value + | Runner -> gameState.Value let stateId = oracle.Predict toPredict afterFirstAIPeek <- true @@ -179,13 +147,77 @@ type internal AISearcher(oracle: Oracle, aiAgentTrainingOptions: Option) = + match gameState with + | None -> Some delta + | Some s -> + let updatedBasicBlocks = delta.GraphVertices |> Array.map (fun b -> b.Id) |> HashSet + let updatedStates = delta.States |> Array.map (fun s -> s.Id) |> HashSet + + let vertices = + s.GraphVertices + |> Array.filter (fun v -> updatedBasicBlocks.Contains v.Id |> not) + |> ResizeArray<_> + + vertices.AddRange delta.GraphVertices + + let edges = + s.Map + |> Array.filter (fun e -> updatedBasicBlocks.Contains e.VertexFrom |> not) + |> ResizeArray<_> + + edges.AddRange delta.Map + let activeStates = vertices |> Seq.collect (fun v -> v.States) |> HashSet + + let states = + let part1 = + s.States + |> Array.filter (fun s -> activeStates.Contains s.Id && (not <| updatedStates.Contains s.Id)) + |> ResizeArray<_> + + part1.AddRange delta.States + + part1.ToArray() + |> Array.map (fun s -> + State( + s.Id, + s.Position, + s.PathCondition, + s.VisitedAgainVertices, + s.VisitedNotCoveredVerticesInZone, + s.VisitedNotCoveredVerticesOutOfZone, + s.StepWhenMovedLastTime, + s.InstructionsVisitedInCurrentBlock, + s.History, + s.Children |> Array.filter activeStates.Contains + )) + + let pathConditionVertices = ResizeArray s.PathConditionVertices - new(pathToONNX: string, useGPU: bool, optimize: bool) = + pathConditionVertices.AddRange delta.PathConditionVertices + + Some + <| GameState(vertices.ToArray(), states, pathConditionVertices.ToArray(), edges.ToArray()) + + static member convertOutputToJson (output: IDisposableReadOnlyCollection) = + seq { 0 .. output.Count - 1 } + |> Seq.map (fun i -> output[i].GetTensorDataAsSpan().ToArray()) + + + + new + ( + pathToONNX: string, + useGPU: bool, + optimize: bool, + aiAgentTrainingModelOptions: Option + ) = let numOfVertexAttributes = 7 let numOfStateAttributes = 7 let numOfHistoryEdgeAttributes = 2 - let createOracle (pathToONNX: string) = + + let createOracleRunner (pathToONNX: string, aiAgentTrainingModelOptions: Option) = let sessionOptions = if useGPU then SessionOptions.MakeSessionOptionWithCudaProvider(0) @@ -199,10 +231,21 @@ type internal AISearcher(oracle: Oracle, aiAgentTrainingOptions: Option + currentGameState <- AISearcher.updateGameState gameStateOrDelta currentGameState + | _ -> currentGameState <- Some gameStateOrDelta + + let gameState = currentGameState.Value let stateIds = Dictionary, int>() let verticesIds = Dictionary, int>() @@ -243,7 +286,7 @@ type internal AISearcher(oracle: Oracle, aiAgentTrainingOptions: Option + aiAgentOptions.stepSaver ( + AIGameStep(gameState = gameStateOrDelta, output = AISearcher.convertOutputToJson output) + ) + | None -> () + + stepsPlayed <- stepsPlayed + 1 + let weighedStates = output[0].GetTensorDataAsSpan().ToArray() let id = weighedStates |> Array.mapi (fun i v -> i, v) |> Array.maxBy snd |> fst @@ -357,7 +411,12 @@ type internal AISearcher(oracle: Oracle, aiAgentTrainingOptions: Option Some(SendModel aiAgentTrainingModelOptions) + | None -> None + + AISearcher(createOracleRunner (pathToONNX, aiAgentTrainingModelOptions), aiAgentTrainingOptions) interface IForwardSearcher with override x.Init states = init states diff --git a/VSharp.Explorer/Explorer.fs b/VSharp.Explorer/Explorer.fs index ec79cca9f..bfd2951b4 100644 --- a/VSharp.Explorer/Explorer.fs +++ b/VSharp.Explorer/Explorer.fs @@ -12,7 +12,6 @@ open VSharp.Core open VSharp.Interpreter.IL open CilState open VSharp.Explorer -open VSharp.ML.GameServer.Messages open VSharp.Solver open VSharp.IL.Serializer @@ -25,19 +24,16 @@ type IReporter = type EntryPointConfiguration(mainArguments: string[]) = - member x.Args = - if mainArguments = null then None else Some mainArguments + member x.Args = if mainArguments = null then None else Some mainArguments type WebConfiguration (mainArguments: string[], environmentName: string, contentRootPath: DirectoryInfo, applicationName: string) = inherit EntryPointConfiguration(mainArguments) - member internal x.ToWebConfig () = - { - environmentName = environmentName - contentRootPath = contentRootPath - applicationName = applicationName - } + member internal x.ToWebConfig() = + { environmentName = environmentName + contentRootPath = contentRootPath + applicationName = applicationName } type private IExplorer = abstract member Reset: seq -> unit @@ -45,15 +41,18 @@ type private IExplorer = type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVMStatistics, reporter: IReporter) = let options = explorationOptions.svmOptions + let folderToStoreSerializationResult = - match options.aiAgentTrainingOptions with - | None -> "" - | Some options -> + match options.aiOptions with + | Some(DatasetGenerator aiOptions) -> + let mapName = aiOptions.mapName + getFolderToStoreSerializationResult (Path.GetDirectoryName explorationOptions.outputDirectory.FullName) - options.mapName - let hasTimeout = - explorationOptions.timeout.TotalMilliseconds > 0 + mapName + | _ -> "" + + let hasTimeout = explorationOptions.timeout.TotalMilliseconds > 0 let solverTimeout = if options.solverTimeout > 0 then @@ -67,8 +66,8 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM -1 let branchReleaseTimeout = - let doubleTimeout = - double explorationOptions.timeout.TotalMilliseconds + let doubleTimeout = double explorationOptions.timeout.TotalMilliseconds + if not hasTimeout then Double.PositiveInfinity elif not options.releaseBranches then doubleTimeout else doubleTimeout * 80.0 / 100.0 @@ -76,29 +75,29 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM let hasStepsLimit = options.stepsLimit > 0u do - API.ConfigureSolver (SolverPool.mkSolver (solverTimeout)) - VSharp.System.SetUp.ConfigureInternalCalls () - API.ConfigureChars (options.prettyChars) + API.ConfigureSolver(SolverPool.mkSolver (solverTimeout)) + VSharp.System.SetUp.ConfigureInternalCalls() + API.ConfigureChars(options.prettyChars) let mutable branchesReleased = false let mutable isStopped = false - let mutable isCoverageAchieved: unit -> bool = - always false + let mutable isCoverageAchieved: unit -> bool = always false - let emptyState = - Memory.EmptyIsolatedState () - let interpreter = ILInterpreter () + let emptyState = Memory.EmptyIsolatedState() + let interpreter = ILInterpreter() do if options.visualize then DotVisualizer explorationOptions.outputDirectory :> IVisualizer |> Application.setVisualizer + SetMaxBuferSize options.maxBufferSize TestGenerator.setMaxBufferSize options.maxBufferSize let isSat pc = // TODO: consider trivial cases emptyState.pc <- pc + match SolverInteraction.checkSat emptyState with | SolverInteraction.SmtSat _ | SolverInteraction.SmtUnknown _ -> true @@ -110,72 +109,78 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM None else Some options.randomSeed + match mode with | AIMode -> - match options.aiAgentTrainingOptions with + let useGPU = options.useGPU + let optimize = options.optimize + + match options.aiOptions with | Some aiOptions -> - match aiOptions.oracle with - | Some oracle -> AISearcher (oracle, options.aiAgentTrainingOptions) :> IForwardSearcher - | None -> failwith "Empty oracle for AI searcher." + match aiOptions with + | Training aiAgentTrainingOptions -> + match aiAgentTrainingOptions with + | SendEachStep aiAgentTrainingEachStepOptions -> + match aiAgentTrainingEachStepOptions.aiAgentTrainingOptions.oracle with + | Some oracle -> AISearcher(oracle, Some aiAgentTrainingOptions) :> IForwardSearcher + | None -> failwith "Empty oracle for AI searcher training (send each step mode)." + | SendModel aiAgentTrainingModelOptions -> + match options.pathToModel with + | Some path -> + AISearcher(path, useGPU, optimize, Some aiAgentTrainingModelOptions) :> IForwardSearcher + | None -> failwith "Empty model for AI searcher training (send model mode)." + | DatasetGenerator aiOptions -> mkForwardSearcher aiOptions.defaultSearchStrategy | None -> match options.pathToModel with - | Some s -> - let useGPU = - options.useGPU.IsSome && options.useGPU.Value - let optimize = - options.optimize.IsSome && options.optimize.Value - AISearcher (s, useGPU, optimize) + | Some s -> AISearcher(s, useGPU, optimize, None) | None -> failwith "Empty model for AI searcher." - | BFSMode -> BFSSearcher () :> IForwardSearcher - | DFSMode -> DFSSearcher () :> IForwardSearcher + | BFSMode -> BFSSearcher() :> IForwardSearcher + | DFSMode -> DFSSearcher() :> IForwardSearcher | ShortestDistanceBasedMode -> ShortestDistanceBasedSearcher statistics :> IForwardSearcher | RandomShortestDistanceBasedMode -> - RandomShortestDistanceBasedSearcher (statistics, getRandomSeedOption ()) :> IForwardSearcher + RandomShortestDistanceBasedSearcher(statistics, getRandomSeedOption ()) :> IForwardSearcher | ContributedCoverageMode -> DFSSortedByContributedCoverageSearcher statistics :> IForwardSearcher - | ExecutionTreeMode -> ExecutionTreeSearcher (getRandomSeedOption ()) + | ExecutionTreeMode -> ExecutionTreeSearcher(getRandomSeedOption ()) | FairMode baseMode -> - FairSearcher ((fun _ -> mkForwardSearcher baseMode), uint branchReleaseTimeout, statistics) + FairSearcher((fun _ -> mkForwardSearcher baseMode), uint branchReleaseTimeout, statistics) :> IForwardSearcher - | InterleavedMode (base1, stepCount1, base2, stepCount2) -> - InterleavedSearcher ( - [ - mkForwardSearcher base1, stepCount1 - mkForwardSearcher base2, stepCount2 - ] - ) + | InterleavedMode(base1, stepCount1, base2, stepCount2) -> + InterleavedSearcher([ mkForwardSearcher base1, stepCount1; mkForwardSearcher base2, stepCount2 ]) :> IForwardSearcher let mutable searcher: IBidirectionalSearcher = match options.explorationMode with - | TestCoverageMode (_, searchMode) -> + | TestCoverageMode(_, searchMode) -> let baseSearcher = if options.recThreshold > 0u then - GuidedSearcher ( + GuidedSearcher( mkForwardSearcher searchMode, - RecursionBasedTargetManager (statistics, options.recThreshold) + RecursionBasedTargetManager(statistics, options.recThreshold) ) :> IForwardSearcher else mkForwardSearcher searchMode - BidirectionalSearcher (baseSearcher, BackwardSearcher (), DummyTargetedSearcher.DummyTargetedSearcher ()) + + BidirectionalSearcher(baseSearcher, BackwardSearcher(), DummyTargetedSearcher.DummyTargetedSearcher()) :> IBidirectionalSearcher | StackTraceReproductionMode _ -> __notImplemented__ () let releaseBranches () = if not branchesReleased then branchesReleased <- true - statistics.OnBranchesReleased () - ReleaseBranches () + statistics.OnBranchesReleased() + ReleaseBranches() + let dfsSearcher = DFSSortedByContributedCoverageSearcher statistics :> IForwardSearcher - let bidirectionalSearcher = - OnlyForwardSearcher (dfsSearcher) - dfsSearcher.Init <| searcher.States () + + let bidirectionalSearcher = OnlyForwardSearcher(dfsSearcher) + dfsSearcher.Init <| searcher.States() searcher <- bidirectionalSearcher let reportStateIncomplete (state: cilState) = searcher.Remove state - statistics.IncompleteStates.Add (state) + statistics.IncompleteStates.Add(state) Application.terminateState state reporter.ReportIIE state.iie.Value @@ -184,33 +189,38 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM let reportState (suite: testSuite) (cilState: cilState) = try let isNewHistory () = - let methodHistory = - Set.filter (fun h -> h.method.InCoverageZone) cilState.history + let methodHistory = Set.filter (fun h -> h.method.InCoverageZone) cilState.history + Set.isEmpty methodHistory || Set.exists (not << statistics.IsBasicBlockCoveredByTest) methodHistory + let isError = suite.IsErrorSuite + let isNewTest = match suite with | Test -> isNewHistory () - | Error (msg, isFatal) -> statistics.IsNewError cilState msg isFatal + | Error(msg, isFatal) -> statistics.IsNewError cilState msg isFatal + if isNewTest then let state = cilState.state - let callStackSize = - Memory.CallStackSize state + let callStackSize = Memory.CallStackSize state let entryMethod = cilState.EntryMethod - let hasException = - cilState.IsUnhandledException + let hasException = cilState.IsUnhandledException + if isError && not hasException then if entryMethod.HasParameterOnStack then Memory.ForcePopFrames (callStackSize - 2) state else Memory.ForcePopFrames (callStackSize - 1) state + match TestGenerator.state2test suite entryMethod state with | Some test -> - statistics.TrackFinished (cilState, isError) + statistics.TrackFinished(cilState, isError) + match suite with | Test -> reporter.ReportFinished test | Error _ -> reporter.ReportException test + if isCoverageAchieved () then isStopped <- true | None -> () @@ -223,10 +233,11 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM | :? InsufficientInformationException as e -> if not state.IsIIEState then state.SetIIE e + reportStateIncomplete state | _ -> searcher.Remove state - statistics.InternalFails.Add (e) + statistics.InternalFails.Add(e) Application.terminateState state reporter.ReportInternalFail state.EntryMethod e @@ -234,7 +245,7 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM match e with | :? InsufficientInformationException as e -> reportIncomplete e | _ -> - statistics.InternalFails.Add (e) + statistics.InternalFails.Add(e) reporter.ReportInternalFail method e let reportFinished (state: cilState) = @@ -246,9 +257,9 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM let wrapOnError isFatal (state: cilState) errorMessage = if not <| String.IsNullOrWhiteSpace errorMessage then Logger.info $"Error in {state.EntryMethod.FullName}: {errorMessage}" + Application.terminateState state - let testSuite = - Error (errorMessage, isFatal) + let testSuite = Error(errorMessage, isFatal) reportState testSuite state let reportError = wrapOnError false @@ -259,55 +270,66 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM hasTimeout && statistics.CurrentExplorationTime.TotalMilliseconds >= explorationOptions.timeout.TotalMilliseconds + let shouldReleaseBranches () = options.releaseBranches && statistics.CurrentExplorationTime.TotalMilliseconds >= branchReleaseTimeout + let isStepsLimitReached () = hasStepsLimit && statistics.StepsCount >= options.stepsLimit static member private AllocateByRefParameters initialState (method: Method) = let allocateIfByRef (pi: ParameterInfo) = let parameterType = pi.ParameterType + if parameterType.IsByRef then if Memory.CallStackSize initialState = 0 then Memory.NewStackFrame initialState None [] - let typ = parameterType.GetElementType () + + let typ = parameterType.GetElementType() let position = pi.Position + 1 + let stackRef = Memory.AllocateTemporaryLocalVariableOfType initialState pi.Name position typ + Some stackRef else None + method.Parameters |> Array.map allocateIfByRef |> Array.toList - member private x.FormIsolatedInitialStates (method: Method, initialState: state) = + member private x.FormIsolatedInitialStates(method: Method, initialState: state) = try initialState.model <- Memory.EmptyModel method let declaringType = method.DeclaringType - let cilState = - cilState.CreateInitial method initialState + let cilState = cilState.CreateInitial method initialState + let this = if method.HasThis then if Types.IsValueType declaringType then Memory.NewStackFrame initialState None [] + Memory.AllocateTemporaryLocalVariableOfType initialState "this" 0 declaringType |> Some else - let this = - Memory.MakeSymbolicThis initialState method + let this = Memory.MakeSymbolicThis initialState method !!(IsNullReference this) |> AddConstraint initialState Some this else None - let parameters = - SVMExplorer.AllocateByRefParameters initialState method + + let parameters = SVMExplorer.AllocateByRefParameters initialState method Memory.InitFunctionFrame initialState method this (Some parameters) + match this with | Some this -> SolveThisType initialState this | _ -> () + let cilStates = ILInterpreter.CheckDisallowNullAttribute method None cilState false id + assert (List.length cilStates = 1) + if not method.IsStaticConstructor then let cilState = List.head cilStates interpreter.InitializeStatics cilState declaringType List.singleton @@ -325,78 +347,85 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM assert method.IsStatic let optionArgs = config.Args let parameters = method.Parameters - let hasParameters = - Array.length parameters > 0 + let hasParameters = Array.length parameters > 0 + let state = { initialState with - complete = not hasParameters || Option.isSome optionArgs - } + complete = not hasParameters || Option.isSome optionArgs } + state.model <- Memory.EmptyModel method + match optionArgs with | Some args -> let stringType = typeof let argsNumber = MakeNumber args.Length - let argsRef = - Memory.AllocateConcreteVectorArray state argsNumber stringType args - let args = - Some (List.singleton (Some argsRef)) + let argsRef = Memory.AllocateConcreteVectorArray state argsNumber stringType args + let args = Some(List.singleton (Some argsRef)) Memory.InitFunctionFrame state method None args | None when hasParameters -> Memory.InitFunctionFrame state method None None // NOTE: if args are symbolic, constraint 'args != null' is added assert (Array.length parameters = 1) let argsParameter = Array.head parameters - let argsParameterTerm = - Memory.ReadArgument state argsParameter + let argsParameterTerm = Memory.ReadArgument state argsParameter AddConstraint state (!!(IsNullReference argsParameterTerm)) // Filling model with default args to match PC let modelState = match state.model with | StateModel modelState -> modelState | _ -> __unreachable__ () + let argsForModel = Memory.AllocateVectorArray modelState (MakeNumber 0) typeof + Memory.WriteStackLocation modelState (ParameterKey argsParameter) argsForModel | None -> let args = Some List.empty Memory.InitFunctionFrame state method None args + Memory.InitializeStaticMembers state method.DeclaringType + let initialState = match config with | :? WebConfiguration as config -> - let webConfig = config.ToWebConfig () + let webConfig = config.ToWebConfig() cilState.CreateWebInitial method state webConfig | _ -> cilState.CreateInitial method state + [ initialState ] with e -> reportInternalFail method e [] - member private x.Forward (s: cilState) = + member private x.Forward(s: cilState) = let loc = s.approximateLoc let ip = s.CurrentIp let stackSize = s.StackSize // TODO: update pobs when visiting new methods; use coverageZone - let goodStates, iieStates, errors = - interpreter.ExecuteOneInstruction s + let goodStates, iieStates, errors = interpreter.ExecuteOneInstruction s + for s in goodStates @ iieStates @ errors do if not s.HasRuntimeExceptionOrError then statistics.TrackStepForward s ip stackSize + let goodStates, toReportFinished = goodStates |> List.partition (fun s -> s.IsExecutable || s.IsIsolated) + toReportFinished |> List.iter reportFinished - let errors, _ = - errors |> List.partition (fun s -> not s.HasReportedError) + let errors, _ = errors |> List.partition (fun s -> not s.HasReportedError) + let errors, toReportExceptions = errors |> List.partition (fun s -> s.IsIsolated || not s.IsStoppedByException) + let runtimeExceptions, userExceptions = toReportExceptions |> List.partition (fun s -> s.HasRuntimeExceptionOrError) + runtimeExceptions |> List.iter (fun state -> reportError state null) userExceptions |> List.iter reportFinished - let iieStates, toReportIIE = - iieStates |> List.partition (fun s -> s.IsIsolated) + let iieStates, toReportIIE = iieStates |> List.partition (fun s -> s.IsIsolated) toReportIIE |> List.iter reportStateIncomplete let mutable sIsStopped = false + let newStates = match goodStates, iieStates, errors with | s' :: goodStates, _, _ when LanguagePrimitives.PhysicalEquality s s' -> goodStates @ iieStates @ errors @@ -410,36 +439,39 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM Application.moveState loc s (Seq.cast<_> newStates) statistics.TrackFork s newStates searcher.UpdateStates s newStates + if sIsStopped then searcher.Remove s member private x.Backward p' (s': cilState) = assert (s'.CurrentLoc = p'.loc) let sLvl = s'.Level + if p'.lvl >= sLvl then let lvl = p'.lvl - sLvl let pc = Memory.WLP s'.state p'.pc + match isSat pc with | true when not s'.IsIsolated -> searcher.Answer p' (Witnessed s') | true -> statistics.TrackStepBackward p' s' + let p = - { - loc = s'.StartingLoc - lvl = lvl - pc = pc - } + { loc = s'.StartingLoc + lvl = lvl + pc = pc } + Logger.trace $"Backward:\nWas: {p'}\nNow: {p}\n\n" Application.addGoal p.loc searcher.UpdatePobs p' p | false -> Logger.trace "UNSAT for pob = %O and s'.PC = %s" p' (API.Print.PrintPC s'.state.pc) - member private x.BidirectionalSymbolicExecution () = + member private x.BidirectionalSymbolicExecution() = let mutable action = Stop let pick () = - match searcher.Pick () with + match searcher.Pick() with | Stop -> false | a -> action <- a @@ -452,21 +484,26 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM && pick () do if shouldReleaseBranches () then releaseBranches () + match action with | GoFront s -> try - if - options.aiAgentTrainingOptions.IsSome - && options.aiAgentTrainingOptions.Value.serializeSteps - then + let needToSerialize = + match options.aiOptions with + | Some(DatasetGenerator _) -> true + | _ -> false + + if needToSerialize then dumpGameState - (Path.Combine (folderToStoreSerializationResult, string firstFreeEpisodeNumber)) + (Path.Combine(folderToStoreSerializationResult, string firstFreeEpisodeNumber)) s.internalId + firstFreeEpisodeNumber <- firstFreeEpisodeNumber + 1 - x.Forward (s) + + x.Forward(s) with e -> reportStateInternalFail s e - | GoBack (s, p) -> + | GoBack(s, p) -> try x.Backward p s with e -> @@ -474,43 +511,46 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM | Stop -> __unreachable__ () member private x.AnswerPobs initialStates = - statistics.ExplorationStarted () + statistics.ExplorationStarted() // For backward compatibility. TODO: remove main pobs at all let mainPobs = [] Application.spawnStates (Seq.cast<_> initialStates) mainPobs |> Seq.map (fun pob -> pob.loc) |> Seq.toArray |> Application.addGoals searcher.Init initialStates mainPobs + initialStates |> Seq.filter (fun s -> s.IsIIEState) |> Seq.iter reportStateIncomplete - x.BidirectionalSymbolicExecution () - searcher.Statuses () + + x.BidirectionalSymbolicExecution() + + searcher.Statuses() |> Seq.iter (fun (pob, status) -> match status with | pobStatus.Unknown -> Logger.warning $"Unknown status for pob at {pob.loc}" - | _ -> () - ) + | _ -> ()) interface IExplorer with member x.Reset entryMethods = HashMap.clear () - API.Reset () + API.Reset() SolverPool.reset () - searcher.Reset () + searcher.Reset() isStopped <- false branchesReleased <- false SolverInteraction.setOnSolverStarted statistics.SolverStarted SolverInteraction.setOnSolverStopped statistics.SolverStopped - AcquireBranches () + AcquireBranches() isCoverageAchieved <- always false + match options.explorationMode with | TestCoverageMode _ -> if options.stopOnCoverageAchieved > 0 then let checkCoverage () = - let cov = - statistics.GetCurrentCoverage entryMethods + let cov = statistics.GetCurrentCoverage entryMethods cov >= options.stopOnCoverageAchieved + isCoverageAchieved <- checkCoverage | StackTraceReproductionMode _ -> __notImplemented__ () @@ -519,30 +559,33 @@ type private SVMExplorer(explorationOptions: ExplorationOptions, statistics: SVM try try interpreter.ConfigureErrorReporter reportError reportFatalError - let isolatedInitialStates = - isolated |> List.collect x.FormIsolatedInitialStates + let isolatedInitialStates = isolated |> List.collect x.FormIsolatedInitialStates + let entryPointsInitialStates = entryPoints |> List.collect x.FormEntryPointInitialStates + let iieStates, initialStates = isolatedInitialStates @ entryPointsInitialStates |> List.partition (fun state -> state.IsIIEState) + iieStates |> List.iter reportStateIncomplete - statistics.SetStatesGetter (fun () -> searcher.States ()) - statistics.SetStatesCountGetter (fun () -> searcher.StatesCount) + statistics.SetStatesGetter(fun () -> searcher.States()) + statistics.SetStatesCountGetter(fun () -> searcher.StatesCount) + if not initialStates.IsEmpty then x.AnswerPobs initialStates with e -> reportCrash e finally try - statistics.ExplorationFinished () - API.Restore () - searcher.Reset () + statistics.ExplorationFinished() + API.Restore() + searcher.Reset() with e -> reportCrash e } - member private x.Stop () = isStopped <- true + member private x.Stop() = isStopped <- true type private FuzzerExplorer(explorationOptions: ExplorationOptions, statistics: SVMStatistics) = @@ -551,22 +594,19 @@ type private FuzzerExplorer(explorationOptions: ExplorationOptions, statistics: member x.Reset _ = () member x.StartExploration isolated _ = - let saveStatistic = - statistics.SetBasicBlocksAsCoveredByTest - let outputDir = - explorationOptions.outputDirectory.FullName - let cancellationTokenSource = - new CancellationTokenSource () - cancellationTokenSource.CancelAfter (explorationOptions.timeout) - let methodsBase = - isolated |> List.map (fun (m, _) -> (m :> IMethod).MethodBase) + let saveStatistic = statistics.SetBasicBlocksAsCoveredByTest + let outputDir = explorationOptions.outputDirectory.FullName + let cancellationTokenSource = new CancellationTokenSource() + cancellationTokenSource.CancelAfter(explorationOptions.timeout) + let methodsBase = isolated |> List.map (fun (m, _) -> (m :> IMethod).MethodBase) + task { try - let targetAssemblyPath = - (Seq.head methodsBase).Module.Assembly.Location + let targetAssemblyPath = (Seq.head methodsBase).Module.Assembly.Location let onCancelled () = Logger.warning "Fuzzer canceled" + let interactor = - Fuzzer.Interactor ( + Fuzzer.Interactor( targetAssemblyPath, methodsBase, cancellationTokenSource.Token, @@ -574,7 +614,8 @@ type private FuzzerExplorer(explorationOptions: ExplorationOptions, statistics: saveStatistic, onCancelled ) - do! interactor.StartFuzzing () + + do! interactor.StartFuzzing() with e -> Logger.error $"Fuzzer unhandled exception: {e}" raise e @@ -582,20 +623,19 @@ type private FuzzerExplorer(explorationOptions: ExplorationOptions, statistics: type public Explorer(options: ExplorationOptions, reporter: IReporter) = - let statistics = - new SVMStatistics (Seq.empty, true) + let statistics = new SVMStatistics(Seq.empty, true) let explorers = let createFuzzer () = - FuzzerExplorer (options, statistics) :> IExplorer + FuzzerExplorer(options, statistics) :> IExplorer let createSVM () = - SVMExplorer (options, statistics, reporter) :> IExplorer + SVMExplorer(options, statistics, reporter) :> IExplorer match options.explorationModeOptions with | Fuzzing _ -> createFuzzer () |> Array.singleton | SVM _ -> createSVM () |> Array.singleton - | Combined _ -> [| createFuzzer () ; createSVM () |] + | Combined _ -> [| createFuzzer (); createSVM () |] let inCoverageZone coverageZone (entryMethods: Method list) = match coverageZone with @@ -610,33 +650,37 @@ type public Explorer(options: ExplorationOptions, reporter: IReporter) = |> List.exists (fun m -> method.Module.ModuleHandle = m.Module.ModuleHandle) member private x.TrySubstituteTypeParameters (state: state) (methodBase: MethodBase) = - let method = - Application.getMethod methodBase + let method = Application.getMethod methodBase + let getConcreteType = function | ConcreteType t -> Some t | _ -> None + try if method.ContainsGenericParameters then match SolveGenericMethodParameters state.typeStorage method with - | Some (classParams, methodParams) -> - let classParams = - classParams |> Array.choose getConcreteType - let methodParams = - methodParams |> Array.choose getConcreteType + | Some(classParams, methodParams) -> + let classParams = classParams |> Array.choose getConcreteType + let methodParams = methodParams |> Array.choose getConcreteType let declaringType = methodBase.DeclaringType + let isSuitableType () = not declaringType.IsGenericType || classParams.Length = declaringType.GetGenericArguments().Length + let isSuitableMethod () = methodBase.IsConstructor || not methodBase.IsGenericMethod || methodParams.Length = methodBase.GetGenericArguments().Length + if isSuitableType () && isSuitableMethod () then let reflectedType = Reflection.concretizeTypeParameters methodBase.ReflectedType classParams + let method = Reflection.concretizeMethodParameters reflectedType methodBase methodParams + Some method else None @@ -650,6 +694,7 @@ type public Explorer(options: ExplorationOptions, reporter: IReporter) = member x.Reset entryMethods = statistics.Reset entryMethods Application.setCoverageZone (inCoverageZone options.coverageZone entryMethods) + for explorer in explorers do explorer.Reset entryMethods @@ -660,20 +705,20 @@ type public Explorer(options: ExplorationOptions, reporter: IReporter) = try let trySubstituteTypeParameters method = - let emptyState = - Memory.EmptyIsolatedState () + let emptyState = Memory.EmptyIsolatedState() (Option.defaultValue method (x.TrySubstituteTypeParameters emptyState method), emptyState) + let isolated = isolated |> Seq.map trySubstituteTypeParameters |> Seq.map (fun (m, s) -> Application.getMethod m, s) |> Seq.toList + let entryPoints = entryPoints |> Seq.map (fun (m, a) -> let m, s = trySubstituteTypeParameters m - (Application.getMethod m, a, s) - ) + (Application.getMethod m, a, s)) |> Seq.toList (List.map fst isolated) @ (List.map (fun (x, _, _) -> x) entryPoints) |> x.Reset @@ -681,8 +726,7 @@ type public Explorer(options: ExplorationOptions, reporter: IReporter) = let explorationTasks = explorers |> Array.map (fun e -> e.StartExploration isolated entryPoints) - let finished = - Task.WaitAll (explorationTasks, options.timeout) + let finished = Task.WaitAll(explorationTasks, options.timeout) if not finished then Logger.warning "Exploration cancelled" @@ -699,4 +743,4 @@ type public Explorer(options: ExplorationOptions, reporter: IReporter) = member x.Statistics = statistics interface IDisposable with - member x.Dispose () = (statistics :> IDisposable).Dispose () + member x.Dispose() = (statistics :> IDisposable).Dispose() diff --git a/VSharp.Explorer/Options.fs b/VSharp.Explorer/Options.fs index 4b29327d8..1d94d900b 100644 --- a/VSharp.Explorer/Options.fs +++ b/VSharp.Explorer/Options.fs @@ -3,6 +3,8 @@ namespace VSharp.Explorer open System.Diagnostics open System.IO open VSharp.ML.GameServer.Messages +open System.Net.Sockets +open Microsoft.ML.OnnxRuntime type searchMode = | DFSMode @@ -27,20 +29,17 @@ type explorationMode = type fuzzerIsolation = | Process type FuzzerOptions = - { - isolation: fuzzerIsolation - coverageZone: coverageZone - } + { isolation: fuzzerIsolation + coverageZone: coverageZone } [] type Oracle = val Predict: GameState -> uint val Feedback: Feedback -> unit + new(predict, feedback) = - { - Predict = predict - Feedback = feedback - } + { Predict = predict + Feedback = feedback } /// /// Options used in AI agent training. @@ -51,34 +50,62 @@ type Oracle = /// Determine whether steps should be serialized. /// Name of map to play. /// Name of map to play. + +[] +type AIGameStep = + interface IRawOutgoingMessageBody + val GameState: GameState + val Output: seq> + + new(gameState, output) = + { GameState = gameState + Output = output } + + +type AIBaseOptions = + { defaultSearchStrategy: searchMode + mapName: string } + type AIAgentTrainingOptions = - { - stepsToSwitchToAI: uint - stepsToPlay: uint - defaultSearchStrategy: searchMode - serializeSteps: bool - mapName: string - oracle: Option - } + { aiBaseOptions: AIBaseOptions + stepsToSwitchToAI: uint + stepsToPlay: uint + oracle: option } + +type AIAgentTrainingEachStepOptions = + { aiAgentTrainingOptions: AIAgentTrainingOptions } + + +type AIAgentTrainingModelOptions = + { aiAgentTrainingOptions: AIAgentTrainingOptions + outputDirectory: string + stepSaver: AIGameStep -> Unit } + + +type AIAgentTrainingMode = + | SendEachStep of AIAgentTrainingEachStepOptions + | SendModel of AIAgentTrainingModelOptions + +type AIOptions = + | Training of AIAgentTrainingMode + | DatasetGenerator of AIBaseOptions type SVMOptions = - { - explorationMode: explorationMode - recThreshold: uint - solverTimeout: int - visualize: bool - releaseBranches: bool - maxBufferSize: int - prettyChars: bool // If true, 33 <= char <= 126, otherwise any char - checkAttributes: bool - stopOnCoverageAchieved: int - randomSeed: int - stepsLimit: uint - aiAgentTrainingOptions: Option - pathToModel: Option - useGPU: Option - optimize: Option - } + { explorationMode: explorationMode + recThreshold: uint + solverTimeout: int + visualize: bool + releaseBranches: bool + maxBufferSize: int + prettyChars: bool // If true, 33 <= char <= 126, otherwise any char + checkAttributes: bool + stopOnCoverageAchieved: int + randomSeed: int + stepsLimit: uint + aiOptions: Option + pathToModel: Option + useGPU: bool + optimize: bool } type explorationModeOptions = | Fuzzing of FuzzerOptions @@ -86,29 +113,27 @@ type explorationModeOptions = | Combined of SVMOptions * FuzzerOptions type ExplorationOptions = - { - timeout: System.TimeSpan - outputDirectory: DirectoryInfo - explorationModeOptions: explorationModeOptions - } + { timeout: System.TimeSpan + outputDirectory: DirectoryInfo + explorationModeOptions: explorationModeOptions } member this.fuzzerOptions = match this.explorationModeOptions with | Fuzzing x -> x - | Combined (_, x) -> x + | Combined(_, x) -> x | _ -> failwith "" member this.svmOptions = match this.explorationModeOptions with | SVM x -> x - | Combined (x, _) -> x + | Combined(x, _) -> x | _ -> failwith "" member this.coverageZone = match this.explorationModeOptions with | SVM x -> match x.explorationMode with - | TestCoverageMode (coverageZone, _) -> coverageZone + | TestCoverageMode(coverageZone, _) -> coverageZone | StackTraceReproductionMode _ -> failwith "" - | Combined (_, x) -> x.coverageZone + | Combined(_, x) -> x.coverageZone | Fuzzing x -> x.coverageZone diff --git a/VSharp.ML.GameServer.Runner/Main.fs b/VSharp.ML.GameServer.Runner/Main.fs index 17ba5e6b7..3d04b64e2 100644 --- a/VSharp.ML.GameServer.Runner/Main.fs +++ b/VSharp.ML.GameServer.Runner/Main.fs @@ -1,4 +1,7 @@ +open System open System.IO +open System.Text.Json +open System.Net.Sockets open System.Reflection open Argu open Microsoft.FSharp.Core @@ -16,7 +19,6 @@ open VSharp.IL open VSharp.ML.GameServer.Messages open VSharp.Runner - [] type ExplorationResult = val ActualCoverage: uint @@ -33,6 +35,10 @@ type ExplorationResult = type Mode = | Server = 0 | Generator = 1 + | SendModel = 2 + +let TIMEOUT_FOR_TRAINING = 15 * 60 +let SOLVER_TIMEOUT_FOR_TRAINING = 2 type CliArguments = | [] Port of int @@ -41,8 +47,15 @@ type CliArguments = | [] Mode of Mode | [] OutFolder of string | [] StepsToSerialize of uint - | [] UseGPU - | [] Optimize + | [] Model of string + | [] StepsToPlay of uint + | [] DefaultSearcher of string + | [] StepsToStart of uint + | [] AssemblyFullName of string + | [] NameOfObjectToCover of string + | [] MapName of string + | [] UseGPU of bool + | [] Optimize of bool interface IArgParserTemplate with member s.Usage = @@ -55,14 +68,22 @@ type CliArguments = "Mode to run application. Server --- to train network, Generator --- to generate data for training." | OutFolder _ -> "Folder to store generated data." | StepsToSerialize _ -> "Maximal number of steps for each method to serialize." - | UseGPU -> "Specifies whether the ONNX execution session should use a CUDA-enabled GPU." - | Optimize -> + | Model _ -> """Path to ONNX model (use it for training in mode "SendModel")""" + | StepsToPlay _ -> """Steps required to play (after `StepsToStart` steps)""" + | DefaultSearcher _ -> """Defines the default searcher algorithm (BFS | DFS)""" + | StepsToStart _ -> """Steps required to start the game""" + | AssemblyFullName _ -> """Path to the DLL that contains the game map implementation""" + | NameOfObjectToCover _ -> """The name of the object that needs to be covered""" + | MapName _ -> """The name of the map used in the game""" + | UseGPU _ -> "Specifies whether the ONNX execution session should use a CUDA-enabled GPU." + | Optimize _ -> "Enabling options like parallel execution and various graph transformations to enhance performance of ONNX." let mutable inTrainMode = true let explore (gameMap: GameMap) options = let assembly = RunnerProgram.TryLoadAssembly <| FileInfo gameMap.AssemblyFullName + let method = RunnerProgram.ResolveMethod(assembly, gameMap.NameOfObjectToCover) let statistics = TestGenerator.Cover(method, options) @@ -92,7 +113,6 @@ let explore (gameMap: GameMap) options = statistics.StepsCount * 1u ) - let loadGameMaps (datasetDescriptionFilePath: string) = let jsonString = File.ReadAllText datasetDescriptionFilePath let maps = ResizeArray() @@ -181,25 +201,28 @@ let ws port outputDirectory (webSocket: WebSocket) (context: HttpContext) = let stepsToPlay = gameMap.StepsToPlay let aiTrainingOptions = - { stepsToSwitchToAI = stepsToStart + { aiBaseOptions = + { defaultSearchStrategy = + match gameMap.DefaultSearcher with + | searcher.BFS -> BFSMode + | searcher.DFS -> DFSMode + | x -> failwithf $"Unexpected searcher {x}. Use DFS or BFS for now." + mapName = gameMap.MapName } + stepsToSwitchToAI = stepsToStart stepsToPlay = stepsToPlay - defaultSearchStrategy = - match gameMap.DefaultSearcher with - | searcher.BFS -> BFSMode - | searcher.DFS -> DFSMode - | x -> failwithf $"Unexpected searcher {x}. Use DFS or BFS for now." - serializeSteps = false - mapName = gameMap.MapName oracle = Some oracle } + let aiOptions: AIOptions = + (Training(SendEachStep { aiAgentTrainingOptions = aiTrainingOptions })) + let options = VSharpOptions( - timeout = 15 * 60, + timeout = TIMEOUT_FOR_TRAINING, outputDirectory = outputDirectory, searchStrategy = SearchStrategy.AI, - aiAgentTrainingOptions = aiTrainingOptions, + aiOptions = aiOptions, stepsLimit = uint (stepsToPlay + stepsToStart), - solverTimeout = 2 + solverTimeout = SOLVER_TIMEOUT_FOR_TRAINING ) let explorationResult = explore gameMap options @@ -231,6 +254,9 @@ let ws port outputDirectory (webSocket: WebSocket) (context: HttpContext) = let app port outputDirectory : WebPart = choose [ path "/gameServer" >=> handShake (ws port outputDirectory) ] +let serializeExplorationResult (explorationResult: ExplorationResult) = + $"{explorationResult.ActualCoverage} {explorationResult.TestsCount} {explorationResult.StepsCount} {explorationResult.ErrorsCount}" + let generateDataForPretraining outputDirectory datasetBasePath (maps: ResizeArray) stepsToSerialize = for map in maps do if map.StepsToStart = 0u then @@ -246,13 +272,9 @@ let generateDataForPretraining outputDirectory datasetBasePath (maps: ResizeArra map.MapName ) - let aiTrainingOptions = - { stepsToSwitchToAI = 0u - stepsToPlay = 0u - defaultSearchStrategy = searchMode.BFSMode - serializeSteps = true - mapName = map.MapName - oracle = None } + let aiBaseOptions = + { defaultSearchStrategy = BFSMode + mapName = map.MapName } let options = VSharpOptions( @@ -261,7 +283,7 @@ let generateDataForPretraining outputDirectory datasetBasePath (maps: ResizeArra searchStrategy = SearchStrategy.ExecutionTreeContributedCoverage, stepsLimit = stepsToSerialize, solverTimeout = 2, - aiAgentTrainingOptions = aiTrainingOptions + aiOptions = DatasetGenerator aiBaseOptions ) let folderForResults = @@ -274,10 +296,7 @@ let generateDataForPretraining outputDirectory datasetBasePath (maps: ResizeArra let explorationResult = explore map options - File.WriteAllText( - Path.Join(folderForResults, "result"), - $"{explorationResult.ActualCoverage} {explorationResult.TestsCount} {explorationResult.StepsCount} {explorationResult.ErrorsCount}" - ) + File.WriteAllText(Path.Join(folderForResults, "result"), serializeExplorationResult explorationResult) printfn $"Generation for {map.MapName} finished with coverage {explorationResult.ActualCoverage}, tests {explorationResult.TestsCount}, steps {explorationResult.StepsCount},errors {explorationResult.ErrorsCount}." @@ -286,49 +305,110 @@ let generateDataForPretraining outputDirectory datasetBasePath (maps: ResizeArra API.Reset() HashMap.hashMap.Clear() +let runTrainingSendModelMode + outputDirectory + (gameMap: GameMap) + (pathToModel: string) + (useGPU: bool) + (optimize: bool) + (port: int) + = + printfn $"Run infer on {gameMap.MapName} have started." + let stepsToStart = gameMap.StepsToStart + let stepsToPlay = gameMap.StepsToPlay + + let aiTrainingOptions = + { aiBaseOptions = + { defaultSearchStrategy = + match gameMap.DefaultSearcher with + | searcher.BFS -> BFSMode + | searcher.DFS -> DFSMode + | x -> failwithf $"Unexpected searcher {x}. Use DFS or BFS for now." + + mapName = gameMap.MapName } + stepsToSwitchToAI = stepsToStart + stepsToPlay = stepsToPlay + oracle = None } + + let steps = ResizeArray() + let stepSaver (aiGameStep: AIGameStep) = steps.Add aiGameStep + + let aiOptions: AIOptions = + Training( + SendModel + { aiAgentTrainingOptions = aiTrainingOptions + outputDirectory = outputDirectory + stepSaver = stepSaver } + ) + + let options = + VSharpOptions( + timeout = TIMEOUT_FOR_TRAINING, + outputDirectory = outputDirectory, + searchStrategy = SearchStrategy.AI, + solverTimeout = SOLVER_TIMEOUT_FOR_TRAINING, + stepsLimit = uint (stepsToPlay + stepsToStart), + aiOptions = aiOptions, + pathToModel = pathToModel, + useGPU = useGPU, + optimize = optimize + ) + + let explorationResult = explore gameMap options + + File.WriteAllText( + Path.Join(outputDirectory, gameMap.MapName + "result"), + serializeExplorationResult explorationResult + ) + + printfn + $"Running for {gameMap.MapName} finished with coverage {explorationResult.ActualCoverage}, tests {explorationResult.TestsCount}, steps {explorationResult.StepsCount},errors {explorationResult.ErrorsCount}." + + let stream = + let host = "localhost" // TODO: working within a local network + let client = new TcpClient() + client.Connect(host, port) + client.SendBufferSize <- 4096 + client.GetStream() + + let needToSendSteps = + let buffer = Array.zeroCreate 1 + let bytesRead = stream.Read(buffer, 0, 1) + + if bytesRead = 0 then + failwith "Connection is closed?!" + + stream.Close() + buffer.[0] <> byte 0 + + if needToSendSteps then + File.WriteAllText(Path.Join(outputDirectory, gameMap.MapName + "_steps"), JsonSerializer.Serialize steps) + [] let main args = let parser = ArgumentParser.Create(programName = "VSharp.ML.GameServer.Runner.exe") let args = parser.Parse args - let mode = args.GetResult <@ Mode @> - let port = - match args.TryGetResult <@ Port @> with - | Some port -> port - | None -> 8100 - - let datasetBasePath = - match args.TryGetResult <@ DatasetBasePath @> with - | Some path -> path - | None -> "" - - let datasetDescription = - match args.TryGetResult <@ DatasetDescription @> with - | Some path -> path - | None -> "" - - let stepsToSerialize = - match args.TryGetResult <@ StepsToSerialize @> with - | Some steps -> steps - | None -> 500u - - let useGPU = (args.TryGetResult <@ UseGPU @>).IsSome + let port = args.GetResult(Port, defaultValue = 8100) - let optimize = (args.TryGetResult <@ Optimize @>).IsSome + let outputDirectory = + args.GetResult(OutFolder, defaultValue = Path.Combine(Directory.GetCurrentDirectory(), string port)) - let outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), string port) + let cleanOutputDirectory () = + if Directory.Exists outputDirectory then + Directory.Delete(outputDirectory, true) - if Directory.Exists outputDirectory then - Directory.Delete(outputDirectory, true) + Directory.CreateDirectory outputDirectory - let testsDirInfo = Directory.CreateDirectory outputDirectory printfn $"outputDir: {outputDirectory}" match mode with | Mode.Server -> + let _ = cleanOutputDirectory () + try startWebServer { defaultConfig with @@ -338,7 +418,46 @@ let main args = with e -> printfn $"Failed on port {port}" printfn $"{e.Message}" + | Mode.SendModel -> + let model = args.GetResult(Model, defaultValue = "models/model.onnx") + + let stepsToPlay = args.GetResult <@ StepsToPlay @> + + let defaultSearcher = + let s = args.GetResult <@ DefaultSearcher @> + let upperedS = String.map System.Char.ToUpper s + + match upperedS with + | "BFS" -> searcher.BFS + | "DFS" -> searcher.DFS + | _ -> failwith "Use BFS or DFS as a default searcher" + + let stepsToStart = args.GetResult <@ StepsToStart @> + let assemblyFullName = args.GetResult <@ AssemblyFullName @> + let nameOfObjectToCover = args.GetResult <@ NameOfObjectToCover @> + let mapName = args.GetResult <@ MapName @> + + let gameMap = + GameMap( + stepsToPlay = stepsToPlay, + stepsToStart = stepsToStart, + assemblyFullName = assemblyFullName, + defaultSearcher = defaultSearcher, + nameOfObjectToCover = nameOfObjectToCover, + mapName = mapName + ) + + let useGPU = args.GetResult(UseGPU, defaultValue = false) + let optimize = args.GetResult(Optimize, defaultValue = false) + + runTrainingSendModelMode outputDirectory gameMap model useGPU optimize port | Mode.Generator -> + let datasetDescription = args.GetResult <@ DatasetDescription @> + let datasetBasePath = args.GetResult <@ DatasetBasePath @> + let stepsToSerialize = args.GetResult(StepsToSerialize, defaultValue = 500u) + + + let _ = cleanOutputDirectory () let maps = loadGameMaps datasetDescription generateDataForPretraining outputDirectory datasetBasePath maps stepsToSerialize | x -> failwithf $"Unexpected mode {x}." diff --git a/VSharp.Test/Benchmarks/Benchmarks.cs b/VSharp.Test/Benchmarks/Benchmarks.cs index 12e022ffd..8e0f49aaa 100644 --- a/VSharp.Test/Benchmarks/Benchmarks.cs +++ b/VSharp.Test/Benchmarks/Benchmarks.cs @@ -93,10 +93,10 @@ public static BenchmarkResult Run( stopOnCoverageAchieved: -1, randomSeed: randomSeed, stepsLimit: stepsLimit, - aiAgentTrainingOptions: null, + aiOptions: null, pathToModel: null, - useGPU: null, - optimize: null + useGPU: false, + optimize: false ); var fuzzerOptions = new FuzzerOptions( diff --git a/VSharp.Test/IntegrationTests.cs b/VSharp.Test/IntegrationTests.cs index 00ed34653..6346f9a57 100644 --- a/VSharp.Test/IntegrationTests.cs +++ b/VSharp.Test/IntegrationTests.cs @@ -136,7 +136,6 @@ static TestSvmAttribute() private readonly bool _useGPU; private readonly bool _optimize; - public TestSvmAttribute( int expectedCoverage = -1, uint recThresholdForTest = 1u, @@ -176,10 +175,10 @@ public TestSvmAttribute( _fuzzerIsolation = fuzzerIsolation; _explorationMode = explorationMode; _randomSeed = randomSeed; + _stepsLimit = stepsLimit; _pathToModel = pathToModel; _useGPU = useGPU; _optimize = optimize; - _stepsLimit = stepsLimit; } public TestCommand Wrap(TestCommand command) @@ -200,10 +199,10 @@ public TestCommand Wrap(TestCommand command) _explorationMode, _randomSeed, _stepsLimit, + _hasExternMocking, _pathToModel, _useGPU, - _optimize, - _hasExternMocking + _optimize ); } @@ -263,10 +262,10 @@ public TestSvmCommand( ExplorationMode explorationMode, int randomSeed, uint stepsLimit, + bool hasExternMocking, string pathToModel, bool useGPU, - bool optimize, - bool hasExternMocking) : base(innerCommand) + bool optimize) : base(innerCommand) { _baseCoverageZone = coverageZone; _baseSearchStrat = TestContext.Parameters[SearchStrategyParameterName] == null ? @@ -478,7 +477,7 @@ private TestResult Explore(TestExecutionContext context) stopOnCoverageAchieved: _expectedCoverage ?? -1, randomSeed: _randomSeed, stepsLimit: _stepsLimit, - aiAgentTrainingOptions: null, + aiOptions: null, pathToModel: _pathToModel, useGPU: _useGPU, optimize: _optimize