Skip to content

Commit 25182fb

Browse files
tmckay-sifivetmckay-sifive
committed
consolidate simulateTests APIs; return a TestResults
1 parent 7230abe commit 25182fb

File tree

3 files changed

+160
-144
lines changed

3 files changed

+160
-144
lines changed

‎src/main/scala/chisel3/simulator/SimulatorAPI.scala

+74-60
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,67 @@ import chisel3.util.simpleClassName
1010
import java.nio.file.Files
1111

1212
trait SimulatorAPI {
13+
object TestChoice {
14+
sealed abstract class Type {
15+
private[simulator] def globs: Seq[String]
16+
}
17+
18+
case object None extends Type {
19+
override def globs = Seq.empty
20+
}
21+
22+
case class Globs(globs: Seq[String]) extends Type
23+
24+
object Glob {
25+
def apply(glob: String) = Names(Seq(glob))
26+
}
27+
28+
case class Names(names: Seq[String]) extends Type {
29+
override def globs = names
30+
}
31+
32+
object Name {
33+
def apply(name: String) = Names(Seq(name))
34+
}
35+
36+
case object All extends Type {
37+
override def globs = Seq("*")
38+
}
39+
}
40+
41+
object TestResult {
42+
sealed trait Type
43+
case object Success extends Type
44+
case class Failure(e: Throwable) extends Type
45+
}
46+
47+
final class TestResults(digests: Seq[(String, Simulator.BackendInvocationDigest[_])]) {
48+
private val results: Map[String, TestResult.Type] = digests.map { case (testName, digest) =>
49+
testName -> {
50+
try {
51+
digest.result
52+
TestResult.Success
53+
} catch {
54+
case e: Throwable => TestResult.Failure(e)
55+
}
56+
}
57+
}.toMap
58+
59+
/** Get the result for the test with this name. */
60+
def apply(testName: String) =
61+
results.lift(testName).getOrElse {
62+
throw new NoSuchElementException(s"Cannot get result for ${testName} as it was not run")
63+
}
64+
65+
def all: Seq[(String, TestResult.Type)] =
66+
results.toSeq
67+
68+
def failed: Seq[(String, Throwable)] =
69+
results.collect { case (name, TestResult.Failure(e)) => name -> e }.toSeq
70+
71+
def passed: Seq[String] =
72+
results.collect { case r @ (name, TestResult.Success) => name }.toSeq
73+
}
1374

1475
/** Simulate a [[RawModule]] without any initialization procedure.
1576
*
@@ -105,7 +166,7 @@ trait SimulatorAPI {
105166

106167
def simulateTests[T <: RawModule with HasTests](
107168
module: => T,
108-
testGlobs: Seq[String],
169+
tests: TestChoice.Type,
109170
timeout: Int,
110171
chiselOpts: Array[String] = Array.empty,
111172
firtoolOpts: Array[String] = Array.empty,
@@ -118,69 +179,22 @@ trait SimulatorAPI {
118179
firtoolOptsModifications: FirtoolOptionsModifications,
119180
commonSettingsModifications: svsim.CommonSettingsModifications,
120181
backendSettingsModifications: svsim.BackendSettingsModifications
121-
): Seq[(String, Simulator.BackendInvocationDigest[_])] = {
182+
): TestResults = {
122183
val modifiedTestingDirectory = subdirectory match {
123184
case Some(subdir) => testingDirectory.withSubdirectory(subdir)
124185
case None => testingDirectory
125186
}
126187

127-
hasSimulator
128-
.getSimulator(modifiedTestingDirectory)
129-
.simulateTests(
130-
module = module,
131-
includeTestGlobs = testGlobs,
132-
chiselOpts = chiselOpts,
133-
firtoolOpts = firtoolOpts,
134-
settings = settings
135-
) { dut => InlineTestStimulus(timeout)(dut.wrapped) }
188+
new TestResults(
189+
hasSimulator
190+
.getSimulator(modifiedTestingDirectory)
191+
.simulateTests(
192+
module = module,
193+
includeTestGlobs = tests.globs,
194+
chiselOpts = chiselOpts,
195+
firtoolOpts = firtoolOpts,
196+
settings = settings
197+
) { dut => InlineTestStimulus(timeout)(dut.wrapped) }
198+
)
136199
}
137-
138-
def simulateAllTests[T <: RawModule with HasTests](
139-
module: => T,
140-
timeout: Int,
141-
chiselOpts: Array[String] = Array.empty,
142-
firtoolOpts: Array[String] = Array.empty,
143-
settings: Settings[TestHarness[T, _]] = Settings.defaultRaw[TestHarness[T, _]],
144-
subdirectory: Option[String] = None
145-
)(
146-
implicit hasSimulator: HasSimulator,
147-
testingDirectory: HasTestingDirectory,
148-
chiselOptsModifications: ChiselOptionsModifications,
149-
firtoolOptsModifications: FirtoolOptionsModifications,
150-
commonSettingsModifications: svsim.CommonSettingsModifications,
151-
backendSettingsModifications: svsim.BackendSettingsModifications
152-
): Seq[(String, Simulator.BackendInvocationDigest[_])] = simulateTests(
153-
module = module,
154-
testGlobs = Seq("*"),
155-
timeout = timeout,
156-
chiselOpts = chiselOpts,
157-
firtoolOpts = firtoolOpts,
158-
settings = settings,
159-
subdirectory = subdirectory
160-
)
161-
162-
def simulateTest[T <: RawModule with HasTests](
163-
module: => T,
164-
testName: String,
165-
timeout: Int,
166-
chiselOpts: Array[String] = Array.empty,
167-
firtoolOpts: Array[String] = Array.empty,
168-
settings: Settings[TestHarness[T, _]] = Settings.defaultRaw[TestHarness[T, _]],
169-
subdirectory: Option[String] = None
170-
)(
171-
implicit hasSimulator: HasSimulator,
172-
testingDirectory: HasTestingDirectory,
173-
chiselOptsModifications: ChiselOptionsModifications,
174-
firtoolOptsModifications: FirtoolOptionsModifications,
175-
commonSettingsModifications: svsim.CommonSettingsModifications,
176-
backendSettingsModifications: svsim.BackendSettingsModifications
177-
): Simulator.BackendInvocationDigest[_] = simulateTests(
178-
module = module,
179-
testGlobs = Seq(testName),
180-
timeout = timeout,
181-
chiselOpts = chiselOpts,
182-
firtoolOpts = firtoolOpts,
183-
settings = settings,
184-
subdirectory = subdirectory
185-
).head._2
186200
}

‎src/main/scala/chisel3/simulator/stimulus/InlineTestStimulus.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ trait InlineTestStimulus extends Stimulus.Type[TestHarness[_, _]] {
4747
cycleCount += 1
4848

4949
if (cycleCount > _timeout) {
50-
throw new Exceptions.Timeout(_timeout, "Timeout reached without test asserting finish")
50+
throw new Exceptions.Timeout(_timeout, s"Test did not assert finish before ${_timeout} timesteps")
5151
}
5252
}
5353

‎src/test/scala/chiselTests/experimental/InlineTestSpec.scala

+85-83
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import chisel3.{assert => _, _}
44
import chisel3.experimental.hierarchy._
55
import chisel3.experimental.inlinetest._
66
import chisel3.simulator.scalatest.ChiselSim
7-
import chisel3.simulator
8-
import chisel3.simulator.{stimulus, Settings}
97
import chisel3.testers._
108
import chisel3.properties.Property
119
import chisel3.testing.scalatest.FileCheck
@@ -460,116 +458,120 @@ class InlineTestSpec extends AnyFlatSpec with FileCheck with ChiselSim {
460458
)
461459
}
462460

463-
def assertPass(digest: simulator.Simulator.BackendInvocationDigest[_]): Unit =
464-
digest.result
461+
def assertPass(result: TestResult.Type): Unit = result match {
462+
case TestResult.Success => ()
463+
case TestResult.Failure(e) => throw e
464+
}
465465

466-
def assertFail(digest: simulator.Simulator.BackendInvocationDigest[_]): Unit =
467-
intercept[simulator.Exceptions.TestFailed](digest.result)
468-
.getMessage()
469-
.fileCheck() {
470-
"""
471-
| CHECK: The test finished and signaled failure
472-
"""
473-
}
466+
def assertFail(result: TestResult.Type): Unit = result match {
467+
case TestResult.Success => fail("Test unexpectedly passed")
468+
case TestResult.Failure(e) =>
469+
e.getMessage()
470+
.fileCheck() {
471+
"""
472+
| CHECK: The test finished and signaled failure
473+
"""
474+
}
475+
}
474476

475-
def assertTimeout(timeout: Int)(digest: simulator.Simulator.BackendInvocationDigest[_]): Unit =
476-
intercept[simulator.Exceptions.Timeout](digest.result)
477-
.getMessage()
478-
.fileCheck() {
479-
s"""
480-
| CHECK: A timeout occurred after ${timeout} timesteps
481-
"""
482-
}
477+
def assertTimeout(timeout: Int)(result: TestResult.Type): Unit = result match {
478+
case TestResult.Success => fail("Test unexpectedly passed")
479+
case TestResult.Failure(e) =>
480+
e.getMessage()
481+
.fileCheck() {
482+
s"""
483+
| CHECK: A timeout occurred after ${timeout} timesteps
484+
"""
485+
}
486+
}
483487

484-
def assertAssertion(message: String)(digest: simulator.Simulator.BackendInvocationDigest[_]) =
485-
intercept[simulator.Exceptions.AssertionFailed](digest.result)
486-
.getMessage()
487-
.fileCheck() {
488-
"""
489-
| CHECK: One or more assertions failed during Chiselsim simulation
490-
| CHECK: counter hit max
491-
"""
492-
}
488+
def assertAssertion(message: String)(result: TestResult.Type): Unit = result match {
489+
case TestResult.Success => fail("Test unexpectedly passed")
490+
case TestResult.Failure(e) =>
491+
e.getMessage()
492+
.fileCheck() {
493+
"""
494+
| CHECK: One or more assertions failed during Chiselsim simulation
495+
| CHECK: counter hit max
496+
"""
497+
}
498+
}
493499

494500
it should "simulate and pass if finish asserted with success=1" in {
495-
assertPass {
496-
simulateTest(
497-
new ModuleWithTests,
498-
testName = "signal_pass",
499-
timeout = 100
500-
)
501-
}
501+
val results = simulateTests(
502+
new ModuleWithTests,
503+
tests = TestChoice.Name("signal_pass"),
504+
timeout = 100
505+
)
506+
assertPass(results("signal_pass"))
502507
}
503508

504509
it should "simulate and fail if finish asserted with success=0" in {
505-
assertFail {
506-
simulateTest(
507-
new ModuleWithTests,
508-
testName = "signal_fail",
509-
timeout = 100
510-
)
511-
}
510+
val results = simulateTests(
511+
new ModuleWithTests,
512+
tests = TestChoice.Name("signal_fail"),
513+
timeout = 100
514+
)
515+
assertFail(results("signal_fail"))
512516
}
513517

514518
it should "simulate and timeout if finish not asserted" in {
515-
assertTimeout(100) {
516-
simulateTest(
517-
new ModuleWithTests,
518-
testName = "timeout",
519-
timeout = 100
520-
)
521-
}
519+
val results = simulateTests(
520+
new ModuleWithTests,
521+
tests = TestChoice.Name("timeout"),
522+
timeout = 100
523+
)
524+
assertTimeout(100)(results("timeout"))
522525
}
523526

524527
it should "simulate and fail early if assertion raised" in {
525-
assertAssertion("counter hit max") {
526-
simulateTest(
527-
new ModuleWithTests,
528-
testName = "assertion",
529-
timeout = 100
530-
)
531-
}
528+
val results = simulateTests(
529+
new ModuleWithTests,
530+
tests = TestChoice.Name("assertion"),
531+
timeout = 100
532+
)
533+
assertAssertion("counter hit max")(results("assertion"))
532534
}
533535

534536
it should "run multiple passing simulations" in {
535-
simulateTests(
537+
val results = simulateTests(
536538
new ModuleWithTests,
537-
testGlobs = Seq("signal_pass", "signal_pass_2"),
539+
tests = TestChoice.Names(Seq("signal_pass", "signal_pass_2")),
538540
timeout = 100
539-
).map(_._2).foreach(assertPass)
541+
)
542+
results.all.foreach { case (name, result) =>
543+
assertPass(result)
544+
}
540545
}
541546

542547
it should "run one passing and one failing simulation" in {
543-
simulateTests(
548+
val results = simulateTests(
544549
new ModuleWithTests,
545-
testGlobs = Seq("signal_pass", "signal_fail"),
550+
tests = TestChoice.Names(Seq("signal_pass", "signal_fail")),
546551
timeout = 100
547-
).map {
548-
case ("signal_pass", digest) => assertPass(digest)
549-
case ("signal_fail", digest) => assertFail(digest)
550-
case _ => fail("unexpected test name")
551-
}
552+
)
553+
assertPass(results("signal_pass"))
554+
assertFail(results("signal_fail"))
552555
}
553556

554557
it should "simulate all tests" in {
555-
val results = simulateAllTests(
558+
val results = simulateTests(
556559
new ModuleWithTests,
560+
tests = TestChoice.All,
557561
timeout = 100
558562
)
559-
assert(results.size == 11)
560-
results.map {
561-
case ("signal_fail", digest) => assertFail(digest)
562-
case ("timeout", digest) => assertTimeout(100)(digest)
563-
case ("assertion", digest) => assertAssertion("counter hit max")(digest)
564-
case ("check1", digest) => assertTimeout(100)(digest)
565-
case ("check2", digest) => assertTimeout(100)(digest)
566-
case ("bar", digest) => assertTimeout(100)(digest)
567-
case ("signal_pass", digest) => assertPass(digest)
568-
case ("signal_pass_2", digest) => assertPass(digest)
569-
case ("with_monitor", digest) => assertTimeout(100)(digest)
570-
case ("with_result", digest) => assertTimeout(100)(digest)
571-
case ("foo", digest) => assertTimeout(100)(digest)
572-
case (testName, digest) => fail(s"unexpected test name: $testName")
573-
}
563+
assert(results.all.size == 11)
564+
565+
assertFail(results("signal_fail"))
566+
assertTimeout(100)(results("timeout"))
567+
assertAssertion("counter hit max")(results("assertion"))
568+
assertTimeout(100)(results("check1"))
569+
assertTimeout(100)(results("check2"))
570+
assertTimeout(100)(results("bar"))
571+
assertPass(results("signal_pass"))
572+
assertPass(results("signal_pass_2"))
573+
assertTimeout(100)(results("with_monitor"))
574+
assertTimeout(100)(results("with_result"))
575+
assertTimeout(100)(results("foo"))
574576
}
575577
}

0 commit comments

Comments
 (0)