Merge pull request #9 from ian-h-chamberlain/feature/run-doctests

Author: ian-h-chamberlain

.github/workflows/ci.yml

@@ -63,18 +63,17 @@ jobs:
           working-directory: test-runner
           args: -- -v
 
-      # TODO(#4): run these suckers
-      # - name: Build and run doc tests
-      #   # Let's still run doc tests even if lib/integration tests fail:
-      #   if: ${{ !cancelled() }}
-      #   env:
-      #     # This ensures the citra logs and video output gets put in a directory
-      #     # where we can upload as artifacts
-      #     RUSTDOCFLAGS: " --persist-doctests ${{ env.GITHUB_WORKSPACE }}/target/armv6k-nintendo-3ds/debug/doctests"
-      #   uses: ./run-tests
-      #   with:
-      #     working-directory: test-runner
-      #     args: --doc -- -v
+      - name: Build and run doc tests
+        # Still run doc tests even if lib/integration tests fail:
+        if: ${{ !cancelled() }}
+        env:
+          # This ensures the citra logs and video output get persisted to a
+          # directory where the artifact upload can find them.
+          RUSTDOCFLAGS: " --persist-doctests target/armv6k-nintendo-3ds/debug/doctests"
+        uses: ./run-tests
+        with:
+          working-directory: test-runner
+          args: --doc -- -v
 
       - name: Upload citra logs and capture videos
         uses: actions/upload-artifact@v3
@@ -83,5 +82,5 @@ jobs:
         with:
           name: citra-logs-${{ matrix.toolchain }}
           path: |
-            target/armv6k-nintendo-3ds/debug/**/*.txt
-            target/armv6k-nintendo-3ds/debug/**/*.webm
+            test-runner/target/armv6k-nintendo-3ds/debug/**/*.txt
+            test-runner/target/armv6k-nintendo-3ds/debug/**/*.webm

README.md

@@ -26,7 +26,7 @@ In `lib.rs` and any integration test files:
 ```
 
 Then use the `setup` and `run-tests` actions in your github workflow. This
-example shows the default value for each of the inputs:
+example shows the default value for each of the inputs.
 
 ```yml
 jobs:
@@ -37,8 +37,6 @@ jobs:
       volumes:
         # This is required so the test action can `docker run` the runner:
         - '/var/run/docker.sock:/var/run/docker.sock'
-        # This is required so doctest artifacts are accessible to the action:
-        - '/tmp:/tmp'
 
     steps:
       - name: Checkout branch
@@ -63,3 +61,6 @@ jobs:
           # https://github.com/actions/runner/issues/2058
           working-directory: ${GITHUB_WORKSPACE}
 ```
+
+See [`ci.yml`](.github/workflows/ci.yml) to see a full lint and test workflow
+using these actions (including uploading output artifacts from the tests).

run-tests/action.yml

@@ -1,9 +1,9 @@
 name: Cargo 3DS Test
 description: >
   Run `cargo 3ds test` executables using Citra. Note that to use this action,
-  you must mount `/var/run/docker.sock:/var/run/docker.sock` and `/tmp:/tmp` into
-  the container so that the runner image can be built and doctest artifacts can
-  be found, respectively.
+  you must use a container image of `devkitpro/devkitarm` and mount
+  `/var/run/docker.sock:/var/run/docker.sock` into the container so that the
+  runner image can be built by the action.
 
 inputs:
   args:
@@ -34,6 +34,8 @@ runs:
         tags: ${{ inputs.runner-image }}:latest
         push: false
         load: true
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
 
     - name: Ensure docker is installed in the container
       shell: bash
@@ -42,20 +44,24 @@ runs:
     - name: Run cargo 3ds test
       shell: bash
       # Set a custom runner for `cargo test` commands to use.
-      # Use ${GITHUB_WORKSPACE} due to
+      # Use ${PWD} and ${RUNNER_TEMP} due to
       # https://github.com/actions/runner/issues/2058, which also means
-      # we have to export this instead of using the env: key
+      # we have to export this in `run` instead of using the `env` key
       run: |
         cd ${{ inputs.working-directory }}
+
+        # Hopefully this still works if the input is an absolute path:
+        mounted_pwd="${{ github.workspace }}/${{ inputs.working-directory }}"
+
         export CARGO_TARGET_ARMV6K_NINTENDO_3DS_RUNNER="
             docker run --rm
-                -v ${{ runner.temp }}:${{ runner.temp }}
-                -v ${{ github.workspace }}/target:/app/target
                 -v ${{ github.workspace }}:${GITHUB_WORKSPACE}
+                -v ${mounted_pwd}/target:/app/target
+                -v ${{ runner.temp }}:${RUNNER_TEMP}
                 ${{ inputs.runner-image }}:latest"
         env
         cargo 3ds -v test ${{ inputs.args }}
       env:
-        # Ensure that doctests get built into a path which is mounted on the host
-        # as well as in this container (via the bind mount in the RUNNER command)
-        TMPDIR: ${{ runner.temp }}
+        # Make sure doctests are built into the shared tempdir instead of the
+        # container's /tmp which will be immediately removed
+        TMPDIR: ${{ env.RUNNER_TEMP }}

test-runner/src/console.rs

@@ -1,3 +1,5 @@
+use std::process::Termination;
+
 use ctru::prelude::*;
 use ctru::services::gfx::{Flush, Swap};
 
@@ -28,11 +30,10 @@ impl TestRunner for ConsoleRunner {
         Console::new(self.gfx.top_screen.borrow_mut())
     }
 
-    fn cleanup(mut self, _test_result: std::io::Result<bool>) {
-        // We don't actually care about the test result, either way we'll stop
-        // and show the results to the user
+    fn cleanup<T: Termination>(mut self, result: T) -> T {
+        // We don't actually care about the output of the test result, either
+        // way we'll stop and show the results to the user.
 
-        // Wait to make sure the user can actually see the results before we exit
         println!("Press START to exit.");
 
         while self.apt.main_loop() {
@@ -47,5 +48,7 @@ impl TestRunner for ConsoleRunner {
                 break;
             }
         }
+
+        result
     }
 }

test-runner/src/gdb.rs

@@ -1,3 +1,5 @@
+use std::process::Termination;
+
 use ctru::error::ResultCode;
 
 use super::TestRunner;
@@ -27,22 +29,10 @@ impl TestRunner for GdbRunner {
         .expect("failed to redirect I/O streams to GDB");
     }
 
-    fn cleanup(self, test_result: std::io::Result<bool>) {
+    fn cleanup<T: Termination>(self, test_result: T) -> T {
         // GDB actually has the opportunity to inspect the exit code,
         // unlike other runners, so let's follow the default behavior of the
         // stdlib test runner.
-        match test_result {
-            Ok(success) => {
-                if success {
-                    std::process::exit(0);
-                } else {
-                    std::process::exit(101);
-                }
-            }
-            Err(err) => {
-                eprintln!("Error: {err}");
-                std::process::exit(101);
-            }
-        }
+        test_result.report().exit_process()
     }
 }

test-runner/src/lib.rs

@@ -6,34 +6,37 @@
 
 #![feature(test)]
 #![feature(custom_test_frameworks)]
+#![feature(exitcode_exit_method)]
 #![test_runner(run_gdb)]
 
 extern crate test;
 
 mod console;
 mod gdb;
+mod macros;
 mod socket;
 
-use console::ConsoleRunner;
-use gdb::GdbRunner;
-use socket::SocketRunner;
+use std::process::{ExitCode, Termination};
 
+pub use console::ConsoleRunner;
+pub use gdb::GdbRunner;
+pub use socket::SocketRunner;
 use test::{ColorConfig, OutputFormat, TestDescAndFn, TestFn, TestOpts};
 
 /// Show test output in GDB, using the [File I/O Protocol] (called HIO in some 3DS
 /// homebrew resources). Both stdout and stderr will be printed to the GDB console.
 ///
 /// [File I/O Protocol]: https://sourceware.org/gdb/onlinedocs/gdb/File_002dI_002fO-Overview.html#File_002dI_002fO-Overview
 pub fn run_gdb(tests: &[&TestDescAndFn]) {
-    run::<GdbRunner>(tests)
+    run::<GdbRunner>(tests);
 }
 
 /// Run tests using the `ctru` [`Console`] (print results to the 3DS screen).
 /// This is mostly useful for running tests manually, especially on real hardware.
 ///
 /// [`Console`]: ctru::console::Console
 pub fn run_console(tests: &[&TestDescAndFn]) {
-    run::<ConsoleRunner>(tests)
+    run::<ConsoleRunner>(tests);
 }
 
 /// Show test output via a network socket to `3dslink`. This runner is only useful
@@ -43,7 +46,7 @@ pub fn run_console(tests: &[&TestDescAndFn]) {
 ///
 /// [`Soc::redirect_to_3dslink`]: ctru::services::soc::Soc::redirect_to_3dslink
 pub fn run_socket(tests: &[&TestDescAndFn]) {
-    run::<SocketRunner>(tests)
+    run::<SocketRunner>(tests);
 }
 
 fn run<Runner: TestRunner>(tests: &[&TestDescAndFn]) {
@@ -71,7 +74,13 @@ fn run<Runner: TestRunner>(tests: &[&TestDescAndFn]) {
 
     drop(ctx);
 
-    runner.cleanup(result);
+    let reportable_result = match result {
+        Ok(true) => Ok(()),
+        // Try to match stdlib console test runner behavior as best we can
+        _ => Err(ExitCode::from(101)),
+    };
+
+    let _ = runner.cleanup(reportable_result);
 }
 
 /// Adapted from [`test::make_owned_test`].
@@ -92,8 +101,16 @@ fn make_owned_test(test: &TestDescAndFn) -> TestDescAndFn {
     }
 }
 
+mod private {
+    pub trait Sealed {}
+
+    impl Sealed for super::ConsoleRunner {}
+    impl Sealed for super::GdbRunner {}
+    impl Sealed for super::SocketRunner {}
+}
+
 /// A helper trait to make the behavior of test runners consistent.
-trait TestRunner: Sized + Default {
+pub trait TestRunner: private::Sealed + Sized + Default {
     /// Any context the test runner needs to remain alive for the duration of
     /// the test. This can be used for things that need to borrow the test runner
     /// itself.
@@ -107,7 +124,11 @@ trait TestRunner: Sized + Default {
 
     /// Handle the results of the test and perform any necessary cleanup.
     /// The [`Context`](Self::Context) will be dropped just before this is called.
-    fn cleanup(self, test_result: std::io::Result<bool>);
+    ///
+    /// This returns `T` so that the result can be used in doctests.
+    fn cleanup<T: Termination>(self, test_result: T) -> T {
+        test_result
+    }
 }
 
 /// This module has stubs needed to link the test library, but they do nothing
@@ -132,17 +153,6 @@ mod link_fix {
     }
 }
 
-/// Verify that doctests work as expected
-/// ```
-/// assert_eq!(2 + 2, 4);
-/// ```
-///
-/// ```should_panic
-/// assert_eq!(2 + 2, 5);
-/// ```
-#[cfg(doctest)]
-struct Dummy;
-
 #[cfg(test)]
 mod tests {
     #[test]

test-runner/src/macros.rs

@@ -0,0 +1,101 @@
+//! Macros for working with test runners.
+
+// Use a neat little trick with cfg(doctest) to make code fences appear in
+// rustdoc output, but still compile normally when doctesting. This raises warnings
+// for invalid code though, so we also silence that lint here.
+#[cfg_attr(not(doctest), allow(rustdoc::invalid_rust_codeblocks))]
+/// Helper macro for writing doctests using this runner. Wrap this macro around
+/// your normal doctest to enable running it with the test runners in this crate.
+///
+/// You may optionally specify a runner before the test body, and may use any of
+/// the various [`fn main()`](https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html#using--in-doc-tests)
+/// signatures allowed by documentation tests.
+///
+/// # Examples
+///
+/// ## Basic usage
+///
+#[cfg_attr(not(doctest), doc = "````")]
+/// ```
+/// test_runner::doctest! {
+///     assert_eq!(2 + 2, 4);
+/// }
+/// ```
+#[cfg_attr(not(doctest), doc = "````")]
+///
+/// ## Custom runner
+///
+#[cfg_attr(not(doctest), doc = "````")]
+/// ```no_run
+/// test_runner::doctest! { SocketRunner,
+///     assert_eq!(2 + 2, 4);
+/// }
+/// ```
+#[cfg_attr(not(doctest), doc = "````")]
+///
+/// ## `should_panic`
+///
+#[cfg_attr(not(doctest), doc = "````")]
+/// ```should_panic
+/// test_runner::doctest! {
+///     assert_eq!(2 + 2, 5);
+/// }
+/// ```
+#[cfg_attr(not(doctest), doc = "````")]
+///
+/// ## Custom `fn main`
+///
+#[cfg_attr(not(doctest), doc = "````")]
+/// ```
+/// test_runner::doctest! {
+///     fn main() {
+///         assert_eq!(2 + 2, 4);
+///     }
+/// }
+/// ```
+#[cfg_attr(not(doctest), doc = "````")]
+///
+#[cfg_attr(not(doctest), doc = "````")]
+/// ```
+/// test_runner::doctest! {
+///     fn main() -> Result<(), Box<dyn std::error::Error>> {
+///         assert_eq!(2 + 2, 4);
+///         Ok(())
+///     }
+/// }
+/// ```
+#[cfg_attr(not(doctest), doc = "````")]
+///
+/// ## Implicit return type
+///
+/// Note that for the rustdoc preprocessor to understand the return type, the
+/// `Ok(())` expression must be written _outside_ the `doctest!` invocation.
+///
+#[cfg_attr(not(doctest), doc = "````")]
+/// ```
+/// test_runner::doctest! {
+///     assert_eq!(2 + 2, 4);
+/// }
+/// Ok::<(), std::io::Error>(())
+/// ```
+#[cfg_attr(not(doctest), doc = "````")]
+#[macro_export]
+macro_rules! doctest {
+    ($runner:ident, fn main() $(-> $ret:ty)? { $($body:tt)* } ) => {
+        fn main() $(-> $ret)? {
+            $crate::doctest!{ $runner, $($body)* }
+        }
+    };
+    ($runner:ident, $($body:tt)*) => {
+        use $crate::TestRunner as _;
+        let mut _runner = $crate::$runner::default();
+        _runner.setup();
+        let _result = { $($body)* };
+        _runner.cleanup(_result)
+    };
+    ($($body:tt)*) => {
+        $crate::doctest!{ GdbRunner,
+            $($body)*
+        }
+    };
+}