Handle grading when output: false option set

live-runtime/src/editor.ts

@@ -1,6 +1,6 @@
 import type { WebR, RFunction } from 'webr';
 import type { PyodideInterface } from 'pyodide';
-import type { OJSElement, EvaluateOptions } from './evaluate';
+import type { EvaluateOptions } from './evaluate';
 import { Indicator } from './indicator';
 import { basicSetup } from 'codemirror';
 import { tagHighlighterTok } from './highlighter';
@@ -11,6 +11,13 @@ import { autocompletion, CompletionContext } from '@codemirror/autocomplete';
 import { python } from "@codemirror/lang-python";
 import { r } from "codemirror-lang-r";
 
+export type EditorValue = {
+  code: string | null;
+  options: ExerciseOptions;
+  indicator?: Indicator;
+}
+export type OJSEditorElement = HTMLElement & { value?: EditorValue };
+
 type ExerciseOptions = EvaluateOptions & {
   autorun: boolean;
   caption: string;
@@ -82,7 +89,7 @@ abstract class ExerciseEditor {
   initialCode: string;
   state: EditorState;
   view: EditorView;
-  container: OJSElement;
+  container: OJSEditorElement;
   options: ExerciseOptions;
   indicator: Indicator;
   completionMethods: Promise<ExerciseCompletionMethods>;

live-runtime/src/evaluate-pyodide.ts

@@ -8,7 +8,8 @@ import {
   EvaluateContext,
   EvaluateOptions,
   ExerciseEvaluator,
-  OJSElement
+  OJSEvaluateElement,
+  EvaluateValue,
 } from "./evaluate";
 import { PyodideInterfaceWorker } from './pyodide-worker';
 
@@ -35,20 +36,21 @@ const requireHtmlManager = {
 };
 
 export class PyodideEvaluator implements ExerciseEvaluator {
-  container: OJSElement;
+  container: OJSEvaluateElement;
   context: EvaluateContext;
   options: EvaluateOptions;
   envLabels: EnvLabels;
   envManager: PyodideEnvironmentManager;
   pyodide: PyodideInterfaceWorker;
-
+  nullResult: EvaluateValue;
   constructor(
     pyodide: PyodideInterfaceWorker,
     environmentManager: PyodideEnvironmentManager,
     context: EvaluateContext
   ) {
     this.container = document.createElement('div');
-    this.container.value = { result: null, evaluator: this };
+    this.nullResult = { result: null, evaluate_result: null, evaluator: this };
+    this.container.value = this.nullResult;
     this.pyodide = pyodide;
     this.context = context;
 
@@ -104,7 +106,7 @@ export class PyodideEvaluator implements ExerciseEvaluator {
     // If we're not evaluating, just print the source directly
     if (!this.options.eval) {
       this.container = this.asSourceHTML(this.context.code);
-      this.container.value = { result: null, evaluator: this };
+      this.container.value = this.nullResult;
       return;
     }
 
@@ -137,12 +139,18 @@ export class PyodideEvaluator implements ExerciseEvaluator {
       const result = await this.evaluate(this.context.code, "result");
 
       // Once we have the evaluate result, render it's contents to HTML
-      if (!this.options.output) {
+      if (!result) {
         this.container.value.result = null;
       } else if (this.options.output === "asis") {
         this.container.innerHTML = await result.stdout;
       } else {
         this.container = await this.asHtml(result);
+        if (!this.options.output) {
+          // Don't show any output in HTML, but return a value
+          const value = this.container.value;
+          this.container = document.createElement("div");
+          this.container.value = value;
+        }
       }
 
       // Grab defined objects from the result environment
@@ -244,8 +252,8 @@ export class PyodideEvaluator implements ExerciseEvaluator {
     result: Awaited<ReturnType<PyodideEvaluator["evaluate"]>>,
     options: EvaluateOptions = this.options
   ) {
-    const container: OJSElement = document.createElement("div");
-    container.value = { result: null, evaluator: this };
+    const container: OJSEvaluateElement = document.createElement("div");
+    container.value = this.nullResult;
 
     if (!result) {
       return container;

live-runtime/src/evaluate-webr.ts

@@ -22,8 +22,9 @@ import {
   EnvLabels,
   EvaluateContext,
   EvaluateOptions,
+  EvaluateValue,
   ExerciseEvaluator,
-  OJSElement
+  OJSEvaluateElement,
 } from "./evaluate";
 import { arrayBufferToBase64 } from './utils';
 
@@ -40,17 +41,19 @@ declare global {
 
 
 export class WebREvaluator implements ExerciseEvaluator {
-  container: OJSElement;
+  container: OJSEvaluateElement;
   shelter: Promise<Shelter>;
   context: EvaluateContext;
   options: EvaluateOptions;
   envLabels: EnvLabels;
   envManager: WebREnvironmentManager;
+  nullResult: EvaluateValue;
   webR: WebR;
 
   constructor(webR: WebR, environmentManager: WebREnvironmentManager, context: EvaluateContext) {
     this.container = document.createElement('div');
-    this.container.value = { result: null, evaluator: this };
+    this.nullResult = { result: null, evaluate_result: null, evaluator: this };
+    this.container.value = this.nullResult;
     this.webR = webR;
     this.context = context;
     this.shelter = new webR.Shelter();
@@ -112,7 +115,7 @@ export class WebREvaluator implements ExerciseEvaluator {
     // If we're not evaluating, just print the source directly
     if (!this.options.eval) {
       this.container = this.asSourceHTML(this.context.code);
-      this.container.value = { result: null, evaluator: this };
+      this.container.value = this.nullResult;
       return;
     }
 
@@ -145,14 +148,20 @@ export class WebREvaluator implements ExerciseEvaluator {
       const result = await this.evaluate(this.context.code, "result");
 
       // Once we have the evaluate result, render it's contents to HTML
-      if (isRNull(result) || !this.options.output) {
+      if (!result) {
         this.container.value.result = null;
       } else if (this.options.output === "asis") {
         const evaluateList = await result.toArray() as RObject[];
         const lastValue = await evaluateList[evaluateList.length - 1].get('value');
         this.container.innerHTML = await lastValue.toString();
       } else {
         this.container = await this.asHtml(result);
+        if (!this.options.output) {
+          // Don't show any output in HTML, but return a value
+          const value = this.container.value;
+          this.container = document.createElement("div");
+          this.container.value = value;
+        }
       }
 
       // Grab objects from the webR OJS environment
@@ -196,7 +205,7 @@ export class WebREvaluator implements ExerciseEvaluator {
   async evaluate(code: string, envLabel: EnvLabel, options: EvaluateOptions = this.options) {
     // Early return if code is undefined, null, or if we're not evaluating
     if (code == null || !options.include) {
-      return this.webR.objs.null;
+      return null;
     }
 
     const shelter = await this.shelter;
@@ -243,8 +252,8 @@ export class WebREvaluator implements ExerciseEvaluator {
 
   async asHtml(value: RList, options: EvaluateOptions = this.options) {
     const sourceLines: string[] = [];
-    const container: OJSElement = document.createElement("div");
-    container.value = { result: null, evaluator: this };
+    const container: OJSEvaluateElement = document.createElement("div");
+    container.value = this.nullResult;
 
     const appendSource = () => {
       if (options.echo && sourceLines.length) {
@@ -460,7 +469,7 @@ export class WebREvaluator implements ExerciseEvaluator {
   }
 
   // Convert webR R object reference to OJS value
-  async asOjs(value: ImageBitmap): Promise<OJSElement>;
+  async asOjs(value: ImageBitmap): Promise<OJSEvaluateElement>;
   async asOjs(value: any): Promise<any>;
   async asOjs(value) {
     if (value instanceof ImageBitmap) {

live-runtime/src/evaluate.ts

@@ -1,7 +1,12 @@
 import { Indicator } from './indicator'
 import { PyodideEnvironmentManager, WebREnvironmentManager } from './environment'
 
-export type OJSElement = HTMLElement & { value?: any };
+export type EvaluateValue = {
+  evaluator: ExerciseEvaluator;
+  result: any;
+  evaluate_result: any;
+}
+export type OJSEvaluateElement = HTMLElement & { value?: EvaluateValue };
 
 export type EvaluateOptions = {
   define?: string[];
@@ -43,10 +48,10 @@ export interface ExerciseEvaluator {
   evaluate(code: string, envir?: EnvLabel, options?: EvaluateOptions): Promise<any>;
   process(inputs: { [key: string]: any }): Promise<void>;
   asOjs(value: any): Promise<any>;
-  asHtml(value: any): Promise<OJSElement>;
+  asHtml(value: any): Promise<OJSEvaluateElement>;
   context: EvaluateContext;
   options: EvaluateOptions;
   envLabels: EnvLabels;
   envManager: EnvManager;
-  container: OJSElement;
+  container: OJSEvaluateElement;
 }

live-runtime/src/grader-pyodide.ts

@@ -169,6 +169,10 @@ export class PyodideGrader extends ExerciseGrader {
     // const globals = await this.envManager.get(this.envLabels.grading);
     // await this.pyodide.runPythonAsync(`print(foo)`, { globals });
     checkerEnvObj.destroy();
+
+    const options = { ...this.options };
+    options.error = false;
+    options.output = true;
     const result = await this.evaluator.evaluate(
       `
         import pyodide
@@ -188,7 +192,7 @@ export class PyodideGrader extends ExerciseGrader {
         feedback
       `,
       "grading",
-      this.options
+      options
     );
     // console.log(result);
     return result;

live-runtime/src/grader-webr.ts

@@ -178,6 +178,7 @@ export class WebRGrader extends ExerciseGrader {
       this.envManager.bind(".checker_args", argsObj, this.envLabels.grading);
       const options = { ...this.options };
       options.error = false;
+      options.output = true;
       const result = await this.evaluator.evaluate(
         `.checker <- getOption('webr.exercise.checker')
         environment(.checker) <- environment()