Merge pull request #486 from open-rpc/fix/meta-schema-version-and-valid-fix

fix: meta-schema version + fix valid setting on errors in example rule

Author: shanejonas

package-lock.json

@@ -20,6 +20,7 @@
   "dependencies": {
     "@open-rpc/html-reporter-react": "^0.0.4",
     "@open-rpc/schema-utils-js": "^1.16.2",
+    "@open-rpc/meta-schema": "^1.14.6",
     "@types/isomorphic-fetch": "0.0.35",
     "@types/lodash": "^4.14.162",
     "ajv": "^7.0.0",
@@ -30,7 +31,6 @@
     "lodash": "^4.17.20"
   },
   "devDependencies": {
-    "@open-rpc/meta-schema": "^1.13.13",
     "@types/jest": "^29.5.12",
     "eslint": "^8.57.0",
     "jest": "^29.7.0",

package.json

@@ -1,5 +1,5 @@
 import colors from "colors";
-import { JSONSchemaObject } from "@open-rpc/meta-schema";
+import { JSONSchemaObject, MethodObject } from "@open-rpc/meta-schema";
 import { Call, IOptions } from "../coverage";
 import _ from "lodash";
 import Reporter from "./reporter";
@@ -85,7 +85,7 @@ class ConsoleRuleReporter implements Reporter {
             console.log(
               colors.magenta("\t\t\t \->"),
               colors.white.underline("Expected result:"),
-              colors.white(expected),
+              colors.white(expected?.toString() || ""),
             );
 
             if (ex.requestError) {
@@ -146,7 +146,7 @@ class ConsoleRuleReporter implements Reporter {
     const failedMethods = Object.keys(failedCallsByMethod);
 
     //take into account failed methods
-    const missingMethods: string[] = options.openrpcDocument.methods
+    const missingMethods: string[] = (options.openrpcDocument.methods as MethodObject[])
       .map(({ name }) => name)
       .filter((methodName) => !passingMethods.includes(methodName))
       .filter((methodName) => !failedMethods.includes(methodName));

src/reporters/console-rule.ts

@@ -1,6 +1,7 @@
 import colors from "colors";
 import {
   JSONSchemaObject,
+  MethodObject,
 } from "@open-rpc/meta-schema";
 import {Call, IOptions} from "../coverage";
 import _ from "lodash";
@@ -65,7 +66,7 @@ class ConsoleStreamingReporter implements Reporter {
       console.log(
         colors.magenta("\t\t\t \->"),
         colors.white.underline("Expected result:"),
-        colors.white(expected),
+        colors.white(expected?.toString() || ""),
       );
 
       if (ex.requestError) {
@@ -123,7 +124,7 @@ class ConsoleStreamingReporter implements Reporter {
     const failedMethods = Object.keys(failedCallsByMethod);
 
     //take into account failed methods
-    const missingMethods: string[] = options.openrpcDocument.methods
+    const missingMethods: string[] = (options.openrpcDocument.methods as MethodObject[])
       .map(({name}) => name)
       .filter((methodName) => !passingMethods.includes(methodName))
       .filter((methodName) => !failedMethods.includes(methodName));

src/reporters/console-streaming.ts

@@ -76,7 +76,7 @@ class ConsoleReporter implements Reporter {
             console.log(
               colors.magenta("\t\t\t \->"),
               colors.white.underline("Expected result:"),
-              colors.white(expected),
+              colors.white(expected?.toString() || ""),
             );
 
             if (ex.requestError) {

src/reporters/console.ts

@@ -162,4 +162,63 @@ describe("ExamplesRule", () => {
     const result = rule.validateCall(calls[0]);
     expect(result.valid).toBe(false);
   });
+  it("should handle errors", () => {
+    const rule = new ExamplesRule();
+    const openrpcDocument = {
+      openrpc: "1.0.0",
+      info: {
+        title: "my api",
+        version: "0.0.0-development",
+      },
+      servers: [
+        {
+          name: "my api",
+          url: "http://localhost:3333",
+        },
+      ],
+      methods: [
+        {
+          name: "foo",
+          params: [],
+          result: {
+            name: "fooResult",
+            schema: {
+              type: "string",
+              unevaluatedProperties: false,
+            },
+          },
+          examples: [
+            {
+              name: "fooExample",
+              summary: "foo example",
+              description: "this is an example of foo",
+              params: [
+                {
+                  name: "barParam",
+                  value: "bar",
+                },
+                {
+                  name: "barParam2",
+                  value: "bar",
+                }
+              ],
+              result: {
+                name: "fooResult",
+                value: "potato",
+              }
+            }
+          ]
+        },
+      ],
+    } as any;
+    const calls = rule.getCalls(openrpcDocument, openrpcDocument.methods[0]);
+    calls[0].error = {
+      code: 123,
+      message: "error",
+      data: {}
+    };
+    const result = rule.validateCall(calls[0]);
+    expect(result.valid).toBe(false);
+    expect(result.reason).toMatch('expected "potato" but got an error: {"code":123,"message":"error","data":{}}');
+  });
 });

src/rules/examples-rule.test.ts

@@ -72,6 +72,11 @@ class ExamplesRule implements Rule {
       if (!call.valid) {
         call.reason = `expected ${JSON.stringify(call.expectedResult)} but got ${JSON.stringify(call.result)}`;
       }
+    } else if (call.error) {
+      call.valid = false;
+      call.reason = `expected ${JSON.stringify(call.expectedResult)} but got an error: ${JSON.stringify(call.error)}`;
+    } else {
+      call.valid = false;
     }
     return call;
   }

src/rules/examples-rule.ts

@@ -1,4 +1,5 @@
 
+import { Call } from "../coverage";
 import JsonSchemaFakerRule from "./json-schema-faker-rule";
 
 describe("JsonSchemaFakerRule", () => {
@@ -37,7 +38,7 @@ describe("JsonSchemaFakerRule", () => {
   });
   it("should handle errors within ajv when validating", () => {
     const rule = new JsonSchemaFakerRule();
-    const Call = {
+    const call: Call = {
       title: 'test call',
       methodName: "foo",
       params: [],
@@ -47,8 +48,26 @@ describe("JsonSchemaFakerRule", () => {
         unevaluatedProperties: false,
       },
     };
-    const result = rule.validateCall(Call);
+    const result = rule.validateCall(call);
     expect(result.valid).toBe(false);
     expect(result.reason).toMatch('unknown keyword: "unevaluatedProperties"');
   });
+  it("should produce a readable error for reason", () => {
+    const rule = new JsonSchemaFakerRule();
+    const call: Call = {
+      title: 'test call',
+      methodName: "foo",
+      params: [],
+      url: "http://localhost:3333",
+      resultSchema: {
+        type: "boolean",
+      },
+      result: "potato",
+    };
+    const result = rule.validateCall(call);
+    expect(result.valid).toBe(false);
+    expect(result.reason).toContain('expected:\n "potato"');
+    expect(result.reason).toContain('to match schema:');
+    expect(result.reason).toContain(JSON.stringify(call.resultSchema, null, 2));
+  });
 });

src/rules/json-schema-faker-rule.test.ts

@@ -66,7 +66,7 @@ class JsonSchemaFakerRule implements Rule {
       ajv.validate(call.resultSchema, call.result);
       if (ajv.errors && ajv.errors.length > 0) {
         call.valid = false;
-        call.reason = JSON.stringify(ajv.errors);
+        call.reason = `expected:\n ${JSON.stringify(call.result, null, 2)}\n to match schema: \n${JSON.stringify(call.resultSchema, null, 2)}\nbut got:\n ${JSON.stringify(ajv.errors, null, 2)}`;
       } else {
         call.valid = true;
       }