diff --git a/.github/workflows/numpy.yml b/.github/workflows/numpy.yml
index 74eeccf4..273ee970 100644
--- a/.github/workflows/numpy.yml
+++ b/.github/workflows/numpy.yml
@@ -75,4 +75,6 @@ jobs:
EOF
- pytest -v -rxXfE --ci
+ pytest -v -rxXfE --ci --json-report
+ - name: Verify JSON report against schema
+ run: ./verify_report.py
diff --git a/report.schema.json b/report.schema.json
new file mode 100644
index 00000000..1e7d04a9
--- /dev/null
+++ b/report.schema.json
@@ -0,0 +1,358 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "properties": {
+ "duration": {
+ "type": "number"
+ },
+ "summary": {
+ "type": "object",
+ "properties": {
+ "total": {
+ "type": "integer"
+ },
+ "collected": {
+ "type": "integer"
+ },
+ "passed": {
+ "type": "integer"
+ },
+ "failed": {
+ "type": "integer"
+ },
+ "skipped": {
+ "type": "integer"
+ },
+ "error": {
+ "type": "integer"
+ },
+ "deselected": {
+ "type": "integer"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "total"
+ ]
+ },
+ "environment": {
+ "type": "object",
+ "properties": {
+ "Platform": {
+ "type": "string"
+ },
+ "Packages": {
+ "type": "object",
+ "properties": {
+ "pytest": {
+ "type": "string"
+ },
+ "pluggy": {
+ "type": "string"
+ },
+ "py": {
+ "type": "string"
+ }
+ },
+ "additionalProperties": {
+ "type": "string"
+ },
+ "required": [
+ "pytest",
+ "pluggy",
+ "py"
+ ]
+ },
+ "array_api_tests_version": {
+ "type": "string"
+ },
+ "array_api_tests_module": {
+ "type": "string"
+ },
+ "Python": {
+ "type": "string"
+ },
+ "Plugins": {
+ "type": "object",
+ "properties": {
+ "metadata": {
+ "type": "string"
+ },
+ "json-report": {
+ "type": "string"
+ },
+ "hypothesis": {
+ "type": "string"
+ }
+ },
+ "additionalProperties": {
+ "type": "string"
+ },
+ "required": [
+ "metadata",
+ "json-report",
+ "hypothesis"
+ ]
+ }
+ },
+ "required": [
+ "Platform",
+ "Packages",
+ "array_api_tests_version",
+ "array_api_tests_module",
+ "Python",
+ "Plugins"
+ ]
+ },
+ "collectors": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "result": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "lineno": {
+ "type": "integer"
+ },
+ "type": {
+ "type": "string"
+ },
+ "nodeid": {
+ "type": "string"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "type",
+ "nodeid"
+ ]
+ }
+ },
+ "nodeid": {
+ "type": "string"
+ },
+ "outcome": {
+ "type": "string"
+ },
+ "longrepr": {
+ "type": "string"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "result",
+ "nodeid",
+ "outcome"
+ ]
+ }
+ },
+ "tests": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "$defs": {
+ "teststage": {
+ "type": "object",
+ "properties": {
+ "duration": {
+ "type": "number"
+ },
+ "longrepr": {
+ "type": "string"
+ },
+ "outcome": {
+ "type": "string"
+ },
+ "traceback": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "path": {
+ "type": "string"
+ },
+ "lineno": {
+ "type": "integer"
+ },
+ "message": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "path",
+ "lineno",
+ "message"
+ ]
+ }
+ },
+ "crash": {
+ "type": "object",
+ "properties": {
+ "path": {
+ "type": "string"
+ },
+ "lineno": {
+ "type": "integer"
+ },
+ "message": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "path",
+ "lineno",
+ "message"
+ ]
+ },
+ "stdout": {
+ "type": "string"
+ },
+ "stderr": {
+ "type": "string"
+ },
+ "stdout": {
+ "type": "string"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "duration",
+ "outcome"
+ ]
+ }
+ },
+ "properties": {
+ "call": {
+ "$ref": "#/properties/tests/items/$defs/teststage"
+ },
+ "crash": {
+ "$ref": "#/properties/tests/items/$defs/teststage"
+ },
+ "teardown": {
+ "$ref": "#/properties/tests/items/$defs/teststage"
+ },
+ "setup": {
+ "$ref": "#/properties/tests/items/$defs/teststage"
+ },
+ "metadata": {
+ "type": "object",
+ "properties": {
+ "test_module": {
+ "type": "string"
+ },
+ "array_api_function_name": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "hypothesis_statistics": {
+ "type": "string"
+ },
+ "hypothesis_report_information": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "test_function": {
+ "type": "string"
+ },
+ "params": {
+ "type": "object"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "test_module",
+ "array_api_function_name",
+ "test_function"
+ ]
+ },
+ "lineno": {
+ "type": "integer"
+ },
+ "keywords": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "nodeid": {
+ "type": "string"
+ },
+ "outcome": {
+ "type": "string"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "lineno",
+ "keywords",
+ "setup",
+ "nodeid",
+ "teardown",
+ "outcome"
+ ]
+ }
+ },
+ "created": {
+ "type": "number"
+ },
+ "root": {
+ "type": "string"
+ },
+ "warnings": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "lineno": {
+ "type": "integer"
+ },
+ "filename": {
+ "type": "string"
+ },
+ "count": {
+ "type": "integer"
+ },
+ "category": {
+ "type": "string"
+ },
+ "message": {
+ "type": "string"
+ },
+ "when": {
+ "type": "string"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "lineno",
+ "filename",
+ "count",
+ "category",
+ "message",
+ "when"
+ ]
+ }
+ },
+ "exitcode": {
+ "type": "integer"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "duration",
+ "summary",
+ "environment",
+ "tests",
+ "created",
+ "root",
+ "warnings",
+ "exitcode"
+ ]
+}
diff --git a/reporting.py b/reporting.py
index f7c7d6b9..bfc1e693 100644
--- a/reporting.py
+++ b/reporting.py
@@ -7,6 +7,7 @@
import dataclasses
import json
import warnings
+import math
from hypothesis.strategies import SearchStrategy
@@ -33,6 +34,13 @@ def to_json_serializable(o):
return tuple(to_json_serializable(i) for i in o)
if isinstance(o, list):
return [to_json_serializable(i) for i in o]
+ if isinstance(o, float):
+ if math.isnan(o):
+ return "NaN"
+ if o == float('inf'):
+ return 'Infinity'
+ if o == -float('inf'):
+ return '-Infinity'
# Ensure everything is JSON serializable. If this warning is issued, it
# means the given type needs to be added above if possible.
diff --git a/verify_report.py b/verify_report.py
new file mode 100755
index 00000000..01e24ee9
--- /dev/null
+++ b/verify_report.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+import sys
+import json
+
+import jsonschema
+
+def main():
+ if len(sys.argv) == 1:
+ report_file = '.report.json'
+ elif len(sys.argv) == 2:
+ report_file = sys.argv[1]
+ else:
+ sys.exit("Usage: verify_report.py [json_report_file]")
+
+ schema = json.load(open('report.schema.json'))
+
+ jsonschema.Validator.check_schema(schema)
+
+ with open(report_file) as f:
+ report = json.load(f)
+
+ # Make sure there are no nans in the report
+ json.dumps(report, allow_nan=False)
+
+ jsonschema.validate(report, schema)
+
+if __name__ == '__main__':
+ main()