json-schema: add schema test suite

CMakeLists.txt

@@ -584,6 +584,10 @@ if (TESTS)
     add_test(NAME util_test_json_compare COMMAND util_test_json_compare.py)
     set_property(TEST util_test_json_compare APPEND PROPERTY DEPENDS util_json-compare)
 
+    configure_file (test/json/schema.py.in util_test_json_schema.py @ONLY)
+    add_test(NAME util_test_json_schema COMMAND util_test_json_schema.py)
+    set_property(TEST util_test_json_schema APPEND PROPERTY DEPENDS util_json-schema)
+
     configure_file (test/cpp.sh.in util_test_cpp.sh @ONLY)
     add_test (NAME util_test_cpp COMMAND util_test_cpp.sh)
     set_property (TEST util_test_cpp APPEND PROPERTY DEPENDS util_macro)

test/json/schema.py.in

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+from glob import glob
+import subprocess
+
+import os.path
+import os
+import tempfile
+
+validate="@CMAKE_CURRENT_BINARY_DIR@/json-schema"
+compare="@CMAKE_CURRENT_BINARY_DIR@/json-compare"
+src="@CMAKE_CURRENT_SOURCE_DIR@/test/json/schema"
+devnull=open(os.devnull, 'w')
+
+
+def test_bad(schema, dir):
+    failures = 0
+
+    for input in glob(os.path.join(dir, "*.json")):
+        code = subprocess.call([validate, schema, input], stdout=devnull, stderr=devnull)
+
+        prefix = ""
+        if code == 0:
+            prefix = "not "
+            failures += 1
+
+        prefix = "not " if code == 0 else ""
+        print(f"{prefix}ok - {input}")
+
+    return failures
+
+
+def test_good(schema, dir):
+    inputs  = glob(os.path.join(dir, "*.input.json"))
+    results = glob(os.path.join(dir, "*.result.json"))
+
+    inputs.sort()
+    results.sort()
+
+    failures = 0
+
+    for (target,truth) in zip(inputs, results):
+        success = False
+
+        try:
+            with tempfile.NamedTemporaryFile(delete=True) as out:
+                subprocess.check_call([validate, schema, target], stdout=out)
+                subprocess.check_call([compare, out.name, truth])
+                success = True
+        except subprocess.CalledProcessError:
+            failures += 1
+
+        prefix = "not " if not success else ""
+        print(f"{prefix}ok - {target}")
+
+    return failures
+
+
+def test_pool(dir):
+    schema = os.path.join(dir, "schema.json")
+    if not os.path.isfile(schema):
+        raise Exception("schema is not present")
+
+    failures  = 0
+    failures += test_good(schema, os.path.join(dir, "good"))
+    failures += test_bad(schema, os.path.join(dir, "bad"))
+
+    return failures
+
+
+if __name__ == "__main__":
+    failures = 0
+
+    for name in os.listdir(src):
+        path = os.path.join(src, name)
+        if not os.path.isdir(path):
+            continue
+        failures += test_pool(path)
+
+    exit(1 if failures else 0)
+

test/json/schema/test.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-tool="/home/danny/src/lobo/build/debug/gcc/cruft/util/json-schema"
-src="/home/danny/src/lobo/cruft/util/test/json/schema"
-
-for base in $(ls "${src}"); do
-    [ -d ${base} ] || continue
-
-    echo "testing ${base}"
-
-    schema="${base}/schema.json"
-
-    for good in $(ls "${base}/good/"*.input.json); do
-        [ -f "${good}" ] || continue
-
-        expected="${good%.input.json}.result.json"
-        echo "${tool} ${schema} ${good} ${expected}"
-    done
-
-    for bad in $(ls "${base}/bad/"*.json); do
-        [ -f "${bad}" ] || continue
-
-        echo "${tool} ${schema} ${bad}"
-    done
-done

test/json/schema/type_any/good/0001.result.json

@@ -0,0 +1 @@
+{}