json/schema: fall back to identity test for .result.json

result.json test cases are only really required when we have default
properties that need to be filled. Fall back to using the input.json
file for other cases.

test/json/schema.py.in

@@ -31,27 +31,45 @@ def test_bad(schema:str, dir:str) -> int:
 
 
 def test_good(schema:str, dir:str) -> int:
+    # extract a list of inputs and truths. the truth list may be incomplete
+    # (if we're not dealing with defaults) so we can't use it directly.
     inputs  = glob(os.path.join(dir, "*.input.json"))
     results = glob(os.path.join(dir, "*.result.json"))
 
+    unused = [x for x in glob(f"{dir}/*") if x not in inputs and x not in results]
+    if unused:
+        raise RuntimeError("unused inputs", unused)
+
     inputs.sort()
-    results.sort()
 
     failures = 0
 
-    for (target,truth) in zip(inputs, results):
+    for test in inputs:
+        # check if we have a corresponding .result.json file as the ground
+        # truth. if not then we assume that there shouldn't be any chance in
+        # the resulting json and set the truth to the test file.
+        (base,_) = os.path.splitext(test)
+        (base,_) = os.path.splitext(base)
+        truth = f"{base}.result.json"
+
+        if not os.path.isfile(truth):
+            truth = test
+
         success = False
 
         try:
-            with tempfile.NamedTemporaryFile(delete=True) as out:
+            # a two stage check:
-                subprocess.check_call([validate, schema, target], stdout=out)
+            # * apply the schema to the test file
+            # * test it matches the truth file
+            with tempfile.NamedTemporaryFile(delete=False) as out:
+                subprocess.check_call([validate, schema, test], stdout=out)
                 subprocess.check_call([compare, out.name, truth])
                 success = True
         except subprocess.CalledProcessError:
             failures += 1
 
         prefix = "not " if not success else ""
-        print(f"{prefix}ok - {target}")
+        print(f"{prefix}ok - {test}")
 
     return failures
 
@@ -59,7 +77,7 @@ def test_good(schema:str, dir:str) -> int:
 def validation_group(dir:str) -> int:
     schema = os.path.join(dir, "schema.json")
     if not os.path.isfile(schema):
-        raise Exception("schema is not present")
+        raise Exception(f"schema is not present, {schema}")
 
     failures  = 0
     failures += test_good(schema, os.path.join(dir, "good"))