libcruft-util/test/json/schema.py.in

109 lines
3.0 KiB
Python
Raw Normal View History

2018-07-05 19:08:58 +10:00
#!/usr/bin/env python3
from glob import glob
import subprocess
import os.path
import os
import tempfile
validate="@CMAKE_CURRENT_BINARY_DIR@/json-schema"
compare="@CMAKE_CURRENT_BINARY_DIR@/json-compare"
src="@CMAKE_CURRENT_SOURCE_DIR@/test/json/schema"
devnull=open(os.devnull, 'w')
def test_bad(schema:str, dir:str) -> int:
2018-07-05 19:08:58 +10:00
failures = 0
for input in glob(os.path.join(dir, "*.json")):
code = subprocess.call([validate, schema, input], stdout=devnull, stderr=devnull)
prefix = ""
if code == 0:
prefix = "not "
failures += 1
prefix = "not " if code == 0 else ""
print(f"{prefix}ok - {input}")
return failures
def test_good(schema:str, dir:str) -> int:
# extract a list of inputs and truths. the truth list may be incomplete
# (if we're not dealing with defaults) so we can't use it directly.
2018-07-05 19:08:58 +10:00
inputs = glob(os.path.join(dir, "*.input.json"))
results = glob(os.path.join(dir, "*.result.json"))
unused = [x for x in glob(f"{dir}/*") if x not in inputs and x not in results]
if unused:
raise RuntimeError("unused inputs", unused)
2018-07-05 19:08:58 +10:00
inputs.sort()
failures = 0
for test in inputs:
# check if we have a corresponding .result.json file as the ground
# truth. if not then we assume that there shouldn't be any chance in
# the resulting json and set the truth to the test file.
(base,_) = os.path.splitext(test)
(base,_) = os.path.splitext(base)
truth = f"{base}.result.json"
if not os.path.isfile(truth):
truth = test
2018-07-05 19:08:58 +10:00
success = False
try:
# a two stage check:
# * apply the schema to the test file
# * test it matches the truth file
with tempfile.NamedTemporaryFile(delete=True) as out:
subprocess.check_call([validate, schema, test], stdout=out)
2018-07-05 19:08:58 +10:00
subprocess.check_call([compare, out.name, truth])
success = True
except subprocess.CalledProcessError:
failures += 1
prefix = "not " if not success else ""
print(f"{prefix}ok - {test}")
2018-07-05 19:08:58 +10:00
return failures
def validation_group(dir:str) -> int:
2018-07-05 19:08:58 +10:00
schema = os.path.join(dir, "schema.json")
if not os.path.isfile(schema):
raise Exception(f"schema is not present, {schema}")
2018-07-05 19:08:58 +10:00
failures = 0
failures += test_good(schema, os.path.join(dir, "good"))
failures += test_bad(schema, os.path.join(dir, "bad"))
return failures
def test_validation(dir:str) -> int:
2018-07-05 19:08:58 +10:00
failures = 0
groups = (x for x in os.listdir(dir))
groups = (os.path.join(dir, x) for x in groups)
groups = (x for x in groups if os.path.isdir(x))
for name in sorted(groups):
2018-07-05 19:08:58 +10:00
path = os.path.join(src, name)
if not os.path.isdir(path):
continue
failures += validation_group(path)
2018-07-05 19:08:58 +10:00
return failures
2018-07-05 19:08:58 +10:00
if __name__ == "__main__":
failures = 0
failures += test_validation(os.path.join(src, "validation"))
exit(1 if failures else 0)