libcruft-util/test/json/schema.py.in

#!/usr/bin/env python3

from glob import glob
import subprocess

import os.path
import os
import tempfile

validate="@CMAKE_CURRENT_BINARY_DIR@/json-schema"
compare="@CMAKE_CURRENT_BINARY_DIR@/json-compare"
src="@CMAKE_CURRENT_SOURCE_DIR@/test/json/schema"
devnull=open(os.devnull, 'w')


def test_bad(schema:str, dir:str) -> int:
    failures = 0

    for input in glob(os.path.join(dir, "*.json")):
        code = subprocess.call([validate, schema, input], stdout=devnull, stderr=devnull)

        prefix = ""
        if code == 0:
            prefix = "not "
            failures += 1

        prefix = "not " if code == 0 else ""
        print(f"{prefix}ok - {input}")

    return failures


def test_good(schema:str, dir:str) -> int:
    # extract a list of inputs and truths. the truth list may be incomplete
    # (if we're not dealing with defaults) so we can't use it directly.
    inputs  = glob(os.path.join(dir, "*.input.json"))
    results = glob(os.path.join(dir, "*.result.json"))

    unused = [x for x in glob(f"{dir}/*") if x not in inputs and x not in results]
    if unused:
        raise RuntimeError("unused inputs", unused)

    inputs.sort()

    failures = 0

    for test in inputs:
        # check if we have a corresponding .result.json file as the ground
        # truth. if not then we assume that there shouldn't be any chance in
        # the resulting json and set the truth to the test file.
        (base,_) = os.path.splitext(test)
        (base,_) = os.path.splitext(base)
        truth = f"{base}.result.json"

        if not os.path.isfile(truth):
            truth = test

        success = False

        try:
            # a two stage check:
            # * apply the schema to the test file
            # * test it matches the truth file
            with tempfile.NamedTemporaryFile(delete=True) as out:
                subprocess.check_call([validate, schema, test], stdout=out)
                subprocess.check_call([compare, out.name, truth])
                success = True
        except subprocess.CalledProcessError:
            failures += 1

        prefix = "not " if not success else ""
        print(f"{prefix}ok - {test}")

    return failures


def validation_group(dir:str) -> int:
    schema = os.path.join(dir, "schema.json")
    if not os.path.isfile(schema):
        raise Exception(f"schema is not present, {schema}")

    failures  = 0
    failures += test_good(schema, os.path.join(dir, "good"))
    failures += test_bad(schema, os.path.join(dir, "bad"))

    return failures

def test_validation(dir:str) -> int:
    failures = 0

    groups = (x for x in os.listdir(dir))
    groups = (os.path.join(dir, x) for x in groups)
    groups = (x for x in groups if os.path.isdir(x))

    for name in sorted(groups):
        path = os.path.join(src, name)
        if not os.path.isdir(path):
            continue
        failures += validation_group(path)

    return failures


if __name__ == "__main__":
    failures = 0

    failures += test_validation(os.path.join(src, "validation"))
    exit(1 if failures else 0)