|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +# Copyright Catch2 Authors |
| 4 | +# Distributed under the Boost Software License, Version 1.0. |
| 5 | +# (See accompanying file LICENSE_1_0.txt or copy at |
| 6 | +# https://www.boost.org/LICENSE_1_0.txt) |
| 7 | + |
| 8 | +# SPDX-License-Identifier: BSL-1.0 |
| 9 | + |
| 10 | +import os |
| 11 | +import re |
| 12 | +import sys |
| 13 | +import xml.etree.ElementTree as ET |
| 14 | +import subprocess |
| 15 | + |
| 16 | +""" |
| 17 | +Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER |
| 18 | +
|
| 19 | +Requires 2 arguments, path to where the output files should be stored |
| 20 | +and the name of the test |
| 21 | +""" |
| 22 | +if len(sys.argv) != 3: |
| 23 | + print("Wrong number of arguments: {}".format(len(sys.argv))) |
| 24 | + print("Usage: {} bin-path bin-name".format(sys.argv[0])) |
| 25 | + exit(1) |
| 26 | + |
| 27 | + |
| 28 | +bin_path = os.path.abspath(sys.argv[1]) |
| 29 | +bin_name = sys.argv[2] |
| 30 | +xml_out_path = os.path.join(bin_path, "{}.xml".format(bin_name)) |
| 31 | +config_path = "Debug" if os.name == "nt" else "" |
| 32 | + |
| 33 | +# Ensure no file exists from previous test runs |
| 34 | +if os.path.isfile(xml_out_path): |
| 35 | + os.remove(xml_out_path) |
| 36 | + |
| 37 | +args = [os.path.join(bin_path, config_path, bin_name)] |
| 38 | +env = os.environ.copy() |
| 39 | +env["XML_OUTPUT_FILE"] = xml_out_path |
| 40 | +test_passing = True |
| 41 | + |
| 42 | +try: |
| 43 | + ret = subprocess.run( |
| 44 | + args, |
| 45 | + stdout=subprocess.PIPE, |
| 46 | + stderr=subprocess.PIPE, |
| 47 | + check=True, |
| 48 | + universal_newlines=True, |
| 49 | + env=env |
| 50 | + ) |
| 51 | + stdout = ret.stdout |
| 52 | +except subprocess.SubprocessError as ex: |
| 53 | + if ex.returncode == 1: |
| 54 | + # The test cases are allowed to fail. |
| 55 | + test_passing = False |
| 56 | + stdout = ex.stdout |
| 57 | + else: |
| 58 | + print('Could not run "{}"'.format(args)) |
| 59 | + print("Return code: {}".format(ex.returncode)) |
| 60 | + print("stdout: {}".format(ex.stdout)) |
| 61 | + print("stderr: {}".format(ex.stdout)) |
| 62 | + raise |
| 63 | + |
| 64 | +# Check for valid XML output |
| 65 | +try: |
| 66 | + tree = ET.parse(xml_out_path) |
| 67 | +except ET.ParseError as ex: |
| 68 | + print("Invalid XML: '{}'".format(ex)) |
| 69 | + raise |
| 70 | +except FileNotFoundError as ex: |
| 71 | + print("Could not find '{}'".format(xml_out_path)) |
| 72 | + raise |
| 73 | + |
| 74 | +# Check for matching testsuite |
| 75 | +if not tree.find('.//testsuite[@name="{}"]'.format(bin_name)): |
| 76 | + print("Could not find '{}' testsuite".format(bin_name)) |
| 77 | + exit(2) |
| 78 | + |
| 79 | +summary_test_cases = re.findall(r'test cases: \d* \| \d* passed \| \d* failed', stdout) |
| 80 | +if len(summary_test_cases) == 0: |
| 81 | + print("Could not find test summary in {}".format(stdout)) |
| 82 | + exit(2) |
| 83 | + |
| 84 | +total, passed, failed = [int(s) for s in summary_test_cases[0].split() if s.isdigit()] |
| 85 | + |
| 86 | +if failed == 0 and not test_passing: |
| 87 | + print("Expected at least 1 test failure!") |
| 88 | + exit(2) |
| 89 | + |
| 90 | +if len(tree.findall('.//testcase')) != total: |
| 91 | + print("Unexpected number of test cases!") |
| 92 | + exit(2) |
| 93 | + |
| 94 | +if len(tree.findall('.//failure')) != failed: |
| 95 | + print("Unexpected number of test failures!") |
| 96 | + exit(2) |
| 97 | + |
| 98 | +if (passed + failed) != total: |
| 99 | + print("Something has gone very wrong, ({} + {}) != {}".format(passed, failed, total)) |
| 100 | + exit(2) |
0 commit comments