Add Test Suite for cTensor

.github/check_all_results.py

@@ -0,0 +1,108 @@
+import csv
+import argparse
+import sys
+import os
+
+def parse_reports(report_paths):
+    """Parses cTensor test reports and identifies failures."""
+    all_failures = []
+    passed_all_reports = True
+
+    print(f"Checking reports: {report_paths}")
+
+    for report_path in report_paths:
+        if not os.path.exists(report_path):
+            print(f"Error: Report file not found: {report_path}", file=sys.stderr)
+            passed_all_reports = False
+            all_failures.append({
+                "file": os.path.basename(report_path),
+                "operator": "N/A",
+                "test_point": "FILE_NOT_FOUND",
+                "details": f"Report file {report_path} was not found."
+            })
+            continue
+
+        try:
+            with open(report_path, 'r', newline='') as csvfile:
+                reader = csv.DictReader(csvfile)
+                expected_base_headers = ['Operator', 'TestPoint']
+                if not reader.fieldnames or not all(header in reader.fieldnames for header in expected_base_headers):
+                    print(f"Error: Report file {report_path} has missing or incorrect base headers.", file=sys.stderr)
+                    print(f"Expected base headers: {expected_base_headers}, Got: {reader.fieldnames}", file=sys.stderr)
+                    passed_all_reports = False
+                    all_failures.append({
+                        "file": os.path.basename(report_path),
+                        "operator": "N/A",
+                        "test_point": "INVALID_BASE_HEADERS",
+                        "details": f"Report file {report_path} has invalid or missing base CSV headers (Operator, TestPoint)."
+                    })
+                    continue
+
+                report_has_failures = False
+                print(f"Processing report: {report_path}")
+
+                sub_test_headers = [h for h in reader.fieldnames if h not in expected_base_headers]
+                if not sub_test_headers:
+                    print(f"Warning: Report file {report_path} has no sub-test columns after 'Operator' and 'TestPoint'.", file=sys.stderr)
+
+                for _, row in enumerate(reader, 1):
+                    operator = row.get('Operator', 'N/A')
+                    test_point = row.get('TestPoint', 'N/A')
+                    row_failed = False
+                    failure_details_for_row = []
+
+                    if not sub_test_headers:
+                        pass
+                    else:
+                        for sub_test_header in sub_test_headers:
+                            result_detail = row.get(sub_test_header, '')
+
+                            if result_detail != '/' and result_detail != '':
+                                row_failed = True
+                                failure_details_for_row.append(f"Sub-test '{sub_test_header}': {result_detail}")
+
+                    if row_failed:
+                        passed_all_reports = False
+                        report_has_failures = True
+                        all_failures.append({
+                            "file": os.path.basename(report_path),
+                            "operator": operator,
+                            "test_point": test_point,
+                            "details": "; ".join(failure_details_for_row)
+                        })
+                if report_has_failures:
+                    print(f"Failures found in {os.path.basename(report_path)}.", file=sys.stderr)
+                else:
+                    print(f"No failures found in {os.path.basename(report_path)}.")
+
+        except Exception as e:
+            print(f"Error reading or parsing {report_path}: {e}", file=sys.stderr)
+            passed_all_reports = False
+            all_failures.append({
+                "file": os.path.basename(report_path),
+                "operator": "N/A",
+                "test_point": "PARSING_ERROR",
+                "details": f"Could not parse {report_path}. Error: {e}"
+            })
+
+    return passed_all_reports, all_failures
+
+def main():
+    parser = argparse.ArgumentParser(description="Parse cTensor test reports and check for failures.")
+    parser.add_argument("report_files", nargs='+', help="Paths to the cten_test_report.csv files.")
+    args = parser.parse_args()
+
+    passed_all, failures = parse_reports(args.report_files)
+
+    if not passed_all:
+        print("\n--- Test Failures Summary ---", file=sys.stderr)
+        for failure in failures:
+            print(f"  File: {failure['file']}, Operator: {failure['operator']}, Test: {failure['test_point']}, Details: {failure['details']}", file=sys.stderr)
+        print("\nOne or more tests failed or reports were invalid.", file=sys.stderr)
+        sys.exit(1)
+    else:
+        print("\nAll tests passed across all reports.")
+        sys.exit(0)
+
+if __name__ == "__main__":
+    main()

.github/check_operator_coverage.py

@@ -0,0 +1,68 @@
+import os
+import re
+import sys
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
+
+OPERATOR_FILE_PATH = os.path.join(PROJECT_ROOT, "src", "operator.c")
+TEST_DIR_PATH = os.path.join(PROJECT_ROOT, "tests", "Operator")
+
+def get_defined_operators(operator_file):
+    """Extracts operator names (Tensor_XXX) from the operator source file."""
+    operators = set()
+    try:
+        with open(operator_file, 'r') as f:
+            content = f.read()
+            matches = re.findall(r"(?:Tensor|static\s+Tensor)\s+(Tensor_([a-zA-Z0-9_]+))\s*\(", content)
+            for match in matches:
+                operators.add(match[1])
+    except FileNotFoundError:
+        print(f"Error: Operator file not found: {operator_file}", file=sys.stderr)
+        sys.exit(1)
+    return operators
+
+def get_existing_test_files(test_dir):
+    """Lists existing test files (test_xxx.c) in the specified directory."""
+    test_files = set()
+    try:
+        for filename in os.listdir(test_dir):
+            if filename.startswith("test_") and filename.endswith(".c"):
+                test_name = filename[len("test_"):-len(".c")]
+                test_files.add(test_name)
+    except FileNotFoundError:
+        print(f"Error: Test directory not found: {test_dir}", file=sys.stderr)
+        sys.exit(1)
+    return test_files
+
+def main():
+    print(f"Checking operator test coverage...")
+    print(f"Operator source file: {os.path.abspath(OPERATOR_FILE_PATH)}")
+    print(f"Test directory: {os.path.abspath(TEST_DIR_PATH)}")
+
+    defined_operators = get_defined_operators(OPERATOR_FILE_PATH)
+    if not defined_operators:
+        print("No operators found in operator file. Exiting.", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Found {len(defined_operators)} operators: {sorted(list(defined_operators))}")
+
+    existing_test_files = get_existing_test_files(TEST_DIR_PATH)
+    print(f"Found {len(existing_test_files)} test files: {sorted(list(existing_test_files))}")
+
+    missing_tests = []
+    for op_name in defined_operators:
+        if op_name not in existing_test_files:
+            missing_tests.append(op_name)
+
+    if not missing_tests:
+        print("\nAll defined operators have corresponding test files.")
+        sys.exit(0)
+    else:
+        print("\nError: The following operators are missing test files in", TEST_DIR_PATH + ":", file=sys.stderr)
+        for missing_op in sorted(missing_tests):
+            print(f"  - Operator: Tensor_{missing_op} (Expected test file: test_{missing_op}.c)", file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

.github/workflows/run-ctests.yml

@@ -0,0 +1,105 @@
+name: Run cTensor Tests
+
+on:
+  push:
+    branches:
+      - test
+  pull_request:
+    branches:
+      - test
+
+jobs:
+  build_and_test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        include:
+          - os: ubuntu-latest
+            artifact_name: report-ubuntu.csv
+            platform_report_suffix: linux
+          - os: macos-latest
+            artifact_name: report-macos.csv
+            platform_report_suffix: macos
+          - os: windows-latest
+            artifact_name: report-windows.csv
+            platform_report_suffix: windows
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up CMake and C Compiler
+      uses: lukka/get-cmake@latest
+
+    - name: Configure CMake
+      run: cmake -B build -S .
+
+    - name: Build tests
+      run: cmake --build build --target cten_tests
+
+    - name: Run cTensor tests
+      run: |
+        cd build
+        ctest -C Debug --output-on-failure 
+        # Fallback if ctest doesn't run or if direct execution is preferred:
+        # if [ ! -f ./cten_test_report.csv ]; then
+        #   if [ "${{ runner.os }}" == "Windows" ]; then
+        #     ./bin/Debug/cten_tests.exe
+        #   else
+        #     ./bin/cten_tests
+        #   fi
+        # fi
+      shell: bash
+
+    - name: Upload test report artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ matrix.artifact_name }}
+        path: build/cten_test_report_${{ matrix.platform_report_suffix }}.csv
+        if-no-files-found: error
+
+  analyze_results:
+    runs-on: ubuntu-latest
+    needs: build_and_test
+    if: always()
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.x'
+
+    - name: Create reports directory
+      run: mkdir -p reports
+
+    - name: Download Ubuntu report
+      uses: actions/download-artifact@v4
+      with:
+        name: report-ubuntu.csv
+        path: reports/
+      continue-on-error: true
+
+    - name: Download macOS report
+      uses: actions/download-artifact@v4
+      with:
+        name: report-macos.csv
+        path: reports/
+      continue-on-error: true
+
+    - name: Download Windows report
+      uses: actions/download-artifact@v4
+      with:
+        name: report-windows.csv
+        path: reports/
+      continue-on-error: true
+
+    - name: List downloaded reports
+      run: ls -R reports/
+
+    - name: Run Python script to check results
+      run: python3 .github/check_all_results.py reports/*.csv

.github/workflows/test-coverage.yml

@@ -0,0 +1,24 @@
+name: Check Operator Test Coverage
+
+on:
+  push:
+    branches:
+      - test
+  pull_request:
+    branches:
+      - test
+
+jobs:
+  verify_test_coverage:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+
+    - name: Run operator test coverage check
+      run: python3 .github/check_operator_coverage.py

.gitignore

@@ -51,5 +51,6 @@ Module.symvers
 Mkfile.old
 dkms.conf
 main
+.cache
 
 build