#!/usr/bin/python3

# Copyright 2022 Julian Gilbey <jdg@debian.org>
# License: MIT License, as found in LICENSE.txt

"""Determine whether every test passes in at least one pytest run.

Some of the tests in the Spyder test suite are flaky: they succeed most
of the time but occasionally fail.  Given the size of the test suite, this
means that a pytest run will often fail because a single test failed.
Using @flaky on every test may or may not help, but it would be quite
invasive.  Furthermore, as some of the tests are dependent on others, it
is sometimes the case that @flaky does not help, whereas running the whole
test suite again succeeds.

This solution is different: we run the command given on the command line
(a pytest command wrapped in an xvfb-run command) and collect the output.
We note all the tests that result in errors or failures.  We run pytest at
most MAX_PYTEST_RUNS, exit with success if every test has succeeded at least
once and otherwise exit with failure.
"""

from enum import Flag, auto
import re
import subprocess
import sys
from typing import IO, cast


MAX_PYTEST_RUNS = 5
PYTEST_FAILURE_MSG = "Pytest run failed (stderr or other error)"


class Status(Flag):
    NOERR = 0
    ERREXIT = auto()
    STDERR = auto()
    UNFINISHED = auto()


def main():
    SUCCESS_MSG = "*** ALL TESTS RUN PASSED/XFAILED AT LEAST ONCE ***"
    FAILURE_MSG = "*** SOME TESTS FAILED/ERRORED EVERY RUN, ABORTING ***"
    RETRY_MSG = "*** RETRYING: THESE TESTS HAVE NOT PASSED AFTER %d RUN%s: ***"

    cmd = sys.argv[1:]
    failures = set()

    for run in range(MAX_PYTEST_RUNS):
        new_successes, new_failures = get_test_results(cmd)
        print()  # clear output line

        # On the first run, note the failures, but on successive runs,
        # remove successful tests from the list of failures
        if run == 0:
            failures |= new_failures
        else:
            failures -= new_successes

        if not failures:
            print(SUCCESS_MSG)
            sys.exit()

        print(RETRY_MSG % (run + 1, ("" if run + 1 == 1 else "s")))
        for test in sorted(failures):
            print(test)

    print(FAILURE_MSG)
    sys.exit(1)


def get_test_results(cmd: list[str]) -> tuple[set[str], set[str]]:
    """Run the tests and return all failures"""
    output, status = run_tests(cmd)
    successes, failures = get_test_results_from_output(output)
    if status & (Status.STDERR | Status.UNFINISHED):
        failures.add(PYTEST_FAILURE_MSG)
    elif status & Status.ERREXIT:
        # We only add a note about this if we have not recorded any failing
        # tests, as these always produce a non-zero exit status, otherwise
        # we clear the flag
        if not failures:
            failures.add(PYTEST_FAILURE_MSG)

    return successes, failures


def tee_output_from_process(proc: subprocess.Popen) -> tuple[list[str], list[str]]:
    """Print the stdout and stderr, returning stdout and stderr as lines"""
    output = []
    errs = []
    for line in cast(IO[str], proc.stdout):
        print(line, end="")
        output.append(line)
    for line in cast(IO[str], proc.stderr):
        print(f"<stderr>{line}", end="")
        errs.append(line)

    return output, errs


def run_tests(cmd: list[str]) -> tuple[list[str], Status]:
    """Run the tests and return the stdout and process failure message"""
    process = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1
    )
    output, errs = tee_output_from_process(process)

    # Check that we haven't missed a failure, for example a segfault
    status = Status.NOERR
    retcode = process.poll()
    if retcode is None:
        # The process has not finished, but process.stdout is empty
        # so we kill the process and add a sentinel to the failures
        process.kill()
        status |= Status.UNFINISHED
    elif errs:
        status |= Status.STDERR
    elif retcode != 0:
        status |= Status.ERREXIT

    return output, status


status_re = re.compile(r"^(.*) (ERROR|FAILED|PASSED|XFAIL) +\[ [0-9 ][0-9]%\]")


def get_test_results_from_output(output: list[str]) -> tuple[set[str], set[str]]:
    successes: set[str] = set()
    failures: set[str] = set()
    for line in output:
        status = status_re.match(line)
        if status:
            if status.group(2) in ["PASSED", "XFAIL"]:
                successes.add(status.group(1))
            else:
                failures.add(status.group(1))
    return successes, failures


if __name__ == "__main__":
    main()
