#!/usr/bin/env python

"""Analyze results from a test root area, comparing non-BFB changes.

Purpose is, instead of re-running tests in compare mode, which is very slow,
allow for very fast analysis of diffs.

Outputs results for each test to stdout (one line per test); possible status
codes are: PASS, FAIL, SKIP. (A SKIP denotes a test that did not make it to the
run phase or a test for which the run phase did not pass: we skip baseline
comparisons in this case.)

In addition, creates files named compare.log.BASELINE_NAME.TIMESTAMP in each
test directory, which contain more detailed output. Also creates
*.cprnc.out.BASELINE_NAME.TIMESTAMP files in each run directory.

Returns a 0 exit status if all tests are bit-for-bit, and a non-zero exit status
(TESTS_FAILED_ERR_CODE) if any tests differed from the baseline.

You may need to load modules for cprnc to work.

"""

from standard_script_setup import *

from CIME.XML.machines import Machines
from CIME.compare_test_results import compare_test_results

import argparse, sys, os, doctest

_MACHINE = Machines()

###############################################################################
def parse_command_line(args, description):
###############################################################################
    parser = argparse.ArgumentParser(
usage="""\n{0} [-r <TESTROOT>] [-b <BRANCH> -c <COMPILER>] [-t <TESTID>] [<TEST> <TEST> ...] [--verbose]
OR
{0} --help
OR
{0} --test

\033[1mEXAMPLES:\033[0m
    \033[1;32m# From most recent run, compare all changes \033[0m
    > {0}
    \033[1;32m# From most recent run, compare only changes for test foo and bar only \033[0m
    > {0} foo bar
    \033[1;32m# For an old run where you know test-id, compare only changes for test foo and bar only \033[0m
    > {0} foo bar -t mytestid
    \033[1;32m# From most recent run of jenkins, compare history changes for next \033[0m
    > {0} -r /home/jenkins/acme/scratch/jenkins -b next
    \033[1;32m# For typical CESM workflow, where baselines are named with tags \033[0m
    > {0} -t TESTID -b BASELINE_TAG
""".format(os.path.basename(args[0])),

description=description,

formatter_class=argparse.ArgumentDefaultsHelpFormatter
)

    default_baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root())
    default_baseline_root = _MACHINE.get_value("BASELINE_ROOT")
    default_compiler      = _MACHINE.get_default_compiler()
    scratch_root          = _MACHINE.get_value("CIME_OUTPUT_ROOT")
    default_testroot      = os.path.join(scratch_root)

    CIME.utils.setup_standard_logging_options(parser)

    parser.add_argument("-n", "--namelists-only", action="store_true",
                        help="Only analyze namelists.")

    parser.add_argument("--hist-only", action="store_true",
                        help="Only analyze history files.")

    parser.add_argument("-b", "--baseline-name", default=default_baseline_name,
                        help="Name of baselines to use, corresponds to branch used.")

    parser.add_argument("--baseline-root", default=default_baseline_root,
                        help="Root of baselines.")

    parser.add_argument("-c", "--compiler", default=default_compiler,
                        help="Compiler of run you want to compare")

    parser.add_argument("-r", "--test-root", default=default_testroot,
                        help="Path to test results that are being compared")

    parser.add_argument("-t", "--test-id",
                        help="Limit processes to case dirs matching this test-id. Can be useful if mutiple runs dumped into the same dir.")

    parser.add_argument("compare_tests", nargs="*",
                        help="When comparing, limit the comparison to tests matching these regex")

    args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)

    return args.baseline_name, args.baseline_root, args.test_root, args.compiler, args.test_id, args.compare_tests, args.namelists_only, args.hist_only

###############################################################################
def _main_func(description):
###############################################################################
    if ("--test" in sys.argv):
        test_results = doctest.testmod(verbose=True)
        sys.exit(1 if test_results.failed > 0 else 0)

    baseline_name, baseline_root, test_root, compiler, test_id, compare_tests, namelists_only, hist_only = \
        parse_command_line(sys.argv, description)

    success = compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id, compare_tests, namelists_only, hist_only)
    sys.exit(0 if success else CIME.utils.TESTS_FAILED_ERR_CODE)

###############################################################################

if (__name__ == "__main__"):
    _main_func(__doc__)
