#!/usr/bin/env python
#
# Fingerprint-based regression tested for the INET Framework.
#
# Accepts one or more CSV files with 4 columns: working directory,
# options to opp_run, simulation time limit, expected fingerprint.
# The program runs the INET simulations in the CSV files, and
# reports fingerprint mismatches as FAILed test cases. To facilitate
# test suite maintenance, the program also creates a new file (or files)
# with the updated fingerprints.
#
# Implementation is based on Python's unit testing library, so it can be
# integrated into larger test suites with minimal effort
#
# Author: Andras Varga
#

import argparse
import os
import re
import glob
import subprocess
import sys
import time
import unittest
import csv

inetRoot = os.path.abspath("../..")
sep = ";" if sys.platform == 'win32' else ':'
nedPath = inetRoot + "/src" + sep + inetRoot + "/examples" + sep + inetRoot + "/tests/networks"
inetLib = inetRoot + "/src/inet"
opp_run = "opp_run"
cpuTimeLimit = "10s"
logFile = "test.out"

class FingerprintTestCaseGenerator():
    fileToSimulationsMap = {}
    def generateFromCSV(self, csvFileList, filterRegexList):
        testcases = []
        for csvFile in csvFileList:
            f = open(csvFile, 'rb')
            simulations = self.parseSimulationsTable(f)
            f.close()
            self.fileToSimulationsMap[csvFile] = simulations
            testcases.extend(self.generateFromDictList(simulations, filterRegexList))
        return testcases

    def generateFromDictList(self, simulations, filterRegexList):
        class StoreFingerprintCallback:
            def __init__(self, simulation):
                self.simulation = simulation
            def __call__(self, fingerprint):
                self.simulation['computedFingerprint'] = fingerprint

        testcases = []
        for simulation in simulations:
            title = simulation['wd'] + " " + simulation['args']
            if not filterRegexList or ['x' for regex in filterRegexList if re.search(regex, title)]: # if any regex matches title
                testcases.append(FingerprintTestCase(title, simulation['wd'], simulation['args'], simulation['simtimelimit'], simulation['fingerprint'], StoreFingerprintCallback(simulation)))
        return testcases

    # parse the CSV into a list of dicts
    def parseSimulationsTable(self, csvFile):
        simulations = []
        csvReader = csv.reader(csvFile, delimiter=',', quotechar='"', skipinitialspace=True)
        for fields in csvReader:
            if (len(fields)==0 or fields[0].startswith("#")):
                continue        # empty line or comment line
            if len(fields) < 4 or (len(fields) > 4 and not fields[4].startswith("#")):
                raise Exception("Line " + str(csvReader.line_num) + " must contain 4 items, but contains " + str(len(fields)) + ": " + '"' + '", "'.join(fields) + '"')
            simulations.append({'wd': fields[0], 'args': fields[1], 'simtimelimit': fields[2], 'fingerprint': fields[3]})
        return simulations

    def writeUpdatedCSVFiles(self):
        for csvFile, simulations in self.fileToSimulationsMap.iteritems():
            updatedContents = self.formatUpdatedSimulationsTable(simulations)
            if updatedContents:
                updatedFile = csvFile + ".UPDATED"
                ff = open(updatedFile, 'w')
                ff.write(updatedContents)
                ff.close()
                print "Check " + updatedFile + " for updated fingerprints"

    def escape(self, str):
        if re.search(r'[\r\n\",]', str):
            str = '"' + re.sub('"','""',str) + '"'
        return str

    def formatUpdatedSimulationsTable(self, simulations):
        # if there is a computed fingerprint, print that instead of existing one
        wdlen = 35
        argslen = 45
        timelen = 15
        txt = "# workingdir".ljust(wdlen) + ", " + "args".ljust(argslen) + ", " + "simtimelimit".ljust(timelen) + ", " + "fingerprint\n"
        containsComputedFingerprint = False
        for simulation in simulations:
            fingerprint = simulation['fingerprint']
            if 'computedFingerprint' in simulation:
                fingerprint = simulation['computedFingerprint']
                containsComputedFingerprint = True
            line = self.escape(simulation['wd']).ljust(wdlen) + ", " + self.escape(simulation['args']).ljust(argslen) + ", " + self.escape(simulation['simtimelimit']).ljust(timelen) + ", " + self.escape(fingerprint)
            txt += line + "\n"
        txt = re.sub("( +),", ",\\1", txt)
        return txt if containsComputedFingerprint else None

class SimulationResult:
    def __init__(self, command, workingdir, exitcode, errorMsg=None, isFingerprintOK=None, computedFingerprint=None, simulatedTime=None, numEvents=None, elapsedTime=None, cpuTimeLimitReached=None):
        self.command = command
        self.workingdir = workingdir
        self.exitcode = exitcode
        self.errorMsg = errorMsg
        self.isFingerprintOK = isFingerprintOK
        self.computedFingerprint = computedFingerprint
        self.simulatedTime = simulatedTime
        self.numEvents = numEvents
        self.elapsedTime = elapsedTime
        self.cpuTimeLimitReached = cpuTimeLimitReached

class SimulationTestCase(unittest.TestCase):
    def runSimulation(self, title, command, workingdir):
        global logFile

        # run the program and log the output
        FILE = open(logFile, "a")
        FILE.write("------------------------------------------------------\n")
        FILE.write("Running: " + title + "\n\n")
        FILE.write("$ cd " + workingdir + "\n");
        FILE.write("$ " + command + "\n\n");
        t0 = time.time()
        (exitcode, out) = self.runProgram(command, workingdir)
        elapsedTime = time.time() - t0
        FILE.write(out.strip() + "\n\n")
        FILE.write("Exit code: " + str(exitcode) + "\n")
        FILE.write("Elapsed time:  " + str(round(elapsedTime,2)) + "s\n\n")
        FILE.close()

        result = SimulationResult(command, workingdir, exitcode, elapsedTime=elapsedTime)

        # process error messages
        errorLines = re.findall("<!>.*", out, re.M)
        errorMsg = ""
        for err in errorLines:
            err = err.strip()
            if re.search("Fingerprint", err):
                if re.search("successfully", err):
                    result.isFingerprintOK = True
                else:
                    m = re.search("(computed|calculated): ([-a-zA-Z0-9]+)", err)
                    if m:
                        result.isFingerprintOK = False
                        result.computedFingerprint = m.group(2)
                    else:
                        raise Exception("Cannot parse fingerprint-related error message: " + err)
            else:
                errorMsg += "\n" + err
                if re.search("CPU time limit reached", err):
                    result.cpuTimeLimitReached = True
                m = re.search("at event #([0-9]+), t=([0-9]*(\\.[0-9]+)?)", err)
                if m:
                    result.numEvents = int(m.group(1))
                    result.simulatedTime = float(m.group(2))

        result.errormsg = errorMsg.strip()
        return result

    def runProgram(self, command, workingdir):
        process = subprocess.Popen(command, shell=True, cwd=workingdir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        out = process.communicate()[0]
        out = re.sub("\r", "", out)
        return (process.returncode, out)

class FingerprintTestCase(SimulationTestCase):
    def __init__(self, title, wd, args, simtimelimit, fingerprint, storeFingerprintCallback):
        SimulationTestCase.__init__(self)
        self.title = title
        self.wd = wd
        self.args = args
        self.simtimelimit = simtimelimit
        self.fingerprint = fingerprint
        self.storeFingerprintCallback = storeFingerprintCallback

    def runTest(self):
        # CPU time limit is a safety guard: fingerprint checks shouldn't take forever
        global inetRoot, opp_run, nedPath, cpuTimeLimit

        # run the simulation
        workingdir = _iif(self.wd.startswith('/'), inetRoot + "/" + self.wd, self.wd)
        command = opp_run + " -n " + nedPath + " -l " + inetLib + " -u Cmdenv " + self.args + \
            _iif(self.simtimelimit!="", " --sim-time-limit=" + self.simtimelimit, "") + \
            " --fingerprint=" + self.fingerprint + " --cpu-time-limit=" + cpuTimeLimit + \
            " --vector-recording=false --scalar-recording=false"

        result = self.runSimulation(self.title, command, workingdir)

        # process the result
        # note: fingerprint mismatch is technically NOT an error in 4.2 or before! (exitcode==0)
        if result.exitcode != 0:
            raise Exception("runtime error:" + result.errormsg)
        elif result.cpuTimeLimitReached:
            raise Exception("cpu time limit exceeded")
        elif result.simulatedTime == 0 and self.simtimelimit != '0s':
            raise Exception("zero time simulated")
        elif result.isFingerprintOK is None:
            raise Exception("other")
        elif result.isFingerprintOK == False:
            self.storeFingerprintCallback(result.computedFingerprint)
            assert False, "fingerprint mismatch; actual " + str(result.computedFingerprint)
        else:
            pass

    def __str__(self):
        return self.title

#
# Copy/paste of TextTestResult, with minor modifications in the output:
# we want to print the error text after ERROR and FAIL, but we don't want
# to print stack traces.
#
class SimulationTextTestResult(unittest.TestResult):
    """A test result class that can print formatted text results to a stream.

    Used by TextTestRunner.
    """
    separator1 = '=' * 70
    separator2 = '-' * 70

    def __init__(self, stream, descriptions, verbosity):
        super(SimulationTextTestResult, self).__init__()
        self.stream = stream
        self.showAll = verbosity > 1
        self.dots = verbosity == 1
        self.descriptions = descriptions

    def getDescription(self, test):
        doc_first_line = test.shortDescription()
        if self.descriptions and doc_first_line:
            return '\n'.join((str(test), doc_first_line))
        else:
            return str(test)

    def startTest(self, test):
        super(SimulationTextTestResult, self).startTest(test)
        if self.showAll:
            self.stream.write(self.getDescription(test))
            self.stream.write(" ... ")
            self.stream.flush()

    def addSuccess(self, test):
        super(SimulationTextTestResult, self).addSuccess(test)
        if self.showAll:
            self.stream.writeln(": PASS")
        elif self.dots:
            self.stream.write('.')
            self.stream.flush()

    def addError(self, test, err):
        # modified
        super(SimulationTextTestResult, self).addError(test, err)
        self.errors[-1] = (test, err[1])  # super class method inserts stack trace; we don't need that, so overwrite it
        if self.showAll:
            (cause, detail) = self._splitMsg(err[1])
            self.stream.writeln(": ERROR (%s)" % cause)
            if detail:
                self.stream.writeln(detail)
        elif self.dots:
            self.stream.write('E')
            self.stream.flush()

    def addFailure(self, test, err):
        # modified
        super(SimulationTextTestResult, self).addFailure(test, err)
        self.failures[-1] = (test, err[1])  # super class method inserts stack trace; we don't need that, so overwrite it
        if self.showAll:
            (cause, detail) = self._splitMsg(err[1])
            self.stream.writeln(": FAIL (%s)" % cause)
            if detail:
                self.stream.writeln(detail)
        elif self.dots:
            self.stream.write('F')
            self.stream.flush()

    def addSkip(self, test, reason):
        super(SimulationTextTestResult, self).addSkip(test, reason)
        if self.showAll:
            self.stream.writeln(": skipped {0!r}".format(reason))
        elif self.dots:
            self.stream.write("s")
            self.stream.flush()

    def addExpectedFailure(self, test, err):
        super(SimulationTextTestResult, self).addExpectedFailure(test, err)
        if self.showAll:
            self.stream.writeln(": expected failure")
        elif self.dots:
            self.stream.write("x")
            self.stream.flush()

    def addUnexpectedSuccess(self, test):
        super(SimulationTextTestResult, self).addUnexpectedSuccess(test)
        if self.showAll:
            self.stream.writeln(": unexpected success")
        elif self.dots:
            self.stream.write("u")
            self.stream.flush()

    def printErrors(self):
        # modified
        if self.dots or self.showAll:
            self.stream.writeln()
        self.printErrorList('Errors', self.errors)
        self.printErrorList('Failures', self.failures)

    def printErrorList(self, flavour, errors):
        # modified
        if errors:
            self.stream.writeln("%s:" % flavour)
        for test, err in errors:
            self.stream.writeln("  %s (%s)" % (self.getDescription(test), self._splitMsg(err)[0]))

    def _splitMsg(self, msg):
        cause = str(msg)
        detail = None
        if cause.count(':'):
            (cause, detail) = cause.split(':',1)
        return (cause, detail)


def _iif(cond,t,f):
    return t if cond else f

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Run the fingerprint tests specified in the input files.')
    parser.add_argument('testspecfiles', nargs='*', metavar='testspecfile', help='CSV files that contain the tests to run (default: *.csv). Expected CSV file columns: workingdir, args, simtimelimit, fingerprint')
    parser.add_argument('-m', '--match', nargs='*', metavar='regex', help='Line filter: a line (more precisely, workingdir+SPACE+args) must match any of the regular expressions in order for that test case to be run')
    args = parser.parse_args()

    if os.path.isfile(logFile):
        FILE = open(logFile, "w")
        FILE.close()

    if not args.testspecfiles:
        args.testspecfiles = glob.glob('*.csv')

    generator = FingerprintTestCaseGenerator()
    testcases = generator.generateFromCSV(args.testspecfiles, args.match)

    testSuite = unittest.TestSuite()
    testSuite.addTests(testcases)

    testRunner = unittest.TextTestRunner(stream=sys.stdout, verbosity=9, resultclass=SimulationTextTestResult)

    testRunner.run(testSuite)

    print
    generator.writeUpdatedCSVFiles()

    print "Log has been saved to %s" % logFile

