#!/bin/sh
# Copyright 1999-2002 Carnegie Mellon University.  
# Portions Copyright 2002 Sun Microsystems, Inc.  
# Portions Copyright 2002 Mitsubishi ElectricResearch Laboratories.
# All Rights Reserved.  Use is subject to license terms.
# 
# See the file "license.terms" for information on usage and
# redistribution of this file, and for a DISCLAIMER OF ALL 
# WARRANTIES.
#
# This script parses the regression log and presents a quick report
#
# Usage: makeReport < regression.log
#
# Field definitions: 
#   1) "test"
#   2) date
#   3) time
#   4) machine
#   5) System
#   6) testName 
#   7) who 
#   8) status
#   9) audioTime 
#  10) procTime 
#  11) words 
#  12) insertions 
#  13) deletions 
#  14) substitutions 
#  15) sentences 
#  16) correctSentences 
#  17) heapSize 
#  18) loadAverage 
#  19) actualUtterances
#  20) foundUtterances
#  21) gapInsertions
#  22) averageResponseTime
#  23) maxResponseTime
#  24) minResponseTime
#  25) correctOutOfGrammar
#  26) falseOutOfGrammar
#  27) correctInGrammar
#  28) falseInGrammar

gawk '

BEGIN {
    FS = "|"
    F_TEST = 1;
    F_DATE = 2;
    F_TIME = 3;
    F_MACHINE = 4;
    F_SYSTEM = 5;
    F_TEST_NAME = 6;
    F_WHO = 7;
    F_STATUS = 8;
    F_AUDIO_TIME = 9;
    F_PROC_TIME = 10;
    F_WORDS = 11;
    F_INSERTIONS = 12;
    F_DELETIONS = 13;
    F_SUBSTITUTIONS = 14;
    F_SENTENCES = 15;
    F_CORRECT_SENTENCES = 16;
    F_HEAP_SIZE = 17;
    F_LOAD_AVERAGE = 18;
    F_ACTUAL_UTTERANCES = 19;
    F_FOUND_UTTERANCES = 20;
    F_GAP_INSERTIONS = 21;
    F_AVG_RESPONSE_TIME = 22;
    F_MAX_RESPONSE_TIME = 23;
    F_MIN_RESPONSE_TIME = 24;
    F_CORRECT_OOG = 25;
    F_FALSE_OOG = 26;
    F_CORRECT_IG = 27;
    F_FALSE_IG = 28;
    "./getHostname" | getline machine
}

$1 == "build" && $4 == machine {
    finalBuildRecord = $0;
    buildRecord = 1;
    buildLiveRecord = 1;
    buildRejectionRecord = 1;
    buildDate = $2;
}

# final test has only the tests from the last build
#

$1 == "test" && buildRecord == 1 {
    buildRecord = 0;
    for (i in finalTest) {
        delete finalTest[i];
    }
}

$1 == "test"  && $4 == machine && $5 == "s4" {
    testCount[$F_TEST_NAME] = testCount[$F_TEST_NAME] + 1;
    testData[$F_TEST_NAME testCount[$F_TEST_NAME]] = $0;
    finalTest[$F_TEST_NAME] = 1;
}

$1 == "live_test" && buildLiveRecord == 1 {
    buildLiveRecord = 0;
    for (i in finalLiveTest) {
	delete finalLiveTest[i];
    }
}

$1 == "live_test" && $4 == machine && $5 == "s4" {
    testCount[$F_TEST_NAME] = testCount[$F_TEST_NAME] + 1;
    testData[$F_TEST_NAME testCount[$F_TEST_NAME]] = $0;
    finalLiveTest[$F_TEST_NAME] = 1;
}

$1 == "environment" && $3 == machine  {
    javaHome = $4;
    javaVersion = $5;
}

$1 == "rejection_test" && buildRejectionRecord == 1 {
    buildRejectionRecord = 0;
    for (i in finalRejectionTest) {
	delete finalRejectionTest[i];
    }
}

$1 == "rejection_test" && $4 == machine && $5 == "s4" {
    testCount[$F_TEST_NAME] = testCount[$F_TEST_NAME] + 1;
    testData[$F_TEST_NAME testCount[$F_TEST_NAME]] = $0;
    finalRejectionTest[$F_TEST_NAME] = 1;
}

#####
# updates the data array with info for the given test
#
function updateData(testName, testCount) {
    split(testData[testName testCount], data);
}

####
# returns the word error rate for the given test
#
function getWER(testName, testCount,     _errors, _words) {
    updateData(testName, testCount);
    _errors = data[F_INSERTIONS] + data[F_DELETIONS] + data[F_SUBSTITUTIONS];
    _words = data[F_WORDS];

    if (_words == "" || _words == 0) {
        return "N/A";
    } else {
	return _errors / _words * 100;
    }
}

####
# gets the speed for the given test
#
function getTime(testName, testCount, _audioTime, _procTime) {
    updateData(testName, testCount);
    _audioTime = data[F_AUDIO_TIME];
    _procTime = data[F_PROC_TIME];

    if (_audioTime == "" || _audioTime == 0 || _procTime == "") {
        return "N/A";
    } else {
	return _procTime / _audioTime;
    }
}

####
# gets the average heap size
#
function getHeap(testName, testCount) {
    updateData(testName, testCount);
    if (data[F_HEAP_SIZE] == "") {
        return "N/A";
    } else {
	return data[F_HEAP_SIZE];
    }
}



####
# returns the accuracy status for the given test
#
function getAccuracyStatus(testName) {

    count = testCount[testName];

    thisWER = getWER(testName, count);
    prevWER = getWER(testName, count - 1);

    return getLessIsBetterMetricStatus(thisWER, prevWER);
}

####
# returns the speed status for the given test
#
function getTimeStatus(testName, _slowThreshold, _fastThreshold) {

  count = testCount[testName];

  thisTime = getTime(testName, count);
  prevTime = getTime(testName, count - 1);

  return getLessIsBetterThresholdMetricStatus(thisTime, prevTime, 0.1);
}

####
# returns the heap average for the given test
#
function getHeapStatus(testName, _minThreshold, _maxThreshold) {
    count = testCount[testName];

    thisHeap = getHeap(testName, count);
    prevHeap = getHeap(testName, count - 1);

    return getLessIsBetterThresholdMetricStatus(thisHeap, prevHeap, 0.1);
}

####
# return the status of the test
#
function getRunStatus(testName) {
    count = testCount[testName];
    updateData(testName, count);
    return data[F_STATUS];
}

function fmtTime(seconds) {
    if (seconds == "N/A") {
	return seconds;
    }
    hours = int(seconds / 3600);
    seconds -= hours * 3600;

    mins = int(seconds / 60);
    seconds -= mins * 60;

    return sprintf("%02.2d:%02.2d:%02.2d", hours, mins, seconds);
}


####
# Returns the status of gap insertion rate
#
function getGapInsertionRateStatus(testName) {
    count = testCount[testName];

    thisGapRate = getGapInsertionRate(testName, count);
    prevGapRate = getGapInsertionRate(testName, count - 1);

    return getLessIsBetterMetricStatus(thisGapRate, prevGapRate);
}

function getGapInsertionRate(testName, testCount) {
    updateData(testName, testCount);
    gapInsertion = data[F_GAP_INSERTIONS];
    totalWords = data[F_WORDS];
    if (gapInsertion == "") {
	gapInsertion = "N/A";
    } else if (totalWords > 0) {
	gapInsertion = (gapInsertion / totalWords) * 100;
    }
    return gapInsertion;
}

####
# Returns the status of the utterance ratio
#
function getUtteranceRatioStatus(testName) {
    count = testCount[testName];

    thisRatio = getUtteranceRatio(testName, count);
    prevRatio = getUtteranceRatio(testName, count - 1);

    if (thisRatio == "N/A") {
	return "N/A";
    }
    if (prevRatio == "N/A") {
	return "OK - " thisRatio "";
    }

    thisDistFromOne = distFromOne(thisRatio);
    prevDistFromOne = distFromOne(prevRatio);
    
    if (thisDistFromOne > prevDistFromOne) {
	return "REGRESSED - increased from " prevRatio " to " thisRatio "";
    }
    if (thisDistFromOne < prevDistFromOne) {
	return "IMPROVED  - decreased from " prevRatio " to " thisRatio "";
    }
    return "OK - " thisRatio "";
}

function getUtteranceRatio(testName, testCount) {
    ratio = "N/A";
    updateData(testName, testCount);
    actualUtterances = data[F_ACTUAL_UTTERANCES];
    foundUtterances = data[F_FOUND_UTTERANCES];
    if (actualUtterances > 0) {
	ratio = foundUtterances / actualUtterances;
    }
    return ratio;
}

function distFromOne(x) {
    dist = x - 1.0;
    if (dist >= 0) {
	return dist;
    } else if (dist < 0) {
	return -dist;
    }
}

####
# Returns the status of average response time
#
function getAvgResponseTimeStatus(testName) {
    count = testCount[testName];

    thisTime = getAvgResponseTime(testName, count);
    prevTime = getAvgResponseTime(testName, count - 1);

    return getLessIsBetterThresholdMetricStatus(thisTime, prevTime, 0.1);
}

function getAvgResponseTime(testName, testCount) {
    updateData(testName, testCount);
    avgResponseTime = data[F_AVG_RESPONSE_TIME];
    if (avgResponseTime == "") {
	avgResponseTime = "N/A";
    }
    return avgResponseTime;    
}


function getRejectionAccuracy(testName, testCount) {
    updateData(testName, testCount);
    ra = (data[F_CORRECT_OOG] + data[F_CORRECT_IG]);
    if (ra == "" || ra == 0) {
	return "N/A";
    } else {
	return (ra * 100 / data[F_SENTENCES]);
    }
}


function getRejectionAccuracyStatus(testName) {
    thisRA = getRejectionAccuracy(testName, testCount[testName]);
    prevRA = getRejectionAccuracy(testName, testCount[testName] - 1);
    return getMoreIsBetterMetricStatus(thisRA, prevRA);
}


function getLessIsBetterMetricStatus(thisMetric, prevMetric) {
    if (thisMetric == "N/A") {
	return "N/A";
    }
    if (prevMetric == "N/A") {
	return "OK - " thisMetric "";
    }
    if (thisMetric > prevMetric) {
	return "REGRESSED - increased from " prevMetric " to " thisMetric "";
    }
    if (thisMetric < prevMetric) {
	return "IMPROVED  - decreased from " prevMetric " to " thisMetric "";
    }
    return "OK - " thisMetric ""
}

function getLessIsBetterThresholdMetricStatus(thisMetric, prevMetric, threshold) {
    if (thisMetric == "N/A") {
	return "N/A";
    }
    if (prevMetric == "N/A") {
	return "OK - " thisMetric "";
    }
    prevMetricPlusTenPercent = prevMetric * (1.0 + threshold);
    prevMetricMinusTenPercent = prevMetric * (1.0 - threshold);
    
    if (thisMetric > prevMetricPlusTenPercent) {
	return "REGRESSED - increased from " prevMetric " to " thisMetric "";
    }
    if (thisMetric < prevMetricMinusTenPercent) {
	return "IMPROVED  - decreased from " prevMetric " to " thisMetric "";
    }
    return "OK - " thisMetric "";
}

function getMoreIsBetterMetricStatus(thisMetric, prevMetric) {
    if (thisMetric == "N/A") {
	return "N/A";
    }
    if (prevMetric == "N/A") {
	return "OK - " thisMetric "";
    }
    if (thisMetric < prevMetric) {
	return "REGRESSED - decreased from " prevMetric " to " thisMetric "";
    }
    if (thisMetric > prevMetric) {
	return "IMPROVED  - increased from " prevMetric " to " thisMetric "";
    }
    return "OK - " thisMetric ""
}

####
# Prints summary info
#
function printSummaryInfo() {
    split(finalBuildRecord, data);
    buildDate = data[2];
    print "Host:             " data[4];
    print "Date:             " data[2];
    print "Build Status:     " data[7];
    print "Build Time:       " fmtTime(data[8]);
    print "Run Time:         " fmtTime(data[9]);
    print "Test Name:        " data[10];
    print "Java Home:        " javaHome;
    print "Java Version:     " javaVersion;
    print "";
    print "";
}

###########################
# Determines if the given test was on the same day as the
# most recent build
#

function isNewTest(testName) {
    updateData(testName, testCount[testName]);
    _date = data[F_DATE];
    if (_date == buildDate) {
        return 1;
    } else {
        return 0;
    }
}

END {

    print ""
    print "Sphinx 4 Regression Test Summary for " buildDate
    print "================================================================="
    printSummaryInfo();

    for (i in finalTest) {
	print i;
	print "    Status                 "  getRunStatus(i);
	print "    Accuracy (WER %)       "  getAccuracyStatus(i);
	print "    Time (X Realtime)      "  getTimeStatus(i);
	print "    Average Heap (MB)      "  getHeapStatus(i);
	printf("\n");
    }

    for (i in finalLiveTest) {
	print i;
	print "    Status                 "  getRunStatus(i);
	print "    Accuracy (WER %)       "  getAccuracyStatus(i);
	print "    Time (X Realtime)      "  getTimeStatus(i);
	print "    Average Heap (MB)      "  getHeapStatus(i);
	print "    Gap Insertion (%)      "  getGapInsertionRateStatus(i);
	print "    Utterance Ratio        "  getUtteranceRatioStatus(i);
	print "    Avg Response Time (s)  "  getAvgResponseTimeStatus(i);
	printf("\n");
    }

    for (i in finalRejectionTest) {
	print i;
	print "    Status                 "  getRunStatus(i);
	print "    Rejection Accuracy (%) "  getRejectionAccuracyStatus(i);
	print "    Time (X Realtime)      "  getTimeStatus(i);
	print "    Average Heap (MB)      "  getHeapStatus(i);
	printf("\n");
    }

    print ""
    print ""
    print "Complete test details at: ";
    print ""
    print "    http://cmusphinx.sourceforge.net/regression_report_" machine ".html"
    print ""
}

'  $*

