runtest.pl

#!/usr/bin/env perl

# runtest [options] FILENAME
#
# Read the file FILENAME. Each line contains a test.
# Convert template to test and crosstest.
# If possilble generate orphaned testversions, too.
# Use make to compile the test

################################################################################
# Global configuration options for the runtestscript itself:
################################################################################

# name of the global configuration file for the testsuite:
$config_file    = "ompts.conf";
$logfile        = "ompts.log"; # overwriteable by value in config file
$env_set_threads_command = 'OMP_NUM_THREADS=%n; export OMP_NUM_THREADS;';
$debug_mode     = 0;
################################################################################
# After this line the script part begins! Do not edit anithing below
################################################################################


# Namespaces:
use Getopt::Long;
#use Unix::PID;
use Data::Dumper;
use ompts_parserFunctions;

# Extracting given options
GetOptions("help",
      "listlanguages",
      "lang=s",
      "list",
      "testinfo=s",
      "numthreads=i",
      "test=s",
      "compile!",
      "run!",
      "orphan!",
      "resultfile=s"
      );

# Get global configuratino options from config file:
if(! -e $config_file){ error ("Could not find config file $config_file\n", 1);}
open (CONFIG, "<$config_file") or error ("Could not open config file $config_file\n", 2);
while (<CONFIG>) { $config .= $_; }
close (CONFIG);

($logfile) = get_tag_values ("logfile", $config);
($timeout) = get_tag_values ("singletesttimeout", $config);
($display_errors) = get_tag_values("displayerrors", $config);
($display_warnings) = get_tag_values ("displaywarnings", $config);
($numthreads) = get_tag_values ("numthreads", $config);
($env_set_threads_command) = get_tag_values("envsetthreadscommand",$config);
$env_set_threads_command =~ s/\%n/$numthreads/g;
@languages = get_tag_values ("language", $config);

if (!defined($opt_compile)) {$opt_compile = 1;}
if (!defined($opt_run))     {$opt_run = 1;}
if (!defined($opt_orphan)) {$opt_orphan = 1;}
if (!defined($opt_resultsfile)) {($opt_resultsfile) = get_tag_values("resultsfile", $config);}
if ( defined($opt_numthreads) && ($opt_numthreads > 0)) {$numthreads = $opt_numthreads;}
if ($debug_mode) {
print <<EOF;
Testsuite configuration:
Logfile = $logfile
Timeout = $timeout seconds
Language:  $opt_lang
Display errors:   $display_errors
Display warnings: $display_warnings
Resultsfile:      $opt_resultsfile
Numthreads: $numthreads
------------------------------
EOF
}

$num_construcs    = 0;
$num_tests        = 0;
$num_failed_tests = 0;
$num_successful_tests   = 0;
$num_verified_tests     = 0;
$num_failed_compilation = 0;

$num_normal_tests_failed = 0;
$num_normal_tests_compile_error = 0;
$num_normal_tests_timed_out = 0;
$num_normal_tests_successful = 0;
$num_normal_tests_verified = 0;

$num_orphaned_tests_failed = 0;
$num_orphaned_tests_compile_error = 0;
$num_orphaned_tests_timed_out = 0;
$num_orphaned_tests_successful = 0;
$num_orphaned_tests_verified = 0;

if ($opt_help)         { print_help_text ();   exit 0; }
if ($opt_listlanguages){ print_avail_langs (); exit 0; }
if ($opt_list)     { print_avail_tests ();   exit 0; }
if ($opt_testinfo) { print_testinfo ();      exit 0; }
if ($opt_test)     { write_result_file_head();
                     execute_single_test (); exit 0; }
if (-e $ARGV[0])   { write_result_file_head();
                     execute_testlist($ARGV[0]); print_results();
                     result_summary(); exit 0;}

################################################################################
# sub function definitions
################################################################################

# Function which prints the results file
sub print_results
{
    system("echo; cat $opt_resultsfile; echo;");
}

# Function which prints a summary of all test
sub result_summary
{
    my $num_directives = @test_results;

    print <<EOF;

Summary:
S Number of tested Open MP constructs: $num_constructs
S Number of used tests:                $num_tests
S Number of failed tests:              $num_failed_tests
S Number of successful tests:          $num_successful_tests
S + from this were verified:           $num_verified_tests

Normal tests:
N Number of failed tests:              $num_normal_tests_failed
N + from this fail compilation:        $num_normal_tests_compile_error
N + from this timed out                $num_normal_tests_timed_out
N Number of successful tests:          $num_normal_tests_successful
N + from this were verified:           $num_normal_tests_verified

Orphaned tests:
O Number of failed tests:              $num_orphaned_tests_failed
O + from this fail compilation:        $num_orphaned_tests_compile_error
O + from this timed out                $num_orphaned_tests_timed_out
O Number of successful tests:          $num_orphaned_tests_successful
O + from this were verified:           $num_orphaned_tests_verified
EOF

}

# Function that executest the tests specified in the given list
sub execute_testlist
{
    my ($filename) = @_;
    # opening testlist
    open(TESTS,$filename) or error ("Could not open  $filename\n", 1);
TEST: while (<TESTS>) {
        if (/^\s*#/) {next TEST;}
        if (/^\s*$/) {next TEST;}
        $opt_test = $_;
        chomp ($opt_test);
        execute_single_test ();
    }
# print Dumper(@test_results);
}

# Function that executes a system command but takes care of the global timeout
# If command did not finish inbetween returns '-' otherwise the exit status of
# the system command
sub timed_sys_command
{
    my ($command) = @_;
    my $exit_status = '-';

# set up the timeout for the command
    eval {
        local $SIG{ALRM} = sub {die "alarm\n"};
        alarm $timeout;
        log_message_add ("Starting command \"$command\"");
        $exit_status = system ($command);
        alarm 0;
    };
# check if command finished during the maximum execution time
    if ($@ eq "alarm\n") {
# test timed out
#		my $pid = Unix::PID->new();
#		$pid->get_pidof($command, 1);
#		$pid->kill();
        if ($debug_mode) {
	    log_message_add ("Command \"$command\" reached max execution time.\n");
        }
        return "TO";
    }
# test finished
    return $exit_status;
}

# Function that runs the tests given as a array containing the testnames
# Returns an array containing the percent values of the passed tests and the
# successful crosstests.
sub run_test
{
    my ($testname, $orphan) = @_;
    my $bin_name, $cbin_name;
    my $cmd, $exit_status, $failed;
    my $resulttest, $resultctest;

# path to test and crosstest either in normal or in orphaned version
    if ($orphan) {
        $bin_name  = "bin/$opt_lang/orph_test_$testname";
        $cbin_name = "bin/$opt_lang/orph_ctest_$testname";
    } else {
        $bin_name  = "bin/$opt_lang/test_$testname";
        $cbin_name = "bin/$opt_lang/ctest_$testname";
    }
# Check if executables exist
    if (! -e $bin_name) {
        test_error ("Could not find executable \"$bin_name\".");
        return ('test' => '-', 'crosstest' => '-');
    }
# run the test
    $cmd = "$env_set_threads_command ./$bin_name >$bin_name.out";
    print "Running test with $numthreads threads .";
    $exit_status = timed_sys_command ($cmd);
############################################################
# Check if test finished within max execution time
    if ($exit_status eq 'TO') {
        print ".... failed (timeout)\n";
        return ('test' => 'TO', 'crosstest' => '-')
    }
############################################################
# check if all tests were successful
    $failed = $exit_status >> 8;
    if ($failed < 0 or $failed > 100) { $failed = 100; }
    $resulttest = 100 - $failed;
    if ($resulttest eq 100) {
        print ".... success ...";
    } else {
        print ".... failed $failed\% of the tests\n";
        return ('test' => $resulttest, 'crosstest' => '-');
    }
############################################################

# do crosstest
# check if executable exist
    if (! -e $cbin_name) {
        test_error ("Could not find executable \"$cbin_name\".");
        print "... not verified (crosstest missing)\n";
        return ('test' => $resulttest, 'crosstest' => '-');
    }
# run crosstest
# Test was successful, so it makes sense to run the crosstest
    $cmd = "$env_set_threads_command ./$cbin_name > $cbin_name.out";
    $exit_status = timed_sys_command ($cmd);
############################################################
# Check if crosstest finished within max execution time
    if ($exit_status eq 'TO') {
        print "... not verified (timeout)\n";
        return ('test' => $result, 'crosstest' => 'TO');
    }
############################################################
# test if crosstests failed as expected
    $resultctest = $exit_status >> 8;
    if ($resultctest > 0) {
        print "... and verified with $resultctest\% certainty\n";
    } else {
        print "... but might be lucky\n";
    }
    return ('test' => $resulttest, 'crosstest' => $resultctest);
############################################################
}

# Function that generates the test binaries out of the sourcecode
sub compile_src
{
    my ($testname, $orphan) = @_;
    print "Compiling soures ............";
    if ($orphan) {
# Make orphaned tests
        $exec_name     = "bin/$opt_lang/orph_test_$testname";
        $crossexe_name = "bin/$opt_lang/orph_ctest_$testname";
        $resulttest  = system ("make $exec_name > $exec_name\_compile.log" );
        $resultctest = system ("make $crossexe_name > $crossexe_name\_compile.log" );
    } else {
# Make test
        $exec_name     = "bin/$opt_lang/test_$testname";
        $crossexe_name = "bin/$opt_lang/ctest_$testname";
        $resulttest  = system ("make $exec_name > $exec_name\_compile.log" );
        $resultctest = system ("make $crossexe_name > $crossexe_name\_compile.log" );
    }
    if ($resulttest) { test_error ("Compilation of the test failed."); }
    if ($resultctest){ test_error ("Compilation of the crosstest failed."); }

    if ($resulttest or $resultctest) {
        print ".... failed\n";
        return 0;
    } else {
        print ".... success\n";
        return 1;
    }
}

# Function which prepare the directory structure:
sub init_directory_structure
{
    my ($language) = @_;
    if (-e "bin" && -d "bin") { warning ("Old binary directory detected!");}
    else { system ("mkdir bin"); }
    if (-e "bin/$language" && -d "bin/$language") {
        warning ("Old binary directory for language $language found.");}
    else { system ("mkdir bin/$language"); }
}

# Function that generates the sourcecode for the given test
sub make_src
{
    my ($testname, $orphan) = @_;
    my $template_file;
    my $src_name;

    $template_file = "$dir/$testname.$extension";
    if (!-e $template_file) { test_error ("Could not find template for \"$testname\""); }

    print "Generating sources ..........";
    if ($orphan) {
# Make orphaned tests
        $src_name = "bin/$opt_lang/orph_test_$testname.$extension";
        $resulttest = system ("./$templateparsername --test --orphan $template_file $src_name");
        $src_name = "bin/$opt_lang/orph_ctest_$testname.$extension";
        $resultctest = system ("./$templateparsername --crosstest --orphan $template_file $src_name");
    } else {
# Make test
        $src_name = "bin/$opt_lang/test_$testname.$extension";
        $resulttest = system ("./$templateparsername --test --noorphan $template_file $src_name");
        $src_name = "bin/$opt_lang/ctest_$testname.$extension";
        $resultctest = system ("./$templateparsername --crosstest --noorphan $template_file $src_name");
    }
    if ($resulttest) { test_error ("Generation of sourcecode for the test failed."); }
    if ($resultctest){ test_error ("Generation of sourcecode for the crosstest failed."); }

    if ($resulttest or $resultctest) {
        print ".... failed\n";
       return 0;
    } else {
       print ".... success\n";
       return 1;
    }
}

# Function which checks if a given test is orphanable
sub test_is_orphanable
{
    my ($testname) = @_;
    my $src;
    my $file = "$dir/$testname.$extension";
    if(! -e $file){ test_error ("Could not find test file $file\n");}
    open (TEST, "<$file") or test_error ("Could not open test file $file\n");
    while (<TEST>) { $src .= $_; }
    close (TEST);
    return $src =~/ompts:orphan/;
}

sub write_result_file_head
{
    open (RESULTS, ">$opt_resultsfile") or error ("Could not open file '$opt_resultsfile' to write results.", 1);
    $resultline = sprintf "%-25s %-s\n", "#Tested Directive", "\tt\tct\tot\toct";
    print RESULTS $resultline;
}

# Function which adds a result to the list of results
sub add_result
{
    my ($testname, $result) = @_;
#	print Dumper(@{$result});

    $num_constructs++;

    open (RESULTS, ">>$opt_resultsfile") or error ("Could not open file '$opt_resultsfile' to write results.", 1);

    if (${$result}[0][0]) {
		$num_tests ++;}

	if ($opt_compile and ${$result}[0][1] eq 0) {
		${$result}[0][2]{test}      = 'ce';
		${$result}[0][2]{crosstest} = '-';
		$num_normal_tests_compile_error++;
	    $num_normal_tests_failed++;
	}

    if ($opt_run and ${$result}[0][2] and ${$result}[0][2]{test} ne 'ce') {
        if (${$result}[0][2]{test} == 100) {
            $num_normal_tests_successful++;
            if (${$result}[0][2]{crosstest} == 100){
				$num_normal_tests_verified++;}
		} elsif (${$result}[0][2]{test} eq 'TO'){
			$num_normal_tests_timed_out++;
			$num_normal_tests_failed++;
		} else {
			$num_normal_tests_failed++;
		}
    }
    $resultline = "${$result}[0][2]{test}\t${$result}[0][2]{crosstest}\t";

    if (${$result}[1][0]) {
		$num_tests ++;}
    else { $resultline .= "-\t-\n"; }

    if ($opt_compile and ${$result}[1][1] eq 0) {
		${$result}[1][2]{test}      = 'ce';
		${$result}[1][2]{crosstest} = '-';
		$num_orphaned_tests_compile_error++;
		$num_orphaned_tests_failed++;
	}

    if ($opt_run and ${$result}[1][2] and ${$result}[1][2]{test} ne 'ce') {
        if (${$result}[1][2]{test} == 100) {
            $num_orphaned_tests_successful++;
            if (${$result}[1][2]{crosstest} == 100){
				$num_orphaned_tests_verified++;}
		} elsif (${$result}[1][2]{test} eq 'TO'){
			$num_orphaned_tests_timed_out++;
			$num_orphaned_tests_failed++;
        } else {
			$num_orphaned_tests_failed++;
		}
    }
    $resultline .= "${$result}[1][2]{test}\t${$result}[1][2]{crosstest}\n";

    $num_failed_tests = $num_normal_tests_failed + $num_orphaned_tests_failed;
	$num_failed_compilation = $num_normal_tests_compile_error + $num_orphaned_tests_compile_error;
	$num_successful_tests = $num_normal_tests_successful + $num_orphaned_tests_successful;
	$num_verified_tests = $num_normal_tests_verified + $num_orphaned_tests_verified;

    $resultline2 = sprintf "%-25s %-s", "$testname", "\t$resultline";
    print RESULTS $resultline2;
}

# Function which executes a single test
sub execute_single_test
{
    my @result;
    init_language_settings ($opt_lang);
    init_directory_structure ($opt_lang);
    log_message_add ("Testing for \"$opt_test\":");
    print "Testing for \"$opt_test\":\n";
# tests in normal mode
    if ($opt_compile){ $result[0][0] = make_src ($opt_test, 0);
                       $result[0][1] = compile_src ($opt_test, 0);}
    if ($opt_run && $result[0][1] == 1) {
                       $result[0][2] = {run_test ($opt_test, 0)};}
# tests in orphaned mode
    if ($opt_orphan && test_is_orphanable($opt_test)){
        log_message_add ("Testing for \"$opt_test\" in orphaned mode:");
        print "+ orphaned mode:\n";
        if ($opt_compile) { $result[1][0] = make_src ($opt_test, 1);
                            $result[1][1] = compile_src ($opt_test, 1);}
        if ($opt_run && $result[1][1] == 1) {
                            $result[1][2] = {run_test ($opt_test, 1)};}
    }
    add_result($opt_test, \@result);
}

# Function that prints info about a given test
sub print_testinfo
{
    init_language_settings($opt_lang);
    my $doc = "";
    my $file = $dir."/".$opt_testinfo.".".$extension;
    if (! -e $file) {error ("Could not find template for test $opt_testinfo", 5);}
    open (TEST,"<$file") or error ("Could not open template file \"$file\" for test $opt_testinfo", 6);
    while (<TEST>) {$doc .= $_;}
    close (TEST);

    (my $omp_version) = get_tag_values ("ompts:ompversion", $doc);
    (my $dependences) = get_tag_values ("ompts:dependences", $doc);
    (my $description) = get_tag_values ("ompts:testdescription", $doc);
    my $orphanable = 'no';
    if ($doc =~ /ompts:orphan/) {$orphanable = 'yes';}
    print <<EOF;
Info for test $opt_testinfo:
Open MP standard: $omp_version
Orphaned mode: $orphanable
Dependencies:  $dependences
Description:   $description
EOF
}

# Function that initializes the settings for the given language
sub init_language_settings
{
    my ($language) = @_;
    foreach my $lang (@languages) {
        (my $name) = get_tag_values ("languagename", $lang);
        if ($name eq $language) {
            ($extension) = get_tag_values ("fileextension", $lang);
            ($dir)       = get_tag_values ("dir", $lang);
            ($templateparsername) = get_tag_values ("templateparsername", $lang);
            last;
        }
    }
    # Check if we found the specified language in the config file
    if (!$extension and !$dir) {
      error ("Language $language could not be found.\n", 3);
    }
}



# Function that prints all available tests for the given language
sub print_avail_tests
{
    init_language_settings($opt_lang);
    my @tests;
    opendir(DIR,"$dir") or error ("Could not open directory $dir", 4);
    while($_ = readdir(DIR)) { if (/\.$extension$/) {s/\.$extension//; push (@tests, $_);}}
    closedir(DIR);
    print "Found ".(@tests)." tests:\n". "-" x 30 . "\n";
    foreach (@tests) { print $_."\n";}
}

# Function that prints all available tests for the given language
sub print_avail_langs
{
    if (@languages > 0) {
        print "Available languages:\n";
        foreach (@languages) {
            (my $name) = get_tag_values ("languagename", $_);
            print "$name\n";
        }
    } else {
        print "No languages available\n";
    }
}

# Function that prints the error message
sub print_help_text
{
    print <<EOF;
runtest.pl [options] [FILE]

Executes the tests listed in FILE. FILE has to contain the names of the tests,
one test per line. Lines starting with '#' will be ignored.
A language has to be specified for all commands except --help and --listlanguages.

Options:
  --help            displays this help message
  --listlanguages   lists all available languages
  --lang=s          select language
  --list            list available tests for a language
  --testinfo=NAME   show info for test NAME
  --numthreads=NUM  set number of threads (overwrites config file settings)
  --test=NAME       execute single test NAME
  --nocompile       do not compile tests
  --norun           do not run tests
  --noorphan        switch of orphaned tests
  --resultfile=NAME use NAME as resultfile (overwrites config file settings)
EOF
}

# Function that writes an error message for a failed test / part of a test
sub test_error
{
   my ($message) = @_;
   log_message_add ("ERROR: $message");
   if ($display_errors eq 1) { print STDERR "ERROR: $message\n"; }
}

# Function that returns an warning message
sub warning {
  my ($message) = @_;
  if ($display_warnings eq 1) { print "Warniong: $message\n"; }
  log_message_add ("Warning: $message");
}

# Function that returns an error message and exits with the specified error code
sub error {
  my ($message, $error_code) = @_;
  if ($display_errors eq 1) { print STDERR "ERROR: $message\n"; }
  log_message_add ("ERROR: $message");
  exit ($error_code);
}

# Function which adds an new entry into the logfile together with a timestamp
sub log_message_add
{
    (my $message) = @_;
    ($sec,$min,$hour,$mday,$mon,$year,$wday,$ydat,$isdst) = localtime();
    if(length($hour) == 1) { $hour="0$hour"; }
    if(length($min) == 1)  { $min="0$min";   }
    if(length($sec) == 1)  { $sec="0$sec";   }
    $mon=$mon+1;
    $year=$year+1900;
    open (LOGFILE,">>$logfile") or die "ERROR: Could not create $logfile\n";
    print LOGFILE "$mday/$mon/$year $hour.$min.$sec: $message\n";
}