shelf_regress

#!/usr/bin/perl

$tools_path="/root/shelf_utils/";

sub check_disk_await;
sub check_disk_qsize;
sub get_errors;
sub start_dd;
sub check_r_speed;
sub check_w_speed;

$SIG{CHLD} = sub { wait };

# Single argument: special device of shelf to test
die "Usage: shelf_disks /dev/sgX" if $#ARGV != 0;

$dev = $ARGV[0];

print "Discover disks...\n";

open SHELF,"$tools_path/shelf_disks $dev |" || die "Cannot spawn shelf_disks utility";
while (<SHELF>) {
    chomp;
    if (m,(/dev/\w+)$,) {
        $d = $1;
        push @disks, $d;
        $sg{$d} = `sg_map26 $d`;
        chomp $sg{$d};
    }
}
close SHELF;

print "Collecting errors stats...\n";
%e = get_errors (\%sg);


# --------------------------------------------------
# Orion
print "Performing orion test...\n";

# generate orion config file
$tmp = "/tmp/orion.$$/";
mkdir $tmp || die "Cannot create temp dir";

open RUN, "> $tmp/test.lun" || die "Cannot create lun file";
foreach $d (@disks) {
    print RUN "$d\n";
}
close RUN;


# run orion test
system ("cd $tmp && orion -run advanced -testname test -num_disks $#disks -size_small 8 -size_large 384 -type seq -write 50 -matrix point -num_small 0 -num_large 12 -simulate raid0 > /dev/null &");

# launch iostat
open IO, "iostat -x 1 |" || die "Cannot start iostat";
while (<IO>) {
    chomp;

    next unless /^(\w+)\s+/;

    $d = $1;
    $f = 0;
    foreach $disk (@disks) {
        $disk =~ m,/dev/(\w+)$,;
        $f = 1 if $d eq $1;
        last if $f;
    }

    next unless $f;

    if (defined $seen{$d}) {
        # we have one of our disks, analyze data
        @d = split /\s+/;
        last if ($d[8] == 0.0 && $d[9] == 0.0 && $d[11] == 0.0);

        push @{$a{"/dev/$d"}->{"qsize"}}, $d[8];
        push @{$a{"/dev/$d"}->{"await"}}, $d[9];
    }
    $seen{$d} = 1;
}
close IO;

foreach $disk (@disks) {
    print "$disk: \n";
    foreach $key (keys %{$a{$disk}}) {
        $avg = $max = 0;
        $min = 1e100;

        foreach $val (@{$a{$disk}->{$key}}) {
            $avg += $val;
            $min = $val if $min > $val;
            $max = $val if $max < $val;
        }
        $avg /= $#{$a{$disk}->{$key}};

        $msg{$disk} .= "$key: ($min, $avg, $max)\n";

        $await{$disk} = check_disk_await ($min, $max, $avg) if $key eq "await";
        $qsize{$disk} = check_disk_qsize ($min, $max, $avg) if $key eq "qsize";
    }
}


# --------------------------------------------------
# dd
print "Performing read test (can take up to an hour, so be patient)...\n";
foreach $disk (@disks) {
    start_dd ($disk, 0);
}
wait_for_dd (@disks);

print "Performing write test (can take up to an hour, so be patient)...\n";
foreach $disk (@disks) {
    start_dd ($disk, 1);
}
wait_for_dd (@disks);

# --------------------------------------------------
# process
print "Collecting errors stats...\n";
%e2 = get_errors (\%sg);

# compare error counters
foreach $disk (@disks) {
    $err{$disk} = 0;
    foreach $key (keys %{$e{$disk}}) {
        $msg{$disk} .= "$key: before $e{$disk}->{$key}, after $e2{$disk}->{$key}\n" if $e{$disk}->{$key} < $e2{$disk}->{$key};
        $err{$disk} ||= $e{$disk}->{$key} < $e2{$disk}->{$key};
    }
}

# show summary
foreach $disk (@disks) {
    print "$disk:\n";

    print "Await test:  ".($await{$disk} ? "FAILED" : "OK")."\n";
    print "Qsize test:  ".($qsize{$disk} ? "FAILED" : "OK")."\n";
    print "Error test:  ".($err{$disk}   ? "FAILED" : "OK")."\n";
    print "$msg{$disk}" if $await{$disk} || $qsize{$disk} || $err{$disk};
    print "\n";
}

system ("rm -rf $tmp");


sub check_disk_await
{
    my ($min, $max, $avg) = @_;
    return $avg > 150 ? 1 : 0;
}


sub check_disk_qsize
{
    my ($min, $max, $avg) = @_;
    return $avg > 15 ? 1 : 0;
}


sub get_errors
{
    my ($sg) = @_;
    my (%e);

    foreach $disk (keys %$sg) {
        # write errors
        open LOG, "sg_logs -p 0x2 ${$sg}{$disk}|";
        while (<LOG>) {
            chomp;
            $e{$disk}->{"w_corr"}   = $1 if /\s*Total errors corrected\s*=\s*(\d+)/;
            $e{$disk}->{"w_uncorr"} = $1 if /\s*Total uncorrected errors\s*=\s*(\d+)/;
        }
        close LOG;

        # read errors
        open LOG, "sg_logs -p 0x3 ${$sg}{$disk}|";
        while (<LOG>) {
            chomp;
            $e{$disk}->{"r_corr"}   = $1 if /\s*Total errors corrected\s*=\s*(\d+)/;
            $e{$disk}->{"r_uncorr"} = $1 if /\s*Total uncorrected errors\s*=\s*(\d+)/;
        }
        close LOG;

        # non-medium errors
        open LOG, "sg_logs -p 0x6 ${$sg}{$disk}|";
        while (<LOG>) {
            chomp;
            $e{$disk}->{"nm"} = $1 if /\s*error count\s*=\s*(\d+)/;
        }
        close LOG;
    }

    return %e;
}



# forks dd for specified disk. Filehandle will be the same as block device name
# Arguments:
# 1. block device name
# 2. start read test if 0, or write test if not
sub start_dd
{
    my ($block, $write) = @_;
    my ($cmd);

    $cmd = $write ? "if=/dev/zero of=$block" : "if=$block of=/dev/null";

    open $block, "dd $cmd bs=10M 2>&1 |" || die "Canot launch dd";
}


sub wait_for_dd
{
    my (@disks) = @_;
    my ($count, %fin, $n, $r);

    $count = 0;
    while ($count < $#disks) {
        $r = "";
        foreach $disk (@disks) {
            vec ($r, fileno ($disk), 1) = 1 unless $fin{$disk};
        }

        $n = select ($r, undef, undef, undef);
        if ($n > 0) {
            foreach $disk (@disks) {
                next if $fin{$disk};
                if (vec ($r, fileno ($disk), 1)) {
                    # read data
                    read $disk, $out{$disk}, 10240;
                    $fin{$disk} = 1;
                    $count++;
                }
            }
        }
    }

    foreach $disk (@disks) {
        close $disk;
    }
}