Skip to content

Commit

Permalink
Ensure single testcases can be run from profile
Browse files Browse the repository at this point in the history
  • Loading branch information
mattias-p committed Dec 6, 2023
1 parent e247073 commit 71278c4
Show file tree
Hide file tree
Showing 4 changed files with 328 additions and 15 deletions.
32 changes: 17 additions & 15 deletions MANIFEST
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
Changes
CONTRIBUTING.md
docs/Implementing_Tests.pod
docs/Translation.pod
docs/Translation-translators.md
docs/Translation.pod
inc/Module/Install.pm
inc/Module/Install/Base.pm
inc/Module/Install/Can.pm
Expand Down Expand Up @@ -87,20 +87,22 @@ t/profiles.t
t/profiles/policy.json
t/profiles/profile.json
t/profiles/Test-address-all.json
t/profiles/Test-all.json
t/profiles/Test-all-levels.json
t/profiles/Test-all.json
t/profiles/Test-basic-all.json
t/profiles/Test-connectivity-all.json
t/profiles/Test-connectivity01-only.json
t/profiles/Test-connectivity02-only.json
t/profiles/Test-connectivity03-only.json
t/profiles/Test-connectivity04-only.json
t/profiles/Test-connectivity-all.json
t/profiles/Test-consistency-all.json
t/profiles/Test-consistency01-only.json
t/profiles/Test-consistency02-only.json
t/profiles/Test-consistency03-only.json
t/profiles/Test-consistency04-only.json
t/profiles/Test-consistency-all.json
t/profiles/Test-delegation-all.json
t/profiles/Test-dnssec-all.json
t/profiles/Test-dnssec-more-all.json
t/profiles/Test-dnssec01-only.json
t/profiles/Test-dnssec02-only.json
t/profiles/Test-dnssec03-only.json
Expand All @@ -117,8 +119,7 @@ t/profiles/Test-dnssec14-only.json
t/profiles/Test-dnssec15-only.json
t/profiles/Test-dnssec17-only.json
t/profiles/Test-dnssec18-only.json
t/profiles/Test-dnssec-all.json
t/profiles/Test-dnssec-more-all.json
t/profiles/Test-nameserver-all.json
t/profiles/Test-nameserver01-only.json
t/profiles/Test-nameserver02-only.json
t/profiles/Test-nameserver03-only.json
Expand All @@ -128,7 +129,7 @@ t/profiles/Test-nameserver06-only.json
t/profiles/Test-nameserver07-only.json
t/profiles/Test-nameserver08-only.json
t/profiles/Test-nameserver09-only.json
t/profiles/Test-nameserver-all.json
t/profiles/Test-syntax-all.json
t/profiles/Test-syntax01-only.json
t/profiles/Test-syntax02-only.json
t/profiles/Test-syntax03-only.json
Expand All @@ -137,7 +138,6 @@ t/profiles/Test-syntax05-only.json
t/profiles/Test-syntax06-only.json
t/profiles/Test-syntax07-only.json
t/profiles/Test-syntax08-only.json
t/profiles/Test-syntax-all.json
t/profiles/Test-zone-all.json
t/recursor.data
t/recursor.t
Expand All @@ -157,14 +157,14 @@ t/Test-connectivity04.data
t/Test-connectivity04.t
t/Test-consistency.data
t/Test-consistency.t
t/Test-consistency05.data
t/Test-consistency05.t
t/Test-consistency05-F.data
t/Test-consistency05-F.t
t/Test-consistency06.data
t/Test-consistency06.t
t/Test-consistency05.data
t/Test-consistency05.t
t/Test-consistency06-B.data
t/Test-consistency06-B.t
t/Test-consistency06.data
t/Test-consistency06.t
t/Test-delegation.data
t/Test-delegation.t
t/Test-delegation01-A.data
Expand Down Expand Up @@ -271,19 +271,21 @@ t/Test-syntax06-K.data
t/Test-syntax06-K.t
t/Test-syntax06-L.data
t/Test-syntax06-L.t
t/TestUtil.pm
t/Test-zone.data
t/Test-zone.t
t/Test-zone01-A.t
t/Test-zone01-A.data
t/Test-zone01-B.t
t/Test-zone01-A.t
t/Test-zone01-B.data
t/Test-zone01-B.t
t/Test-zone09-1.data
t/Test-zone09-1.t
t/Test-zone09.data
t/Test-zone09.t
t/Test-zone11.data
t/Test-zone11.t
t/Test.data
t/Test.t
t/TestUtil.pm
t/translator.t
t/undelegated.data
t/undelegated.t
Expand Down
7 changes: 7 additions & 0 deletions lib/Zonemaster/Engine/Util.pm
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ BEGIN {
should_run_test
scramble_case
test_levels
zone
];
our %EXPORT_TAGS = ( all => \@EXPORT_OK );

Expand All @@ -44,6 +45,12 @@ sub info {
return Zonemaster::Engine->logger->add( $tag, $argref );
}

sub zone {
my ( $name ) = @_;

return Zonemaster::Engine::Zone->new( { name => Zonemaster::Engine::DNSName->new( $name ) } );
}

sub should_run_test {
my ( $test_name ) = @_;
my %test_names = map { $_ => 1 } @{ Zonemaster::Engine::Profile->effective->get( q{test_cases} ) };
Expand Down
232 changes: 232 additions & 0 deletions t/Test.data

Large diffs are not rendered by default.

72 changes: 72 additions & 0 deletions t/Test.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!perl
use v5.26.3;
use warnings;
use Test::More tests => 2;
use Test::NoWarnings;

use Zonemaster::Engine;
use Zonemaster::Engine::Nameserver;
use Zonemaster::Engine::Profile;
use Zonemaster::Engine::Test;
use Zonemaster::Engine::Util qw( zone );
use Test::Differences;
use Test::Exception;

my $datafile = q{t/Test.data};
if ( not $ENV{ZONEMASTER_RECORD} ) {
die "Stored data file missing" if not -r $datafile;
Zonemaster::Engine::Nameserver->restore( $datafile );
Zonemaster::Engine::Profile->effective->set( q{no_network}, 1 );
}

subtest 'Everything but Test::NoWarnings' => sub {
subtest 'Enabling single test cases in profile' => sub {
my %all_methods = Zonemaster::Engine->all_methods;
my @testcases =
map { fc $_ }
map { @{ $all_methods{$_} } } Zonemaster::Engine->modules;
my $zone = zone( 'se' );

my $effective_profile = Zonemaster::Engine::Profile->effective;
$effective_profile->merge( Zonemaster::Engine::Profile->default );
$effective_profile->set( 'net.ipv6', 0 );

for my $testcase ( @testcases ) {
subtest "Testcase '$testcase'" => sub {
$effective_profile->set( 'test_cases', [$testcase] );

my @logentries;
lives_ok { @logentries = Zonemaster::Engine::Test->run_all_for( $zone ) }
"Test suite does not crash when running only this test case";

my $actual = {
'A) started testcases' => [
grep { $_ ne 'unspecified' }
map { fc $_->args->{testcase} }
grep { $_->tag eq 'TEST_CASE_START' } @logentries
],
'B) ended testcases' => [
grep { $_ ne 'unspecified' }
map { fc $_->args->{testcase} }
grep { $_->tag eq 'TEST_CASE_END' } @logentries
],
'C) internal errors' => [
map { $_->args->{msg} =~ s/\n$//r }
grep { $_->tag eq 'MODULE_ERROR' } @logentries
],
};
my $expected = {
'A) started testcases' => [$testcase],
'B) ended testcases' => [$testcase],
'C) internal errors' => [],
};
eq_or_diff $actual, $expected,
"Test suite emits expected messages from this testcase and no others";
};
} ## end for my $testcase ( @testcases)
};
};

if ( $ENV{ZONEMASTER_RECORD} ) {
Zonemaster::Engine::Nameserver->save( $datafile );
}

0 comments on commit 71278c4

Please sign in to comment.