Skip to content

Commit

Permalink
Added RepeatAttribute that is aware of RandomizedContext (#1125)
Browse files Browse the repository at this point in the history
* Lucene.Net.Util.LuceneTestCase.LuceneDelegatingTestCommand::Execute(): Only create an instance of TestResult if it is not already instantiated.

* Lucene.Net.Util.LuceneTestCase: Added RepeatAttribute that is RandomizedContext aware. Added tests from NUnit to verify it passes RepeatAttribute tests.

* SWEEP: Added URLs of source code from randomizedtesting and NUnit to files that were copied from those libraries

* Lucene.Net.TestData.RepeatingTests.RepeatingTestsFixtureBase: Added asserts to ensure the random seed and test seed in RandomizedContext are being updated consistently and correctly

* Lucene.Net.Util.LuceneTestCase.RepeatAttribute: Added more robust subclass checking and a test to ensure that we receive the correct result

* publish-test-results-for-test-projects.yml: Added Lucene.Net.Tests.TestFramework.NUnitExtensions

* .github/workflows: Generated Lucene-Net-Tests-TestFramework-NUnitExtensions workflow file

* LICENSE.txt: Added attribution of new test projects for NUnit extensions

* Lucene.Net.Util.LuceneRandomSeedInitializer: Removed TODO

* Lucene.Net.Util.LuceneTestCase.RepeatAttribute: Fixed method name of SetResultErrorNonLuceneTestCaseSubclass
  • Loading branch information
NightOwl888 authored Jan 27, 2025
1 parent b1d309f commit fa630da
Show file tree
Hide file tree
Showing 24 changed files with 1,501 additions and 35 deletions.
10 changes: 10 additions & 0 deletions .build/azure-templates/publish-test-results-for-test-projects.yml
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,16 @@ steps:
testResultsArtifactName: '${{ parameters.testResultsArtifactName }}'
testResultsFileName: '${{ parameters.testResultsFileName }}'

- template: publish-test-results.yml
parameters:
testProjectName: 'Lucene.Net.Tests.TestFramework.NUnitExtensions'
osName: '${{ parameters.osName }}'
framework: '${{ parameters.framework }}'
vsTestPlatform: '${{ parameters.vsTestPlatform }}'
testResultsFormat: '${{ parameters.testResultsFormat }}'
testResultsArtifactName: '${{ parameters.testResultsArtifactName }}'
testResultsFileName: '${{ parameters.testResultsFileName }}'

- template: publish-test-results.yml
parameters:
testProjectName: 'Lucene.Net.Tests.TestFramework'
Expand Down
123 changes: 123 additions & 0 deletions .github/workflows/Lucene-Net-Tests-TestFramework-NUnitExtensions.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
####################################################################################
# DO NOT EDIT: This file was automatically generated by Generate-TestWorkflows.ps1
####################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: 'Lucene.Net.Tests.TestFramework.NUnitExtensions'

on:
workflow_dispatch:
pull_request:
paths:
- 'src/Lucene.Net.Tests.TestFramework.NUnitExtensions/**/*'
- '.build/dependencies.props'
- '.build/TestReferences.Common.*'
- 'TestTargetFrameworks.*'
- '.github/**/*.yml'
- '*.sln'
- 'src/Lucene.Net.Tests.TestFramework.NUnitExtensions/Directory.Build.*'
- 'src/Directory.Build.*'
- 'Directory.Build.*'

# Dependencies
- 'src/Lucene.Net/**/*'
- 'src/Lucene.Net.Analysis.Common/**/*'
- 'src/Lucene.Net.Codecs/**/*'
- 'src/Lucene.Net.TestFramework/**/*'
- 'src/dotnet/Lucene.Net.TestFramework.TestData.NUnit/**/*'

- '!**/*.md'

jobs:

Test:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [windows-latest, ubuntu-latest]
framework: [net9.0, net8.0, net6.0, net48, net472]
platform: [x64]
configuration: [Release]
exclude:
- os: ubuntu-latest
framework: net48
- os: ubuntu-latest
framework: net472
- os: macos-latest
framework: net48
- os: macos-latest
framework: net472
env:
project_path: './src/Lucene.Net.Tests.TestFramework.NUnitExtensions/Lucene.Net.Tests.TestFramework.NUnitExtensions.csproj'
run_slow_tests: 'false'
trx_file_name: 'TestResults.trx'
md_file_name: 'TestResults.md' # Report file name for LiquidTestReports.Markdown

steps:
- name: Checkout Source Code
uses: actions/checkout@v3

- name: Disable .NET SDK Telemetry and Logo
run: |
echo "DOTNET_NOLOGO=1" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
echo "DOTNET_CLI_TELEMETRY_OPTOUT=1" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
shell: pwsh

- name: Setup .NET 6 SDK
uses: actions/setup-dotnet@v3
with:
dotnet-version: '6.0.x'
if: ${{ startswith(matrix.framework, 'net6.') }}

- name: Setup .NET 8 SDK
uses: actions/setup-dotnet@v3
with:
dotnet-version: '8.0.x'

- name: Setup .NET 9 SDK
uses: actions/setup-dotnet@v3
with:
dotnet-version: '9.0.x'

- name: Setup Environment Variables
run: |
$project_name = [System.IO.Path]::GetFileNameWithoutExtension($env:project_path)
$test_results_artifact_name = "testresults_${{matrix.os}}_${{matrix.framework}}_${{matrix.platform}}_${{matrix.configuration}}"
$working_directory = "$env:GITHUB_WORKSPACE"
Write-Host "Project Name: $project_name"
Write-Host "Results Artifact Name: $test_results_artifact_name"
Write-Host "Working Directory: $working_directory"
echo "project_name=$project_name" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
echo "test_results_artifact_name=$test_results_artifact_name" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
# Set the Azure DevOps default working directory env variable, so our tests only need to deal with a single env variable
echo "SYSTEM_DEFAULTWORKINGDIRECTORY=$working_directory" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
# Title for LiquidTestReports.Markdown
echo "title=Test Run for $project_name - ${{matrix.framework}} - ${{matrix.platform}} - ${{matrix.os}}" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
shell: pwsh
- run: dotnet build ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} /p:TestFrameworks=true
- run: dotnet test ${{env.project_path}} --configuration ${{matrix.configuration}} --framework ${{matrix.framework}} --no-build --no-restore --blame-hang --blame-hang-dump-type mini --blame-hang-timeout 20minutes --logger:"console;verbosity=normal" --logger:"trx;LogFileName=${{env.trx_file_name}}" --logger:"liquid.md;LogFileName=${{env.md_file_name}};Title=${{env.title}};" --results-directory:"${{github.workspace}}/${{env.test_results_artifact_name}}/${{env.project_name}}" -- RunConfiguration.TargetPlatform=${{matrix.platform}} NUnit.DisplayName=FullName TestRunParameters.Parameter\(name=\"tests:slow\",\ value=\"\${{env.run_slow_tests}}\"\)
shell: bash
# upload reports as build artifacts
- name: Upload a Build Artifact
uses: actions/upload-artifact@v4
if: ${{always()}}
with:
name: '${{env.test_results_artifact_name}}'
path: '${{github.workspace}}/${{env.test_results_artifact_name}}'

4 changes: 3 additions & 1 deletion LICENSE.txt
Original file line number Diff line number Diff line change
Expand Up @@ -735,6 +735,8 @@ Some code in
src/Lucene.Net.TestFramework/Support/Util/DefaultNamespaceTypeWrapper.cs
src/Lucene.Net.TestFramework/Support/Util/LuceneTestCase.TestFixtureAttribute.cs
src/Lucene.Net.TestFramework/Support/Util/NUnitTestFixtureBuilder.cs
src/dotnet/Lucene.Net.TestFramework.TestData.NUnit/*
src/Lucene.Net.Tests.TestFramework.NUnitExtensions/*
falls under the following license:

// Copyright (c) 2021 Charlie Poole, Rob Prouse
Expand Down Expand Up @@ -798,4 +800,4 @@ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
13 changes: 13 additions & 0 deletions Lucene.Net.sln
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,10 @@ Project("{13B669BE-BB05-4DDF-9536-439F39A36129}") = "build", "proj\build.msbuild
EndProject
Project("{13B669BE-BB05-4DDF-9536-439F39A36129}") = "github", "proj\github.msbuildproj", "{E71152A0-48CC-4334-981F-F5FBFFA50891}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.TestFramework.TestData.NUnit", "src\dotnet\Lucene.Net.TestFramework.TestData.NUnit\Lucene.Net.TestFramework.TestData.NUnit.csproj", "{6E131ACF-D27A-45AF-923B-A1932EB11E2D}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Tests.TestFramework.NUnitExtensions", "src\Lucene.Net.Tests.TestFramework.NUnitExtensions\Lucene.Net.Tests.TestFramework.NUnitExtensions.csproj", "{724CFF92-5ED6-4F7D-9D2C-53ABA0589F37}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -488,6 +492,14 @@ Global
{E71152A0-48CC-4334-981F-F5FBFFA50891}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E71152A0-48CC-4334-981F-F5FBFFA50891}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E71152A0-48CC-4334-981F-F5FBFFA50891}.Release|Any CPU.Build.0 = Release|Any CPU
{6E131ACF-D27A-45AF-923B-A1932EB11E2D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6E131ACF-D27A-45AF-923B-A1932EB11E2D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6E131ACF-D27A-45AF-923B-A1932EB11E2D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6E131ACF-D27A-45AF-923B-A1932EB11E2D}.Release|Any CPU.Build.0 = Release|Any CPU
{724CFF92-5ED6-4F7D-9D2C-53ABA0589F37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{724CFF92-5ED6-4F7D-9D2C-53ABA0589F37}.Debug|Any CPU.Build.0 = Debug|Any CPU
{724CFF92-5ED6-4F7D-9D2C-53ABA0589F37}.Release|Any CPU.ActiveCfg = Release|Any CPU
{724CFF92-5ED6-4F7D-9D2C-53ABA0589F37}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -506,6 +518,7 @@ Global
{4D0ED7D9-ABEE-4890-B06C-477E3A32B9A0} = {E5E8C5DC-7048-4818-B884-FB2D037D2EF2}
{FED4A824-1F32-4948-8D37-2B7610804DB5} = {42599646-275F-4970-BC60-A3349F6498CC}
{C0448DD3-68D2-485F-B31A-D2806E589FA7} = {42599646-275F-4970-BC60-A3349F6498CC}
{6E131ACF-D27A-45AF-923B-A1932EB11E2D} = {8CA61D33-3590-4024-A304-7B1F75B50653}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {9F2179CC-CFD2-4419-AB74-D72856931F36}
Expand Down
2 changes: 2 additions & 0 deletions src/Lucene.Net.TestFramework/Lucene.Net.TestFramework.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,8 @@
<InternalsVisibleTo Include="Lucene.Net.Tests.Suggest" />
<InternalsVisibleTo Include="Lucene.Net.Tests.TestFramework" />
<InternalsVisibleTo Include="Lucene.Net.Tests.TestFramework.DependencyInjection" />

<InternalsVisibleTo Include="Lucene.Net.TestFramework.TestData.NUnit" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using NUnit.Framework.Interfaces;
// Based on: https://github.com/nunit/nunit/blob/v3.14.0/src/NUnitFramework/framework/Internal/TypeWrapper.cs

using NUnit.Framework.Interfaces;
using NUnit.Framework.Internal;
using System;
using System.Linq;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using System;
// Source: https://github.com/randomizedtesting/randomizedtesting/blob/release/2.7.8/randomized-runner/src/main/java/com/carrotsearch/randomizedtesting/CloseableResourceInfo.java

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
Expand Down
4 changes: 3 additions & 1 deletion src/Lucene.Net.TestFramework/Support/Util/LifecycleScope.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using System;
// Source: https://github.com/randomizedtesting/randomizedtesting/blob/release/2.7.8/randomized-runner/src/main/java/com/carrotsearch/randomizedtesting/LifecycleScope.java

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ internal class LuceneRandomSeedInitializer
#region Messages

const string RANDOM_SEED_PARAMS_MSG =
"\"tests:seed\" parameter must be a valid long hexadecimal value or the word \"random\".";
"\"tests:seed\" parameter must be one or two valid long hexadecimal values separated by a \":\" or the word \"random\".";

#endregion

Expand All @@ -47,7 +47,6 @@ internal class LuceneRandomSeedInitializer
/// <returns><c>true</c> if the seed was found in context; <c>false</c> if the seed was generated.</returns>
private static bool TryGetRandomSeedsFromContext(Test test, out long seed, out long? testSeed)
{
//bool generate;
seed = default;
testSeed = default;
string seedAsString;
Expand All @@ -58,7 +57,6 @@ private static bool TryGetRandomSeedsFromContext(Test test, out long seed, out l
if (randomSeedAttribute != null)
{
seedAsString = randomSeedAttribute.RandomSeed;
//generate = false;
}
else
{
Expand All @@ -68,31 +66,25 @@ private static bool TryGetRandomSeedsFromContext(Test test, out long seed, out l

if (seedAsString is null || "random".Equals(seedAsString, StringComparison.OrdinalIgnoreCase))
{
//generate = true;
return false;
}

int colonIndex = seedAsString.IndexOf(':');
if (colonIndex != -1)
{
if (!J2N.Numerics.Int64.TryParse(seedAsString, 0, colonIndex, radix: 16, out seed))
if (!J2N.Numerics.Int64.TryParse(seedAsString.AsSpan(0, colonIndex), radix: 16, out seed))
{
test.MakeInvalid(RANDOM_SEED_PARAMS_MSG);
return false;
}

// LUCENENET TODO: For now, we are ignoring anything after a colon in the string, but logically it seems like
// a second seed would be to set the a test so the RandomAttribute fails on the first iteration. Lucene uses a compound
// seed, but it isn't clear from analyzing the source how it is used, just that it can contain any number of colon delimited
// values. If we ignore now, we leave the door open for adding a compound seed in the most sensible way later without breaking
// the current version when the change is introduced.
//if (!J2N.Numerics.Int64.TryParse(seedAsString, colonIndex + 1, seedAsString.Length - (colonIndex + 1), radix: 16, out long testSeedValue))
//{
// test.MakeInvalid(RANDOM_SEED_PARAMS_MSG);
// return false;
//}

//testSeed = testSeedValue;
if (!J2N.Numerics.Int64.TryParse(seedAsString.AsSpan(colonIndex + 1, seedAsString.Length - (colonIndex + 1)), radix: 16, out long testSeedValue))
{
test.MakeInvalid(RANDOM_SEED_PARAMS_MSG);
return false;
}

testSeed = testSeedValue;
return true;
}
else if (J2N.Numerics.Int64.TryParse(seedAsString, radix: 16, out seed))
Expand Down
Loading

0 comments on commit fa630da

Please sign in to comment.