Skip to content

Commit

Permalink
Add ability to retry failed Scenarios (#223, #212)
Browse files Browse the repository at this point in the history
- add `--retry`, `--retry-after` and `--retry-tag-filter` CLI options
- describe retrying in a separate Book chapter
- add `writer::Stats::retried_steps()` method
- wrap `event::Scenario` into `event::RetryableScenario` for storing in other `event`s

Co-authored-by: Kai Ren <tyranron@gmail.com>
  • Loading branch information
ilslv and tyranron authored Sep 8, 2022
1 parent 3dc3236 commit c1d919a
Show file tree
Hide file tree
Showing 59 changed files with 3,115 additions and 563 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ All user visible changes to `cucumber` crate will be documented in this file. Th
- Reworked `writer::Failure`/`writer::discard::Failure` as `writer::Stats`/`writer::discard::Stats`. ([#220])
- Renamed `WriterExt::discard_failure_writes()` to `WriterExt::discard_stats_writes()`. ([#220])
- Added `Option<step::Location>` field to `event::Step::Passed` and `event::Step::Failed`. ([#221])
- Wrapped `event::Scenario` into `event::RetryableScenario` for storing in other `event`s. ([#223], [#212])
- Added `retried_steps()` method to `writer::Stats`. ([#223], [#212])

### Added

Expand All @@ -28,18 +30,22 @@ All user visible changes to `cucumber` crate will be documented in this file. Th
- `writer::Stats::passed_steps()` and `writer::Stats::skipped_steps()` methods. ([#220])
- `FeatureExt::count_steps()` method. ([#220])
- Location of the `fn` matching a failed `Step` in output. ([#221])
- Ability to retry failed `Scenario`s. ([#223], [#212])
- `--retry`, `--retry-after` and `--retry-tag-filter` CLI options. ([#223], [#212])

### Changed

- Provided default CLI options are now global (allowed to be specified after custom subcommands). ([#216], [#215])
- Stripped `CARGO_MANIFEST_DIR` from output paths whenever is possible. ([#221])

[#212]: /../../issues/212
[#215]: /../../issues/215
[#216]: /../../pull/216
[#217]: /../../issues/217
[#219]: /../../pull/219
[#220]: /../../pull/220
[#221]: /../../pull/221
[#223]: /../../pull/223
[8ad5cc86]: /../../commit/8ad5cc866bb9d6b49470790e3b0dd40690f63a09
[cf055ac0]: /../../commit/cf055ac06c7b72f572882ce15d6a60da92ad60a0
[fbd08ec2]: /../../commit/fbd08ec24dbd036c89f5f0af4d936b616790a166
Expand Down
9 changes: 8 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,12 @@ atty = "0.2.14"
clap = { version = "3.0", features = ["derive"] }
console = "0.15"
derive_more = { version = "0.99.17", features = ["as_ref", "deref", "deref_mut", "display", "error", "from", "into"], default_features = false }
drain_filter_polyfill = "0.1.2"
either = "1.6"
futures = "0.3.17"
gherkin = "0.12"
globwalk = "0.8.1"
humantime = "2.1"
itertools = "0.10"
linked-hash-map = "0.5.3"
once_cell = "1.8"
Expand All @@ -71,8 +73,9 @@ junit-report = { version = "0.7", optional = true }
[dev-dependencies]
derive_more = "0.99.17"
humantime = "2.1"
once_cell = "1.13"
tempfile = "3.2"
tokio = { version = "1.12", features = ["macros", "rt-multi-thread", "time"] }
tokio = { version = "1.12", features = ["macros", "rt-multi-thread", "sync", "time"] }

[[test]]
name = "after_hook"
Expand All @@ -93,6 +96,10 @@ name = "libtest"
required-features = ["libtest"]
harness = false

[[test]]
name = "retry"
harness = false

[[test]]
name = "wait"
required-features = ["libtest"]
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ book.tests: test.book
# https://docs.docker.com/get-docker
#
# Usage:
# make record [name=(<current-datetime>|<file-name>)]
# make record.gif [name=(<current-datetime>|<file-name>)]

record-gif-dir := book/src/rec
record-gif-name := $(or $(name),$(shell date +%y"-"%m"-"%d"_"%H"-"%M"-"%S))
Expand Down
1 change: 1 addition & 0 deletions book/src/SUMMARY.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
- [Scenario hooks](writing/hooks.md)
- [Spoken languages](writing/languages.md)
- [Tags](writing/tags.md)
- [Retrying failed scenarios](writing/retries.md)
- [Modules organization](writing/modules.md)
- [CLI (command-line interface)](cli.md)
- [Output](output/index.md)
Expand Down
5 changes: 4 additions & 1 deletion book/src/architecture/runner.md
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,10 @@ impl CustomRunner {
]
})))
.chain(stream::once(future::ready(event::Scenario::Finished)))
.map(move |ev| event::Feature::Scenario(scenario.clone(), ev))
.map(move |event| event::Feature::Scenario(
scenario.clone(),
event::RetryableScenario { event, retries: None },
))
}

fn execute_feature(
Expand Down
9 changes: 6 additions & 3 deletions book/src/architecture/writer.md
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,10 @@ Finally, let's implement a custom [`Writer`] which simply outputs [cucumber even
# ]
# })))
# .chain(stream::once(future::ready(event::Scenario::Finished)))
# .map(move |ev| event::Feature::Scenario(scenario.clone(), ev))
# .map(move |event| event::Feature::Scenario(
# scenario.clone(),
# event::RetryableScenario { event, retries: None },
# ))
# }
#
# fn execute_feature(
Expand Down Expand Up @@ -250,7 +253,7 @@ impl<W: 'static> cucumber::Writer<W> for CustomWriter {
event::Feature::Started => {
println!("{}: {}", feature.keyword, feature.name)
}
event::Feature::Scenario(scenario, ev) => match ev {
event::Feature::Scenario(scenario, ev) => match ev.event {
event::Scenario::Started => {
println!("{}: {}", scenario.keyword, scenario.name)
}
Expand Down Expand Up @@ -419,7 +422,7 @@ async fn main() {
# event::Feature::Started => {
# println!("{}: {}", feature.keyword, feature.name)
# }
# event::Feature::Scenario(scenario, ev) => match ev {
# event::Feature::Scenario(scenario, ev) => match ev.event {
# event::Scenario::Started => {
# println!("{}: {}", scenario.keyword, scenario.name)
# }
Expand Down
19 changes: 18 additions & 1 deletion book/src/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ OPTIONS:
Coloring policy for a console output
[default: auto]
--fail-fast
Run tests until the first failure
Expand All @@ -41,6 +41,23 @@ OPTIONS:
[aliases: scenario-name]
--retry <int>
Number of times a scenario will be retried in case of a failure
--retry-after <duration>
Delay between each scenario retry attempt.
Duration is represented in a human-readable format like `12min5s`.
Supported suffixes:
- `nsec`, `ns` — nanoseconds.
- `usec`, `us` — microseconds.
- `msec`, `ms` — milliseconds.
- `seconds`, `second`, `sec`, `s` - seconds.
- `minutes`, `minute`, `min`, `m` - minutes.
--retry-tag-filter <tagexpr>
Tag expression to filter retried scenarios
-t, --tags <tagexpr>
Tag expression to filter scenarios by.
Expand Down
Binary file added book/src/rec/writing_retries.gif
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 2 additions & 1 deletion book/src/writing/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ Also, it's worth to become familiar with [Gherkin language][1].
8. [Scenario hooks](hooks.md)
9. [Spoken languages](languages.md)
10. [Tags](tags.md)
11. [Modules organization](modules.md)
11. [Retrying failed scenarios](retries.md)
12. [Modules organization](modules.md)



Expand Down
148 changes: 148 additions & 0 deletions book/src/writing/retries.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
Retrying failed scenarios
=========================

Often, it's nearly impossible to create fully-deterministic test case, especially when you are relying on environments like external services, browsers, file system, networking etc. That's why there is an ability to retry failed [scenario]s.

> __WARNING__: Although this feature is supported, we highly recommend to use it as the _last resort only_. First, consider implementing in-[step] retries with your own needs (like [exponential backoff]). Other ways of dealing with flaky tests include, but not limited to: reducing number of concurrently executed scenarios (maybe even using `@serial` [tag]), mocking external environment, [controlling time in tests] or even [simulation testing]. It's always better to move towards tests determinism, rather than trying to tame their flakiness.



## Tags

Recommended way to specify retried [scenario]s is using [tags][tag] ([inheritance] is supported too):
```gherkin
Feature: Heads and tails
# Attempts a single retry immediately.
@retry
Scenario: Tails
Given a coin
When I flip the coin
Then I see tails
# Attempts a single retry in 1 second.
@retry.after(1s)
Scenario: Heads
Given a coin
When I flip the coin
Then I see heads
# Attempts to retry 5 times with no delay between them.
@retry(5)
Scenario: Edge
Given a coin
When I flip the coin
Then I see edge
# Attempts to retry 10 times with 100 milliseconds delay between them.
@retry(10).after(100ms)
Scenario: Levitating
Given a coin
When I flip the coin
Then the coin never lands
```
```rust,should_panic
# use std::time::Duration;
#
# use cucumber::{given, then, when, World};
# use rand::Rng as _;
# use tokio::time::sleep;
#
# #[derive(Debug, Default, World)]
# pub struct FlipWorld {
# flipped: &'static str,
# }
#
#[given("a coin")]
async fn coin(_: &mut FlipWorld) {
sleep(Duration::from_secs(2)).await;
}
#[when("I flip the coin")]
async fn flip(world: &mut FlipWorld) {
sleep(Duration::from_secs(2)).await;
world.flipped = match rand::thread_rng().gen_range(0.0..1.0) {
p if p < 0.2 => "edge",
p if p < 0.5 => "heads",
_ => "tails",
}
}
#[then(regex = r#"^I see (heads|tails|edge)$"#)]
async fn see(world: &mut FlipWorld, what: String) {
sleep(Duration::from_secs(2)).await;
assert_eq!(what, world.flipped);
}
#[then("the coin never lands")]
async fn never_lands(_: &mut FlipWorld) {
sleep(Duration::from_secs(2)).await;
unreachable!("coin always lands")
}
#
# #[tokio::main]
# async fn main() {
# FlipWorld::cucumber()
# .fail_on_skipped()
# .run_and_exit("tests/features/book/writing/retries.feature")
# .await;
# }
```
![record](../rec/writing_retries.gif)

> __NOTE__: On failure, the whole [scenario] is re-executed with a new fresh [`World`] instance.



## CLI

The following [CLI option]s are related to the [scenario] retries:
```
--retry <int>
Number of times a scenario will be retried in case of a failure
--retry-after <duration>
Delay between each scenario retry attempt.
Duration is represented in a human-readable format like `12min5s`.
Supported suffixes:
- `nsec`, `ns` — nanoseconds.
- `usec`, `us` — microseconds.
- `msec`, `ms` — milliseconds.
- `seconds`, `second`, `sec`, `s` - seconds.
- `minutes`, `minute`, `min`, `m` - minutes.
--retry-tag-filter <tagexpr>
Tag expression to filter retried scenarios
```

- `--retry` [CLI option] is similar to `@retry(<number-of-retries>)` [tag], but is applied to all [scenario]s matching the `--retry-tag-filter` (if not provided, all possible [scenario]s are matched).
- `--retry-after` [CLI option] is similar to `@retry.after(<delay-after-each-retry>)` [tag] in the same manner.


### Precedence of tags and CLI options

- Just `@retry` [tag] takes the number of retries and the delay from `--retry` and `--retry-after` [CLI option]s respectively, if they're specified, otherwise defaults to a single retry attempt with no delay.
- `@retry(3)` [tag] always retries failed [scenario]s at most 3 times, even if `--retry` [CLI option] provides a greater value. Delay is taken from `--retry-after` [CLI option], if it's specified, otherwise defaults to no delay.
- `@retry.after(1s)` [tag] always delays 1 second before next retry attempt, even if `--retry-after` [CLI option] provides another value. Number of retries is taken from `--retry-after` [CLI option], if it's specified, otherwise defaults a single retry attempt.
- `@retry(3).after(1s)` always retries failed scenarios at most 3 times with 1 second delay before each attempt, ignoring `--retry` and `--retry-after` [CLI option]s.

> __TIP__: It could be handy to specify `@retry` [tags][tag] only, without any explicit values, and use `--retry=n --retry-after=d --retry-tag-filter=@retry` [CLI option]s to overwrite retrying parameters without affecting any other [scenario]s.



[`World`]: https://docs.rs/cucumber/latest/cucumber/trait.World.html
[CLI option]: ../cli.md
[controlling time in tests]: https://docs.rs/tokio/1.0/tokio/time/fn.pause.html
[exponential backoff]: https://en.wikipedia.org/wiki/Exponential_backoff
[inheritance]: tags.md#inheritance
[scenario]: https://cucumber.io/docs/gherkin/reference#example
[simulation testing]: https://github.com/madsys-dev/madsim
[step]: https://cucumber.io/docs/gherkin/reference#steps
[tag]: https://cucumber.io/docs/cucumber/api#tags
1 change: 1 addition & 0 deletions book/tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ cucumber = { path = "../..", features = ["libtest", "output-json", "output-junit
futures = "0.3"
humantime = "2.1"
once_cell = "1.8"
rand = "0.8"
skeptic = "0.13.7"
tokio = { version = "1.12", features = ["macros", "rt-multi-thread", "time"] }

Expand Down
10 changes: 7 additions & 3 deletions src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ where
#[clap(
short = 'n',
long = "name",
name = "regex",
value_name = "regex",
visible_alias = "scenario-name",
global = true
)]
Expand All @@ -108,8 +108,8 @@ where
#[clap(
short = 't',
long = "tags",
name = "tagexpr",
conflicts_with = "regex",
value_name = "tagexpr",
conflicts_with = "re-filter",
global = true
)]
pub tags_filter: Option<TagOperation>,
Expand Down Expand Up @@ -245,6 +245,10 @@ impl Colored for Empty {}
/// self.0.failed_steps()
/// }
///
/// fn retried_steps(&self) -> usize {
/// self.0.retried_steps()
/// }
///
/// fn parsing_errors(&self) -> usize {
/// self.0.parsing_errors()
/// }
Expand Down
Loading

0 comments on commit c1d919a

Please sign in to comment.