[plugin-web-app-to-rest-api] Introduce variable name as HTML source i…

…n `FROM_HTML` transformer (#4652)

docs/modules/plugins/pages/plugin-web-app-to-rest-api.adoc

@@ -250,7 +250,18 @@ Examples:
 |Description
 
 |`pageUrl`
-|The URL of the page to build the table upon.
+a|The URL of the page to build the table upon.
+
+WARNING: The `pageUrl` parameter is deprecated and will be removed in VIVIDUS 0.7.0, please use `variableName` instead.
+
+IMPORTANT: The `pageUrl` parameter can not be used together with the `variableName` parameter.
+
+|`variableName`
+a|The name of the variable containing source HTML, only variables of scopes `global` and `next_batches` are allowed.
+Exceptions are cases when the transformer using in xref:commons:vividus-steps.adoc#_initialize_variable_with_a_table[step]
+which initializes a variable with a table.
+
+IMPORTANT: The `variableName` parameter can not be used together with the `pageUrl` parameter.
 
 |`column`
 |The column name in the generated table.

vividus-plugin-web-app-to-rest-api/src/main/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformer.java

@@ -26,32 +26,60 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Properties;
 import java.util.function.Function;
+import java.util.function.Supplier;
 
+import org.apache.commons.lang3.Validate;
 import org.jbehave.core.model.ExamplesTable.TableProperties;
 import org.jbehave.core.model.TableParsers;
 import org.jbehave.core.model.TableTransformers.TableTransformer;
 import org.jsoup.Connection;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.vividus.context.VariableContext;
 import org.vividus.util.ExamplesTableProcessor;
 
 public class HtmlDocumentTableTransformer implements TableTransformer
 {
+ private static final Logger LOGGER = LoggerFactory.getLogger(HtmlDocumentTableTransformer.class);
+
  private static final String SLASH = "/";
 
  private final Optional<HttpConfiguration> httpConfiguration;
+ private final VariableContext variableContext;
 
- public HtmlDocumentTableTransformer(Optional<HttpConfiguration> httpConfiguration)
+ public HtmlDocumentTableTransformer(Optional<HttpConfiguration> httpConfiguration, VariableContext variableContext)
  {
  this.httpConfiguration = httpConfiguration;
+ this.variableContext = variableContext;
  }
 
  @Override
  public String transform(String table, TableParsers parsers, TableProperties tableProperties)
  {
- String pageUrl = tableProperties.getMandatoryNonBlankProperty("pageUrl", String.class);
+ Properties propertes = tableProperties.getProperties();
+ String pageUrl = propertes.getProperty("pageUrl");
+ String variableName = propertes.getProperty("variableName");
+
+ Validate.isTrue(pageUrl != null && variableName == null || pageUrl == null && variableName != null,
+ "Either 'pageUrl' or 'variableName' should be specified.");
+
+ Supplier<Document> documentSuppler;
+ if (pageUrl == null)
+ {
+ documentSuppler = () -> Jsoup.parse((String) variableContext.getVariable(variableName));
+ }
+ else
+ {
+ LOGGER.atWarn().log("The 'pageUrl' transformer parameter is deprecated and will be removed VIVIDUS 0.7.0, "
+ + "please use 'variableName' parameter instead.");
+ documentSuppler = () -> createDocument(pageUrl);
+ }
+
  String column = tableProperties.getMandatoryNonBlankProperty("column", String.class);
 
  String xpathSelector = tableProperties.getMandatoryNonBlankProperty("xpathSelector", String.class);
@@ -74,12 +102,12 @@ else if (lastSegment.startsWith("@"))
  getter = Element::outerHtml;
  }
 
- return createDocument(pageUrl).selectXpath(xpathSelector)
-  .stream()
-  .map(getter)
-  .collect(collectingAndThen(toList(), attrs -> ExamplesTableProcessor
-  .buildExamplesTableFromColumns(List.of(column), List.of(attrs),
-  tableProperties)));
+ return documentSuppler.get().selectXpath(xpathSelector)
+ .stream()
+ .map(getter)
+ .collect(collectingAndThen(toList(), attrs -> ExamplesTableProcessor
+ .buildExamplesTableFromColumns(List.of(column), List.of(attrs),
+ tableProperties)));
  }
 
  private Document createDocument(String pageUrl)

vividus-plugin-web-app-to-rest-api/src/test/java/org/vividus/crawler/transformer/HtmlDocumentTableTransformerTests.java

@@ -17,6 +17,7 @@
 package org.vividus.crawler.transformer;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.mockStatic;
 import static org.mockito.Mockito.verify;
@@ -34,14 +35,21 @@
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.mockito.Mock;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.vividus.context.VariableContext;
 import org.vividus.crawler.transformer.HtmlDocumentTableTransformer.HttpConfiguration;
 
+@ExtendWith(MockitoExtension.class)
 class HtmlDocumentTableTransformerTests
 {
  private static final String PAGE_URL = "https://example.com";
- private static final Document HTML_DOC = Jsoup.parse("""
+ private static final String DOC = """
  <!DOCTYPE html>
  <html>
  <body>
@@ -50,19 +58,49 @@ class HtmlDocumentTableTransformerTests
  <a href="/b">B</a>
  </body>
  </html>
- """);
+ """;
+ private static final Document HTML_DOC = Jsoup.parse(DOC);
+ private static final String RELS = """
+ |col|
+ |/r|
+ |/g|
+ |/b|""";
+
+ @Mock private VariableContext variableContext;
+
+ @ParameterizedTest
+ @CsvSource(value = {
+ "pageUrl=https://example.com, variableName=html",
+ "column=col"
+ }, delimiterString = "none")
+ void shouldFailOnInvalidInputParameters(String parameters)
+ {
+ TableProperties tableProperties = new TableProperties(parameters, new Keywords(), new ParameterConverters());
+ HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(Optional.empty(), variableContext);
+ IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class,
+ () -> transformer.transform("", null, tableProperties));
+ assertEquals("Either 'pageUrl' or 'variableName' should be specified.", thrown.getMessage());
+ }
+
+ @Test
+ void shouldBuildTableByElementAttributeFromVariableValue() throws IOException
+ {
+ when(variableContext.getVariable("html")).thenReturn(DOC);
+ TableProperties tableProperties = new TableProperties("column=col, variableName=html, xpathSelector=//a/@href",
+ new Keywords(), new ParameterConverters());
+ HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(Optional.empty(),
+ variableContext);
+ String table = transformer.transform("", null, tableProperties);
+ assertEquals(RELS, table);
+ }
 
  @Test
  void shouldBuildTableByElementAttribute() throws IOException
  {
  HttpConfiguration cfg = new HttpConfiguration();
  Map<String, String> headers = Map.of("credit-card-pin", "1234");
  cfg.setHeaders(headers);
- performTest(Optional.of(cfg), "//a/@href", t -> assertEquals("""
- |col|
- |/r|
- |/g|
- |/b|""", t), con -> verify(con).headers(headers));
+ performTest(Optional.of(cfg), "//a/@href", t -> assertEquals(RELS, t), con -> verify(con).headers(headers));
  }
 
  @Test
@@ -98,7 +136,8 @@ private void performTest(Optional<HttpConfiguration> httpConfiguration, String x
  TableProperties tableProperties = new TableProperties(
  "column=col, pageUrl=https://example.com, xpathSelector=%s".formatted(xpathSelector),
  new Keywords(), new ParameterConverters());
- HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(httpConfiguration);
+ HtmlDocumentTableTransformer transformer = new HtmlDocumentTableTransformer(httpConfiguration,
+ variableContext);
  String table = transformer.transform("", null, tableProperties);
  tableConsumer.accept(table);
  verify(connection).get();

vividus-tests/src/main/resources/story/integration/TableTransformers.story

@@ -449,11 +449,22 @@ Then `${documentTable}` is equal to table:
 |Link to unexistent element|
 |Link with tooltip |
 
-Scenario: Verify FROM_HTML transformer with HTML
+Scenario: [Deprecated] Verify FROM_HTML transformer with page HTML
 When I initialize scenario variable `documentTable` with values:
 {transformer=FROM_HTML, column=col, pageUrl=$\{vividus-test-site-url\}/links.html, xpathSelector=//a}
 Then `${documentTable}` is equal to table:
 |col |
 |<a href="#ElementId">Link to an element</a> |
 |<a href="#notFound">Link to unexistent element</a> |
 |<a href="#" title="Link title" onclick="onLinkClick(event)">Link with tooltip</a>|
+
+Scenario: Verify FROM_HTML transformer with variable HTML
+When I execute HTTP GET request for resource with URL `${vividus-test-site-url}/links.html`
+Given I initialize story variable `pageSource` with value `${response}`
+When I initialize scenario variable `documentTable` with values:
+{transformer=FROM_HTML, column=col, variableName=pageSource, xpathSelector=//a}
+Then `${documentTable}` is equal to table:
+|col |
+|<a href="#ElementId">Link to an element</a> |
+|<a href="#notFound">Link to unexistent element</a> |
+|<a href="#" title="Link title" onclick="onLinkClick(event)">Link with tooltip</a>|