Add CLI-level integration tests for common commands (#211)

* Add integration tests for #178. * Use only paths, not URLs Adjust method to load files PR per @david-waltermire-nist's rec. * Parameterize tests and fix bad profiles for #178. * Add junit-jupiter-params to POM test scope Per discussion with Dave and re-reviewing the tutorial section on steps for installation I had missed, add that dependency to the POM for #178. Also, per his recommendation, add it to the test scope and do not set an explicit version number because the core JUnit5 version is handled by the support POM for oss-maven project shared across the different Java projects.
usnistgov · Nov 30, 2023 · 4b6b205 · 4b6b205
1 parent a5cd2e7
commit 4b6b205
Show file tree

Hide file tree

Showing 44 changed files with 1,761 additions and 8 deletions.
diff --git a/pom.xml b/pom.xml
@@ -236,6 +236,11 @@
 			<artifactId>junit-jupiter-engine</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter-params</artifactId>
+			<scope>test</scope>
+		</dependency>		
 	</dependencies>
 	<build>
 		<resources>

diff --git a/src/test/java/gov/nist/secauto/oscal/tools/cli/core/CLITest.java b/src/test/java/gov/nist/secauto/oscal/tools/cli/core/CLITest.java
@@ -30,21 +30,26 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
+import gov.nist.secauto.metaschema.binding.io.Format;
 import gov.nist.secauto.metaschema.cli.processor.ExitCode;
 import gov.nist.secauto.metaschema.cli.processor.ExitStatus;
+import gov.nist.secauto.oscal.lib.profile.resolver.ProfileResolutionException;
 
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
 
 import edu.umd.cs.findbugs.annotations.NonNull;
 
 public class CLITest {
-
-  @Test
-  void testVersionInfo() {
-    String[] args = { "--version" };
-    evaluateResult(CLI.runCli(args), ExitCode.OK);
-  }
-
   void evaluateResult(@NonNull ExitStatus status, @NonNull ExitCode expectedCode) {
     status.generateMessage(true);
     assertAll(
@@ -61,4 +66,67 @@ void evaluateResult(@NonNull ExitStatus status, @NonNull ExitCode expectedCode,
         () -> assertEquals(expectedCode, status.getExitCode(), "exit code mismatch"),
         () -> assertEquals(thrownClass, thrown.getClass(), "expected Throwable mismatch"));
   }
+
+  private static Stream<Arguments> providesValues() {
+    final String[] commands = { "ap", "ar", "catalog", "component-definition", "profile", "poam", "ssp" };
+    final Map<Format, List<Format>> formatEntries = Map.of(
+        Format.XML, Arrays.asList(Format.JSON, Format.YAML),
+        Format.JSON, Arrays.asList(Format.XML, Format.JSON),
+        Format.YAML, Arrays.asList(Format.XML, Format.JSON));
+    List<Arguments> values = new ArrayList<>();
+
+    values.add(Arguments.of(new String[] { "--version" }, ExitCode.OK, null));
+    // TODO: Test all data formats once usnistgov/oscal-cli#216 fix merged.
+    Path path = Paths.get("src/test/resources/cli/example_profile_invalid" + Format.XML.getDefaultExtension());
+    values.add(
+        Arguments.of(new String[] { "profile", "resolve", "--to=" + Format.XML.name().toLowerCase(), path.toString() },
+            ExitCode.PROCESSING_ERROR, ProfileResolutionException.class));
+
+    for (String cmd : commands) {
+      values.add(Arguments.of(new String[] { cmd, "validate", "-h" }, ExitCode.OK, null));
+      // TODO: Update when usnistgov/oscal-cli#210 fix merged.
+      values.add(Arguments.of(new String[] { cmd, "convert", "-h" }, ExitCode.INVALID_COMMAND, null));
+
+      for (Format format : Format.values()) {
+        path = Paths.get("src/test/resources/cli/example_" + cmd + "_invalid" + format.getDefaultExtension());
+        values.add(Arguments.of(new String[] { cmd, "validate", path.toString() }, ExitCode.FAIL, null));
+        path = Paths.get("src/test/resources/cli/example_" + cmd + "_valid" + format.getDefaultExtension());
+        values.add(Arguments.of(new String[] { cmd, "validate", path.toString() }, ExitCode.OK, null));
+        path = Paths.get("src/test/resources/cli/example_profile_valid" + format.getDefaultExtension());
+        List<Format> targetFormats = formatEntries.get(format);
+        for (Format targetFormat : targetFormats) {
+          path = Paths.get("src/test/resources/cli/example_" + cmd + "_valid" + format.getDefaultExtension());
+          String outputPath = path.toString().replace(format.getDefaultExtension(),
+              "_converted" + targetFormat.getDefaultExtension());
+          values.add(Arguments.of(new String[] { cmd, "convert", "--to=" + targetFormat.name().toLowerCase(),
+              path.toString(), outputPath, "--overwrite" }, ExitCode.OK, null));
+          // TODO: Update when usnistgov/oscal#217 fix merged.
+          path = Paths.get("src/test/resources/cli/example_" + cmd + "_invalid" + format.getDefaultExtension());
+          outputPath = path.toString().replace(format.getDefaultExtension(),
+              "_converted" + targetFormat.getDefaultExtension());
+          values.add(Arguments.of(new String[] { cmd, "convert", "--to=" + targetFormat.name().toLowerCase(),
+              path.toString(), outputPath, "--overwrite" }, ExitCode.OK, null));
+        }
+        if (cmd == "profile") {
+          path = Paths.get("src/test/resources/cli/example_profile_valid" + format.getDefaultExtension());
+          values
+              .add(Arguments.of(new String[] { cmd, "resolve", "--to=" + format.name().toLowerCase(), path.toString() },
+                  ExitCode.OK, null));
+        }
+      }
+    }
+
+    return values.stream();
+  }
+
+  @ParameterizedTest
+  @MethodSource("providesValues")
+  void testAllSubCommands(@NonNull String[] args, @NonNull ExitCode expectedExitCode,
+      Class<? extends Throwable> expectedThrownClass) {
+    if (expectedThrownClass == null) {
+      evaluateResult(CLI.runCli(args), expectedExitCode);
+    } else {
+      evaluateResult(CLI.runCli(args), expectedExitCode, expectedThrownClass);
+    }
+  }
 }
diff --git a/src/test/resources/cli/example_ap_invalid.json b/src/test/resources/cli/example_ap_invalid.json
@@ -0,0 +1,39 @@
+{
+  "assessment-plan" : {
+    "uuid" : "bd49e556-187a-458a-9000-0b557f553dae",
+    "metadata" : {
+      "title" : "Invalid OSCAL Document",
+      "last-modified" : "2023-10-24T00:00:00Z",
+      "version" : "1.0",
+      "oscal-version" : "1.1.1",
+      "roles" : [ {
+        "id" : "maintainer",
+        "title" : "Maintainer of oscal-cli"
+      } ],
+      "parties" : [ {
+        "uuid" : "4ba3f2b7-e894-48d7-b940-91c68661df55",
+        "type" : "person",
+        "name" : "NIST ITL CSD Developer"
+      } ],
+      "responsible-parties" : [ {
+        "role-id" : "maintainer",
+        "party-uuids" : [ "4ba3f2b7-e894-48d7-b940-91c68661df55" ]
+      } ]
+    },
+    "import-ssp" : {
+      "href" : "#1db9626b-1ef9-47f9-944f-a71ce8494322"
+    },
+    "back-matter" : {
+      "resources" : [ {
+        "uuid" : "1db9626b-1ef9-47f9-944f-a71ce8494322",
+        "rlinks" : [ {
+          "href" : "example_ssp_invalid.xml"
+        }, {
+          "href" : "example_ssp_invalid.json"
+        }, {
+          "href" : "example_ssp_invalid.yaml"
+        } ]
+      } ]
+    }
+  }
+}
diff --git a/src/test/resources/cli/example_ap_invalid.xml b/src/test/resources/cli/example_ap_invalid.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="https://github.com/usnistgov/OSCAL/releases/download/v1.1.1/oscal_complete_schema.xsd" type="application/xml" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<assessment-plan xmlns="http://csrc.nist.gov/ns/oscal/1.0" uuid="bd49e556-187a-458a-9000-0b557f553dae">
+    <metadata>
+        <title>Invalid OSCAL Document</title>
+        <last-modified>2023-10-24T00:00:00.000000-00:00</last-modified>
+        <version>1.0</version>
+        <oscal-version>1.1.1</oscal-version>
+        <role id="maintainer">
+            <title>Maintainer of oscal-cli</title>
+        </role>
+        <party uuid="4ba3f2b7-e894-48d7-b940-91c68661df55" type="person">
+            <name>NIST ITL CSD Developer</name>
+        </party>
+        <responsible-party role-id="maintainer">
+            <party-uuid>4ba3f2b7-e894-48d7-b940-91c68661df55</party-uuid>
+        </responsible-party>
+    </metadata>
+    <import-ssp href="#1db9626b-1ef9-47f9-944f-a71ce8494322"/>
+    <back-matter>
+        <resource uuid="1db9626b-1ef9-47f9-944f-a71ce8494322">
+            <rlink href="example_ssp_invalid.xml"/>
+            <rlink href="example_ssp_invalid.json"/>
+            <rlink href="example_ssp_invalid.yaml"/>
+        </resource>
+    </back-matter>
+</assessment-plan>
diff --git a/src/test/resources/cli/example_ap_invalid.yml b/src/test/resources/cli/example_ap_invalid.yml
@@ -0,0 +1,28 @@
+---
+assessment-plan:
+  uuid: bd49e556-187a-458a-9000-0b557f553dae
+  metadata:
+    title: Invalid OSCAL Document
+    last-modified: 2023-10-24T00:00:00Z
+    version: "1.0"
+    oscal-version: 1.1.1
+    roles:
+    - id: maintainer
+      title: Maintainer of oscal-cli
+    parties:
+    - uuid: 4ba3f2b7-e894-48d7-b940-91c68661df55
+      type: person
+      name: NIST ITL CSD Developer
+    responsible-parties:
+    - role-id: maintainer
+      party-uuids:
+      - 4ba3f2b7-e894-48d7-b940-91c68661df55
+  import-ssp:
+    href: '#1db9626b-1ef9-47f9-944f-a71ce8494322'
+  back-matter:
+    resources:
+    - uuid: 1db9626b-1ef9-47f9-944f-a71ce8494322
+      rlinks:
+      - href: example_ssp_invalid.xml
+      - href: example_ssp_invalid.json
+      - href: example_ssp_invalid.yaml
diff --git a/src/test/resources/cli/example_ap_valid.json b/src/test/resources/cli/example_ap_valid.json
@@ -0,0 +1,44 @@
+{
+  "assessment-plan" : {
+    "uuid" : "948a30b0-a706-43b0-ac5a-4c64fda1c3d0",
+    "metadata" : {
+      "title" : "Valid OSCAL Document",
+      "last-modified" : "2023-10-24T00:00:00Z",
+      "version" : "1.0",
+      "oscal-version" : "1.1.1",
+      "roles" : [ {
+        "id" : "maintainer",
+        "title" : "Maintainer of oscal-cli"
+      } ],
+      "parties" : [ {
+        "uuid" : "4ba3f2b7-e894-48d7-b940-91c68661df55",
+        "type" : "person",
+        "name" : "NIST ITL CSD Developer"
+      } ],
+      "responsible-parties" : [ {
+        "role-id" : "maintainer",
+        "party-uuids" : [ "4ba3f2b7-e894-48d7-b940-91c68661df55" ]
+      } ]
+    },
+    "import-ssp" : {
+      "href" : "#4a8f42e8-1357-4664-bd88-aac54cb9aeb8"
+    },
+    "reviewed-controls" : {
+      "control-selections" : [ {
+        "include-all" : { }
+      } ]
+    },
+    "back-matter" : {
+      "resources" : [ {
+        "uuid" : "4a8f42e8-1357-4664-bd88-aac54cb9aeb8",
+        "rlinks" : [ {
+          "href" : "example_ssp_valid.xml"
+        }, {
+          "href" : "example_ssp_valid.json"
+        }, {
+          "href" : "example_ssp_valid.yaml"
+        } ]
+      } ]
+    }
+  }
+}
diff --git a/src/test/resources/cli/example_ap_valid.xml b/src/test/resources/cli/example_ap_valid.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="https://github.com/usnistgov/OSCAL/releases/download/v1.1.1/oscal_complete_schema.xsd" type="application/xml" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<assessment-plan xmlns="http://csrc.nist.gov/ns/oscal/1.0" uuid="948a30b0-a706-43b0-ac5a-4c64fda1c3d0">
+    <metadata>
+        <title>Valid OSCAL Document</title>
+        <last-modified>2023-10-24T00:00:00.000000-00:00</last-modified>
+        <version>1.0</version>
+        <oscal-version>1.1.1</oscal-version>
+        <role id="maintainer">
+            <title>Maintainer of oscal-cli</title>
+        </role>
+        <party uuid="4ba3f2b7-e894-48d7-b940-91c68661df55" type="person">
+            <name>NIST ITL CSD Developer</name>
+        </party>
+        <responsible-party role-id="maintainer">
+            <party-uuid>4ba3f2b7-e894-48d7-b940-91c68661df55</party-uuid>
+        </responsible-party>
+    </metadata>
+    <import-ssp href="#4a8f42e8-1357-4664-bd88-aac54cb9aeb8"/>
+    <reviewed-controls>
+        <control-selection>
+            <include-all />
+        </control-selection>
+    </reviewed-controls>
+    <back-matter>
+        <resource uuid="4a8f42e8-1357-4664-bd88-aac54cb9aeb8">
+            <rlink href="example_ssp_valid.xml"/>
+            <rlink href="example_ssp_valid.json"/>
+            <rlink href="example_ssp_valid.yaml"/>
+        </resource>
+    </back-matter>
+</assessment-plan>
diff --git a/src/test/resources/cli/example_ap_valid.yml b/src/test/resources/cli/example_ap_valid.yml
@@ -0,0 +1,31 @@
+---
+assessment-plan:
+  uuid: 948a30b0-a706-43b0-ac5a-4c64fda1c3d0
+  metadata:
+    title: Valid OSCAL Document
+    last-modified: 2023-10-24T00:00:00Z
+    version: "1.0"
+    oscal-version: 1.1.1
+    roles:
+    - id: maintainer
+      title: Maintainer of oscal-cli
+    parties:
+    - uuid: 4ba3f2b7-e894-48d7-b940-91c68661df55
+      type: person
+      name: NIST ITL CSD Developer
+    responsible-parties:
+    - role-id: maintainer
+      party-uuids:
+      - 4ba3f2b7-e894-48d7-b940-91c68661df55
+  import-ssp:
+    href: '#4a8f42e8-1357-4664-bd88-aac54cb9aeb8'
+  reviewed-controls:
+    control-selections:
+    - include-all: {}
+  back-matter:
+    resources:
+    - uuid: 4a8f42e8-1357-4664-bd88-aac54cb9aeb8
+      rlinks:
+      - href: example_ssp_valid.xml
+      - href: example_ssp_valid.json
+      - href: example_ssp_valid.yaml
diff --git a/src/test/resources/cli/example_ar_invalid.json b/src/test/resources/cli/example_ar_invalid.json
@@ -0,0 +1,39 @@
+{
+  "assessment-results" : {
+    "uuid" : "33bef05b-dda4-4c4e-aaec-2721acaee3dc",
+    "metadata" : {
+      "title" : "Invalid OSCAL Document",
+      "last-modified" : "2023-10-24T00:00:00Z",
+      "version" : "1.0",
+      "oscal-version" : "1.1.1",
+      "roles" : [ {
+        "id" : "maintainer",
+        "title" : "Maintainer of oscal-cli"
+      } ],
+      "parties" : [ {
+        "uuid" : "4ba3f2b7-e894-48d7-b940-91c68661df55",
+        "type" : "person",
+        "name" : "NIST ITL CSD Developer"
+      } ],
+      "responsible-parties" : [ {
+        "role-id" : "maintainer",
+        "party-uuids" : [ "4ba3f2b7-e894-48d7-b940-91c68661df55" ]
+      } ]
+    },
+    "import-ap" : {
+      "href" : "#f1a6f923-ca07-4d61-8415-40e873aef6ea"
+    },
+    "back-matter" : {
+      "resources" : [ {
+        "uuid" : "f1a6f923-ca07-4d61-8415-40e873aef6ea",
+        "rlinks" : [ {
+          "href" : "example_ap_invalid.xml"
+        }, {
+          "href" : "example_ap_invalid.json"
+        }, {
+          "href" : "example_ap_invalid.yaml"
+        } ]
+      } ]
+    }
+  }
+}
diff --git a/src/test/resources/cli/example_ar_invalid.xml b/src/test/resources/cli/example_ar_invalid.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="https://github.com/usnistgov/OSCAL/releases/download/v1.1.1/oscal_complete_schema.xsd" type="application/xml" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<assessment-results xmlns="http://csrc.nist.gov/ns/oscal/1.0" uuid="33bef05b-dda4-4c4e-aaec-2721acaee3dc">
+    <metadata>
+        <title>Invalid OSCAL Document</title>
+        <last-modified>2023-10-24T00:00:00.000000-00:00</last-modified>
+        <version>1.0</version>
+        <oscal-version>1.1.1</oscal-version>
+        <role id="maintainer">
+            <title>Maintainer of oscal-cli</title>
+        </role>
+        <party uuid="4ba3f2b7-e894-48d7-b940-91c68661df55" type="person">
+            <name>NIST ITL CSD Developer</name>
+        </party>
+        <responsible-party role-id="maintainer">
+            <party-uuid>4ba3f2b7-e894-48d7-b940-91c68661df55</party-uuid>
+        </responsible-party>
+    </metadata>
+    <import-ap href="#f1a6f923-ca07-4d61-8415-40e873aef6ea"/>
+    <back-matter>
+        <resource uuid="f1a6f923-ca07-4d61-8415-40e873aef6ea">
+            <rlink href="example_ap_invalid.xml"/>
+            <rlink href="example_ap_invalid.json"/>
+            <rlink href="example_ap_invalid.yaml"/>
+        </resource>
+    </back-matter>    
+</assessment-results>