diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index e8ec26c0..e9eaedb4 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -13,7 +13,7 @@ jobs:
image: fedora:32
steps:
- name: Install Deps
- run: dnf install -y make anaconda openscap-python3 python3-cpio python3-mock python3-pytest python3-pycurl
+ run: dnf install -y make anaconda openscap-scanner openscap-python3 python3-cpio python3-pytest python3-pycurl
- name: Checkout
uses: actions/checkout@v2
- name: Test
diff --git a/create_update_image.sh b/create_update_image.sh
index fe5a5891..64eb7705 100755
--- a/create_update_image.sh
+++ b/create_update_image.sh
@@ -204,8 +204,8 @@ install_addon_from_repo() {
else
install_po_files="DEFAULT_INSTALL_OF_PO_FILES=yes"
fi
- # "copy files" to new root, sudo needed because we may overwrite files installed by rpm
- sudo make install "$install_po_files" DESTDIR="${tmp_root}" >&2 || die "Failed to install the addon to $tmp_root."
+ # "copy files" to new root, sudo may be needed because we may overwrite files installed by rpm
+ $SUDO make install "$install_po_files" DESTDIR="${tmp_root}" >&2 || die "Failed to install the addon to $tmp_root."
}
@@ -216,12 +216,16 @@ create_image() {
cleanup() {
- # cleanup, sudo needed because former RPM installs
- sudo rm -rf "$tmp_root"
+ # cleanup, sudo may be needed because former RPM installs
+ $SUDO rm -rf "$tmp_root"
}
-
-sudo true || die "Unable to get sudo working, bailing out."
+if test $_arg_start_with_index -gt 1; then
+ SUDO=
+else
+ SUDO=sudo
+ $SUDO true || die "Unable to get sudo working, bailing out."
+fi
for (( action_index=_arg_start_with_index; action_index < ${#actions[*]}; action_index++ )) do
"${actions[$action_index]}"
diff --git a/org_fedora_oscap/common.py b/org_fedora_oscap/common.py
index 27415714..884bbc88 100644
--- a/org_fedora_oscap/common.py
+++ b/org_fedora_oscap/common.py
@@ -307,7 +307,10 @@ def extract_data(archive, out_dir, ensure_has_files=None):
ensure_has_files = []
# get rid of empty file paths
- ensure_has_files = [fpath for fpath in ensure_has_files if fpath]
+ if not ensure_has_files:
+ ensure_has_files = []
+ else:
+ ensure_has_files = [fpath for fpath in ensure_has_files if fpath]
msg = "OSCAP addon: Extracting {archive}".format(archive=archive)
if ensure_has_files:
diff --git a/org_fedora_oscap/content_discovery.py b/org_fedora_oscap/content_discovery.py
new file mode 100644
index 00000000..63930865
--- /dev/null
+++ b/org_fedora_oscap/content_discovery.py
@@ -0,0 +1,346 @@
+import threading
+import logging
+import pathlib
+import shutil
+from glob import glob
+
+from pyanaconda.core import constants
+from pyanaconda.threading import threadMgr
+from pykickstart.errors import KickstartValueError
+
+from org_fedora_oscap import data_fetch, utils
+from org_fedora_oscap import common
+from org_fedora_oscap import content_handling
+
+from org_fedora_oscap.common import _
+
+log = logging.getLogger("anaconda")
+
+
+def is_network(scheme):
+ return any(
+ scheme.startswith(net_prefix)
+ for net_prefix in data_fetch.NET_URL_PREFIXES)
+
+
+class ContentBringer:
+ CONTENT_DOWNLOAD_LOCATION = pathlib.Path(common.INSTALLATION_CONTENT_DIR)
+ DEFAULT_SSG_DATA_STREAM_PATH = f"{common.SSG_DIR}/{common.SSG_CONTENT}"
+
+ def __init__(self, addon_data):
+ self.content_uri_scheme = ""
+ self.content_uri_path = ""
+ self.fetched_content = ""
+
+ self.activity_lock = threading.Lock()
+ self.now_fetching_or_processing = False
+
+ self.CONTENT_DOWNLOAD_LOCATION.mkdir(parents=True, exist_ok=True)
+
+ self._addon_data = addon_data
+
+ def get_content_type(self, url):
+ if url.endswith(".rpm"):
+ return "rpm"
+ elif any(url.endswith(arch_type) for arch_type in common.SUPPORTED_ARCHIVES):
+ return "archive"
+ else:
+ return "file"
+
+ @property
+ def content_uri(self):
+ return self.content_uri_scheme + "://" + self.content_uri_path
+
+ @content_uri.setter
+ def content_uri(self, uri):
+ scheme, path = uri.split("://", 1)
+ self.content_uri_path = path
+ self.content_uri_scheme = scheme
+
+ def fetch_content(self, what_if_fail, ca_certs_path=""):
+ """
+ Initiate fetch of the content into an appropriate directory
+
+ Args:
+ what_if_fail: Callback accepting exception as an argument that
+ should handle them in the calling layer.
+ ca_certs_path: Path to the HTTPS certificate file
+ """
+ self.content_uri = self._addon_data.content_url
+ shutil.rmtree(self.CONTENT_DOWNLOAD_LOCATION, ignore_errors=True)
+ self.CONTENT_DOWNLOAD_LOCATION.mkdir(parents=True, exist_ok=True)
+ fetching_thread_name = self._fetch_files(
+ self.content_uri_scheme, self.content_uri_path,
+ self.CONTENT_DOWNLOAD_LOCATION, ca_certs_path, what_if_fail)
+ return fetching_thread_name
+
+ def _fetch_files(self, scheme, path, destdir, ca_certs_path, what_if_fail):
+ with self.activity_lock:
+ if self.now_fetching_or_processing:
+ msg = "Strange, it seems that we are already fetching something."
+ log.warn(msg)
+ return
+ self.now_fetching_or_processing = True
+
+ fetching_thread_name = None
+ try:
+ fetching_thread_name = self._start_actual_fetch(scheme, path, destdir, ca_certs_path)
+ except Exception as exc:
+ with self.activity_lock:
+ self.now_fetching_or_processing = False
+ what_if_fail(exc)
+
+ # We are not finished yet with the fetch
+ return fetching_thread_name
+
+ def _start_actual_fetch(self, scheme, path, destdir, ca_certs_path):
+ fetching_thread_name = None
+ url = scheme + "://" + path
+
+ if "/" not in path:
+ msg = f"Missing the path component of the '{url}' URL"
+ raise KickstartValueError(msg)
+ basename = path.rsplit("/", 1)[1]
+ if not basename:
+ msg = f"Unable to deduce basename from the '{url}' URL"
+ raise KickstartValueError(msg)
+
+ dest = destdir / basename
+
+ if is_network(scheme):
+ fetching_thread_name = data_fetch.wait_and_fetch_net_data(
+ url,
+ dest,
+ ca_certs_path
+ )
+ else: # invalid schemes are handled down the road
+ fetching_thread_name = data_fetch.fetch_local_data(
+ url,
+ dest,
+ )
+ return fetching_thread_name
+
+ def finish_content_fetch(self, fetching_thread_name, fingerprint, report_callback, dest_filename,
+ what_if_fail):
+ """
+ Finish any ongoing fetch and analyze what has been fetched.
+
+ After the fetch is completed, it analyzes verifies fetched content if applicable,
+ analyzes it and compiles into an instance of ObtainedContent.
+
+ Args:
+ fetching_thread_name: Name of the fetching thread
+ or None if we are only after the analysis
+ fingerprint: A checksum for downloaded file verification
+ report_callback: Means for the method to send user-relevant messages outside
+ dest_filename: The target of the fetch operation. Can be falsy -
+ in this case there is no content filename defined
+ what_if_fail: Callback accepting exception as an argument
+ that should handle them in the calling layer.
+
+ Returns:
+ Instance of ObtainedContent if everything went well, or None.
+ """
+ try:
+ content = self._finish_actual_fetch(fetching_thread_name, fingerprint, report_callback, dest_filename)
+ except Exception as exc:
+ what_if_fail(exc)
+ content = None
+ finally:
+ with self.activity_lock:
+ self.now_fetching_or_processing = False
+
+ return content
+
+ def _verify_fingerprint(self, dest_filename, fingerprint=""):
+ if not fingerprint:
+ return
+
+ hash_obj = utils.get_hashing_algorithm(fingerprint)
+ digest = utils.get_file_fingerprint(dest_filename,
+ hash_obj)
+ if digest != fingerprint:
+ log.error(
+ f"File {dest_filename} failed integrity check - assumed a "
+ f"{hash_obj.name} hash and '{fingerprint}', got '{digest}'"
+ )
+ msg = _(f"Integrity check of the content failed - {hash_obj.name} hash didn't match")
+ raise content_handling.ContentCheckError(msg)
+
+ def _finish_actual_fetch(self, wait_for, fingerprint, report_callback, dest_filename):
+ threadMgr.wait(wait_for)
+ actually_fetched_content = wait_for is not None
+
+ if fingerprint and dest_filename:
+ self._verify_fingerprint(dest_filename, fingerprint)
+
+ fpaths = self._gather_available_files(actually_fetched_content, dest_filename)
+
+ structured_content = ObtainedContent(self.CONTENT_DOWNLOAD_LOCATION)
+ content_type = self.get_content_type(str(dest_filename))
+ if content_type in ("archive", "rpm"):
+ structured_content.add_content_archive(dest_filename)
+
+ labelled_files = content_handling.identify_files(fpaths)
+ for fname, label in labelled_files.items():
+ structured_content.add_file(fname, label)
+
+ if fingerprint and dest_filename:
+ structured_content.record_verification(dest_filename)
+
+ return structured_content
+
+ def _gather_available_files(self, actually_fetched_content, dest_filename):
+ fpaths = []
+ if not actually_fetched_content:
+ if not dest_filename: # using scap-security-guide
+ fpaths = [self.DEFAULT_SSG_DATA_STREAM_PATH]
+ else: # Using downloaded XCCDF/OVAL/DS/tailoring
+ fpaths = glob(str(self.CONTENT_DOWNLOAD_LOCATION / "*.xml"))
+ else:
+ dest_filename = pathlib.Path(dest_filename)
+ # RPM is an archive at this phase
+ content_type = self.get_content_type(str(dest_filename))
+ if content_type in ("archive", "rpm"):
+ try:
+ fpaths = common.extract_data(
+ str(dest_filename),
+ str(dest_filename.parent)
+ )
+ except common.ExtractionError as err:
+ msg = f"Failed to extract the '{dest_filename}' archive: {str(err)}"
+ log.error(msg)
+ raise err
+
+ elif content_type == "file":
+ fpaths = [str(dest_filename)]
+ else:
+ raise common.OSCAPaddonError("Unsupported content type")
+ return fpaths
+
+ def use_downloaded_content(self, content):
+ preferred_content = self.get_preferred_content(content)
+
+ # We know that we have ended up with a datastream-like content,
+ # but if we can't convert an archive to a datastream.
+ # self._addon_data.content_type = "datastream"
+ self._addon_data.content_path = str(preferred_content.relative_to(content.root))
+
+ preferred_tailoring = self.get_preferred_tailoring(content)
+ if content.tailoring:
+ self._addon_data.tailoring_path = str(preferred_tailoring.relative_to(content.root))
+
+ def use_system_content(self, content=None):
+ self._addon_data.clear_all()
+ self._addon_data.content_type = "scap-security-guide"
+ self._addon_data.content_path = common.get_ssg_path()
+
+ def get_preferred_content(self, content):
+ if self._addon_data.content_path:
+ preferred_content = content.find_expected_usable_content(self._addon_data.content_path)
+ else:
+ preferred_content = content.select_main_usable_content()
+ return preferred_content
+
+ def get_preferred_tailoring(self, content):
+ if self._addon_data.tailoring_path:
+ if self._addon_data.tailoring_path != str(content.tailoring.relative_to(content.root)):
+ msg = f"Expected a tailoring {self.tailoring_path}, but it couldn't be found"
+ raise content_handling.ContentHandlingError(msg)
+ return content.tailoring
+
+
+class ObtainedContent:
+ """
+ This class aims to assist the gathered files discovery -
+ the addon can downloaded files directly, or they can be extracted for an archive.
+ The class enables user to quickly understand what is available,
+ and whether the current set of contents is usable for further processing.
+ """
+ def __init__(self, root):
+ self.labelled_files = dict()
+ self.datastream = ""
+ self.xccdf = ""
+ self.ovals = []
+ self.tailoring = ""
+ self.archive = ""
+ self.verified = ""
+ self.root = pathlib.Path(root)
+
+ def record_verification(self, path):
+ """
+ Declare a file as verified (typically by means of a checksum)
+ """
+ path = pathlib.Path(path)
+ assert path in self.labelled_files
+ self.verified = path
+
+ def add_content_archive(self, fname):
+ """
+ If files come from an archive, record this information using this function.
+ """
+ path = pathlib.Path(fname)
+ self.labelled_files[path] = None
+ self.archive = path
+
+ def _assign_content_type(self, attribute_name, new_value):
+ old_value = getattr(self, attribute_name)
+ if old_value:
+ msg = (
+ f"When dealing with {attribute_name}, "
+ f"there was already the {old_value.name} when setting the new {new_value.name}")
+ raise content_handling.ContentHandlingError(msg)
+ setattr(self, attribute_name, new_value)
+
+ def add_file(self, fname, label):
+ path = pathlib.Path(fname)
+ if label == content_handling.CONTENT_TYPES["TAILORING"]:
+ self._assign_content_type("tailoring", path)
+ elif label == content_handling.CONTENT_TYPES["DATASTREAM"]:
+ self._assign_content_type("datastream", path)
+ elif label == content_handling.CONTENT_TYPES["OVAL"]:
+ self.ovals.append(path)
+ elif label == content_handling.CONTENT_TYPES["XCCDF_CHECKLIST"]:
+ self._assign_content_type("xccdf", path)
+ self.labelled_files[path] = label
+
+ def _datastream_content(self):
+ if not self.datastream:
+ return None
+ if not self.datastream.exists():
+ return None
+ return self.datastream
+
+ def _xccdf_content(self):
+ if not self.xccdf or not self.ovals:
+ return None
+ some_ovals_exist = any([path.exists() for path in self.ovals])
+ if not (self.xccdf.exists() and some_ovals_exist):
+ return None
+ return self.xccdf
+
+ def find_expected_usable_content(self, relative_expected_content_path):
+ content_path = self.root / relative_expected_content_path
+ eligible_main_content = (self._datastream_content(), self._xccdf_content())
+
+ if content_path in eligible_main_content:
+ return content_path
+ else:
+ if not content_path.exists():
+ msg = f"Couldn't find '{content_path}' among the available content"
+ else:
+ msg = (
+ f"File '{content_path}' is not a valid datastream "
+ "or a valid XCCDF of a XCCDF-OVAL file tuple")
+ raise content_handling.ContentHandlingError(msg)
+
+ def select_main_usable_content(self):
+ if self._datastream_content():
+ return self._datastream_content()
+ elif self._xccdf_content():
+ return self._xccdf_content()
+ else:
+ msg = (
+ "Couldn't find a valid datastream or a valid XCCDF-OVAL file tuple "
+ "among the available content")
+ raise content_handling.ContentHandlingError(msg)
diff --git a/org_fedora_oscap/content_handling.py b/org_fedora_oscap/content_handling.py
index fddcf580..f2af22f3 100644
--- a/org_fedora_oscap/content_handling.py
+++ b/org_fedora_oscap/content_handling.py
@@ -27,6 +27,8 @@
import os.path
from collections import namedtuple
+import multiprocessing
+
from pyanaconda.core.util import execReadlines
try:
from html.parser import HTMLParser
@@ -37,6 +39,29 @@
log = logging.getLogger("anaconda")
+CONTENT_TYPES = dict(
+ DATASTREAM="Source Data Stream",
+ XCCDF_CHECKLIST="XCCDF Checklist",
+ OVAL="OVAL Definitions",
+ CPE_DICT="CPE Dictionary",
+ TAILORING="XCCDF Tailoring",
+)
+
+
+class ContentHandlingError(Exception):
+ """Exception class for errors related to SCAP content handling."""
+
+ pass
+
+
+class ContentCheckError(ContentHandlingError):
+ """
+ Exception class for errors related to content (integrity,...) checking.
+ """
+
+ pass
+
+
class ParseHTMLContent(HTMLParser):
"""Parser class for HTML tags within content"""
@@ -85,6 +110,33 @@ def parse_HTML_from_content(content):
ContentFiles = namedtuple("ContentFiles", ["xccdf", "cpe", "tailoring"])
+def identify_files(fpaths):
+ with multiprocessing.Pool(os.cpu_count()) as p:
+ labels = p.map(get_doc_type, fpaths)
+ return {path: label for (path, label) in zip(fpaths, labels)}
+
+
+def get_doc_type(file_path):
+ content_type = "unknown"
+ try:
+ for line in execReadlines("oscap", ["info", file_path]):
+ if line.startswith("Document type:"):
+ _prefix, _sep, type_info = line.partition(":")
+ content_type = type_info.strip()
+ break
+ except OSError:
+ # 'oscap info' exitted with a non-zero exit code -> unknown doc
+ # type
+ pass
+ except UnicodeDecodeError:
+ # 'oscap info' supplied weird output, which happens when it tries
+ # to explain why it can't examine e.g. a JPG.
+ return None
+ log.info("OSCAP addon: Identified {file_path} as {content_type}"
+ .format(file_path=file_path, content_type=content_type))
+ return content_type
+
+
def explore_content_files(fpaths):
"""
Function for finding content files in a list of file paths. SIMPLY PICKS
@@ -99,23 +151,6 @@ def explore_content_files(fpaths):
:rtype: ContentFiles
"""
-
- def get_doc_type(file_path):
- content_type = "unknown"
- try:
- for line in execReadlines("oscap", ["info", file_path]):
- if line.startswith("Document type:"):
- _prefix, _sep, type_info = line.partition(":")
- content_type = type_info.strip()
- break
- except OSError:
- # 'oscap info' exitted with a non-zero exit code -> unknown doc
- # type
- pass
- log.info("OSCAP addon: Identified {file_path} as {content_type}"
- .format(file_path=file_path, content_type=content_type))
- return content_type
-
xccdf_file = ""
cpe_file = ""
tailoring_file = ""
diff --git a/org_fedora_oscap/data_fetch.py b/org_fedora_oscap/data_fetch.py
index eefaed51..1ce7a260 100644
--- a/org_fedora_oscap/data_fetch.py
+++ b/org_fedora_oscap/data_fetch.py
@@ -75,7 +75,27 @@ class FetchError(DataFetchError):
pass
-def wait_and_fetch_net_data(url, out_file, ca_certs=None):
+def fetch_local_data(url, out_file):
+ """
+ Function that fetches data locally.
+
+ :see: org_fedora_oscap.data_fetch.fetch_data
+ :return: the name of the thread running fetch_data
+ :rtype: str
+
+ """
+ fetch_data_thread = AnacondaThread(name=common.THREAD_FETCH_DATA,
+ target=fetch_data,
+ args=(url, out_file, None),
+ fatal=False)
+
+ # register and run the thread
+ threadMgr.add(fetch_data_thread)
+
+ return common.THREAD_FETCH_DATA
+
+
+def wait_and_fetch_net_data(url, out_file, ca_certs_path=None):
"""
Function that waits for network connection and starts a thread that fetches
data over network.
@@ -99,7 +119,7 @@ def wait_and_fetch_net_data(url, out_file, ca_certs=None):
log.info(f"Fetching data from {url}")
fetch_data_thread = AnacondaThread(name=common.THREAD_FETCH_DATA,
target=fetch_data,
- args=(url, out_file, ca_certs),
+ args=(url, out_file, ca_certs_path),
fatal=False)
# register and run the thread
@@ -123,9 +143,9 @@ def can_fetch_from(url):
return any(url.startswith(prefix) for prefix in resources)
-def fetch_data(url, out_file, ca_certs=None):
+def fetch_data(url, out_file, ca_certs_path=None):
"""
- Fetch data from a given URL. If the URL starts with https://, ca_certs can
+ Fetch data from a given URL. If the URL starts with https://, ca_certs_path can
be a path to PEM file with CA certificate chain to validate server
certificate.
@@ -133,10 +153,10 @@ def fetch_data(url, out_file, ca_certs=None):
:type url: str
:param out_file: path to the output file
:type out_file: str
- :param ca_certs: path to a PEM file with CA certificate chain
- :type ca_certs: str
+ :param ca_certs_path: path to a PEM file with CA certificate chain
+ :type ca_certs_path: str
:raise WrongRequestError: if a wrong combination of arguments is passed
- (ca_certs file path given and url starting with
+ (ca_certs_path file path given and url starting with
http://) or arguments don't have required format
:raise CertificateValidationError: if server certificate validation fails
:raise FetchError: if data fetching fails (usually due to I/O errors)
@@ -148,13 +168,14 @@ def fetch_data(url, out_file, ca_certs=None):
utils.ensure_dir_exists(out_dir)
if can_fetch_from(url):
- _curl_fetch(url, out_file, ca_certs)
+ _curl_fetch(url, out_file, ca_certs_path)
else:
msg = "Cannot fetch data from '%s': unknown URL format" % url
raise UnknownURLformatError(msg)
+ log.info(f"Data fetch from {url} completed")
-def _curl_fetch(url, out_file, ca_certs=None):
+def _curl_fetch(url, out_file, ca_certs_path=None):
"""
Function that fetches data and writes it out to the given file path. If a
path to the file with CA certificates is given and the url starts with
@@ -164,11 +185,11 @@ def _curl_fetch(url, out_file, ca_certs=None):
:type url: str
:param out_file: path to the output file
:type out_file: str
- :param ca_certs: path to the file with CA certificates for server
+ :param ca_certs_path: path to the file with CA certificates for server
certificate validation
- :type ca_certs: str
+ :type ca_certs_path: str
:raise WrongRequestError: if a wrong combination of arguments is passed
- (ca_certs file path given and url starting with
+ (ca_certs_path file path given and url starting with
http://) or arguments don't have required format
:raise CertificateValidationError: if server certificate validation fails
:raise FetchError: if data fetching fails (usually due to I/O errors)
@@ -202,18 +223,18 @@ def _curl_fetch(url, out_file, ca_certs=None):
if not out_file:
raise WrongRequestError("out_file cannot be an empty string")
- if ca_certs and protocol != "https":
+ if ca_certs_path and protocol != "https":
msg = "Cannot verify server certificate when using plain HTTP"
raise WrongRequestError(msg)
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
- if ca_certs and protocol == "https":
+ if ca_certs_path and protocol == "https":
# the strictest verification
curl.setopt(pycurl.SSL_VERIFYHOST, 2)
curl.setopt(pycurl.SSL_VERIFYPEER, 1)
- curl.setopt(pycurl.CAINFO, ca_certs)
+ curl.setopt(pycurl.CAINFO, ca_certs_path)
# may be turned off by flags (specified on command line, take precedence)
if not conf.payload.verify_ssl:
diff --git a/org_fedora_oscap/gui/spokes/oscap.py b/org_fedora_oscap/gui/spokes/oscap.py
index 56de4b9e..c57b1cdb 100644
--- a/org_fedora_oscap/gui/spokes/oscap.py
+++ b/org_fedora_oscap/gui/spokes/oscap.py
@@ -21,6 +21,7 @@
import threading
import logging
from functools import wraps
+import pathlib
# the path to addons is in sys.path so we can import things
# from org_fedora_oscap
@@ -31,6 +32,7 @@
from org_fedora_oscap import scap_content_handler
from org_fedora_oscap import utils
from org_fedora_oscap.common import dry_run_skip
+from org_fedora_oscap.content_discovery import ContentBringer
from pyanaconda.threading import threadMgr, AnacondaThread
from pyanaconda.ui.gui.spokes import NormalSpoke
from pyanaconda.ui.communication import hubQ
@@ -250,6 +252,8 @@ def __init__(self, data, storage, payload):
self._anaconda_spokes_initialized = threading.Event()
self.initialization_controller.init_done.connect(self._all_anaconda_spokes_initialized)
+ self.content_bringer = ContentBringer(self._addon_data)
+
def _all_anaconda_spokes_initialized(self):
log.debug("OSCAP addon: Anaconda init_done signal triggered")
self._anaconda_spokes_initialized.set()
@@ -333,6 +337,27 @@ def initialize(self):
# else fetch data
self._fetch_data_and_initialize()
+ def _handle_error(self, exception):
+ log.error(str(exception))
+ if isinstance(exception, KickstartValueError):
+ self._invalid_url()
+ elif isinstance(exception, common.OSCAPaddonNetworkError):
+ self._network_problem()
+ elif isinstance(exception, data_fetch.DataFetchError):
+ self._data_fetch_failed()
+ elif isinstance(exception, common.ExtractionError):
+ self._extraction_failed(str(exception))
+ elif isinstance(exception, content_handling.ContentHandlingError):
+ self._invalid_content()
+ elif isinstance(exception, content_handling.ContentCheckError):
+ self._integrity_check_failed()
+ else:
+ self._general_content_problem()
+
+ def _end_fetching(self, fetched_content):
+ # stop the spinner in any case
+ fire_gtk_action(self._progress_spinner.stop)
+
def _render_selected(self, column, renderer, model, itr, user_data=None):
if model[itr][2]:
renderer.set_property("stock-id", "gtk-apply")
@@ -349,24 +374,9 @@ def _fetch_data_and_initialize(self):
self._fetching = True
thread_name = None
- if any(self._addon_data.content_url.startswith(net_prefix)
- for net_prefix in data_fetch.NET_URL_PREFIXES):
- # need to fetch data over network
- try:
- thread_name = data_fetch.wait_and_fetch_net_data(
- self._addon_data.content_url,
- self._addon_data.raw_preinst_content_path,
- self._addon_data.certificates)
- except common.OSCAPaddonNetworkError:
- self._network_problem()
- with self._fetch_flag_lock:
- self._fetching = False
- return
- except KickstartValueError:
- self._invalid_url()
- with self._fetch_flag_lock:
- self._fetching = False
- return
+ if self._addon_data.content_url and self._addon_data.content_type != "scap-security-guide":
+ thread_name = self.content_bringer.fetch_content(
+ self._handle_error, self._addon_data.certificates)
# pylint: disable-msg=E1101
hubQ.send_message(self.__class__.__name__,
@@ -388,62 +398,40 @@ def _init_after_data_fetch(self, wait_for):
:type wait_for: str or None
"""
+ def update_progress_label(msg):
+ fire_gtk_action(self._progress_label.set_text(msg))
- try:
- threadMgr.wait(wait_for)
- except data_fetch.DataFetchError:
- self._data_fetch_failed()
+ content_path = None
+ actually_fetched_content = wait_for is not None
+
+ if actually_fetched_content:
+ content_path = self._addon_data.raw_preinst_content_path
+
+ content = self.content_bringer.finish_content_fetch(
+ wait_for, self._addon_data.fingerprint, update_progress_label,
+ content_path, self._handle_error)
+ if not content:
with self._fetch_flag_lock:
self._fetching = False
- return
- finally:
- # stop the spinner in any case
- fire_gtk_action(self._progress_spinner.stop)
-
- if self._addon_data.fingerprint:
- hash_obj = utils.get_hashing_algorithm(self._addon_data.fingerprint)
- digest = utils.get_file_fingerprint(self._addon_data.raw_preinst_content_path,
- hash_obj)
- if digest != self._addon_data.fingerprint:
- self._integrity_check_failed()
- # fetching done
- with self._fetch_flag_lock:
- self._fetching = False
- return
- # RPM is an archive at this phase
- if self._addon_data.content_type in ("archive", "rpm"):
- # extract the content
- try:
- fpaths = common.extract_data(self._addon_data.raw_preinst_content_path,
- common.INSTALLATION_CONTENT_DIR,
- [self._addon_data.content_path])
- except common.ExtractionError as err:
- self._extraction_failed(str(err))
- # fetching done
- with self._fetch_flag_lock:
- self._fetching = False
- return
+ return
- # and populate missing fields
- files = content_handling.explore_content_files(fpaths)
- files = common.strip_content_dir(files)
+ try:
+ if actually_fetched_content:
+ self.content_bringer.use_downloaded_content(content)
- # pylint: disable-msg=E1103
- self._addon_data.content_path = self._addon_data.content_path or files.xccdf
- self._addon_data.cpe_path = self._addon_data.cpe_path or files.cpe
- self._addon_data.tailoring_path = (self._addon_data.tailoring_path or
- files.tailoring)
- elif self._addon_data.content_type not in (
- "datastream", "scap-security-guide"):
- raise common.OSCAPaddonError("Unsupported content type")
+ msg = f"Opening SCAP content at {self._addon_data.preinst_content_path}"
+ if self._addon_data.tailoring_path:
+ msg += f" with tailoring {self._addon_data.preinst_tailoring_path}"
+ else:
+ msg += " without considering tailoring"
+ log.info(msg)
- try:
self._content_handler = scap_content_handler.SCAPContentHandler(
self._addon_data.preinst_content_path,
self._addon_data.preinst_tailoring_path)
- except scap_content_handler.SCAPContentHandlerError as e:
- log.warning(str(e))
+ except Exception as e:
+ log.error(str(e))
self._invalid_content()
# fetching done
with self._fetch_flag_lock:
@@ -713,6 +701,8 @@ def _select_profile(self, profile_id):
# no profile specified, nothing to do
return False
+ ds = None
+ xccdf = None
if self._using_ds:
ds = self._current_ds_id
xccdf = self._current_xccdf_id
@@ -720,16 +710,12 @@ def _select_profile(self, profile_id):
if not all((ds, xccdf, profile_id)):
# something is not set -> do nothing
return False
- else:
- ds = None
- xccdf = None
# get pre-install fix rules from the content
try:
- rules = common.get_fix_rules_pre(profile_id,
- self._addon_data.preinst_content_path,
- ds, xccdf,
- self._addon_data.preinst_tailoring_path)
+ self._rule_data = rule_handling.get_rule_data_from_content(
+ profile_id, self._addon_data.preinst_content_path,
+ ds, xccdf, self._addon_data.preinst_tailoring_path)
except common.OSCAPaddonError as exc:
log.error(
"Failed to get rules for the profile '{}': {}"
@@ -745,11 +731,6 @@ def _select_profile(self, profile_id):
self._profiles_store.set_value(itr, 2, True)
itr = self._profiles_store.iter_next(itr)
- # parse and store rules with a clean RuleData instance
- self._rule_data = rule_handling.RuleData()
- for rule in rules.splitlines():
- self._rule_data.new_rule(rule)
-
# remember the active profile
self._active_profile = profile_id
@@ -788,6 +769,12 @@ def _set_error(self, msg):
self._error = None
self.clear_info()
+ @async_action_wait
+ def _general_content_problem(self):
+ msg = _("There was an unexpected problem with the supplied content.")
+ self._progress_label.set_markup("%s" % msg)
+ self._wrong_content(msg)
+
@async_action_wait
def _invalid_content(self):
"""Callback for informing user about provided content invalidity."""
@@ -1133,6 +1120,7 @@ def on_fetch_button_clicked(self, *args):
with self._fetch_flag_lock:
if self._fetching:
# some other fetching/pre-processing running, give up
+ log.warn("Clicked the fetch button, although the GUI is in the fetching mode.")
return
# prevent user from changing the URL in the meantime
@@ -1172,7 +1160,5 @@ def on_change_content_clicked(self, *args):
self.refresh()
def on_use_ssg_clicked(self, *args):
- self._addon_data.clear_all()
- self._addon_data.content_type = "scap-security-guide"
- self._addon_data.content_path = common.SSG_DIR + common.SSG_CONTENT
+ self.content_bringer.use_system_content()
self._fetch_data_and_initialize()
diff --git a/org_fedora_oscap/ks/oscap.py b/org_fedora_oscap/ks/oscap.py
index ff805fc9..2172293e 100644
--- a/org_fedora_oscap/ks/oscap.py
+++ b/org_fedora_oscap/ks/oscap.py
@@ -25,6 +25,7 @@
import os
import time
import logging
+import pathlib
from pyanaconda.addons import AddonData
from pyanaconda.core.configuration.anaconda import conf
@@ -35,6 +36,8 @@
from pykickstart.errors import KickstartParseError, KickstartValueError
from org_fedora_oscap import utils, common, rule_handling, data_fetch
from org_fedora_oscap.common import SUPPORTED_ARCHIVES, _
+from org_fedora_oscap.content_handling import ContentCheckError, ContentHandlingError
+from org_fedora_oscap import content_discovery
log = logging.getLogger("anaconda")
@@ -47,7 +50,7 @@
"scap-security-guide",
)
-SUPPORTED_URL_PREFIXES = ("http://", "https://", "ftp://"
+SUPPORTED_URL_PREFIXES = ("http://", "https://", "ftp://", "file://"
# LABEL:?, hdaX:?,
)
@@ -101,6 +104,8 @@ def __init__(self, name, just_clear=False):
self.rule_data = rule_handling.RuleData()
self.dry_run = False
+ self.content_bringer = content_discovery.ContentBringer(self)
+
def __str__(self):
"""
What should end up in the resulting kickstart file, i.e. string
@@ -367,27 +372,42 @@ def postinst_tailoring_path(self):
return utils.join_paths(common.TARGET_CONTENT_DIR,
self.tailoring_path)
- def _fetch_content_and_initialize(self):
- """Fetch content and initialize from it"""
-
- data_fetch.fetch_data(self.content_url, self.raw_preinst_content_path,
- self.certificates)
- # RPM is an archive at this phase
- if self.content_type in ("archive", "rpm"):
- # extract the content
- common.extract_data(self.raw_preinst_content_path,
- common.INSTALLATION_CONTENT_DIR,
- [self.content_path])
-
- rules = common.get_fix_rules_pre(self.profile_id,
- self.preinst_content_path,
- self.datastream_id, self.xccdf_id,
- self.preinst_tailoring_path)
+ def _terminate(self, message):
+ message += "\n" + _("The installation should be aborted.")
+ message += " " + _("Do you wish to continue anyway?")
+ if flags.flags.automatedInstall and not flags.flags.ksprompt:
+ # cannot have ask in a non-interactive kickstart
+ # installation
+ raise errors.CmdlineError(message)
+
+ answ = errors.errorHandler.ui.showYesNoQuestion(message)
+ if answ == errors.ERROR_CONTINUE:
+ # prevent any futher actions here by switching to the dry
+ # run mode and let things go on
+ self.dry_run = True
+ return
+ else:
+ # Let's sleep forever to prevent any further actions and
+ # wait for the main thread to quit the process.
+ progressQ.send_quit(1)
+ while True:
+ time.sleep(100000)
+
+ def _handle_error(self, exception):
+ log.error("Failed to fetch and initialize SCAP content!")
+
+ if isinstance(exception, ContentCheckError):
+ msg = _("The integrity check of the security content failed.")
+ self._terminate(msg)
+ elif (isinstance(exception, common.OSCAPaddonError)
+ or isinstance(exception, data_fetch.DataFetchError)):
+ msg = _("There was an error fetching and loading the security content:\n" +
+ f"{str(exception)}")
+ self._terminate(msg)
- # parse and store rules with a clean RuleData instance
- self.rule_data = rule_handling.RuleData()
- for rule in rules.splitlines():
- self.rule_data.new_rule(rule)
+ else:
+ msg = _("There was an unexpected problem with the supplied content.")
+ self._terminate(msg)
def setup(self, storage, ksdata, payload):
"""
@@ -408,86 +428,41 @@ def setup(self, storage, ksdata, payload):
# selected
return
+ thread_name = None
if not os.path.exists(self.preinst_content_path) and not os.path.exists(self.raw_preinst_content_path):
# content not available/fetched yet
- try:
- self._fetch_content_and_initialize()
- except (common.OSCAPaddonError, data_fetch.DataFetchError) as e:
- log.error("Failed to fetch and initialize SCAP content!")
- msg = _("There was an error fetching and loading the security content:\n" +
- "%s\n" +
- "The installation should be aborted. Do you wish to continue anyway?") % e
-
- if flags.flags.automatedInstall and not flags.flags.ksprompt:
- # cannot have ask in a non-interactive kickstart
- # installation
- raise errors.CmdlineError(msg)
-
- answ = errors.errorHandler.ui.showYesNoQuestion(msg)
- if answ == errors.ERROR_CONTINUE:
- # prevent any futher actions here by switching to the dry
- # run mode and let things go on
- self.dry_run = True
- return
- else:
- # Let's sleep forever to prevent any further actions and
- # wait for the main thread to quit the process.
- progressQ.send_quit(1)
- while True:
- time.sleep(100000)
-
- # check fingerprint if given
- if self.fingerprint:
- hash_obj = utils.get_hashing_algorithm(self.fingerprint)
- digest = utils.get_file_fingerprint(self.raw_preinst_content_path,
- hash_obj)
- if digest != self.fingerprint:
- log.error("Failed to fetch and initialize SCAP content!")
- msg = _("The integrity check of the security content failed.\n" +
- "The installation should be aborted. Do you wish to continue anyway?")
-
- if flags.flags.automatedInstall and not flags.flags.ksprompt:
- # cannot have ask in a non-interactive kickstart
- # installation
- raise errors.CmdlineError(msg)
-
- answ = errors.errorHandler.ui.showYesNoQuestion(msg)
- if answ == errors.ERROR_CONTINUE:
- # prevent any futher actions here by switching to the dry
- # run mode and let things go on
- self.dry_run = True
- return
- else:
- # Let's sleep forever to prevent any further actions and
- # wait for the main thread to quit the process.
- progressQ.send_quit(1)
- while True:
- time.sleep(100000)
+ thread_name = self.content_bringer.fetch_content(self._handle_error, self.certificates)
+
+ content_dest = None
+ if self.content_type != "scap-security-guide":
+ content_dest = self.raw_preinst_content_path
+
+ content = self.content_bringer.finish_content_fetch(
+ thread_name, self.fingerprint, lambda msg: log.info(msg), content_dest, self._handle_error)
+
+ if not content:
+ return
+
+ try:
+ # just check that preferred content exists
+ _ = self.content_bringer.get_preferred_content(content)
+ except Exception as exc:
+ self._terminate(str(exc))
+ return
+
+ self.rule_data = rule_handling.get_rule_data_from_content(
+ self.profile_id, self.preinst_content_path,
+ self.datastream_id, self.xccdf_id, self.preinst_tailoring_path)
# evaluate rules, do automatic fixes and stop if something that cannot
# be fixed automatically is wrong
fatal_messages = [message for message in self.rule_data.eval_rules(ksdata, storage)
if message.type == common.MESSAGE_TYPE_FATAL]
if any(fatal_messages):
- msg = "Wrong configuration detected!\n"
- msg += "\n".join(message.text for message in fatal_messages)
- msg += "\nThe installation should be aborted. Do you wish to continue anyway?"
- if flags.flags.automatedInstall and not flags.flags.ksprompt:
- # cannot have ask in a non-interactive kickstart installation
- raise errors.CmdlineError(msg)
-
- answ = errors.errorHandler.ui.showYesNoQuestion(msg)
- if answ == errors.ERROR_CONTINUE:
- # prevent any futher actions here by switching to the dry
- # run mode and let things go on
- self.dry_run = True
- return
- else:
- # Let's sleep forever to prevent any further actions and wait
- # for the main thread to quit the process.
- progressQ.send_quit(1)
- while True:
- time.sleep(100000)
+ msg_lines = [_("Wrong configuration detected!")]
+ msg_lines.extend(fatal_messages)
+ self._terminate("\n".join(msg_lines))
+ return
# add packages needed on the target system to the list of packages
# that are requested to be installed
diff --git a/org_fedora_oscap/rule_handling.py b/org_fedora_oscap/rule_handling.py
index 80d86c7c..d7729f71 100644
--- a/org_fedora_oscap/rule_handling.py
+++ b/org_fedora_oscap/rule_handling.py
@@ -68,6 +68,17 @@
_ = common._
+def get_rule_data_from_content(profile_id, content_path, ds_id="", xccdf_id="", tailoring_path=""):
+ rules = common.get_fix_rules_pre(
+ profile_id, content_path, ds_id, xccdf_id, tailoring_path)
+
+ # parse and store rules with a clean RuleData instance
+ rule_data = RuleData()
+ for rule in rules.splitlines():
+ rule_data.new_rule(rule)
+ return rule_data
+
+
# TODO: use set instead of list for mount options?
def parse_csv(option, opt_str, value, parser):
for item in value.split(","):
diff --git a/testing_files/cpe-dict.xml b/testing_files/cpe-dict.xml
new file mode 100644
index 00000000..394cae8a
--- /dev/null
+++ b/testing_files/cpe-dict.xml
@@ -0,0 +1,9 @@
+
+
+
+ Applicable example platform
+ oval:x:def:1
+
+
diff --git a/tests/test_content_handling.py b/tests/test_content_handling.py
new file mode 100644
index 00000000..56dfe686
--- /dev/null
+++ b/tests/test_content_handling.py
@@ -0,0 +1,34 @@
+import os
+import glob
+
+import pytest
+
+from org_fedora_oscap import content_handling as ch
+
+
+TESTING_FILES_PATH = os.path.join(
+ os.path.dirname(__file__), os.path.pardir, "testing_files")
+DS_FILEPATH = os.path.join(
+ TESTING_FILES_PATH, "testing_ds.xml")
+
+DS_IDS = "scap_org.open-scap_datastream_tst"
+CHK_FIRST_ID = "scap_org.open-scap_cref_first-xccdf.xml"
+CHK_SECOND_ID = "scap_org.open-scap_cref_second-xccdf.xml"
+
+PROFILE1_ID = "xccdf_com.example_profile_my_profile"
+PROFILE2_ID = "xccdf_com.example_profile_my_profile2"
+PROFILE3_ID = "xccdf_com.example_profile_my_profile3"
+
+
+def test_identify_files():
+ filenames = glob.glob(TESTING_FILES_PATH + "/*")
+ identified = ch.identify_files(filenames)
+ assert identified[DS_FILEPATH] == ch.CONTENT_TYPES["DATASTREAM"]
+ assert identified[
+ os.path.join(TESTING_FILES_PATH, "scap-mycheck-oval.xml")] == ch.CONTENT_TYPES["OVAL"]
+ assert identified[
+ os.path.join(TESTING_FILES_PATH, "tailoring.xml")] == ch.CONTENT_TYPES["TAILORING"]
+ assert identified[
+ os.path.join(TESTING_FILES_PATH, "testing_xccdf.xml")] == ch.CONTENT_TYPES["XCCDF_CHECKLIST"]
+ assert identified[
+ os.path.join(TESTING_FILES_PATH, "cpe-dict.xml")] == ch.CONTENT_TYPES["CPE_DICT"]
diff --git a/tests/test_data_fetch.py b/tests/test_data_fetch.py
index e2bb573d..cfc56b9a 100644
--- a/tests/test_data_fetch.py
+++ b/tests/test_data_fetch.py
@@ -59,3 +59,11 @@ def test_supported_url():
def test_unsupported_url():
assert not data_fetch.can_fetch_from("aaaaa")
+
+
+def test_fetch_local(tmp_path):
+ source_path = pathlib.Path(__file__).absolute()
+ dest_path = tmp_path / "dest"
+ data_fetch.fetch_data("file://" + str(source_path), dest_path)
+ with open(dest_path, "r") as copied_file:
+ assert "This line is here and in the copied file as well" in copied_file.read()
diff --git a/tests/test_ks_oscap.py b/tests/test_ks_oscap.py
index 4baafec0..ac9bc2d2 100644
--- a/tests/test_ks_oscap.py
+++ b/tests/test_ks_oscap.py
@@ -311,12 +311,11 @@ def test_valid_fingerprints(blank_oscap_data):
def test_invalid_fingerprints(blank_oscap_data):
# invalid character
- with pytest.raises(
- KickstartValueError, message="Unsupported or invalid fingerprint"):
+ with pytest.raises(KickstartValueError, match="Unsupported or invalid fingerprint"):
blank_oscap_data.handle_line("fingerprint = %s?" % ("a" * 31))
# invalid lengths (odd and even)
for repetitions in (31, 41, 54, 66, 98, 124):
with pytest.raises(
- KickstartValueError, message="Unsupported fingerprint"):
+ KickstartValueError, match="Unsupported fingerprint"):
blank_oscap_data.handle_line("fingerprint = %s" % ("a" * repetitions))