CORTX-34143: k8 support bundle script fails to collect cortx-rgw files.

py-utils/src/utils/support_framework/log_filters.py

@@ -20,7 +20,6 @@
 import re
 import errno
 import shutil
-import gzip
 import pathlib
 from datetime import datetime, timedelta
 from cortx.utils.log import Log
@@ -178,12 +177,15 @@ def limit_time(src_dir, dest_dir, duration, file_name_reg_ex,
             log_timestamp_regex:
             '[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}'
         """
-        supported_file_types = ['.log', '.gz']
         invalid_chars = re.findall("[^PTDHMS0-9-:]+", duration)
         if invalid_chars:
             raise BundleError(errno.EINVAL, "Invalid duration passed! "
             + f"unexpected characters: {invalid_chars}")
 
+        include_lines_without_timestamp = False
+        supported_file_types = ['.log', '.gz']
+        start_time, end_time = FilterLog._parse_duration(duration)
+        Log.info(f'start_time = {start_time}, end_time = {end_time}')
         # sort files based on timestamp
         list_of_files = filter(lambda f: os.path.isfile(os.path.join(src_dir, f)),
             os.listdir(src_dir))
@@ -192,6 +194,8 @@ def limit_time(src_dir, dest_dir, duration, file_name_reg_ex,
             key=lambda f: os.path.getmtime(os.path.join(src_dir, f)),
             reverse=True)
         for file in list_of_files:
+            log_scope_exceeded = False
+            is_log_written_to_file = False
             file_extension = pathlib.Path(file).suffix
             # Ignore processing of other file format.
             if file_extension not in supported_file_types:
@@ -201,70 +205,57 @@ def limit_time(src_dir, dest_dir, duration, file_name_reg_ex,
             if file.startswith(file_name_reg_ex):
                 in_file = os.path.join(src_dir, file)
                 if file.endswith('.gz'):
-                    # Reading .gz file content without extracting.
-                    with gzip.open(in_file, 'rt') as fd_in:
-                        lines = fd_in.readlines()
-                else:
-                    with open(in_file, 'r') as fd_in:
-                        lines = fd_in.readlines()
-            if not lines:
-                continue
-            if file.endswith('.gz'):
-                with gzip.open(op_file, 'a') as fd_out:
-                    log_scope_exceeded, is_log_written_to_file = FilterLog._add_logs_in_dest_file(
-                        lines, log_timestamp_regex, datetime_format,
-                        duration, fd_out, encode=True)
-            else:
-                with open(op_file, 'a') as fd_out:
-                    log_scope_exceeded, is_log_written_to_file = FilterLog._add_logs_in_dest_file(
-                        lines, log_timestamp_regex, datetime_format,
-                        duration, fd_out)
-            try:
-                if is_log_written_to_file:
-                    final_op_file = os.path.join(dest_dir, file)
-                    os.rename(op_file, final_op_file)
-                else:
-                    os.remove(op_file)
-                if log_scope_exceeded:
-                    break
-            except OSError as e:
-                raise BundleError(errno.EINVAL, "Failed to filter log " +
-                    f"file based on time duration. ERROR:{e}")
-
-    @staticmethod
-    def _add_logs_in_dest_file(contents: list, timestamp_regex: str,
-        datetime_format: str, duration: str, fd_out, encode=False):
-        """The logs which are in given time duration add those logs in dest_dir."""
-        include_lines_without_timestamp = False
-        log_scope_exceeded = False
-        is_log_written_to_file = False
-        start_time, end_time = FilterLog._parse_duration(duration)
-        for line in contents:
-            regex_res = re.search(timestamp_regex, line)
-            log_duration = regex_res.group(0) if regex_res else None
-            if log_duration:
-                # convert log timestamp to datetime object
+                    # File modification time is in
+                    FilterLog._collect_rotated_log_file(
+                        in_file, dest_dir, start_time, end_time)
+                    continue
+                with open(in_file, 'r') as fd_in, open(op_file, 'a') as fd_out:
+                    line = fd_in.readline()
+                    while (line):
+                        regex_res = re.search(log_timestamp_regex, line)
+                        log_duration = regex_res.group(0) if regex_res else None
+                        if log_duration:
+                            # convert log timestamp to datetime object
+                            try:
+                                log_time = datetime.strptime(log_duration, datetime_format)
+                                if start_time <= log_time and log_time <= end_time:
+                                    include_lines_without_timestamp = True
+                                    is_log_written_to_file = True
+                                    fd_out.write(line)
+                                elif log_time > end_time and include_lines_without_timestamp:
+                                    include_lines_without_timestamp = False
+                                    log_scope_exceeded = True
+                                    break
+                            # There will be some log lines which lies under passed
+                            # duration, but has no timestamp, those lines will
+                            # throw ValueError while trying to fetch log
+                            # timestamp, to capture such logs, we have set a
+                            # flag include_lines_without_timestamp = True
+                            except ValueError:
+                                if include_lines_without_timestamp:
+                                    is_log_written_to_file = True
+                                    fd_out.write(line)
+                        line = fd_in.readline()
                 try:
-                    log_time = datetime.strptime(log_duration, datetime_format)
-                    if start_time <= log_time and log_time <= end_time:
-                        include_lines_without_timestamp = True
-                        is_log_written_to_file = True
-                        if encode:
-                            line = line.encode('utf-8')
-                        fd_out.write(line)
-                    elif log_time > end_time and include_lines_without_timestamp:
-                        include_lines_without_timestamp = False
-                        log_scope_exceeded = True
+                    if is_log_written_to_file:
+                        final_op_file = os.path.join(dest_dir, file)
+                        os.rename(op_file, final_op_file)
+                    else:
+                        os.remove(op_file)
+                    if log_scope_exceeded:
                         break
-                # There will be some log lines which lies under passed duration,
-                # but has no timestamp, those lines will throw ValueError while
-                # trying to fetch log timestamp,
-                # to capture such logs, we have set a flag
-                # include_lines_without_timestamp = True
-                except ValueError:
-                    if include_lines_without_timestamp:
-                        is_log_written_to_file = True
-                        if encode:
-                            line = line.encode('utf-8')
-                        fd_out.write(line)
-        return log_scope_exceeded, is_log_written_to_file
+                except OSError as e:
+                    raise BundleError(errno.EINVAL, "Failed to filter log " +
+                        f"file based on time duration. ERROR:{e}")
+
+    @staticmethod
+    def _collect_rotated_log_file(in_file: str, dest_dir: str,
+            start_time: str, end_time: str):
+        """Collect rotated log files(.gz) in support bundle if its
+           creation time is in given time duration."""
+        creation_time = os.path.getctime(in_file)
+        # convert creation_time into datetime format e.g %Y-%m-%d %H:%M:%S.
+        c_time = datetime.fromtimestamp(creation_time)
+        if start_time <= c_time and c_time <= end_time:
+            dest_file = os.path.join(dest_dir, os.path.basename(in_file))
+            shutil.copyfile(in_file, dest_file)