diff --git a/src/rgw/support/rgw_support_bundle b/src/rgw/support/rgw_support_bundle index bd7086d..b6e94c8 100755 --- a/src/rgw/support/rgw_support_bundle +++ b/src/rgw/support/rgw_support_bundle @@ -32,7 +32,8 @@ from cortx.utils.conf_store.conf_store import Conf, MappedConf from cortx.utils.support_framework.log_filters import FilterLog from cortx.rgw.const import ( CONFIG_PATH_KEY, LOG_PATH_KEY, COMPONENT_NAME, RGW_CONF_FILE, - SVC_FILE_PERCENTAGE) + SVC_FILE_PERCENTAGE, LOG_DATE_REGEX, LOG_TIME_REGEX, + DATETIME_DATE_REGEX, DATETIME_TIME_REGEX) class SupportBundleError(BaseError): @@ -98,34 +99,22 @@ class RGWSupportBundle: # files in delim_T use timestamp syntax as # 2022-07-10T05:28:35.570+0000 from cortx.utils.support_framework.log_filters import FilterLog - from cortx.rgw.const import (RGW_STARTUP_LOG, RGW_SETUP_LOG, - RGW_SUPPORT_BUNDLE_LOG, RADOSGW_ADMIN_LOG, RGW_1_LOG, - LOG_DATE_REGEX, LOG_TIME_REGEX, DATETIME_DATE_REGEX, - DATETIME_TIME_REGEX) - - # log files with timestamp syntax "YYYY-mm-DD H:M:S" - delim_space = [RGW_STARTUP_LOG, RGW_SETUP_LOG, RGW_SUPPORT_BUNDLE_LOG] - # log files with timestamp syntax "YYYY-mm-DDTH:M:S" - delim_T = [RADOSGW_ADMIN_LOG, RGW_1_LOG] + temp_path_src = os.path.join('/tmp', 'rgw_src') + os.makedirs(temp_path_src, exist_ok=True) + # Move text logs and rotated log files to temp dir. for file in os.listdir(RGWSupportBundle._tmp_src): - if file in delim_space: - delim = ' ' - elif file in delim_T: - delim = 'T' - else: + if file.endswith('.conf') or file == "installed-cortx-rpms.txt": continue - log_timestamp_regex = LOG_DATE_REGEX + delim + LOG_TIME_REGEX - datetime_format = DATETIME_DATE_REGEX + delim + DATETIME_TIME_REGEX - # apply log filter to individual file by passing exact filename - # to file_name_reg_ex - FilterLog.limit_time( - src_dir=RGWSupportBundle._tmp_src, - dest_dir=RGWSupportBundle._tmp_src, - duration=duration, - file_name_reg_ex=file, - log_timestamp_regex=log_timestamp_regex, - datetime_format=datetime_format) - + shutil.move(os.path.join(RGWSupportBundle._tmp_src, file), + os.path.join(temp_path_src, file)) + temp_path = os.path.join('/tmp', 'rgw_time_based_log') + RGWSupportBundle._apply_limit_time(temp_path_src, temp_path, 'rgw-*', duration) + RGWSupportBundle._apply_limit_time(temp_path_src, temp_path, 'rgw_*', duration) + RGWSupportBundle._apply_limit_time(temp_path_src, temp_path, 'radosgw*', duration) + # delete temp paths. + shutil.rmtree(temp_path) + shutil.rmtree(temp_path_src) + Log.info('Collected log files based on time duration.') else: Log.error("RGW log file does not exists hence skipping log file collection.") @@ -233,6 +222,30 @@ class RGWSupportBundle: RGWSupportBundle._generate_tar(bundle_id, target_path) RGWSupportBundle._cleanup() + @staticmethod + def _apply_limit_time(temp_path_src: str, path: str, reg_ex: str, duration: str): + """Apply limit_time filter.""" + if os.path.exists(path): + shutil.rmtree(path) + os.makedirs(path, exist_ok=True) + for file in os.listdir(temp_path_src): + regex = re.compile(f'({reg_ex})') + if regex.match(file) and file.startswith(reg_ex[:-1]): + shutil.move(os.path.join(temp_path_src, file), + os.path.join(path, file)) + if reg_ex == 'rgw-*' or reg_ex == 'radosgw*': + delim = 'T' + else: + delim = ' ' + log_timestamp_regex = LOG_DATE_REGEX + delim + LOG_TIME_REGEX + datetime_format = DATETIME_DATE_REGEX + delim + DATETIME_TIME_REGEX + Log.info(f'apply limit-time filter on {reg_ex} files.') + # apply limit_time log filter to file names started by reg_ex + FilterLog.limit_time(src_dir=path, dest_dir=RGWSupportBundle._tmp_src, + duration=duration, file_name_reg_ex=reg_ex[:-1], + log_timestamp_regex=log_timestamp_regex, + datetime_format=datetime_format) + @staticmethod def _get_dir_size(dir_path: str): """Calculate total directory size.""" @@ -285,7 +298,7 @@ class RGWSupportBundle: if size_limit: remaining_size_limit = RGWSupportBundle._get_remaining_folder_size( RGWSupportBundle._tmp_src, size_limit) - if (os.path.getsize(addb_dir) > int(remaining_size_limit[:-1])): + if (RGWSupportBundle._get_dir_size(addb_dir) > int(remaining_size_limit[:-1])): Log.warn('Exhausted support bundle size limit while ' 'collecting addb files.') break