diff --git a/docs/archive/docs-1.x/conf.py b/docs/archive/docs-1.x/conf.py index cfe1b57be..549d335c5 100644 --- a/docs/archive/docs-1.x/conf.py +++ b/docs/archive/docs-1.x/conf.py @@ -10,17 +10,17 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. +# documentation root, use str(Path().absolute().resolve()) to make it absolute, like shown here. # -import os +from pathlib import Path import subprocess as sp import sys -sys.path.insert(0, os.path.abspath("..")) +sys.path.insert(0, str(Path("..").absolute().resolve())) repo_version = "unknown" # Determine short version by file in repo -if os.path.isfile("./VERSION"): +if Path("./VERSION").is_file(): with open("./VERSION") as f: repo_version = f.readline().strip() diff --git a/docs/archive/docs-2.x/conf.py b/docs/archive/docs-2.x/conf.py index e637a9803..543b96feb 100644 --- a/docs/archive/docs-2.x/conf.py +++ b/docs/archive/docs-2.x/conf.py @@ -10,17 +10,17 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. +# documentation root, use str(Path().absolute().resolve()) to make it absolute, like shown here. # -import os +from pathlib import Path import subprocess as sp import sys -sys.path.insert(0, os.path.abspath("..")) +sys.path.insert(0, str(Path("..").absolute().resolve())) repo_version = "unknown" # Determine short version by file in repo -if os.path.isfile("./VERSION"): +if Path("./VERSION").is_file(): with open("./VERSION") as f: repo_version = f.readline().strip() diff --git a/src/argparser.py b/src/argparser.py index a05c7f808..20a07fb40 100644 --- a/src/argparser.py +++ b/src/argparser.py @@ -24,6 +24,7 @@ import argparse import os +from pathlib import Path import shutil @@ -123,7 +124,7 @@ def omniarg_parser( metavar="", type=str, dest="path", - default=os.path.join(os.getcwd(), "workloads"), + default=str(Path(os.getcwd()).joinpath("workloads")), required=False, help="\t\t\tSpecify path to save workload.\n\t\t\t(DEFAULT: {}/workloads/)".format( os.getcwd() diff --git a/src/rocprof-compute b/src/rocprof-compute index 90d16af30..b9ff410e6 100755 --- a/src/rocprof-compute +++ b/src/rocprof-compute @@ -27,13 +27,12 @@ ##############################################################################el # import logging -import os +from pathlib import Path import re import sys try: from importlib import metadata - from pathlib import Path from rocprof_compute_base import RocProfCompute from utils.utils import console_error @@ -73,12 +72,12 @@ def verify_deps(): ) sys.exit(1) - bindir = Path(__file__).resolve().parent + bindir = str(Path(__file__).resolve().parent) depsLocation = ["requirements.txt", "../requirements.txt"] for location in depsLocation: - checkFile = os.path.join(bindir, location) - if os.path.exists(checkFile): + checkFile = str(Path(bindir).joinpath(location)) + if Path(checkFile).exists(): with open(checkFile, "r", encoding="utf-8") as file_in: dependencies = file_in.read().splitlines() diff --git a/src/rocprof_compute_analyze/analysis_base.py b/src/rocprof_compute_analyze/analysis_base.py index ff206a7aa..4f8b939d3 100644 --- a/src/rocprof_compute_analyze/analysis_base.py +++ b/src/rocprof_compute_analyze/analysis_base.py @@ -192,9 +192,9 @@ def sanitize(self): ) # ensure absolute path for dir in self.__args.path: - full_path = os.path.abspath(dir[0]) + full_path = str(Path(dir[0]).absolute().resolve()) dir[0] = full_path - if not os.path.isdir(dir[0]): + if not Path(dir[0]).is_dir(): console_error("Invalid directory {}\nPlease try again.".format(dir[0])) # validate profiling data diff --git a/src/rocprof_compute_analyze/analysis_webui.py b/src/rocprof_compute_analyze/analysis_webui.py index 8acdf68ca..ddcaa3c82 100644 --- a/src/rocprof_compute_analyze/analysis_webui.py +++ b/src/rocprof_compute_analyze/analysis_webui.py @@ -24,6 +24,7 @@ import copy import os +from pathlib import Path import random import dash @@ -45,7 +46,7 @@ def __init__(self, args, supported_archs): self.app = dash.Dash( __name__, title=PROJECT_NAME, external_stylesheets=[dbc.themes.CYBORG] ) - self.dest_dir = os.path.abspath(args.path[0][0]) + self.dest_dir = str(Path(args.path[0][0]).absolute().resolve()) self.arch = None self.__hidden_sections = ["Memory Chart", "Roofline"] @@ -167,7 +168,7 @@ def generate_from_filter( div_children.append( get_memchart(panel_configs[300]["data source"], base_data[base_run]) ) - has_roofline = os.path.isfile(os.path.join(self.dest_dir, "roofline.csv")) + has_roofline = Path(self.dest_dir).joinpath("roofline.csv").is_file() if has_roofline and hasattr(self.get_socs()[self.arch], "roofline_obj"): # update roofline for visualization in GUI self.get_socs()[self.arch].analysis_setup( diff --git a/src/rocprof_compute_base.py b/src/rocprof_compute_base.py index 9b964c6ef..4b19acbcf 100644 --- a/src/rocprof_compute_base.py +++ b/src/rocprof_compute_base.py @@ -205,9 +205,13 @@ def parse_args(self): # FIXME: # Might want to get host name from detected spec if self.__args.subpath == "node_name": - self.__args.path = os.path.join(self.__args.path, socket.gethostname()) + self.__args.path = str( + Path(self.__args.path).joinpath(socket.gethostname()) + ) elif self.__args.subpath == "gpu_model": - self.__args.path = os.path.join(self.__args.path, self.__mspec.gpu_model) + self.__args.path = str( + Path(self.__args.path).joinpath(self.__mspec.gpu_model) + ) p = Path(self.__args.path) if not p.exists(): @@ -227,9 +231,9 @@ def run_profiler(self): # unless there is a specific reason to do here. # Update default path - if self.__args.path == os.path.join(os.getcwd(), "workloads"): - self.__args.path = os.path.join( - self.__args.path, self.__args.name, self.__mspec.gpu_model + if self.__args.path == str(Path(os.getcwd()).joinpath("workloads")): + self.__args.path = str( + Path(self.__args.path).joinpath(self.__args.name, self.__mspec.gpu_model) ) # instantiate desired profiler diff --git a/src/rocprof_compute_profile/profiler_base.py b/src/rocprof_compute_profile/profiler_base.py index 2e078ce21..cabc7fc38 100644 --- a/src/rocprof_compute_profile/profiler_base.py +++ b/src/rocprof_compute_profile/profiler_base.py @@ -25,6 +25,7 @@ import glob import logging import os +from pathlib import Path import re import sys import time @@ -53,8 +54,10 @@ def __init__(self, args, profiler_mode, soc): self.__args = args self.__profiler = profiler_mode self._soc = soc # OmniSoC obj - self.__perfmon_dir = os.path.join( - str(config.rocprof_compute_home), "rocprof_compute_soc", "profile_configs" + self.__perfmon_dir = str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", "profile_configs" + ) ) def get_args(self): @@ -263,7 +266,7 @@ def pre_processing(self): # verify correct formatting for application binary self.__args.remaining = self.__args.remaining[1:] if self.__args.remaining: - if not os.path.isfile(self.__args.remaining[0]): + if not Path(self.__args.remaining[0]).is_file(): console_error( "Your command %s doesn't point to a executable. Please verify." % self.__args.remaining[0] @@ -290,7 +293,7 @@ def run_profiling(self, version: str, prog: str): # log basic info console_log(str(prog).title() + " version: " + str(version)) console_log("Profiler choice: %s" % self.__profiler) - console_log("Path: " + str(os.path.abspath(self.__args.path))) + console_log("Path: " + str(Path(self.__args.path).absolute().resolve())) console_log("Target: " + str(self._soc._mspec.gpu_model)) console_log("Command: " + str(self.__args.remaining)) console_log("Kernel Selection: " + str(self.__args.kernel)) diff --git a/src/rocprof_compute_profile/profiler_rocprof_v1.py b/src/rocprof_compute_profile/profiler_rocprof_v1.py index 4ee1c141c..213fe4ac6 100644 --- a/src/rocprof_compute_profile/profiler_rocprof_v1.py +++ b/src/rocprof_compute_profile/profiler_rocprof_v1.py @@ -22,6 +22,7 @@ # SOFTWARE. ##############################################################################el +from pathlib import Path import os from rocprof_compute_profile.profiler_base import RocProfCompute_Base @@ -33,12 +34,12 @@ def __init__(self, profiling_args, profiler_mode, soc): super().__init__(profiling_args, profiler_mode, soc) self.ready_to_profile = ( self.get_args().roof_only - and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv")) + and not Path(self.get_args().path).joinpath("pmc_perf.csv").is_file() or not self.get_args().roof_only ) def get_profiler_options(self, fname): - fbase = os.path.splitext(os.path.basename(fname))[0] + fbase = Path(fname).stem app_cmd = self.get_args().remaining args = [ # v1 requires request for timestamps diff --git a/src/rocprof_compute_profile/profiler_rocprof_v2.py b/src/rocprof_compute_profile/profiler_rocprof_v2.py index 5f2709097..cb9a421df 100644 --- a/src/rocprof_compute_profile/profiler_rocprof_v2.py +++ b/src/rocprof_compute_profile/profiler_rocprof_v2.py @@ -23,6 +23,7 @@ ##############################################################################el import os +from pathlib import Path import shlex from rocprof_compute_profile.profiler_base import RocProfCompute_Base @@ -34,12 +35,12 @@ def __init__(self, profiling_args, profiler_mode, soc): super().__init__(profiling_args, profiler_mode, soc) self.ready_to_profile = ( self.get_args().roof_only - and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv")) + and not Path(self.get_args().path).joinpath("pmc_perf.csv").is_file() or not self.get_args().roof_only ) def get_profiler_options(self, fname): - fbase = os.path.splitext(os.path.basename(fname))[0] + fbase = Path(fname).stem app_cmd = shlex.split(self.get_args().remaining) args = [ # v2 requires output directory argument diff --git a/src/rocprof_compute_profile/profiler_rocprof_v3.py b/src/rocprof_compute_profile/profiler_rocprof_v3.py index e0d7b4b84..43f9a7ad4 100644 --- a/src/rocprof_compute_profile/profiler_rocprof_v3.py +++ b/src/rocprof_compute_profile/profiler_rocprof_v3.py @@ -23,6 +23,7 @@ ##############################################################################el import os +from pathlib import Path import shlex from rocprof_compute_profile.profiler_base import RocProfCompute_Base @@ -34,7 +35,7 @@ def __init__(self, profiling_args, profiler_mode, soc): super().__init__(profiling_args, profiler_mode, soc) self.ready_to_profile = ( self.get_args().roof_only - and not os.path.isfile(os.path.join(self.get_args().path, "pmc_perf.csv")) + and not Path(self.get_args().path).joinpath("pmc_perf.csv").is_file() or not self.get_args().roof_only ) diff --git a/src/rocprof_compute_soc/soc_base.py b/src/rocprof_compute_soc/soc_base.py index e6053c838..b73c72e19 100644 --- a/src/rocprof_compute_soc/soc_base.py +++ b/src/rocprof_compute_soc/soc_base.py @@ -53,9 +53,11 @@ def __init__( self.populate_mspec() # In some cases (i.e. --specs) path will not be given if hasattr(self.__args, "path"): - if self.__args.path == os.path.join(os.getcwd(), "workloads"): - self.__workload_dir = os.path.join( - self.__args.path, self.__args.name, self._mspec.gpu_model + if self.__args.path == str(Path(os.getcwd()).joinpath("workloads")): + self.__workload_dir = str( + Path(self.__args.path).joinpath( + self.__args.name, self._mspec.gpu_model + ) ) else: self.__workload_dir = self.__args.path @@ -196,16 +198,17 @@ def populate_mspec(self): @demarcate def perfmon_filter(self, roofline_perfmon_only: bool): """Filter default performance counter set based on user arguments""" - if roofline_perfmon_only and os.path.isfile( - os.path.join(self.get_args().path, "pmc_perf.csv") + if ( + roofline_perfmon_only + and Path(self.get_args().path).joinpath("pmc_perf.csv").is_file() ): return workload_perfmon_dir = self.__workload_dir + "/perfmon" # Initialize directories - if not os.path.isdir(self.__workload_dir): + if not Path(self.__workload_dir).is_dir(): os.makedirs(self.__workload_dir) - elif not os.path.islink(self.__workload_dir): + elif not Path(self.__workload_dir).is_symlink(): shutil.rmtree(self.__workload_dir) else: os.unlink(self.__workload_dir) @@ -226,7 +229,7 @@ def perfmon_filter(self, roofline_perfmon_only: bool): pmc_files_list = [] for fname in ref_pmc_files_list: - fbase = os.path.splitext(os.path.basename(fname))[0] + fbase = Path(fname).stem ip = re.match(mpattern, fbase).group(1) if ip in self.__args.ipblocks: pmc_files_list.append(fname) @@ -469,9 +472,10 @@ def perfmon_coalesce(pmc_files_list, perfmon_config, workload_dir, spatial_multi ) for f_idx in range(groups_per_bucket): - file_name = os.path.join( - workload_perfmon_dir, - "pmc_perf_" + "node_" + str(node_idx) + "_" + str(f_idx) + ".txt", + file_name = str( + Path(workload_perfmon_dir).joinpath( + "pmc_perf_" + "node_" + str(node_idx) + "_" + str(f_idx) + ".txt" + ) ) pmc = [] @@ -494,7 +498,7 @@ def perfmon_coalesce(pmc_files_list, perfmon_config, workload_dir, spatial_multi else: # Output to files for f in output_files: - file_name = os.path.join(workload_perfmon_dir, f.file_name) + file_name = str(Path(workload_perfmon_dir).joinpath(f.file_name)) pmc = [] for block_name in f.blocks.keys(): @@ -529,7 +533,7 @@ def perfmon_coalesce(pmc_files_list, perfmon_config, workload_dir, spatial_multi # Add a timestamp file # TODO: Does v3 need this? if not using_v3(): - fd = open(os.path.join(workload_perfmon_dir, "timestamps.txt"), "w") + fd = open(str(Path(workload_perfmon_dir).joinpath("timestamps.txt")), "w") fd.write("pmc:\n\n") fd.write("gpu:\n") fd.write("range:\n") diff --git a/src/rocprof_compute_soc/soc_gfx906.py b/src/rocprof_compute_soc/soc_gfx906.py index 367a0a6cf..ce5c488be 100644 --- a/src/rocprof_compute_soc/soc_gfx906.py +++ b/src/rocprof_compute_soc/soc_gfx906.py @@ -22,7 +22,7 @@ # SOFTWARE. ##############################################################################el -import os +from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base @@ -34,11 +34,12 @@ def __init__(self, args, mspec): super().__init__(args, mspec) self.set_arch("gfx906") self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - self.get_arch(), + str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + self.get_arch(), + ) ) ) self.set_compatible_profilers(["rocprofv1", "rocscope"]) diff --git a/src/rocprof_compute_soc/soc_gfx908.py b/src/rocprof_compute_soc/soc_gfx908.py index d58a7d392..b77e20a87 100644 --- a/src/rocprof_compute_soc/soc_gfx908.py +++ b/src/rocprof_compute_soc/soc_gfx908.py @@ -22,7 +22,7 @@ # SOFTWARE. ##############################################################################el -import os +from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base @@ -34,11 +34,12 @@ def __init__(self, args, mspec): super().__init__(args, mspec) self.set_arch("gfx908") self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - self.get_arch(), + str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + self.get_arch(), + ) ) ) self.set_compatible_profilers(["rocprofv1", "rocscope"]) diff --git a/src/rocprof_compute_soc/soc_gfx90a.py b/src/rocprof_compute_soc/soc_gfx90a.py index ceb904bcf..d9cf55feb 100644 --- a/src/rocprof_compute_soc/soc_gfx90a.py +++ b/src/rocprof_compute_soc/soc_gfx90a.py @@ -22,7 +22,7 @@ # SOFTWARE. ##############################################################################el -import os +from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base @@ -36,21 +36,23 @@ def __init__(self, args, mspec): self.set_arch("gfx90a") if hasattr(self.get_args(), "roof_only") and self.get_args().roof_only: self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - self.get_arch(), - "roofline", + str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + self.get_arch(), + "roofline", + ) ) ) else: self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - self.get_arch(), + str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + self.get_arch(), + ) ) ) self.set_compatible_profilers(["rocprofv1", "rocscope", "rocprofv2"]) @@ -96,7 +98,7 @@ def post_profiling(self): console_log( "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) - if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): + if not Path(self.get_args().path).joinpath("roofline.csv").is_file(): mibench(self.get_args(), self._mspec) self.roofline_obj.post_processing() else: diff --git a/src/rocprof_compute_soc/soc_gfx940.py b/src/rocprof_compute_soc/soc_gfx940.py index 3e5df980d..fccf551b2 100644 --- a/src/rocprof_compute_soc/soc_gfx940.py +++ b/src/rocprof_compute_soc/soc_gfx940.py @@ -22,7 +22,7 @@ # SOFTWARE. ##############################################################################el -import os +from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base @@ -36,22 +36,24 @@ def __init__(self, args, mspec): self.set_arch("gfx940") if hasattr(self.get_args(), "roof_only") and self.get_args().roof_only: self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - "gfx940", - "roofline", + str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + "gfx940", + "roofline", + ) ) ) else: # NB: We're using generalized Mi300 perfmon configs self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - "gfx940", + set( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + "gfx940", + ) ) ) self.set_compatible_profilers(["rocprofv1", "rocprofv2"]) @@ -97,7 +99,7 @@ def post_profiling(self): console_log( "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) - if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): + if not Path(self.get_args().path).joinpath("roofline.csv").is_file(): mibench(self.get_args(), self._mspec) self.roofline_obj.post_processing() else: diff --git a/src/rocprof_compute_soc/soc_gfx941.py b/src/rocprof_compute_soc/soc_gfx941.py index 412e3b15f..750975a1e 100644 --- a/src/rocprof_compute_soc/soc_gfx941.py +++ b/src/rocprof_compute_soc/soc_gfx941.py @@ -22,7 +22,7 @@ # SOFTWARE. ##############################################################################el -import os +from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base @@ -36,22 +36,24 @@ def __init__(self, args, mspec): self.set_arch("gfx941") if hasattr(self.get_args(), "roof_only") and self.get_args().roof_only: self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - "gfx940", - "roofline", + str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + "gfx940", + "roofline", + ) ) ) else: # NB: We're using generalized Mi300 perfmon configs self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - "gfx940", + str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + "gfx940", + ) ) ) self.set_compatible_profilers(["rocprofv1", "rocprofv2"]) @@ -97,7 +99,7 @@ def post_profiling(self): console_log( "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) - if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): + if not Path(self.get_args().path).joinpath("roofline.csv").is_file(): mibench(self.get_args(), self._mspec) self.roofline_obj.post_processing() else: diff --git a/src/rocprof_compute_soc/soc_gfx942.py b/src/rocprof_compute_soc/soc_gfx942.py index b2b756672..41bb6c7ad 100644 --- a/src/rocprof_compute_soc/soc_gfx942.py +++ b/src/rocprof_compute_soc/soc_gfx942.py @@ -22,7 +22,7 @@ # SOFTWARE. ##############################################################################el -import os +from pathlib import Path import config from rocprof_compute_soc.soc_base import OmniSoC_Base @@ -36,22 +36,24 @@ def __init__(self, args, mspec): self.set_arch("gfx942") if hasattr(self.get_args(), "roof_only") and self.get_args().roof_only: self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - "gfx940", - "roofline", + str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + "gfx940", + "roofline", + ) ) ) else: # NB: We're using generalized Mi300 perfmon configs self.set_perfmon_dir( - os.path.join( - str(config.rocprof_compute_home), - "rocprof_compute_soc", - "profile_configs", - "gfx940", + str( + Path(str(config.rocprof_compute_home)).joinpath( + "rocprof_compute_soc", + "profile_configs", + "gfx940", + ) ) ) self.set_compatible_profilers(["rocprofv1", "rocprofv2", "rocprofv3"]) @@ -102,7 +104,7 @@ def post_profiling(self): console_log( "roofline", "Checking for roofline.csv in " + str(self.get_args().path) ) - if not os.path.isfile(os.path.join(self.get_args().path, "roofline.csv")): + if not Path(self.get_args().path).joinpath("roofline.csv").is_file(): mibench(self.get_args(), self._mspec) self.roofline_obj.post_processing() else: diff --git a/src/roofline.py b/src/roofline.py index 61d734fe4..3ced336e0 100644 --- a/src/roofline.py +++ b/src/roofline.py @@ -23,6 +23,7 @@ ##############################################################################el import os +from pathlib import Path import time from abc import ABC, abstractmethod @@ -90,16 +91,17 @@ def validate_parameters(self): def roof_setup(self): # set default workload path if not specified - if self.__run_parameters["workload_dir"] == os.path.join( - os.getcwd(), "workloads" + if self.__run_parameters["workload_dir"] == str( + Path(os.getcwd()).joinpath("workloads") ): - self.__run_parameters["workload_dir"] = os.path.join( - self.__run_parameters["workload_dir"], - self.__args.name, - self.__mspec.gpu_series, + self.__run_parameters["workload_dir"] = str( + Path(self.__run_parameters["workload_dir"]).joinpath( + self.__args.name, + self.__mspec.gpu_model, + ) ) # create new directory for roofline if it doesn't exist - if not os.path.isdir(self.__run_parameters["workload_dir"]): + if not Path(self.__run_parameters["workload_dir"]).is_dir(): os.makedirs(self.__run_parameters["workload_dir"]) @demarcate @@ -369,8 +371,10 @@ def standalone_roofline(self): self.__run_parameters["mem_level"].remove("vL1D") self.__run_parameters["mem_level"].append("L1") - app_path = os.path.join(self.__run_parameters["workload_dir"], "pmc_perf.csv") - roofline_exists = os.path.isfile(app_path) + app_path = str( + Path(self.__run_parameters["workload_dir"]).joinpath("pmc_perf.csv") + ) + roofline_exists = Path(app_path).is_file() if not roofline_exists: console_error("roofline", "{} does not exist".format(app_path)) t_df = OrderedDict() @@ -385,8 +389,8 @@ def pre_processing(self): console_log( "roofline", "Checking for sysinfo.csv in " + str(self.__args.path) ) - sysinfo_path = os.path.join(self.__args.path, "sysinfo.csv") - if not os.path.isfile(sysinfo_path): + sysinfo_path = str(Path(self.__args.path).joinpath("sysinfo.csv")) + if not Path(sysinfo_path).is_file(): console_log("roofline", "sysinfo.csv not found. Generating...") class Dummy_SoC: @@ -410,16 +414,16 @@ def profile(self): console_log( "roofline", "Checking for roofline.csv in " + str(self.__args.path) ) - roof_path = os.path.join(self.__args.path, "roofline.csv") - if not os.path.isfile(roof_path): + roof_path = str(Path(self.__args.path).joinpath("roofline.csv")) + if not Path(roof_path).is_file(): mibench(self.__args, self.__mspec) # check for profiling data console_log( "roofline", "Checking for pmc_perf.csv in " + str(self.__args.path) ) - app_path = os.path.join(self.__args.path, "pmc_perf.csv") - if not os.path.isfile(app_path): + app_path = str(Path(self.__args.path).joinpath("pmc_perf.csv")) + if not Path(app_path).is_file(): console_log("roofline", "pmc_perf.csv not found. Generating...") if not self.__args.remaining: console_error( diff --git a/src/utils/db_connector.py b/src/utils/db_connector.py index df21f0772..26c2ba618 100644 --- a/src/utils/db_connector.py +++ b/src/utils/db_connector.py @@ -24,6 +24,7 @@ import getpass import os +from pathlib import Path from abc import ABC, abstractmethod import pandas as pd @@ -64,8 +65,8 @@ def __init__(self, args): @demarcate def prep_import(self): # Extract SoC and workload name from sysinfo.csv - sys_info = os.path.join(self.connection_info["workload"], "sysinfo.csv") - if os.path.isfile(sys_info): + sys_info = str(Path(self.connection_info["workload"]).joinpath("sysinfo.csv")) + if Path(sys_info).is_file(): sys_info = pd.read_csv(sys_info) try: soc = sys_info["gpu_model"][0].strip() @@ -193,7 +194,7 @@ def pre_processing(self): % self.interaction_type ) - if os.path.isdir(os.path.abspath(self.connection_info["workload"])): + if Path(self.connection_info["workload"]).absolute().is_dir(): is_workload_empty(self.connection_info["workload"]) else: console_error( @@ -204,8 +205,8 @@ def pre_processing(self): console_error("--team exceeds 13 character limit. Try again.") # format path properly - self.connection_info["workload"] = os.path.abspath( - self.connection_info["workload"] + self.connection_info["workload"] = str( + Path(self.connection_info["workload"]).absolute().resolve() ) # Detect password diff --git a/src/utils/file_io.py b/src/utils/file_io.py index 22998cd70..c1932f95d 100644 --- a/src/utils/file_io.py +++ b/src/utils/file_io.py @@ -73,7 +73,7 @@ def load_panel_configs(dir): for root, dirs, files in os.walk(dir): for f in files: if f.endswith(".yaml"): - with open(os.path.join(root, f)) as file: + with open(str(Path(root).joinpath(f))) as file: config = yaml.safe_load(file) d[config["Panel Config"]["id"]] = config["Panel Config"] @@ -129,7 +129,9 @@ def create_df_kernel_top_stats( if "Node" in df.columns else df.loc[:, ["Dispatch_ID", "Kernel_Name", "GPU_ID"]] ) - dispatch_info.to_csv(os.path.join(raw_data_dir, "pmc_dispatch_info.csv"), index=False) + dispatch_info.to_csv( + str(Path(raw_data_dir).joinpath("pmc_dispatch_info.csv")), index=False + ) time_stats = pd.concat( [df["Kernel_Name"], (df["End_Timestamp"] - df["Start_Timestamp"])], @@ -163,10 +165,14 @@ def create_df_kernel_top_stats( # Sort by total time as default. if sortby == "sum": grouped = grouped.sort_values(by=("Sum" + time_unit_str), ascending=False) - grouped.to_csv(os.path.join(raw_data_dir, "pmc_kernel_top.csv"), index=False) + grouped.to_csv( + str(Path(raw_data_dir).joinpath("pmc_kernel_top.csv")), index=False + ) elif sortby == "kernel": grouped = grouped.sort_values("Kernel_Name") - grouped.to_csv(os.path.join(raw_data_dir, "pmc_kernel_top.csv"), index=False) + grouped.to_csv( + str(Path(raw_data_dir).joinpath("pmc_kernel_top.csv")), index=False + ) @demarcate @@ -187,7 +193,7 @@ def create_single_df_pmc(raw_data_dir, node_name, kernel_verbose, verbose): if (f.endswith(".csv") and f.startswith("SQ")) or ( f == schema.pmc_perf_file_prefix + ".csv" ): - tmp_df = pd.read_csv(os.path.join(root, f)) + tmp_df = pd.read_csv(str(Path(root).joinpath(f))) # Demangle original KernelNames kernel_name_shortener(tmp_df, kernel_verbose) diff --git a/src/utils/kernel_name_shortener.py b/src/utils/kernel_name_shortener.py index 797024c63..831c731ae 100644 --- a/src/utils/kernel_name_shortener.py +++ b/src/utils/kernel_name_shortener.py @@ -22,7 +22,7 @@ # SOFTWARE. ##############################################################################el -import os +from pathlib import Path import re import subprocess @@ -119,8 +119,8 @@ def shorten_file(df, level): # Only shorten if valid shortening level if level < 5: - cpp_filt = os.path.join("/usr", "bin", "c++filt") - if not os.path.isfile(cpp_filt): + cpp_filt = str(Path("/usr").joinpath("bin", "c++filt")) + if not Path(cpp_filt).is_file(): console_error( "Could not resolve c++filt in expected directory: %s" % cpp_filt ) diff --git a/src/utils/logger.py b/src/utils/logger.py index 7b40c6eda..0a0ba13b6 100644 --- a/src/utils/logger.py +++ b/src/utils/logger.py @@ -24,6 +24,7 @@ import logging import os +from pathlib import Path import sys from utils.utils import trace_logger @@ -111,7 +112,7 @@ def setup_console_handler(): # Setup file handler - enabled in profile mode def setup_file_handler(loglevel, workload_dir): - filename = os.path.join(workload_dir, "log.txt") + filename = str(Path(workload_dir).joinpath("log.txt")) file_handler = logging.FileHandler(filename, "w") file_loglevel = min([loglevel, logging.INFO]) file_handler.setLevel(file_loglevel) diff --git a/src/utils/parser.py b/src/utils/parser.py index 8be10a296..85c891c79 100644 --- a/src/utils/parser.py +++ b/src/utils/parser.py @@ -23,7 +23,6 @@ ##############################################################################el import ast -import os import re import warnings from pathlib import Path @@ -873,7 +872,7 @@ def apply_filters(workload, dir, is_gui, debug): if workload.filter_kernel_ids: if all(type(kid) == int for kid in workload.filter_kernel_ids): # Verify valid kernel filter - kernels_df = pd.read_csv(os.path.join(dir, "pmc_kernel_top.csv")) + kernels_df = pd.read_csv(str(Path(dir).joinpath("pmc_kernel_top.csv"))) for kernel_id in workload.filter_kernel_ids: if kernel_id >= len(kernels_df["Kernel_Name"]): console_error( diff --git a/src/utils/roofline_calc.py b/src/utils/roofline_calc.py index c318b66e3..1aabbce52 100644 --- a/src/utils/roofline_calc.py +++ b/src/utils/roofline_calc.py @@ -23,7 +23,7 @@ ##############################################################################el import csv -import os +from pathlib import Path from dataclasses import dataclass from utils.utils import console_debug @@ -500,7 +500,9 @@ def calc_ai(mspec, sort_type, ret_df): def constuct_roof(roofline_parameters, dtype): - benchmark_results = os.path.join(roofline_parameters["workload_dir"], "roofline.csv") + benchmark_results = str( + Path(roofline_parameters["workload_dir"]).joinpath("roofline.csv") + ) # ----------------------------------------------------- # Initialize roofline data dictionary from roofline.csv # ----------------------------------------------------- diff --git a/src/utils/specs.py b/src/utils/specs.py index 70e454a85..a43bc2add 100644 --- a/src/utils/specs.py +++ b/src/utils/specs.py @@ -594,8 +594,8 @@ def __repr__(self): def get_rocm_ver(): rocm_found = False for itr in VERSION_LOC: - _path = os.path.join(os.getenv("ROCM_PATH", "/opt/rocm"), ".info", itr) - if os.path.exists(_path): + _path = str(path(os.getenv("ROCM_PATH", "/opt/rocm")).joinpath(".info", itr)) + if path(_path).exists(): rocm_ver = path(_path).read_text() rocm_found = True break diff --git a/src/utils/utils.py b/src/utils/utils.py index 8928b2a92..0911e7fcf 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -104,7 +104,7 @@ def get_version(rocprof_compute_home) -> dict: versionDir = None for dir in searchDirs: - version = os.path.join(dir, "VERSION") + version = str(path(dir).joinpath("VERSION")) try: with open(version, "r") as file: VER = file.read().replace("\n", "") @@ -117,8 +117,8 @@ def get_version(rocprof_compute_home) -> dict: console_error("Cannot find VERSION file at {}".format(searchDirs)) # git version info - gitDir = os.path.join(rocprof_compute_home.parent, ".git") - if (shutil.which("git") is not None) and os.path.exists(gitDir): + gitDir = str(path(rocprof_compute_home.parent).joinpath(".git")) + if (shutil.which("git") is not None) and path(gitDir).exists(): gitQuery = subprocess.run( ["git", "log", "--pretty=format:%h", "-n", "1"], stdout=subprocess.PIPE, @@ -131,7 +131,7 @@ def get_version(rocprof_compute_home) -> dict: SHA = gitQuery.stdout.decode("utf-8") MODE = "dev" else: - shaFile = os.path.join(versionDir, "VERSION.sha") + shaFile = str(path(versionDir).joinpath("VERSION.sha")) try: with open(shaFile, "r") as file: SHA = file.read().replace("\n", "") @@ -179,7 +179,7 @@ def detect_rocprof(): ) else: # Resolve any sym links in file path - rocprof_path = os.path.realpath(rocprof_path.rstrip("\n")) + rocprof_path = str(path(rocprof_path.rstrip("\n")).resolve()) console_debug("ROC Profiler: " + str(rocprof_path)) console_debug("rocprof_cmd is {}".format(str(rocprof_cmd))) @@ -562,9 +562,9 @@ def run_prof( fname, profiler_options, workload_dir, mspec, loglevel, format_rocprof_output ): time_0 = time.time() - fbase = os.path.splitext(os.path.basename(fname))[0] + fbase = path(fname).stem - console_debug("pmc file: %s" % str(os.path.basename(fname))) + console_debug("pmc file: %s" % path(fname).name) # standard rocprof options default_options = ["-i", fname] @@ -578,11 +578,11 @@ def run_prof( or mspec.gpu_model.lower() == "mi300a_a0" or mspec.gpu_model.lower() == "mi300a_a1" ) and ( - os.path.basename(fname) == "pmc_perf_13.txt" - or os.path.basename(fname) == "pmc_perf_14.txt" - or os.path.basename(fname) == "pmc_perf_15.txt" - or os.path.basename(fname) == "pmc_perf_16.txt" - or os.path.basename(fname) == "pmc_perf_17.txt" + path(fname).name == "pmc_perf_13.txt" + or path(fname).name == "pmc_perf_14.txt" + or path(fname).name == "pmc_perf_15.txt" + or path(fname).name == "pmc_perf_16.txt" + or path(fname).name == "pmc_perf_17.txt" ): new_env = os.environ.copy() new_env["ROCPROFILER_INDIVIDUAL_XCC_MODE"] = "1" @@ -642,30 +642,32 @@ def run_prof( workload_dir + "/out/pmc_1/*/*_counter_collection.csv" ) existing_counter_files_csv = [ - d for d in counter_info_csvs if os.path.isfile(d) + d for d in counter_info_csvs if path(d).is_file() ] if len(existing_counter_files_csv) > 0: for counter_file in existing_counter_files_csv: - current_dir = os.path.dirname(counter_file) - agent_info_filepath = os.path.join( - current_dir, - os.path.basename(counter_file).replace( - "_counter_collection", "_agent_info" - ), + current_dir = str(path(counter_file).parent) + agent_info_filepath = str( + path(current_dir).joinpath( + path(counter_file).name.replace( + "_counter_collection", "_agent_info" + ) + ) ) - if not os.path.isfile(agent_info_filepath): + if not path(agent_info_filepath).is_file(): raise ValueError( '{} has no coresponding "agent info" file'.format( counter_file ) ) - converted_csv_file = os.path.join( - current_dir, - os.path.basename(counter_file).replace( - "_counter_collection", "_converted" - ), + converted_csv_file = str( + path(current_dir).joinpath( + path(counter_file).name.replace( + "_counter_collection", "_converted" + ) + ) ) v3_counter_csv_to_v2_csv( @@ -702,7 +704,7 @@ def run_prof( df = flatten_tcc_info_across_xcds(f, xcds, int(mspec._l2_banks)) df.to_csv(f, index=False) - if os.path.exists(workload_dir + "/out"): + if path(workload_dir + "/out").exists(): # copy and remove out directory if needed shutil.copyfile( workload_dir + "/out/pmc_1/results_" + fbase + ".csv", @@ -791,7 +793,7 @@ def detect_roofline(mspec): if "ROOFLINE_BIN" in os.environ.keys(): rooflineBinary = os.environ["ROOFLINE_BIN"] - if os.path.exists(rooflineBinary): + if path(rooflineBinary).exists(): console_warning("roofline", "Detected user-supplied binary") return { "rocm_ver": "override", @@ -886,7 +888,7 @@ def mibench(args, mspec): # Distro is valid but cant find rocm ver found = False for path in binary_paths: - if os.path.exists(path): + if pathlib.Path(path).exists(): found = True path_to_binary = path break @@ -1040,7 +1042,7 @@ def get_submodules(package_name): def is_workload_empty(path): """Peek workload directory to verify valid profiling output""" pmc_perf_path = path + "/pmc_perf.csv" - if os.path.isfile(pmc_perf_path): + if pathlib.Path(pmc_perf_path).is_file(): temp_df = pd.read_csv(pmc_perf_path) if temp_df.dropna().empty: console_error( diff --git a/tests/test_analyze_commands.py b/tests/test_analyze_commands.py index 5bcee3ad8..1ef861b55 100644 --- a/tests/test_analyze_commands.py +++ b/tests/test_analyze_commands.py @@ -1,7 +1,6 @@ -import os.path +import os import shutil from importlib.machinery import SourceFileLoader -from pathlib import Path from unittest.mock import patch import pandas as pd diff --git a/tests/test_analyze_workloads.py b/tests/test_analyze_workloads.py index 3acc7b920..3b0b315ee 100644 --- a/tests/test_analyze_workloads.py +++ b/tests/test_analyze_workloads.py @@ -1,11 +1,9 @@ import inspect -import os.path import re import shutil import subprocess import sys from importlib.machinery import SourceFileLoader -from pathlib import Path from unittest.mock import patch import pandas as pd diff --git a/tests/test_profile_general.py b/tests/test_profile_general.py index 3051ced56..05a583dae 100644 --- a/tests/test_profile_general.py +++ b/tests/test_profile_general.py @@ -1,5 +1,5 @@ import inspect -import os.path +import os import re import shutil import subprocess @@ -321,7 +321,7 @@ def gpu_soc(): if "MI300" in soc: os.environ["ROCPROF"] = "rocprofv2" -Baseline_dir = os.path.realpath("tests/workloads/vcopy/" + soc) +Baseline_dir = str(Path("tests/workloads/vcopy/" + soc).resolve()) def log_counter(file_dict, test_name): @@ -334,9 +334,9 @@ def log_counter(file_dict, test_name): errors = counter_compare(test_name, pd.DataFrame(), df_1, df_2, 5) if not errors.empty: - if os.path.exists( + if Path( Baseline_dir + "/" + file.split(".")[0] + "_error_log.csv" - ): + ).exists(): error_log = pd.read_csv( Baseline_dir + "/" + file.split(".")[0] + "_error_log.csv", index_col=0, @@ -376,7 +376,7 @@ def baseline_compare_metric(test_name, workload_dir, args=[]): if "DEBUG ERROR" in captured_output: error_df = pd.DataFrame() - if os.path.exists(Baseline_dir + "/metric_error_log.csv"): + if Path(Baseline_dir + "/metric_error_log.csv").exists(): error_df = pd.read_csv( Baseline_dir + "/metric_error_log.csv", index_col=0, diff --git a/tests/test_utils.py b/tests/test_utils.py index 131032cb7..cfdc1e157 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -25,6 +25,7 @@ import inspect import os +from pathlib import Path import shutil from unittest.mock import patch @@ -62,7 +63,7 @@ def get_output_dir(suffix="_output", clean_existing=True): output_dir = inspect.stack()[1].function + suffix if clean_existing: - if os.path.exists(output_dir): + if Path(output_dir).exists(): shutil.rmtree(output_dir) return output_dir @@ -79,7 +80,7 @@ def setup_workload_dir(input_dir, suffix="_tmp", clean_existing=True): output_dir = inspect.stack()[1].function + suffix if clean_existing: - if os.path.exists(output_dir): + if Path(output_dir).exists(): shutil.rmtree(output_dir) shutil.copytree(input_dir, output_dir) @@ -94,7 +95,7 @@ def clean_output_dir(cleanup, output_dir): output_dir (string): name of directory to remove """ if cleanup: - if os.path.exists(output_dir): + if Path(output_dir).exists(): try: shutil.rmtree(output_dir) except OSError as e: diff --git a/utils/update_license.py b/utils/update_license.py index 594b8dc44..6cf7ffb48 100755 --- a/utils/update_license.py +++ b/utils/update_license.py @@ -8,6 +8,7 @@ import glob import logging import os +from pathlib import Path import re import shutil import sys @@ -18,7 +19,7 @@ def cacheLicenseFile(infile, comment="#"): - if not os.path.isfile(infile): + if not Path(infile).is_file(): logging.error("Unable to access license file - >%s" % infile) sys.exit(1) @@ -66,7 +67,7 @@ def cacheLicenseFile(infile, comment="#"): # Scan files in provided source directory... for filename in glob.iglob(srcDir + "/**", recursive=True): # skip directories - if os.path.isdir(filename): + if Path(filename).is_dir(): continue # File matching options: @@ -80,7 +81,7 @@ def cacheLicenseFile(infile, comment="#"): if specificFiles: found = False for file in specificFiles: - fullPath = os.path.join(srcDir, file) + fullPath = str(Path(srcDir).joinpath(file)) if fullPath == filename: found = True break @@ -91,8 +92,8 @@ def cacheLicenseFile(infile, comment="#"): # Update license header contents if delimiters are found with open(filename, "r") as file_in: - baseName = os.path.basename(filename) - dirName = os.path.dirname(filename) + baseName = Path(filename).name + dirName = str(Path(filename).parent) tmpFile = dirName + "/." + baseName + ".tmp" file_out = open(tmpFile, "w") diff --git a/utils/ver_check.py b/utils/ver_check.py index f30afe59f..f34169fe8 100755 --- a/utils/ver_check.py +++ b/utils/ver_check.py @@ -4,13 +4,13 @@ # release pipeline. import argparse -import os +from pathlib import Path parser = argparse.ArgumentParser() parser.add_argument("--tag", type=str, required=True, help="tagname to check") args = parser.parse_args() -execPath = os.path.dirname(__file__) +execPath = str(Path(__file__).parent) with open(execPath + "/../VERSION") as f: repoVer = f.readline().strip()