Skip to content

Commit a7090eb

Browse files
jay-taujay-tau
committed
feat: Export directory prefix
1 parent 3f83dcc commit a7090eb

File tree

4 files changed

+113
-80
lines changed

4 files changed

+113
-80
lines changed

‎drishti/handlers/handle_darshan.py

+12-14
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
#!/usr/bin/env python3
22

33
import io
4-
import sys
5-
import time
64
import shlex
75
import shutil
86
import subprocess
9-
import pandas as pd
7+
import sys
8+
import time
9+
1010
import darshan
1111
import darshan.backend.cffi_backend as darshanll
12-
13-
from rich import print
12+
import pandas as pd
1413
from packaging import version
14+
from rich import print
15+
1516
from drishti.includes.module import *
1617

1718

@@ -752,15 +753,12 @@ def handler():
752753
display_thresholds(console)
753754
display_footer(console, insights_start_time, insights_end_time)
754755

755-
filename = '{}.html'.format(args.log_path)
756-
export_html(console, filename)
756+
input_filename = os.path.basename(args.log_path).replace('.darshan', '')
757+
out_dir = args.export_dir if args.export_dir != "" else os.getcwd()
757758

758-
filename = '{}.svg'.format(args.log_path)
759-
export_svg(console, filename)
759+
print(f"DEBUG: outfile_name: {input_filename}")
760760

761-
filename = '{}-summary.csv'.format(
762-
args.log_path.replace('.darshan', '')
763-
)
764-
765-
export_csv(filename, job['job']['jobid'])
761+
export_html(console, out_dir, input_filename)
762+
export_svg(console, out_dir, input_filename)
766763

764+
export_csv(out_dir, input_filename, job['job']['jobid'])

‎drishti/includes/config.py

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
#!/usr/bin/env python3
22

3-
import os
43
import json
4+
import os
55

66
from rich.console import Console, Group
77
from rich.padding import Padding
88
from rich.panel import Panel
9-
from rich.terminal_theme import TerminalTheme
10-
from rich.terminal_theme import MONOKAI
9+
from rich.terminal_theme import MONOKAI, TerminalTheme
1110

1211
from drishti.includes.parser import *
1312

‎drishti/includes/module.py

+88-63
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
#!/usr/bin/env python3
22

3-
import datetime
43
import csv
4+
import datetime
55
import time
6+
67
import pandas as pd
78
from rich import box
89
from rich.syntax import Syntax
10+
911
from drishti.includes.config import *
1012

1113
'''
@@ -1823,76 +1825,99 @@ def display_footer(console, insights_start_time, insights_end_time):
18231825
)
18241826
)
18251827

1826-
def export_html(console, filename):
1828+
def export_html(console, export_dir, filename):
18271829
'''
18281830
'''
18291831

1830-
if args.export_html:
1831-
console.save_html(
1832-
filename,
1833-
theme=set_export_theme(),
1834-
clear=False
1835-
)
1832+
if not args.export_html:
1833+
print("DEBUG: export_html() - return")
1834+
return
18361835

1836+
os.makedirs(export_dir, exist_ok=True)
1837+
filepath = os.path.join(export_dir, f"{filename}.html")
18371838

1838-
def export_svg(console, filename):
1839-
if args.export_svg:
1840-
console.save_svg(
1841-
filename,
1842-
title='Drishti',
1843-
theme=set_export_theme(),
1844-
clear=False
1845-
)
1839+
console.save_html(
1840+
filepath,
1841+
theme=set_export_theme(),
1842+
clear=False
1843+
)
18461844

1845+
print("DEBUG: END export_html()")
18471846

1848-
def export_csv(filename, jobid=None):
1849-
if args.export_csv:
1850-
issues = [
1851-
'JOB',
1852-
INSIGHTS_STDIO_HIGH_USAGE,
1853-
INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE,
1854-
INSIGHTS_POSIX_READ_COUNT_INTENSIVE,
1855-
INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE,
1856-
INSIGHTS_POSIX_READ_SIZE_INTENSIVE,
1857-
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE,
1858-
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE,
1859-
INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE,
1860-
INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE,
1861-
INSIGHTS_POSIX_REDUNDANT_READ_USAGE,
1862-
INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE,
1863-
INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE,
1864-
INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE,
1865-
INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE,
1866-
INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE,
1867-
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE,
1868-
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE,
1869-
INSIGHTS_POSIX_HIGH_METADATA_TIME,
1870-
INSIGHTS_POSIX_SIZE_IMBALANCE,
1871-
INSIGHTS_POSIX_TIME_IMBALANCE,
1872-
INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE,
1873-
INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE,
1874-
INSIGHTS_MPI_IO_NO_USAGE,
1875-
INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE,
1876-
INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE,
1877-
INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE,
1878-
INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE,
1879-
INSIGHTS_MPI_IO_BLOCKING_READ_USAGE,
1880-
INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE,
1881-
INSIGHTS_MPI_IO_AGGREGATORS_INTRA,
1882-
INSIGHTS_MPI_IO_AGGREGATORS_INTER,
1883-
INSIGHTS_MPI_IO_AGGREGATORS_OK
1884-
]
1885-
if codes:
1886-
issues.extend(codes)
18871847

1888-
detected_issues = dict.fromkeys(issues, False)
1889-
detected_issues['JOB'] = jobid
1848+
def export_svg(console, export_dir, filename):
1849+
if not args.export_svg:
1850+
return
1851+
1852+
os.makedirs(export_dir, exist_ok=True)
1853+
filepath = os.path.join(export_dir, f"{filename}.svg")
1854+
1855+
console.save_svg(
1856+
filepath,
1857+
title='Drishti',
1858+
theme=set_export_theme(),
1859+
clear=False
1860+
)
18901861

1891-
for report in csv_report:
1892-
detected_issues[report] = True
18931862

1894-
with open(filename, 'w') as f:
1895-
w = csv.writer(f)
1896-
w.writerow(detected_issues.keys())
1897-
w.writerow(detected_issues.values())
1863+
def export_csv(export_dir, filename, jobid=None):
1864+
if not args.export_csv:
1865+
return
1866+
1867+
issues = [
1868+
'JOB',
1869+
INSIGHTS_STDIO_HIGH_USAGE,
1870+
INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE,
1871+
INSIGHTS_POSIX_READ_COUNT_INTENSIVE,
1872+
INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE,
1873+
INSIGHTS_POSIX_READ_SIZE_INTENSIVE,
1874+
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE,
1875+
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE,
1876+
INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE,
1877+
INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE,
1878+
INSIGHTS_POSIX_REDUNDANT_READ_USAGE,
1879+
INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE,
1880+
INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE,
1881+
INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE,
1882+
INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE,
1883+
INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE,
1884+
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE,
1885+
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE,
1886+
INSIGHTS_POSIX_HIGH_METADATA_TIME,
1887+
INSIGHTS_POSIX_SIZE_IMBALANCE,
1888+
INSIGHTS_POSIX_TIME_IMBALANCE,
1889+
INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE,
1890+
INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE,
1891+
INSIGHTS_MPI_IO_NO_USAGE,
1892+
INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE,
1893+
INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE,
1894+
INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE,
1895+
INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE,
1896+
INSIGHTS_MPI_IO_BLOCKING_READ_USAGE,
1897+
INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE,
1898+
INSIGHTS_MPI_IO_AGGREGATORS_INTRA,
1899+
INSIGHTS_MPI_IO_AGGREGATORS_INTER,
1900+
INSIGHTS_MPI_IO_AGGREGATORS_OK
1901+
]
1902+
if codes:
1903+
issues.extend(codes)
1904+
1905+
detected_issues = dict.fromkeys(issues, False)
1906+
detected_issues['JOB'] = jobid
1907+
1908+
for report in csv_report:
1909+
detected_issues[report] = True
1910+
1911+
# ensure dir exists
1912+
os.makedirs(export_dir, exist_ok=True)
1913+
filepath = os.path.join(export_dir, f"{filename}.csv")
1914+
1915+
print(f"DEBUG: export_dir: {export_dir}")
1916+
print(f"DEBUG: filename: {filename}")
1917+
print(f"DEBUG: filepath: {filepath}")
1918+
1919+
with open(filepath, 'w') as f:
1920+
w = csv.writer(f)
1921+
w.writerow(detected_issues.keys())
1922+
w.writerow(detected_issues.values())
18981923

‎drishti/includes/parser.py

+11
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@
9696
help='Export a CSV with the code of all issues that were triggered'
9797
)
9898

99+
parser.add_argument(
100+
'--export_dir',
101+
default="",
102+
dest='export_dir',
103+
help='Specify the directory prefix for the output files (if any)'
104+
)
105+
99106
parser.add_argument(
100107
'--json',
101108
default=False,
@@ -119,3 +126,7 @@
119126
)
120127

121128
args = parser.parse_args()
129+
130+
print(f"DEBUG: log_path: {args.log_path}")
131+
print(f"DEBUG: export_path: {args.export_dir}")
132+
print(f"DEBUG: export_csv: {args.export_csv}")

0 commit comments

Comments
 (0)