Skip to content

Commit

Permalink
allow to use YAML files to configure the entire plotting procedures
Browse files Browse the repository at this point in the history
add sample csv, yml and svg files
  • Loading branch information
skaupper committed Feb 4, 2021
1 parent 78c17d7 commit a69056e
Show file tree
Hide file tree
Showing 14 changed files with 2,808 additions and 66 deletions.
7 changes: 4 additions & 3 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@
"console": "integratedTerminal",
"args": [
"plot",
"~/dumps/0409-4-enc0.csv",
"master_angle"
]
"-c",
"angle_data_cfg.yml"
],
"cwd": "${workspaceFolder}/doc"
}
]
}
3 changes: 2 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@
"editor.formatOnSave": true,
"editor.insertSpaces": true,
"editor.tabSize": 4,
"python.venvPath": "${workspaceFolder}/venv"
"python.venvPath": "${workspaceFolder}/venv",
"python.pythonPath": "${workspaceFolder}/venv/bin/python"
}
68 changes: 49 additions & 19 deletions CsvPlotter/entrypoints.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import signal
import yaml
from math import log10, ceil

from .internal import argument_parser, csv_handling, plotting
from .internal import configuration as cfg
from .internal.util_classes import SampleRange
from .internal.samplerange import SampleRange


#
Expand Down Expand Up @@ -32,32 +32,62 @@ def __resolve_column_id(data_obj, col_id):


def __handle_plot_args(args):
# Make sure Ctrl+C in the terminal closes the plot
signal.signal(signal.SIGINT, signal.SIG_DFL)

sample_range = SampleRange(args.region[0], args.region[1],
args.divider)
config = cfg.PlotConfig(args.filename, sample_range, args.output_file)

data_obj = csv_handling.CsvData.from_file(config)
if 'yaml_config' in args and args.yaml_config is not None:
# Construct config from YAML file and update values with command line arguments
try:
with open(args.yaml_config, 'r') as f:
yaml_config = yaml.safe_load(f)
except yaml.YAMLError as ex:
print(f'Failed to load config file "{args.yaml_config}": {ex}')
return

plot_cfg = cfg.PlotConfig.from_obj(yaml_config)

if 'input_file' in args and args.input_file is not None:
plot_cfg.input_file = args.input_file
if 'output_file' in args and args.output_file is not None:
plot_cfg.output_file = args.output_file
if 'range' in args and args.range is not None:
plot_cfg.range.start = args.range[0]
plot_cfg.range.end = args.range[1]
if 'divider' in args and args.divider is not None:
plot_cfg.range.divider = args.divider

else:
# Construct config from command line arguments
plot_cfg = cfg.PlotConfig.from_obj({
'input_file': args.input_file,
'output_file': args.output_file,
'range_start': args.range[0],
'range_end': args.range[1],
'divider': args.divider,
})

# Load data into RAM
data_obj = csv_handling.CsvData.from_file(plot_cfg)
if data_obj.size == 0:
return

for col_id in args.columns:
col_name = __resolve_column_id(data_obj, col_id)
if col_name is None:
print(f'Unable to resolve column "{col_id}"! '
'Make sure you either pass a column name or an index!')
continue
# Resolve column identifiers given as command line arguments and add the constructed subplot
if len(args.columns) > 0:
subplot_cfg = cfg.SubplotConfig()
for col_id in args.columns:
col_name = __resolve_column_id(data_obj, col_id)
if col_name is None:
print(f'Unable to resolve column "{col_id}"! '
'Make sure you either pass a column name or an index!')
continue

config.add_column_to_subplot(cfg.ColumnConfig(col_name))
subplot_cfg.add_column(cfg.ColumnConfig(col_name))
plot_cfg.add_subplot(subplot_cfg)

plotting.plot_csv_data(data_obj, config)
# Plot data
plotting.plot_csv_data(data_obj, plot_cfg)


def __handle_util_args(args):
if args.list_headers:
headers = csv_handling.read_headers(args.filename)
headers = csv_handling.read_headers(args.input_file)
print(f'Column headers found: {len(headers)}')
idx_width = ceil(log10(len(headers)))
for i, header in enumerate(headers):
Expand Down
21 changes: 15 additions & 6 deletions CsvPlotter/internal/argument_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ def __region_check(v):
# sets END to the index of the last sample).
# Omitting either of the values will set its value to default (0 for START and -1 for END).

if v is None:
return None

error = True
try:
v = str(v)
Expand Down Expand Up @@ -52,7 +55,8 @@ def __region_check(v):

def __create_common_parser():
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument('filename', help='CSV debug data')
parser.add_argument('-i', '--input-file',
help='CSV debug data', required=False)
return parser


Expand All @@ -64,16 +68,21 @@ def create_plot_parser(generate_help=True):
parser = argparse.ArgumentParser(description='Analyse the given CSV file.', parents=[
__create_common_parser()], add_help=generate_help)

parser.add_argument('columns', type=str, nargs='+',
help='A list of columns that should be plotted. Either the column name or index are valid.', default=[])
parser.add_argument('columns', type=str, nargs='*',
help='A list of columns that should be plotted. Both the column name and index are valid.', default=[])
parser.add_argument('-o', '--output-file', type=str,
help='If specified, stores the plot in a file and prevents opening a plot window.\nThe file '
'format is determined using the default behaviour of matplotlib.pyplot.savefig!', default=None)
'format is determined using the default behaviour of matplotlib.pyplot.savefig!', required=False)
parser.add_argument('-d', '--divider', type=__divide_check,
help='Divides the input data to only take each nth packet', default=1)
help='Divides the input data to only take each nth packet', required=False)
parser.add_argument('-r', '--region', type=__region_check, help='Specifies the desired data range in format \'START:END\' (both inclusive). '
'START and END (or both) my be omitted to specify an open range '
'e.g. use \'100:\' to plot all data from the 100th sample until the last one.', default=':')
'e.g. use \'100:\' to plot all data from the 100th sample until the last one.', required=False)
parser.add_argument('-c', '--config', dest='yaml_config', type=str,
help='Specifies a YAML plot configuration file to be used. This file is able to set all other settings'
' like range and divider settings as well as the input and output files if desired.\nIf a configuration'
' file is specified, passing additional columns plot them in a separate subplot. Explicitely passing'
' other arguments to the script will override their values set in the configuration file.', required=False)
return parser


Expand Down
103 changes: 81 additions & 22 deletions CsvPlotter/internal/configuration.py
Original file line number Diff line number Diff line change
@@ -1,43 +1,102 @@
from .util_classes import SampleRange
from .samplerange import SampleRange


class PlotConfig(object):
def __init__(self, input_filename, range_cfg=SampleRange(), output_filename=None):
self.input_filename = input_filename
self.output_filename = output_filename
self.range = range_cfg
self.subplots = [SubplotConfig()]
def __init__(self):
self.input_file = None
self.output_file = None
self.range = SampleRange()
self.share_x_axis = True
self.subplots = []

def create_empty_subplots(self, subplot_count):
self.subplots = [SubplotConfig() for _ in range(subplot_count)]
@classmethod
def from_obj(cls, cfg_obj):
def get_or_default(key, default=None, conv=None):
if key not in cfg_obj or cfg_obj[key] is None:
return default
v = cfg_obj[key]
if conv is not None:
return conv(v)
return v

def add_column_to_subplot(self, column_cfg, subplot_idx=0):
if subplot_idx >= len(self.subplots):
raise KeyError(
f'Given subplot index ({subplot_idx}) does not exist')
plot_cfg = cls()
plot_cfg.input_file = get_or_default('input_file', conv=str)
plot_cfg.output_file = get_or_default('output_file', conv=str)
plot_cfg.range.start = get_or_default('range_start', 0, conv=int)
plot_cfg.range.end = get_or_default('range_end', -1, conv=int)
plot_cfg.range.divider = get_or_default('divider', 1, conv=int)
plot_cfg.share_x_axis = get_or_default('share_x_axis', True, conv=bool)

self.subplots[subplot_idx].add_column(column_cfg)
for p in get_or_default('plots', []):
plot_cfg.add_subplot(SubplotConfig.from_obj(p))

def __repr__(self):
return f'PlotConfig{{input_filename={self.input_filename!r}, output_filename={self.output_filename!r}, range={self.range!r}, subplots={self.subplots!r}}}'
return plot_cfg


class ColumnConfig(object):
def __init__(self, name=None, label=None, alt_y_axis=False):
self.name = name
self.alt_y_axis = alt_y_axis
self.label = label if label is not None else name
def add_subplot(self, subplot_cfg):
self.subplots.append(subplot_cfg)

def __repr__(self):
return f'ColumnConfig{{name={self.name!r}, label={self.label!r}, alt_y_axis={self.alt_y_axis!r}}}'
return f'PlotConfig{{input_file={self.input_file!r}, output_file={self.output_file!r}, range={self.range!r}, subplots={self.subplots!r}}}'


class SubplotConfig(object):
def __init__(self):
self.title = None
self.xlabel = 'X'
self.ylabel = 'Y'
self.alt_ylabel = None
self.columns = []

@classmethod
def from_obj(cls, cfg_obj):
def get_or_default(key, default=None, conv=None):
if key not in cfg_obj or cfg_obj[key] is None:
return default
v = cfg_obj[key]
if conv is not None:
return conv(v)
return v

subplot_cfg = cls()
subplot_cfg.title = get_or_default('title', conv=str)
subplot_cfg.xlabel = get_or_default('xlabel', 'X', conv=str)
subplot_cfg.ylabel = get_or_default('ylabel', 'Y', conv=str)
subplot_cfg.alt_ylabel = get_or_default('alt_ylabel', conv=str)

for c in get_or_default('columns', []):
subplot_cfg.add_column(ColumnConfig.from_obj(c))

return subplot_cfg

def add_column(self, column_cfg):
self.columns.append(column_cfg)

def __repr__(self):
return f'SubplotConfig{{columns={self.columns!r}}}'


class ColumnConfig(object):
def __init__(self, name=None, label=None, alt_y_axis=False):
self.name = name
self.alt_y_axis = alt_y_axis
self.label = label if label is not None else name

@classmethod
def from_obj(cls, cfg_obj):
def get_or_default(key, default=None, conv=None):
if key not in cfg_obj or cfg_obj[key] is None:
return default
v = cfg_obj[key]
if conv is not None:
return conv(v)
return v

column_cfg = cls()
column_cfg.name = get_or_default('name', conv=str)
column_cfg.label = get_or_default('label', conv=str)
column_cfg.alt_y_axis = get_or_default('alt_y_axis', False, conv=bool)

return column_cfg

def __repr__(self):
return f'ColumnConfig{{name={self.name!r}, label={self.label!r}, alt_y_axis={self.alt_y_axis!r}}}'
33 changes: 27 additions & 6 deletions CsvPlotter/internal/csv_handling.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import numpy as np
import csv

from .utils import str2bool


def read_headers(filename):
with open(filename, 'r') as f:
Expand All @@ -18,13 +20,14 @@ def __init__(self, headers):

csv_data = {}
for h in headers:
csv_data[h] = np.array([], dtype=np.int32)
csv_data[h] = np.array([], dtype=np.float32)
self.data = csv_data

def __increase_capacity(self, incr):
self.capacity += incr
for h in self.headers:
self.data[h] = np.concatenate((self.data[h], np.full(incr, None)))
self.data[h] = np.concatenate(
(self.data[h], np.full(incr, None, dtype=np.float32)))

def add_row(self, row):
if self.size >= self.capacity:
Expand All @@ -33,7 +36,25 @@ def add_row(self, row):

for idx, col in enumerate(row):
h = self.headers[idx]
self.data[h][self.size] = int(col.strip())
val_str = col.strip()
val = None

# Try parsing the given entry
try:
val = 1 if str2bool(val_str) else 0
except ValueError:
pass
try:
val = float(val_str)
val = int(val_str)
except ValueError:
pass

if val is None:
print(f'Unsupported value type of "{val_str}"!')
return

self.data[h][self.size] = val

self.size += 1

Expand All @@ -42,13 +63,13 @@ def __repr__(self):

@classmethod
def from_file(cls, config):
print(f'Extract data from file: {config.input_filename}')
print(f'Extract data from file: {config.input_file}')

PRINT_THRESHOLD = 1000000

data_obj = cls(read_headers(config.input_filename))
data_obj = cls(read_headers(config.input_file))

with open(config.input_filename, 'r') as f:
with open(config.input_file, 'r') as f:
plots = csv.reader(f, delimiter=',')
for i, row in enumerate(plots):
if i == 0:
Expand Down
Loading

0 comments on commit a69056e

Please sign in to comment.