Skip to content

records.yaml - Make some changes on the convert script. #9466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
brbzull0 merged 1 commit into from
Feb 27, 2023
Merged

records.yaml - Make some changes on the convert script. #9466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions doc/admin-guide/tools/converting-records-to-yaml.en.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ Converting a file with a detailed output.
.. code-block:: bash
:linenos:

$ python3 convert2yaml.py -f records.config -S records.yaml -y
$ python3 convert2yaml.py -f records.config -o records.yaml
[████████████████████████████████████████] 494/494

┌■ 8 Renamed records:
Expand All @@ -171,13 +171,12 @@ Converting a file with no output. If any, errors are displayed.

.. code-block:: bash

$ convert2yaml.py -f records.config -S records.yaml -y -m
$ convert2yaml.py -f records.config -o records.yaml -m

.. note::

Use -m, --mute to mute the output.


Non core records
================

Expand Down Expand Up @@ -221,7 +220,7 @@ non core records, for instance:

.. code-block:: bash

$ convert2yaml.py -f records.config -S records.yaml -y -t float,int
$ convert2yaml.py -f records.config -o records.yaml -t float,int

$ cat records.yaml
ts:
Expand Down
8 changes: 4 additions & 4 deletions tests/gold_tests/records/records_config_to_yaml.test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@

tr.Setup.Copy(os.path.join(Test.Variables.BuildRoot, "tools/records/convert2yaml.py"))
tr.Setup.Copy('legacy_config/full_records.config')
tr.Processes.Default.Command = f'python3 convert2yaml.py -f full_records.config --save2 generated{file_suffix}.yaml --yaml --mute'
tr.Processes.Default.Command = f'python3 convert2yaml.py -f full_records.config --output generated{file_suffix}.yaml --yaml --mute'
f = tr.Disk.File(f"generated{file_suffix}.yaml")
f.Content = "gold/full_records.yaml"

Expand All @@ -37,7 +37,7 @@
tr = Test.AddTestRun("Test records to yaml convert script -only renamed records.")
tr.Setup.Copy(os.path.join(Test.Variables.BuildRoot, "tools/records/convert2yaml.py"))
tr.Setup.Copy('legacy_config/old_records.config')
tr.Processes.Default.Command = f'python3 convert2yaml.py -f old_records.config --save2 generated{file_suffix}.yaml --yaml'
tr.Processes.Default.Command = f'python3 convert2yaml.py -f old_records.config --output generated{file_suffix}.yaml --yaml'
tr.Processes.Default.Stream = 'gold/renamed_records.out'
f = tr.Disk.File(f"generated{file_suffix}.yaml")
f.Content = "gold/renamed_records.yaml"
Expand All @@ -46,15 +46,15 @@
tr = Test.AddTestRun("Test errors when trying to override values ")
tr.Setup.Copy(os.path.join(Test.Variables.BuildRoot, "tools/records/convert2yaml.py"))
tr.Setup.Copy('legacy_config/override_value.config')
tr.Processes.Default.Command = f'python3 convert2yaml.py -f override_value.config --save2 generated{file_suffix}.yaml --yaml -m'
tr.Processes.Default.Command = f'python3 convert2yaml.py -f override_value.config --output generated{file_suffix}.yaml --yaml -m'
tr.Processes.Default.Streams.stdout += Testers.ContainsExpression(
"We cannot continue with 'proxy.config.ssl.client.verify.server.policy' at line '3' as a value node will be overridden",
"Error should be present")

tr = Test.AddTestRun("Test errors when trying to override maps")
tr.Setup.Copy(os.path.join(Test.Variables.BuildRoot, "tools/records/convert2yaml.py"))
tr.Setup.Copy('legacy_config/override_map.config')
tr.Processes.Default.Command = f'python3 convert2yaml.py -f override_map.config --save2 generated{file_suffix}.yaml --yaml -m'
tr.Processes.Default.Command = f'python3 convert2yaml.py -f override_map.config --output generated{file_suffix}.yaml --yaml -m'
tr.Processes.Default.Streams.stdout += Testers.ContainsExpression(
"We cannot continue with 'proxy.config.ssl.client.verify.server' at line '3' as an existing YAML map will be overridden.",
"Error should be present")
22 changes: 12 additions & 10 deletions tools/records/convert2yaml.py
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@
import yaml
import tempfile
import fileinput

import time
import os.path
import io

from io import StringIO
Expand Down Expand Up @@ -135,6 +136,10 @@ def bool_representer(dumper, value):
def null_representer(dumper, value):
return dumper.represent_scalar(u'tag:yaml.org,2002:null', str(value), style="'")

# Make sure we do not wipe out an existing file.
if os.path.exists(filename):
os.popen(f'cp {filename} {filename}.{int(time.time())}')

with open(filename, 'w') as f:
if is_json:
json.dump(data, f, indent=4, sort_keys=True)
Expand Down Expand Up @@ -235,9 +240,6 @@ def fix_record_names(file):


def handle_file_input(args):
if args.file is None:
raise Exception("Hey!! there is no file!")

config = {}

# Fix the names first.
Expand Down Expand Up @@ -276,29 +278,29 @@ def handle_file_input(args):
if err:
raise Exception("Schema failed.")

save_to_file(args.save2, args.json, args.typerepr, ts)
save_to_file(args.output, args.json, args.typerepr, ts)

f.close()
return


if __name__ == '__main__':
parser = argparse.ArgumentParser(description='records.config to YAML/JSON convert tool')
parser.add_argument('-f', '--file', help='records.config input file.')
parser.add_argument('-f', '--file', help='records.config input file.', required=False, default="records.config")
parser.add_argument('-n', '--node', help="Include 'proxy.node' variables in the parser.", action='store_true')
parser.add_argument('-t', '--typerepr', help="Use type representer (list)", required=False, default=[''])
parser.add_argument('-s', '--schema', help="Validate the output using a json schema file.")
parser.add_argument('-S', '--save2', help="Save to file.", required=True)
parser.add_argument('-o', '--output', help="Save to output file.", required=False, default="records.yaml")
parser.add_argument(
'-m',
'--mute',
help="Be quiet, do not output anything, except for errors",
required=False,
action='store_true')
parser.add_argument('-e', '--error', help="Show traceback", required=False, action='store_true', default=False)
kk = parser.add_mutually_exclusive_group(required=True)
kk.add_argument('-j', '--json', help="Output as json", action='store_true')
kk.add_argument('-y', '--yaml', help="Output as yaml", action='store_true')
kk = parser.add_mutually_exclusive_group(required=False)
kk.add_argument('-j', '--json', help="Output as json", action='store_true', default=False)
kk.add_argument('-y', '--yaml', help="Output as yaml", action='store_true', default=True)
parser.set_defaults(func=handle_file_input)
args = parser.parse_args()

Expand Down