Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Re-enable Rosie ID. #2510

Merged
merged 6 commits into from
Dec 7, 2021
Merged

Re-enable Rosie ID. #2510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c2ed511
remove duplicate _command-rose-stem
wxtim Nov 29, 2021
ea4fa30
make rosie id work with Cylc 8 VC file
wxtim Nov 30, 2021
334cccf
refactor
wxtim Nov 30, 2021
9212912
Apply suggestions from code review
wxtim Dec 1, 2021
1e0915b
Update suite_id.py
wxtim Dec 1, 2021
66aa67c
Update t/rosie-id/00-basic.t
wxtim Dec 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 80 additions & 60 deletions metomi/rosie/suite_id.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
main - CLI interface function.

"""

from ast import literal_eval
import os
from pathlib import Path
import re
Expand Down Expand Up @@ -118,6 +118,7 @@ class SuiteId:
REC_IDX = re.compile(r"\A(?:(\w+)-)?(\w+)(?:/([^\@/]+))?(?:@([^\@/]+))?\Z")
BRANCH_TRUNK = "trunk"
REV_HEAD = "HEAD"
VC_FILENAME = "log/version/vcs.conf"
svn = SvnCaller()

STATUS_CR = "X"
Expand Down Expand Up @@ -370,71 +371,18 @@ def _from_id_text(self, id_text):

def _from_location(self, location):
"""Return the ID of a location (origin URL or local copy path)."""
suite_engine_proc = SuiteEngineProcessor.get_processor()
suite_dir_rel_root = getattr(
suite_engine_proc, "SUITE_DIR_REL_ROOT", None
)

# Cylc8 run directory
# TODO: extract version control information
loc = Path(location)
sdrr = Path('~', suite_dir_rel_root).expanduser().resolve()
try:
loc.relative_to(sdrr)
except ValueError:
# Not an installed Cylc8 workflow run directory
pass
else:
if (loc / 'rose-suite.info').is_file():
# This is an installed workflow with a rose-suite.info file
# (most likely a Cylc8 run directory)

# TODO: extract version control information written by
# Cylc install, see:
# https://github.com/metomi/rose/issues/2432
# https://github.com/cylc/cylc-flow/issues/3849
raise SuiteIdLocationError(location)

# Cylc7 run directory
# Use a hacky way to read the "log/rose-suite-run.version" file
suite_dir_rel_root = getattr(
suite_engine_proc, "SUITE_DIR_REL_ROOT", None
)
if suite_dir_rel_root and "/" + suite_dir_rel_root + "/" in location:
loc = location
while "/" + suite_dir_rel_root + "/" in loc:
suite_version_file_name = os.path.join(
loc, "log/rose-suite-run.version"
)
loc = os.path.dirname(loc)
if not os.access(suite_version_file_name, os.F_OK | os.R_OK):
continue
state = None
url = None
rev = None
for line in open(suite_version_file_name):
line = line.strip()
if state is None:
if line.startswith("# svn info"):
state = line
elif state.startswith("# svn info"):
if line.startswith("URL:"):
url = line.split(":", 1)[1].strip()
elif line.startswith("Revision:"):
rev = line.split(":", 1)[1].strip()
elif not line:
break
if url and rev:
location = url + "@" + rev
break
# Get Version control file location and convert to a URL if rqd.
loc = self._find_vc_file_from_location(location)
if loc:
location = self._parse_cylc_vc_file(loc)

# Assume location is a Subversion working copy of a Rosie suite
info_parser = SvnInfoXMLParser()
try:
info_entry = info_parser.parse(self.svn("info", "--xml", location))
info_entry = info_parser.parse(
self.svn("info", "--xml", location))
except RosePopenError:
raise SuiteIdLocationError(location)

if "url" not in info_entry:
raise SuiteIdLocationError(location)
root = info_entry["repository:root"]
Expand All @@ -456,6 +404,78 @@ def _from_location(self, location):
if "commit:revision" in info_entry:
self.revision = info_entry["commit:revision"]

@staticmethod
def _find_vc_file_from_location(location):
"""Search a location and parents for a version control log file.

Args:
location: Path to search.

Returns:
If a file is found, a path, else None.
"""
suite_engine_proc = SuiteEngineProcessor.get_processor()
suite_dir_rel_root = getattr(
suite_engine_proc, "SUITE_DIR_REL_ROOT", None
)
loc = Path(location).expanduser().resolve()
if suite_dir_rel_root:
sdrr = Path('~', suite_dir_rel_root).expanduser().resolve()
try:
loc.relative_to(sdrr)
except ValueError:
# Not an installed Cylc8 workflow run directory
pass
else:
# Slightly odd construction = loc + parents
for loc in (loc.relative_to(sdrr) / '_').parents:
vcfilepath = sdrr / loc / SuiteId.VC_FILENAME
if os.access(vcfilepath, os.F_OK | os.R_OK):
return vcfilepath
return None

@staticmethod
def _parse_cylc_vc_file(fpath):
"""Take a path to a Cylc VC file and returns an svn URL.

Args:
fpath: Location of Cylc Version Control log file.

Returns: SVN location, or None

Examples:
>>> class MockFpath():
... def read_text(self):
... return (
... 'version control system="svn"\\n'
... 'url="/a/b/c"\\n'
... 'revision="4242"'
... )
>>> mypath = MockFpath()
>>> SuiteId.parse_cylc_vc_file(mypath)
'/a/b/c@4242'
"""
location = None
vcsystem = None
url = None
rev = None
for line in fpath.read_text().split('\n'):
Copy link
Contributor

@MetRonnie MetRonnie Dec 6, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it not possible to parse this using Rose's parser? I guess not because Rose doesn't allow multiline strings like

status = """
 M flow.cylc
 M whatever.txt
"""

?

Then perhaps it would make sense to save VCS info as JSON instead, as you suggested. Which I think would have to be a Cylc 8.0rc1 PR...

Copy link
Contributor Author

@wxtim wxtim Dec 7, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's an internal which users shouldn't see: so IMO we should create an issue to change this beyond rc1. I don't think that it needs to be a blocker.
If we want to be failsafe I can't think that it'd be massively expensive to write this + json both.

cylc/cylc-flow#4546

line = line.strip()
if vcsystem is None:
if line.startswith("version control system"):
vcsystem = literal_eval(
line.split('=', 1)[1].strip())
elif 'svn' in vcsystem:
if line.startswith("url"):
url = literal_eval(line.split("=", 1)[1].strip())
elif line.startswith("revision"):
rev = literal_eval(line.split("=", 1)[1].strip())
elif not line:
break
if url and rev:
location = url + "@" + rev
return location

def get_status(self, user=None, force_mode=False):
"""Determine and return local status for this suite.

Expand Down
1 change: 0 additions & 1 deletion sphinx/api/command-reference.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ This command has been replaced by ``cylc play``.

.. TODO: This is here to allow the documentation tests to pass


----

.. _command-rose-test-battery:
Expand Down
49 changes: 23 additions & 26 deletions t/rosie-id/00-basic.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
#-------------------------------------------------------------------------------
. $(dirname $0)/test_header
#-------------------------------------------------------------------------------
tests 36
tests 39
#-------------------------------------------------------------------------------
svnadmin create foo
URL=file://$PWD/foo
Expand Down Expand Up @@ -114,31 +114,26 @@ foo-aa000
__OUT__
file_cmp "$TEST_KEY.err" "$TEST_KEY.err" </dev/null

# TODO: Cylc8 support for rosie id
# https://github.com/metomi/rose/issues/2432

#TEST_KEY="${TEST_KEY_BASE}-run"
#get_reg
#svn co -q "${URL}/a/a/0/0/0/trunk" 'foo-aa000'
#touch 'foo-aa000/rose-suite.conf'
#cat >'foo-aa000/flow.cylc' <<'__SUITE_RC__'
#[scheduling]
# [[dependencies]]
# graph='t1'
#[runtime]
# [[t1]]
#__SUITE_RC__
#cylc install \
# -C "${PWD}/foo-aa000" \
# --flow-name="${FLOW}" \
# --no-run-name
#run_pass "$TEST_KEY" rosie id "${HOME}/cylc-run/${FLOW}"
#file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__
#foo-aa000
#__OUT__
#file_cmp "$TEST_KEY.err" "$TEST_KEY.err" </dev/null
#purge
#rm -fr 'foo-aa000'
TEST_KEY="${TEST_KEY_BASE}-run"
get_reg
svn co -q "${URL}/a/a/0/0/0/trunk" 'foo-aa000'
touch 'foo-aa000/rose-suite.conf'
cat >'foo-aa000/flow.cylc' <<'__SUITE_RC__'
[scheduling]
[[dependencies]]
graph='t1'
[runtime]
[[t1]]
__SUITE_RC__
cylc install \
-C "${PWD}/foo-aa000" \
--flow-name="${FLOW}" \
--no-run-name
run_pass "$TEST_KEY" rosie id "${HOME}/cylc-run/${FLOW}"
file_cmp "$TEST_KEY.out" "$TEST_KEY.out" <<__OUT__
foo-aa000
__OUT__
file_cmp "$TEST_KEY.err" "$TEST_KEY.err" </dev/null
#-------------------------------------------------------------------------------
# Latest and next should still be correct if latest suite removed from HEAD
TEST_KEY=$TEST_KEY_BASE-latest-not-at-head
Expand All @@ -155,4 +150,6 @@ foo-aa001
__OUT__
file_cmp "$TEST_KEY.err" "$TEST_KEY.err" </dev/null
#-------------------------------------------------------------------------------
purge
rm -fr 'foo-aa000'
exit 0