Remove CYLC_DIR

.gitignore

@@ -15,11 +15,6 @@ doc/src/appendices/command-ref.rst
 # ordereddict-0.4.5 build 
 ext/ordereddict-0.4.5/build
 
-# global configuration and processed configuration
-etc/job-init-env.sh
-etc/global.rc
-etc/global-tests.rc
-
 # processed suite definitions
 *.rc.processed
 

.travis/cover.py

@@ -17,18 +17,21 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import sys
-
+from os.path import dirname, abspath, join
 from subprocess import call
 
 
 def main():
     # Run tests with virtual frame buffer for X support.
-    if call('xvfb-run -a cylc test-battery --chunk $CHUNK --state=save -j 5',
-            shell=True) != 0:
-        # Non-zero return code
+    pdir = dirname(dirname(abspath(__file__)))
+    fn_tests = join(pdir, 'etc', 'bin', 'run-functional-tests.sh')
+    # shell=True is no threat here: this is test code, not production.
+    if call('xvfb-run -a ' + fn_tests + ' --chunk $CHUNK --state=save -j 5',
+            shell=True) != 0:  # nosec
         sys.stderr.write('\n\nRerunning Failed Tests...\n\n')
-        # Exit with final return code
-        sys.exit(call('cylc test-battery --state=failed -j 5', shell=True))
+        sys.exit(
+            call(fn_tests + ' --state=failed -j 5', shell=True)  # nosec
+        )
 
 
 if __name__ == '__main__':

.travis/shellchecker

@@ -77,7 +77,7 @@ default () {
     main '.' \
         --exclude 'tests/' \
         --exclude 'cylc/flow/tests/parsec' \
-        --exclude 'etc/dev-bin/live-graph-movie.sh' \
+        --exclude 'etc/bin/live-graph-movie.sh' \
         -- -e SC1090
 
     # run a lenient check on all test scripts

bin/cylc

@@ -22,11 +22,14 @@ CATEGORIES=('control' 'con' 'information' 'info' 'all' 'task' 'license' \
 HELP_OPTS=('help' '--help' '-h' 'h' '?')
 
 print_version() {
+    VN=$(python3 -c "from cylc.flow import __version__; print(__version__)")
     if [[ "$#" -eq 0 ]]; then
-        python3 -c "from cylc.flow import __version__; print(__version__)"
+        echo "$VN"
     fi
     if [[ "$*" == 'long' || "$*" == '--long' ]]; then
-        python3 -c """import cylc.flow;import os;print(f'{cylc.flow.__version__} ({os.path.dirname(os.path.realpath(cylc.flow.__file__))})')"""
+	LOC1=$(dirname "$0")
+	LOC2=$(cd "${LOC1}"; pwd -P)
+	echo  "${VN}(${LOC2})"
     fi
 }
 

bin/cylc-extract-pkg-resources

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+"""cylc [info] get-pkg-resources [OPTIONS] DIR [RESOURCES]
+
+Extract resources from the cylc.flow package and write them to DIR.
+
+Options:
+    --list      List available resources
+Arguments:
+    DIR         Target Directory
+    [RESOURCES] Specific resources to extract (default all).
+"""
+
+import os
+import sys
+from cylc.flow.pkg_resources import extract_pkg_resources, list_pkg_resources
+from cylc.flow.terminal import cli_function
+
+
+@cli_function
+def main(args):
+    target_dir = args[0]
+    if len(args) > 1:
+        resources = args[1:]
+    else:
+        resources = None
+    extract_pkg_resources(target_dir, resources)
+
+
+if __name__ == '__main__':
+    if {'help', '--help', "-h"} & set(sys.argv):
+        print(__doc__)
+    elif len(sys.argv) < 2:
+        sys.exit("ERROR: wrong number of arguments, see '%s --help'." % (
+            os.path.basename(sys.argv[0])))
+    elif '--list' in sys.argv:
+        print('\n'.join(list_pkg_resources()))
+    else:
+        main(sys.argv[1:])

bin/cylc-help

@@ -17,7 +17,6 @@
 
 import sys
 
-# Import cylc to initialise CYLC_DIR and the path for python (__init__.py).
 from cylc.flow import __version__ as CYLC_VERSION
 from cylc.flow.exceptions import CylcError
 from cylc.flow.terminal import cli_function
@@ -217,6 +216,7 @@ information_commands['list'] = ['list', 'ls']
 information_commands['dump'] = ['dump']
 information_commands['show'] = ['show']
 information_commands['cat-log'] = ['cat-log', 'log']
+information_commands['extract-pkg-resources'] = ['extract-pkg-resources']
 information_commands['get-suite-contact'] = [
     'get-suite-contact', 'get-contact']
 information_commands['get-suite-version'] = [
@@ -266,8 +266,6 @@ hook_commands = {}
 hook_commands['check-triggering'] = ['check-triggering']
 
 admin_commands = {}
-admin_commands['test-battery'] = ['test-battery']
-admin_commands['import-examples'] = ['import-examples']
 admin_commands['check-software'] = ['check-software']
 
 preparation_commands = {}
@@ -328,8 +326,6 @@ catsum['utility'] = "Cycle arithmetic and templating, etc."
 # command summaries
 comsum = {}
 # admin
-comsum['test-battery'] = 'Run a battery of self-diagnosing test suites'
-comsum['import-examples'] = 'Import example suites your suite run directory'
 comsum['check-software'] = 'Check required software is installed'
 # preparation
 comsum['register'] = 'Register a suite for use'
@@ -347,6 +343,7 @@ comsum['list'] = 'List suite tasks and family namespaces'
 comsum['dump'] = 'Print the state of tasks in a running suite'
 comsum['show'] = 'Print task state (prerequisites and outputs etc.)'
 comsum['cat-log'] = 'Print various suite and task log files'
+comsum['extract-pkg-resources'] = 'Extract cylc.flow library package resources'
 comsum['documentation'] = 'Display cylc documentation (User Guide etc.)'
 comsum['monitor'] = 'An in-terminal suite monitor'
 comsum['get-suite-config'] = 'Print suite configuration items'

cylc/flow/__init__.py

@@ -27,8 +27,6 @@
 
 def environ_init():
     """Initialise cylc environment."""
-    cylc_dir_lib = os.path.dirname(os.path.realpath(__file__))
-    os.environ['CYLC_DIR'] = cylc_dir_lib
     if os.getenv('CYLC_SUITE_DEF_PATH', ''):
         environ_path_add([os.getenv('CYLC_SUITE_DEF_PATH')])
 

cylc/flow/batch_sys_manager.py

@@ -190,14 +190,11 @@ class BatchSysManager(object):
     CYLC_BATCH_SYS_JOB_ID = "CYLC_BATCH_SYS_JOB_ID"
     CYLC_BATCH_SYS_JOB_SUBMIT_TIME = "CYLC_BATCH_SYS_JOB_SUBMIT_TIME"
     CYLC_BATCH_SYS_EXIT_POLLED = "CYLC_BATCH_SYS_EXIT_POLLED"
-    LINE_PREFIX_CYLC_DIR = "export CYLC_DIR="
     LINE_PREFIX_BATCH_SYS_NAME = "# Job submit method: "
     LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL = "# Job submit command template: "
     LINE_PREFIX_EXECUTION_TIME_LIMIT = "# Execution time limit: "
     LINE_PREFIX_EOF = "#EOF: "
     LINE_PREFIX_JOB_LOG_DIR = "# Job log directory: "
-    LINE_UPDATE_CYLC_DIR = (
-        "# N.B. CYLC_DIR has been updated on the remote host\n")
     OUT_PREFIX_COMMAND = "[TASK JOB COMMAND]"
     OUT_PREFIX_MESSAGE = "[TASK JOB MESSAGE]"
     OUT_PREFIX_SUMMARY = "[TASK JOB SUMMARY]"
@@ -724,9 +721,8 @@ def _jobs_submit_prep_by_args(self, job_log_root, job_log_dirs):
     def _jobs_submit_prep_by_stdin(self, job_log_root, job_log_dirs):
         """Prepare job files for submit by reading from STDIN.
 
-        Job files are uploaded via STDIN in remote mode. Modify job
-        files' CYLC_DIR for this host. Extract job submission methods
-        and job submission command templates from each job file.
+        Job files are uploaded via STDIN in remote mode. Extract job submission
+        methods and job submission command templates from each job file.
 
         Return a list, where each element contains something like:
         (job_log_dir, batch_sys_name, submit_opts)
@@ -742,7 +738,6 @@ def _jobs_submit_prep_by_stdin(self, job_log_root, job_log_dirs):
         job_log_dir = None
         lines = []
         # Get job files from STDIN.
-        # Modify CYLC_DIR in job file, if necessary.
         # Get batch system name and batch submit command template from each job
         # file.
         # Write job file in correct location.
@@ -752,14 +747,7 @@ def _jobs_submit_prep_by_stdin(self, job_log_root, job_log_dirs):
                 if handle is not None:
                     handle.close()
                 break
-
-            if cur_line.startswith(self.LINE_PREFIX_CYLC_DIR):
-                old_line = cur_line
-                cur_line = "%s'%s'\n" % (
-                    self.LINE_PREFIX_CYLC_DIR, os.environ["CYLC_DIR"])
-                if old_line != cur_line:
-                    lines.append(self.LINE_UPDATE_CYLC_DIR)
-            elif cur_line.startswith(self.LINE_PREFIX_BATCH_SYS_NAME):
+            if cur_line.startswith(self.LINE_PREFIX_BATCH_SYS_NAME):
                 batch_sys_name = cur_line.replace(
                     self.LINE_PREFIX_BATCH_SYS_NAME, "").strip()
             elif cur_line.startswith(self.LINE_PREFIX_BATCH_SUBMIT_CMD_TMPL):

cylc/flow/cfgspec/globalcfg.py

@@ -88,7 +88,7 @@
     },
 
     'documentation': {
-        'local': [VDR.V_STRING, '$CYLC_DIR/doc/built-sphinx/index.html'],
+        'local': [VDR.V_STRING, ''],
         'online': [VDR.V_STRING,
                    'http://cylc.github.io/doc/built-sphinx/index.html'],
         'cylc homepage': [VDR.V_STRING, 'http://cylc.github.io/'],
@@ -273,14 +273,9 @@ class GlobalConfig(ParsecConfig):
 
     _DEFAULT = None
     _HOME = os.getenv('HOME') or get_user_home()
-    CONF_BASE = "global.rc"
-    # TODO: do we need new values for these variables?
-    # Site global.rc loc preference: if not in etc/ look in old conf/.
-    # SITE_CONF_DIR = os.path.join(os.environ["CYLC_DIR"], "etc")
-    # SITE_CONF_DIR_OLD = os.path.join(os.environ["CYLC_DIR"], "conf")
-    # User global.rc loc preference: if not in .cylc/x.y.z/ look in .cylc/.
-    USER_CONF_DIR_1 = os.path.join(_HOME, '.cylc', CYLC_VERSION)
-    USER_CONF_DIR_2 = os.path.join(_HOME, '.cylc')
+    CONF_BASENAME = "flow.rc"
+    SITE_CONF_DIR = os.path.join(os.sep, 'etc', 'cylc', 'flow', CYLC_VERSION)
+    USER_CONF_DIR = os.path.join(_HOME, '.cylc', 'flow', CYLC_VERSION)
 
     @classmethod
     def get_inst(cls, cached=True):
@@ -306,20 +301,20 @@ def load(self):
         """Load or reload configuration from files."""
         self.sparse.clear()
         self.dense.clear()
-        LOG.debug("Loading site/user global config files")
+        LOG.debug("Loading site/user config files")
         conf_path_str = os.getenv("CYLC_CONF_PATH")
-        if conf_path_str is None:
-            # CYLC_CONF_PATH not defined, use default locations.
-            for conf_dir_1, conf_dir_2, conf_type in [
-                    (self.USER_CONF_DIR_1, self.USER_CONF_DIR_2,
-                     upgrader.USER_CONFIG)]:
-                fname1 = os.path.join(conf_dir_1, self.CONF_BASE)
-                fname2 = os.path.join(conf_dir_2, self.CONF_BASE)
-                if os.access(fname1, os.F_OK | os.R_OK):
-                    fname = fname1
-                elif os.access(fname2, os.F_OK | os.R_OK):
-                    fname = fname2
-                else:
+        if conf_path_str:
+            # Explicit config file override.
+            fname = os.path.join(conf_path_str, self.CONF_BASENAME)
+            if os.access(fname, os.F_OK | os.R_OK):
+                self.loadcfg(fname, upgrader.USER_CONFIG)
+        elif conf_path_str is None:
+            # Use default locations.
+            for conf_dir, conf_type in [
+                    (self.SITE_CONF_DIR, upgrader.SITE_CONFIG),
+                    (self.USER_CONF_DIR, upgrader.USER_CONFIG)]:
+                fname = os.path.join(conf_dir, self.CONF_BASENAME)
+                if not os.access(fname, os.F_OK | os.R_OK):
                     continue
                 try:
                     self.loadcfg(fname, conf_type)
@@ -332,14 +327,7 @@ def load(self):
                         # Abort on bad user file (users can fix it).
                         LOG.error('bad %s %s', conf_type, fname)
                         raise
-                    break
-        elif conf_path_str:
-            # CYLC_CONF_PATH defined with a value
-            for path in conf_path_str.split(os.pathsep):
-                fname = os.path.join(path, self.CONF_BASE)
-                if os.access(fname, os.F_OK | os.R_OK):
-                    self.loadcfg(fname, upgrader.USER_CONFIG)
-        # (OK if no global.rc is found, just use system defaults).
+        # (OK if no flow.rc is found, just use system defaults).
         self.transform()
 
     def get_derived_host_item(

cylc/flow/etc/README

@@ -0,0 +1,2 @@
+This directory contains package resources to be installed with the cylc.flow
+library. If new files are added, please update bin/cylc-get-pkg-resources.