feat/configurable_config (#74)

Co-authored-by: jarbasal <jarbasai@mailfence.com>

ovos_utils/__init__.py

@@ -114,7 +114,7 @@ def resolve_resource_file(res_name, root_path=None, config=None):
         return filename
 
     # Next look for /opt/mycroft/res/res_name
-    data_dir = os.path.expanduser(config['data_dir'])
+    data_dir = os.path.expanduser(config.get('data_dir', "/opt/mycroft"))
     filename = os.path.expanduser(os.path.join(data_dir, res_name))
     if os.path.isfile(filename):
         return filename

ovos_utils/configuration.py

@@ -1,38 +1,140 @@
-from os.path import isfile, exists, expanduser, join, dirname, isdir
-from os import makedirs
 import json
-from ovos_utils.log import LOG
-from ovos_utils.json_helper import merge_dict, load_commented_json
-from ovos_utils.system import search_mycroft_core_location
+import os
+from os import makedirs
+from os.path import isfile, exists, expanduser, join, dirname, isdir
+
 from xdg import BaseDirectory as XDG
+
 from ovos_utils.fingerprinting import core_supports_xdg
+from ovos_utils.json_helper import merge_dict, load_commented_json
+from ovos_utils.log import LOG
+from ovos_utils.system import search_mycroft_core_location
 
+# for downstream support, all XDG paths should respect this
+_BASE_FOLDER = "mycroft"
+_CONFIG_FILE_NAME = "mycroft.conf"
+
+_DEFAULT_CONFIG = None
+_SYSTEM_CONFIG = os.environ.get('MYCROFT_SYSTEM_CONFIG',
+                                f'/etc/{_BASE_FOLDER}/{_CONFIG_FILE_NAME}')
+# Make sure we support the old location until mycroft moves to XDG
+_OLD_USER_CONFIG = join(expanduser('~'), '.' + _BASE_FOLDER, _CONFIG_FILE_NAME)
+_USER_CONFIG = join(XDG.xdg_config_home, _BASE_FOLDER, _CONFIG_FILE_NAME)
+_WEB_CONFIG_CACHE = join(XDG.xdg_config_home, _BASE_FOLDER, 'web_cache.json')
+
+
+def get_xdg_base():
+    global _BASE_FOLDER
+    return _BASE_FOLDER
+
+
+def set_xdg_base(folder_name):
+    global _BASE_FOLDER, _WEB_CONFIG_CACHE
+    LOG.info(f"XDG base folder set to: '{folder_name}'")
+    _BASE_FOLDER = folder_name
+    _WEB_CONFIG_CACHE = join(XDG.xdg_config_home, _BASE_FOLDER,
+                             'web_cache.json')
+
+
+def set_config_filename(file_name, core_folder=None):
+    global _CONFIG_FILE_NAME, _SYSTEM_CONFIG, _OLD_USER_CONFIG, _USER_CONFIG, \
+        _BASE_FOLDER
+    if core_folder:
+        _BASE_FOLDER = core_folder
+        set_xdg_base(core_folder)
+    LOG.info(f"config filename set to: '{file_name}'")
+    _CONFIG_FILE_NAME = file_name
+    _SYSTEM_CONFIG = os.environ.get('MYCROFT_SYSTEM_CONFIG',
+                                    f'/etc/{_BASE_FOLDER}/{_CONFIG_FILE_NAME}')
+    # Make sure we support the old location still
+    # Deprecated and will be removed eventually
+    _OLD_USER_CONFIG = join(expanduser('~'), '.' + _BASE_FOLDER,
+                            _CONFIG_FILE_NAME)
+    _USER_CONFIG = join(XDG.xdg_config_home, _BASE_FOLDER, _CONFIG_FILE_NAME)
+
+
+def set_default_config(file_path=None):
+    global _DEFAULT_CONFIG
+    _DEFAULT_CONFIG = file_path or find_default_config()
+    LOG.info(f"default config file changed to: {file_path}")
+
+
+def find_default_config():
+    if _DEFAULT_CONFIG:
+        # previously set, otherwise None
+        return _DEFAULT_CONFIG
+    mycroft_root = search_mycroft_core_location()
+    if not mycroft_root:
+        raise FileNotFoundError("Couldn't find mycroft core root folder.")
+    return join(mycroft_root, _BASE_FOLDER, "configuration", _CONFIG_FILE_NAME)
+
+
+def find_user_config():
+    # ideally it will have been set by downstream using util methods
+    old, path = get_config_locations(default=False, web_cache=False,
+                                     system=False, old_user=True,
+                                     user=True)
+    if isfile(path):
+        return path
+
+    if core_supports_xdg():
+        path = join(XDG.xdg_config_home, _BASE_FOLDER, _CONFIG_FILE_NAME)
+    else:
+        path = old
+        # mark1 runs as a different user
+        sysconfig = MycroftSystemConfig()
+        platform_str = sysconfig.get("enclosure", {}).get("platform", "")
+        if platform_str == "mycroft_mark_1":
+            path = "/home/mycroft/.mycroft/mycroft.conf"
 
-MYCROFT_DEFAULT_CONFIG = join("{ROOT_PATH}", "mycroft",
-                              "configuration", "mycroft.conf")
-MYCROFT_SYSTEM_CONFIG = "/etc/mycroft/mycroft.conf"
-MYCROFT_OLD_USER_CONFIG = join(expanduser("~"), ".mycroft", "mycroft.conf")
-MYCROFT_XDG_USER_CONFIG = join(XDG.xdg_config_home, 'mycroft', 'mycroft.conf')
-MYCROFT_USER_CONFIG = MYCROFT_XDG_USER_CONFIG
+    if not isfile(path) and isfile(old):
+        # xdg might be disabled in HolmesV compatibility mode
+        # or migration might be in progress
+        # (user action required when updated from a no xdg install)
+        path = old
+
+    return path
+
+
+def get_config_locations(default=True, web_cache=True, system=True,
+                         old_user=True, user=True):
+    locs = []
+    if default:
+        locs.append(_DEFAULT_CONFIG)
+    if system:
+        locs.append(_SYSTEM_CONFIG)
+    if web_cache:
+        locs.append(_WEB_CONFIG_CACHE)
+    if old_user:
+        locs.append(_OLD_USER_CONFIG)
+    if user:
+        locs.append(_USER_CONFIG)
+
+    return locs
+
+
+def get_webcache_location():
+    return join(XDG.xdg_config_home, _BASE_FOLDER, _CONFIG_FILE_NAME)
+
+
+def get_xdg_config_locations():
+    # This includes both the user config and
+    # /etc/xdg/mycroft/mycroft.conf
+    xdg_paths = list(reversed(
+        [join(p, get_config_filename())
+         for p in XDG.load_config_paths(get_xdg_base())]
+    ))
+    return xdg_paths
+
+
+def get_config_filename():
+    return _CONFIG_FILE_NAME
 
 
 def set_config_name(name, core_folder=None):
-    global MYCROFT_USER_CONFIG, MYCROFT_SYSTEM_CONFIG, \
-        MYCROFT_XDG_USER_CONFIG, MYCROFT_OLD_USER_CONFIG, \
-        MYCROFT_DEFAULT_CONFIG
-
-    core_folder = core_folder or name
-    MYCROFT_DEFAULT_CONFIG = join("{ROOT_PATH}", core_folder,
-                                  "configuration", f"{name}.conf")
-    MYCROFT_SYSTEM_CONFIG = f"/etc/{name}/{name}.conf"
-    MYCROFT_OLD_USER_CONFIG = join(expanduser("~"), f".{name}", f"{name}.conf")
-    MYCROFT_XDG_USER_CONFIG = join(XDG.xdg_config_home, name, f'{name}.conf')
-    MYCROFT_USER_CONFIG = MYCROFT_XDG_USER_CONFIG
-    LOG.info("config paths changed:\n"
-             f"DEFAULT: {MYCROFT_DEFAULT_CONFIG}\n"
-             f"SYSTEM: {MYCROFT_SYSTEM_CONFIG}\n"
-             f"USER: {MYCROFT_USER_CONFIG}\n"
-             f"OLD_USER: {MYCROFT_OLD_USER_CONFIG}")
+    # TODO deprecate, was only out in a couple versions
+    # renamed to match HolmesV
+    set_config_filename(name, core_folder)
 
 
 def read_mycroft_config():
@@ -53,7 +155,6 @@ def update_mycroft_config(config, path=None):
     return conf
 
 
-# TODO consider using json_database.JsonStorage
 class LocalConf(dict):
     """
         Config dict from file.
@@ -146,31 +247,13 @@ def store(self, path=None):
 
 class MycroftUserConfig(LocalConf):
     def __init__(self):
-        if core_supports_xdg():
-            path = MYCROFT_XDG_USER_CONFIG
-        else:
-            path = MYCROFT_USER_CONFIG
-            # mark1 runs as a different user
-            sysconfig = MycroftSystemConfig()
-            platform_str = sysconfig.get("enclosure", {}).get("platform", "")
-            if platform_str == "mycroft_mark_1":
-                path = "/home/mycroft/.mycroft/mycroft.conf"
-
-        if not isfile(path) and isfile(MYCROFT_OLD_USER_CONFIG):
-            # xdg might be disabled in HolmesV compatibility mode
-            # or migration might be in progress
-            # (user action required when updated from a no xdg install)
-            path = MYCROFT_OLD_USER_CONFIG
-
+        path = find_user_config()
         super().__init__(path)
 
 
 class MycroftDefaultConfig(ReadOnlyConfig):
     def __init__(self):
-        mycroft_root = search_mycroft_core_location()
-        if not mycroft_root:
-            raise FileNotFoundError("Couldn't find mycroft core root folder.")
-        path = MYCROFT_DEFAULT_CONFIG.replace("{ROOT_PATH}", mycroft_root)
+        path = find_default_config()
         super().__init__(path)
         if not self.path or not isfile(self.path):
             LOG.error("mycroft root path not found, could not load default "
@@ -184,10 +267,15 @@ def set_root_config_path(self, root_config):
 
 class MycroftSystemConfig(ReadOnlyConfig):
     def __init__(self, allow_overwrite=False):
-        super().__init__(MYCROFT_SYSTEM_CONFIG, allow_overwrite)
+        path = get_config_locations(default=False, web_cache=False,
+                                    system=True, old_user=False,
+                                    user=False)[0]
+        super().__init__(path, allow_overwrite)
 
 
 class MycroftXDGConfig(LocalConf):
     def __init__(self):
-        path = MYCROFT_XDG_USER_CONFIG
+        path = get_config_locations(default=False, web_cache=False,
+                                    system=False, old_user=False,
+                                    user=True)[0]
         super().__init__(path)

ovos_utils/intents/__init__.py

@@ -275,7 +275,6 @@ def sync_with_intent_service(cls):
         WARNING
             we don't have the timestamps so order might be messed up!!
             avoid calling this until
-            https://github.com/MycroftAI/mycroft-core/pull/2786 is merged
         """
         skill_ids = cls.intent_api.get_active_skills(include_timestamps=True)
         if skill_ids:

ovos_utils/skills/__init__.py

@@ -1,7 +1,9 @@
-from ovos_utils.configuration import read_mycroft_config, update_mycroft_config
+from ovos_utils.configuration import read_mycroft_config, \
+    update_mycroft_config, get_xdg_base
 from ovos_utils.messagebus import wait_for_reply
-from os.path import join, isdir
-import xdg
+from os.path import join, isdir, isfile
+from os import listdir
+from xdg import BaseDirectory as XDG
 
 
 def skills_loaded(bus=None):
@@ -65,18 +67,39 @@ def get_skills_folder(config=None):
     # once XDG PR is merged skills folder will no longer be configurable,
     # skills are moved automatically to new locations
     # this is already live in mycroft-lib
-    xdg_skills = xdg.BaseDirectory.save_data_path('mycroft/skills')
+    xdg_skills = join(XDG.xdg_data_home, get_xdg_base(), 'skills')
     if isdir(xdg_skills):
         return xdg_skills
-    config = config or read_mycroft_config()
+
     # read user defined location
+    config = config or read_mycroft_config()
     if config:
         skill_folder = config["skills"]["msm"]["directory"]
         return join(config["data_dir"], skill_folder)
+
     # check if default path exists
     elif isdir("/opt/mycroft/skills"):
         return "/opt/mycroft/skills"
 
     # .conf not found, xdg directory not detected, default path not
     # detected, doesn't look like we are running mycroft-core
     return None
+
+
+def get_installed_skills(config=None):
+    skills_dir = get_skills_folder(config)
+    installed_skills = []
+    if skills_dir:
+        for skill_id in listdir(skills_dir):
+            skill_path = join(skills_dir, skill_id)
+            if not isdir(skill_path):
+                continue
+            skill_file = join(skill_path, "__init__.py")
+            if not isfile(skill_file):
+                continue
+            with open(skill_file) as f:
+                if "def create_skill(" not in f.read():
+                    continue
+            installed_skills.append(skill_id)
+
+    return installed_skills

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='ovos_utils',
-    version='0.0.10',
+    version='0.0.12',
     packages=['ovos_utils',
               'ovos_utils.intents',
               'ovos_utils.sound',