feat: use url from gitsubmodules

ai4papi/routers/v1/catalog/common.py

@@ -3,6 +3,7 @@
 
 Both modules and tools share similar workflows so they will inherit from a common
 Catalog class. We only finetune methods wheen needed (eg. /config).
+Both modules and tools are referred in the common code as "items".
 
 Implementation notes:
 =====================
@@ -38,16 +39,23 @@ def __init__(self) -> None:
 
 
  @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60))
- def get_list(
+ def get_items(
  self,
  ):
  """
- Retrieve a list of *all* items.
-
+ Retrieve a dict of *all* items.
+ ```
+ {'module 1': {
+ 'url': ***,
+ 'branch': ***,
+ },
+ ...
+ }
+ ```
  This is implemented in a separate function as many functions from this router
  are using this function, so we need to avoid infinite recursions.
  """
- return []
+ return {}
 
 
  @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60))
@@ -75,7 +83,7 @@ def get_filtered_list(
 
  """
  # Retrieve all modules
- modules = self.get_list()
+ modules = self.get_items().keys()
 
  if any([tags, tags_any, not_tags, not_tags_any]): # apply filtering
 
@@ -194,7 +202,7 @@ def get_tags(
  Retrieve a list of all the existing tags.
  """
  tags = []
- for m in self.get_list():
+ for m in self.get_items().keys():
  meta = self.get_metadata(m)
  tags += meta['keywords']
  tags = sorted(set(tags))
@@ -209,37 +217,20 @@ def get_metadata(
  """
  Get the item's full metadata.
  """
-
- # Check the module is in the modules list
- items = self.get_list()
- if item_name not in items:
+ # Check if item is in the items list
+ items = self.get_items()
+ if item_name not in items.keys():
  raise HTTPException(
  status_code=400,
- detail="Item {item_name} not in catalog: {items}",
+ detail=f"Item {item_name} not in catalog: {items.keys()}",
  )
 
- # Read the index of modules from Github
- gitmodules_url = "https://raw.githubusercontent.com/deephdc/deep-oc/master/.gitmodules"
- r = requests.get(gitmodules_url)
-
- cfg = configparser.ConfigParser()
- cfg.read_string(r.text)
-
- # Convert ConfigParser to cleaner dict
- # and retrieve default branch (if no branch use master)
- modules_conf = {
- re.search(r'submodule "(.*)"', s).group(1).lower():
- # 'submodule "DEEP-OC-..."' --> 'deep-oc-...'
- dict(cfg.items(s))
- for s in cfg.sections()
- }
- branch = modules_conf[item_name].get("branch", "master")
-
- # Retrieve metadata from that branch
+ # Retrieve metadata from default branch
  # Use try/except to avoid that a single module formatting error could take down
  # all the Dashboard
- metadata_url = f"https://raw.githubusercontent.com/deephdc/{item_name}/{branch}/metadata.json"
-
+ branch = items[item_name].get("branch", "master")
+ url = items[item_name]['url'].replace('github.com', 'raw.githubusercontent.com')
+ metadata_url = f"{url}/{branch}/metadata.json"
  try:
  r = requests.get(metadata_url)
  metadata = json.loads(r.text)
@@ -269,14 +260,19 @@ def get_metadata(
 
  def get_config(
  self,
- ):
+ ):
+ """
+ Returns the default configuration (dict) for creating a deployment
+ for a specific item. It is prefilled with the appropriate
+ docker image and the available docker tags.
+ """
  return {}
 
 
 def retrieve_docker_tags(
  image: str,
  repo: str = 'deephdc',
-):
+ ):
  """
  Retrieve tags from Dockerhub image
  """

ai4papi/routers/v1/catalog/modules.py

@@ -1,61 +1,104 @@
 import configparser
 from copy import deepcopy
-import re
+import json
 
 from cachetools import cached, TTLCache
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
 import requests
 
 from ai4papi import quotas, nomad
 import ai4papi.conf as papiconf
 from .common import Catalog, retrieve_docker_tags
 
 
-
 @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60))
-def get_list(
+def get_items(
  ):
- """
- Retrieve a list of *all* modules.
-
- This is implemented in a separate function as many functions from this router
- are using this function, so we need to avoid infinite recursions.
- """
-
  gitmodules_url = "https://raw.githubusercontent.com/deephdc/deep-oc/master/.gitmodules"
  r = requests.get(gitmodules_url)
 
  cfg = configparser.ConfigParser()
  cfg.read_string(r.text)
 
- # Convert 'submodule "DEEP-OC-..."' --> 'deep-oc-...'
- modules = [
- re.search(r'submodule "(.*)"', s).group(1).lower() for s in cfg.sections()
- ]
+ modules = {}
+ for section in cfg.sections():
+ items = dict(cfg.items(section))
+ key = items.pop('path').lower()
+ modules[key] = items
 
  return modules
 
 
+@cached(cache=TTLCache(maxsize=1024, ttl=6*60*60))
+def get_metadata(
+ item_name: str,
+ ):
+ # Check if item is in the items list
+ items = get_items()
+ if item_name not in items.keys():
+ raise HTTPException(
+ status_code=400,
+ detail=f"Item {item_name} not in catalog: {items.keys()}",
+ )
+
+ # Retrieve metadata from default branch
+ # Use try/except to avoid that a single module formatting error could take down
+ # all the Dashboard
+ branch = items[item_name].get("branch", "master")
+ url = items[item_name]['url'].replace('github.com', 'raw.githubusercontent.com')
+ metadata_url = f"{url}/{branch}/metadata.json"
+ try:
+ r = requests.get(metadata_url)
+ metadata = json.loads(r.text)
+ except Exception:
+ print(f'Error parsing metadata: {item_name}')
+ metadata = {
+ "title": item_name,
+ "summary": "",
+ "description": [
+ "The metadata of this module could not be retrieved probably due to a ",
+ "JSON formatting error from the module maintainer."
+ ],
+ "keywords": [],
+ "license": "",
+ "date_creation": "",
+ "sources": {
+ "dockerfile_repo": f"https://github.com/deephdc/{item_name}",
+ "docker_registry_repo": f"deephdc/{item_name}",
+ "code": "",
+ }
+ }
+
+ # Format "description" field nicely for the Dashboards Markdown parser
+ metadata["description"] = "\n".join(metadata["description"])
+
+ return metadata
+
+
 def get_config(
  item_name: str,
  vo: str,
-):
- """
- Returns the default configuration (dict) for creating a deployment
- for a specific module. It is prefilled with the appropriate
- docker image and the available docker tags.
- """
- #TODO: We are not checking if module exists in the marketplace because
- # we are treating each route as independent. In the future, this can
- # be done as an API call to the other route.
-
+ ):
+ # Check if module exists
+ modules = get_items()
+ if item_name not in modules.keys():
+  raise HTTPException(
+  status_code=400,
+  detail=f"{item_name} is not an available module.",
+  )
+
+ # Retrieve module configuration
  conf = deepcopy(papiconf.MODULES['user']['full'])
 
- # Fill with correct Docker image
- conf["general"]["docker_image"]["value"] = f"deephdc/{item_name}"
+ # Retrieve module metadata
+ metadata = get_metadata(item_name)
 
  # Add available Docker tags
- tags = retrieve_docker_tags(item_name)
+ registry = metadata['sources']['docker_registry_repo']
+ repo = registry.split('/')[0]
+ if repo not in ['deephdc', 'ai4oshub']:
+ repo = 'deephdc'
+ tags = retrieve_docker_tags(image=item_name, repo=repo)
  conf["general"]["docker_tag"]["options"] = tags
  conf["general"]["docker_tag"]["value"] = tags[0]
 
@@ -79,10 +122,10 @@ def get_config(
  return conf
 
 
-
 Modules = Catalog()
-Modules.get_list = get_list
+Modules.get_items = get_items
 Modules.get_config = get_config
+Modules.get_metadata = get_metadata # TODO: inherit the common one, because it's the same for modules and tools
 
 
 router = APIRouter(

ai4papi/routers/v1/catalog/tools.py

@@ -3,51 +3,55 @@
 
 from cachetools import cached, TTLCache
 from fastapi import APIRouter, HTTPException
-import requests
 import secrets
+import requests
 
 from ai4papi import quotas
 import ai4papi.conf as papiconf
 from .common import Catalog, retrieve_docker_tags
 
 
-
 @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60))
-def get_list(
+def get_items(
  ):
- """
- Retrieve a list of *all* modules.
+ # Set default branch manually (because we are not yet reading this from submodules)
+ tools_branches= {
+ 'deep-oc-federated-server': 'main',
+ }
 
- This is implemented in a separate function as many functions from this router
- are using this function, so we need to avoid infinite recursions.
- """
+ tools = {}
+ for k in papiconf.TOOLS.keys():
+ tools[k] = {
+ 'url': f'https://github.com/deephdc/{k}',
+ 'branch': tools_branches[k],
+ }
 
- return list(papiconf.TOOLS.keys())
+ return tools
 
 
 @cached(cache=TTLCache(maxsize=1024, ttl=6*60*60))
 def get_metadata(
  item_name: str,
  ):
- """
- Get the module's full metadata.
- """
- # Get default branch
- tools_branches= {
- 'deep-oc-federated-server': 'main',
- }
- branch = tools_branches[item_name]
+ # Check if item is in the items list
+ items = get_items()
+ if item_name not in items.keys():
+ raise HTTPException(
+ status_code=400,
+ detail=f"Item {item_name} not in catalog: {items.keys()}",
+ )
 
- # Retrieve metadata from that branch
+ # Retrieve metadata from default branch
  # Use try/except to avoid that a single module formatting error could take down
  # all the Dashboard
- metadata_url = f"https://raw.githubusercontent.com/deephdc/{item_name}/{branch}/metadata.json"
-
+ branch = items[item_name].get("branch", "master")
+ url = items[item_name]['url'].replace('github.com', 'raw.githubusercontent.com')
+ metadata_url = f"{url}/{branch}/metadata.json"
  try:
  r = requests.get(metadata_url)
  metadata = json.loads(r.text)
-
  except Exception:
+ print(f'Error parsing metadata: {item_name}')
  metadata = {
  "title": item_name,
  "summary": "",
@@ -75,11 +79,6 @@ def get_config(
  item_name: str,
  vo: str,
  ):
- """
- Returns the default configuration (dict) for creating a deployment
- for a specific module. It is prefilled with the appropriate
- docker image and the available docker tags.
- """
  # Retrieve tool configuration
  try:
  conf = deepcopy(papiconf.TOOLS[item_name]['user']['full'])
@@ -89,8 +88,15 @@ def get_config(
  detail=f"{item_name} is not an available tool.",
  )
 
+ # Retrieve tool metadata
+ metadata = get_metadata(item_name)
+
  # Add available Docker tags
- tags = retrieve_docker_tags(item_name)
+ registry = metadata['sources']['docker_registry_repo']
+ repo = registry.split('/')[0]
+ if repo not in ['deephdc', 'ai4oshub']:
+ repo = 'deephdc'
+ tags = retrieve_docker_tags(image=item_name, repo=repo)
  conf["general"]["docker_tag"]["options"] = tags
  conf["general"]["docker_tag"]["value"] = tags[0]
 
@@ -108,12 +114,10 @@ def get_config(
  return conf
 
 
-
 Tools = Catalog()
-Tools.get_list = get_list
+Tools.get_items = get_items
 Tools.get_config = get_config
-Tools.get_metadata = get_metadata
-
+Tools.get_metadata = get_metadata # TODO: inherit the common one, because it's the same for modules and tools
 
 router = APIRouter(
  prefix="/tools",