Restapi 1453 commands log

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.2.1]
+
+### Added
+
+- Added log tracing of SSH commands, filesystem and scheduler interactions with standard JSON format.
+- Added f7tlog for debugging and general usage.
+
 
 ## [2.2.0]
 
@@ -13,9 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `/filesystem/{system_name}/transfer/compress` and `/filesystem/{system_name}/transfer/extract`
   - `compress` operations (on `transfer` and `ops` endpoints) accept `match_pattern` parameter to compress files using `regex` syntax.
 - Added new FirecREST demo image.
-- Added support for private key passphrase. 
+- Added support for private key passphrase.
 ### Changed
-- Images are now built for multiple platforms: inux/amd64, linux/arm64
+- Images are now built for multiple platforms: linux/amd64, linux/arm64
 
 ### Fixed
 

build/demo-launcher/src/launcher/sinfo_command.py

@@ -19,6 +19,9 @@ def __init__(
     ) -> None:
         super().__init__()
 
+    def get_log(self) -> str:
+        return "sinfo"
+
     def get_command(self) -> str:
         return f"timeout {UTILITIES_TIMEOUT} sinfo -V"
 

build/docker/firecrest-api/Dockerfile

@@ -25,20 +25,20 @@ COPY ./requirements.txt ./requirements.txt
 COPY ./requirements-testing.txt ./requirements-testing.txt
 
 
-
 # Build the dependencies
 RUN pip install --no-cache-dir --upgrade -r ./requirements.txt
 RUN if [ "$ENVIRONMENT" = "development" ] ; then pip install --no-cache-dir --upgrade -r ./requirements-testing.txt ; fi
 
-
 # Copy the code
 COPY ./src/lib ./lib
 COPY ./src/firecrest ./firecrest
 
+COPY ./build/environment/firecrest-api-config ./config
+
 # Creates a non-root user with an explicit UID and adds permission to access the /app folder
 # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
 RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
 USER appuser
 
 # Run the app with uvicorn
-CMD ["uvicorn", "firecrest.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000"]
+CMD ["uvicorn", "firecrest.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-config=./config/log-config.yaml"]

build/environment/firecrest-api-config/log-config.yaml

@@ -0,0 +1,57 @@
+version: 1
+disable_existing_loggers: False
+formatters:
+  default:
+    # "()": uvicorn.logging.DefaultFormatter
+    format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+  access:
+    # "()": uvicorn.logging.AccessFormatter
+    format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+  f7tlogformatter:
+    format: '%(asctime)s - F7TLOG - %(levelname)s - %(message)s'
+  jsonformatter:
+    format: '%(asctime)s %(levelname)s %(name)s %(message)s'
+    class: pythonjsonlogger.jsonlogger.JsonFormatter
+handlers:
+  default:
+    formatter: default
+    class: logging.StreamHandler
+    stream: ext://sys.stderr
+  access:
+    formatter: access
+    class: logging.StreamHandler
+    stream: ext://sys.stdout
+  f7tlog_handler:
+    formatter: f7tlogformatter
+    class: logging.StreamHandler
+    stream: ext://sys.stdout
+  json_handler:
+    formatter: jsonformatter
+    class: logging.StreamHandler
+    stream: ext://sys.stdout
+loggers:
+  uvicorn.error:
+    level: INFO
+    handlers:
+      - default
+    propagate: no
+  uvicorn.access:
+    level: INFO
+    handlers:
+      - access
+    propagate: no
+  f7t_v2_log:
+    level: DEBUG
+    handlers:
+      - f7tlog_handler
+    propagate: no
+  f7t_v2_tracing_log:
+    level: INFO
+    handlers:
+      - json_handler
+    propagate: no
+root:
+  level: ERROR
+  handlers:
+    - default
+  propagate: no

src/firecrest/compute/router.py

@@ -8,10 +8,11 @@
 
 # helpers
 from lib.helpers.api_auth_helper import ApiAuthHelper
-
-
 from lib.helpers.router_helper import create_router
 
+# logs
+from lib.loggers.tracing_logs import tracing_log_command
+
 # dependencies
 from firecrest.dependencies import (
     APIAuthDependency,
@@ -57,6 +58,11 @@ async def post_job_submit(
         username=username,
         jwt_token=access_token,
     )
+    tracing_log_command(
+        username,
+        "submit_job",
+        0 if job_id is not None else status.HTTP_500_INTERNAL_SERVER_ERROR,
+    )
     if job_id is None:
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -81,6 +87,7 @@ async def get_jobs(
     username = ApiAuthHelper.get_auth().username
     access_token = ApiAuthHelper.get_access_token()
     jobs = await scheduler_client.get_jobs(username=username, jwt_token=access_token)
+    tracing_log_command(username, "list_jobs", 0)
     return {"jobs": jobs}
 
 
@@ -103,6 +110,11 @@ async def get_job(
     jobs = await scheduler_client.get_job(
         job_id=job_id, username=username, jwt_token=access_token
     )
+    tracing_log_command(
+        username,
+        "job_info",
+        0 if jobs is not None else status.HTTP_404_NOT_FOUND,
+    )
     if jobs is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND, detail="Job not found."
@@ -129,6 +141,11 @@ async def get_job_metadata(
     jobs = await scheduler_client.get_job_metadata(
         job_id=job_id, username=username, jwt_token=access_token
     )
+    tracing_log_command(
+        username,
+        "job_metadata",
+        0 if jobs is not None else status.HTTP_404_NOT_FOUND,
+    )
     if jobs is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND, detail="Job not found."
@@ -158,6 +175,7 @@ async def attach(
         username=username,
         jwt_token=access_token,
     )
+    tracing_log_command(username, "attach_job_process", 0)
     return None
 
 
@@ -179,6 +197,11 @@ async def delete_job_cancel(
     confirmed = await scheduler_client.cancel_job(
         job_id=job_id, username=username, jwt_token=access_token
     )
+    tracing_log_command(
+        username,
+        "cancel_job",
+        0 if confirmed is not None else status.HTTP_404_NOT_FOUND,
+    )
     if confirmed is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND, detail="Job not found"

src/firecrest/config.py

@@ -203,6 +203,11 @@ class Auth(CamelModel):
     authorization: Optional[OpenFGA] = None
 
 
+class Logs(CamelModel):
+    enable_tracing_log: Optional[bool] = False
+    f7tlog_level: Optional[str] = "DEBUG"
+
+
 class Settings(BaseSettings):
     # FastAPI App variables
     app_debug: bool = False
@@ -217,6 +222,7 @@ class Settings(BaseSettings):
     clusters: List[HPCCluster] = []
     # HPC Storage definition
     storage: Optional[Storage] = None
+    logs: Optional[Logs] = Logs()
 
     model_config = SettingsConfigDict(
         env_file=".env",

src/firecrest/filesystem/ops/commands/base64_command.py

@@ -18,6 +18,9 @@ def __init__(self, path: str | None = None, decode: bool = False) -> None:
         self.path = path
         self.decode = decode
 
+    def get_log(self) -> str:
+        return "base64"
+
     def get_command(
         self,
     ) -> str:

src/firecrest/filesystem/ops/commands/checksum_command.py

@@ -29,6 +29,9 @@ def __init__(self, target_path: str = None, algorithm: str = "SHA256") -> None:
         self.selected_algorithm = algorithm
         self.target_path = target_path
 
+    def get_log(self) -> str:
+        return "checksum"
+
     def get_command(self) -> str:
         return f"timeout {UTILITIES_TIMEOUT} {available_algorithms[self.selected_algorithm]} -- '{self.target_path}'"
 

src/firecrest/filesystem/ops/commands/chmod_command.py

@@ -19,6 +19,9 @@ def __init__(self, target_path: str = None, mode: str = None) -> None:
         self.target_path = target_path
         self.mode = mode
 
+    def get_log(self) -> str:
+        return "chmod"
+
     def get_command(self) -> str:
         ls_command = super().get_command()
         return f"timeout {UTILITIES_TIMEOUT} chmod -v '{self.mode}' -- '{self.target_path}' && {ls_command}"

src/firecrest/filesystem/ops/commands/chown_command.py

@@ -22,6 +22,9 @@ def __init__(
         self.owner = owner
         self.group = group
 
+    def get_log(self) -> str:
+        return "chown"
+
     def get_command(self) -> str:
         ls_command = super().get_command()
         return f"timeout {UTILITIES_TIMEOUT} chown -v '{self.owner}':'{self.group}' -- '{self.target_path}' && {ls_command}"

src/firecrest/filesystem/ops/commands/file_command.py

@@ -21,6 +21,9 @@ def __init__(
         super().__init__()
         self.target_path = target_path
 
+    def get_log(self) -> str:
+        return "file"
+
     def get_command(self) -> str:
         return f"timeout {UTILITIES_TIMEOUT} file -b -- '{self.target_path}'"
 

src/firecrest/filesystem/ops/commands/head_command.py

@@ -27,6 +27,9 @@ def __init__(
         self.lines = lines
         self.skip_trailing = skip_trailing
 
+    def get_log(self) -> str:
+        return "head"
+
     def get_command(self) -> str:
         options = ""
         if self.file_bytes:

src/firecrest/filesystem/ops/commands/ls_base_command.py

@@ -43,6 +43,9 @@ def __init__(
                 detail="Either the recursion or the no-recursion option can be set",
             )
 
+    def get_log(self) -> str:
+        return "ls"
+
     def get_command(self) -> str:
         options = "-l --quoting-style=c --time-style='+%Y-%m-%dT%H:%M:%S' "
 

src/firecrest/filesystem/ops/commands/mkdir_command.py

@@ -20,6 +20,9 @@ def __init__(self, target_path: str = None, parent: bool = False) -> None:
         self.target_path = target_path
         self.parent = parent
 
+    def get_log(self) -> str:
+        return "mkdir"
+
     def get_command(self) -> str:
         ls_command = super().get_command()
         options = ""

src/firecrest/filesystem/ops/commands/rm_command.py

@@ -18,6 +18,9 @@ def __init__(self, target_path: str = None) -> None:
         super().__init__()
         self.target_path = target_path
 
+    def get_log(self) -> str:
+        return "rm"
+
     def get_command(self) -> str:
         return f"timeout {UTILITIES_TIMEOUT} rm -r --interactive=never -- '{self.target_path}'"
 

src/firecrest/filesystem/ops/commands/stat_command.py

@@ -20,6 +20,9 @@ def __init__(self, target_path: str = None, dereference: bool = False) -> None:
         self.target_path = target_path
         self.dereference = dereference
 
+    def get_log(self) -> str:
+        return "stat"
+
     def get_command(
         self,
     ) -> str:

src/firecrest/filesystem/ops/commands/symlink_command.py

@@ -19,6 +19,9 @@ def __init__(self, target_path: str = None, link_path: str = None) -> None:
         self.target_path = target_path
         self.link_path = link_path
 
+    def get_log(self) -> str:
+        return "ln"
+
     def get_command(self) -> str:
         ls_command = super().get_command()
         return f"timeout {UTILITIES_TIMEOUT} ln -s -- '{self.target_path}' '{self.link_path}' && {ls_command}"

src/firecrest/filesystem/ops/commands/tail_command.py

@@ -27,6 +27,9 @@ def __init__(
         self.lines = lines
         self.skip_heading = skip_heading
 
+    def get_log(self) -> str:
+        return "tail"
+
     def get_command(
         self,
     ) -> str:

src/firecrest/filesystem/ops/commands/tar_command.py

@@ -34,6 +34,9 @@ def __init__(
         self.dereference = dereference
         self.operation = operation
 
+    def get_log(self) -> str:
+        return "tar"
+
     def get_command(
         self,
     ) -> str:
@@ -57,7 +60,6 @@ def get_compress_command(self) -> str:
 
         return f"timeout {UTILITIES_TIMEOUT} tar {options} -czvf '{self.target_path}' -C '{source_dir}' '{source_file}'"
 
-
     def get_extract_command(self) -> str:
 
         return f"timeout {UTILITIES_TIMEOUT} tar -xzf '{self.source_path}' -C '{self.target_path}'"

src/firecrest/filesystem/ops/commands/view_command.py

@@ -19,6 +19,9 @@ def __init__(self, target_path: str = None) -> None:
         super().__init__()
         self.target_path = target_path
 
+    def get_log(self) -> str:
+        return "view"
+
     def get_command(self) -> str:
         return f"timeout {UTILITIES_TIMEOUT} head --bytes {SIZE_LIMIT} -- '{self.target_path}'"