Feature/2023.07 os test refactor test cases

.gitignore

@@ -0,0 +1,2 @@
+# Credentials / Secrets
+credentials.env

README.md

@@ -1,2 +1,2 @@
 # corporate_data_extraction
-The code in this repo is undergoing testing.  It is not verified as fully functional yet.
+The code in this repo has been verified as working in Linux and Windows environments.  Team is actively testing it in an Openshift evironment.

data_extractor/.gitignore

@@ -0,0 +1 @@
+__pycache__

data_extractor/code/coordinator/Dockerfile

@@ -0,0 +1,41 @@
+FROM ubuntu:20.04
+SHELL ["/bin/bash", "-c"]
+
+# no prompt during installation:
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update
+RUN apt-get install -y apt-utils wget python3 python3-pip vim
+
+COPY ./code/coordinator/requirements_coordinator.txt /app/code/requirements.txt
+COPY ./code/coordinator/entry_coordinator.sh /app/code/entry.sh
+COPY ./code/coordinator/server_coordinator.py /app/code/server_coordinator.py
+COPY ./code/infer_on_pdf.py /app/code/infer_on_pdf.py
+COPY ./code/train_on_pdf.py /app/code/train_on_pdf.py
+COPY ./code/kpi_mapping.csv /app/code/kpi_mapping.csv
+COPY ./code/setup_project.py /app/code/setup_project.py
+COPY ./code/config_path.py /app/code/config_path.py
+COPY ./code/s3_communication.py /app/code/s3_communication.py
+COPY ./data /app/data
+COPY ./models /app/models
+
+WORKDIR /app/code
+
+RUN pip install -r requirements.txt
+
+RUN chgrp -R 0 /app/code && chmod -R g=u /app/code
+RUN chmod -R a+x /app/code
+
+RUN mkdir -p /app/server_logs
+
+RUN chmod -R 777 /app/server_logs
+RUN chmod -R 777 /app/data
+RUN chmod -R 777 /app/models
+
+# Adding vs code server
+RUN mkdir -p /.vscode-server
+RUN mkdir -p /.vscode-remote
+RUN chmod -R 777 /.vscode-server
+RUN chmod -R 777 /.vscode-remote
+
+CMD ./entry.sh

data_extractor/code/coordinator/entry_coordinator.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+date_time=$(date +'%d_%h_%Y:%H_%M')
+log_file_path="/app/server_logs/coordinator_logs_${date_time}.txt"
+cd /app/code
+python3 server_coordinator.py > $log_file_path 2>&1 &
+
+sleep infinity

data_extractor/code/coordinator/requirements_coordinator.txt

@@ -0,0 +1,6 @@
+requests~=2.31.0
+Flask==2.3.2
+pandas~=2.0.3
+pyyaml~=6.0.1
+boto3~=1.28.8
+openpyxl~=3.1.2

data_extractor/code/coordinator/server_coordinator.py

@@ -0,0 +1,150 @@
+import argparse
+import pathlib
+import json
+import os
+import traceback
+
+from flask import Flask, Response, request
+
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent.parent.parent
+DATA_FOLDER = ROOT / "data"
+MODEL_FOLDER = ROOT / "models"
+
+app = Flask(__name__)
+
+
+@app.route("/liveness")
+def liveness():
+    return Response(response={}, status=200)
+
+
+@app.route("/train")
+def train():
+    """ This function should start the train_on_pdf.py with given parameters as a web access point.
+
+    :return:
+    """
+    parser_train = argparse.ArgumentParser(description='End-to-end training')
+
+    parser_train.add_argument('--project_name',
+                              type=str,
+                              default=None,
+                              help='Name of the Project')
+
+    parser_train.add_argument('--s3_usage',
+                              type=str,
+                              default=None,
+                              help='Do you want to use S3? Type either Y or N.')
+
+    # Read arguments from direct python call
+    args_train = parser_train.parse_args()
+    try:
+        project_name = args_train.project_name
+        s3_usage = args_train.s3_usage
+    except AttributeError:
+        pass
+
+    # Read arguments from wget call
+    if project_name is None or s3_usage is None:
+        try:
+            project_name = request.args.get("project_name")
+            s3_usage = request.args.get("s3_usage")
+        except AttributeError:
+            pass
+
+    # Read arguments from payload if given
+    if project_name is None or s3_usage is None:
+        try:
+            args_train = json.loads(request.args['payload'])
+            project_name = args_train["project_name"]
+            s3_usage = args_train["s3_usage"]
+        except Exception:
+            msg = "Project name or s3_usage where not given via command or payload. Please recheck your call."
+            return Response(msg, status=500)
+
+    cmd = 'python3 train_on_pdf.py' + \
+          ' --project_name "' + project_name + '"' + \
+          ' --s3_usage "' + s3_usage + '"'
+    print("Running command: " + cmd)
+    try:
+        os.system(cmd)
+    except Exception as e:
+        msg = "Error during train_on_pdf.py \nException:" + str(repr(e) + traceback.format_exc())
+        return Response(msg, status=500)
+    msg = "train_on_pdf.py was executed without any error. Check the results please."
+    return Response(msg, status=200)
+
+
+@app.route("/infer")
+def infer():
+    """ This function should start the infer_on_pdf.py with given parameters (either via cli arguments or via
+     payload) as a web access point.
+
+    :return: Response type containing a message and the int for the type of message (200 if ok, 500 if error)
+    """
+    parser_infer = argparse.ArgumentParser(description='End-to-end inference')
+
+    parser_infer.add_argument('--project_name',
+                              type=str,
+                              default=None,
+                              help='Name of the Project')
+
+    parser_infer.add_argument('--s3_usage',
+                              type=str,
+                              default=None,
+                              help='Do you want to use S3? Type either Y or N.')
+
+    parser_infer.add_argument('--mode',
+                              type=str,
+                              default='both',
+                              help='Inference Mode (RB, ML, both, or none - for just doing postprocessing)')
+
+    args_infer = parser_infer.parse_args()
+    project_name = args_infer.project_name
+    s3_usage = args_infer.s3_usage
+    mode = args_infer.mode
+
+    # Read arguments from wget call
+    if project_name is None or s3_usage is None:
+        try:
+            project_name = request.args.get("project_name")
+            s3_usage = request.args.get("s3_usage")
+            mode = request.args.get("mode")
+        except AttributeError:
+            pass
+
+    # Read arguments from payload if given
+    if project_name is None or s3_usage is None:
+        try:
+            args_infer = json.loads(request.args['payload'])
+            project_name = args_infer["project_name"]
+            s3_usage = args_infer["s3_usage"]
+            mode = args_infer["mode"]
+        except Exception:
+            msg = "Project name, mode or s3_usage where not given via command or payload. Please recheck your call."
+            return Response(msg, status=500)
+
+    cmd = 'python3 infer_on_pdf.py' + \
+          ' --project_name "' + project_name + '"' + \
+          ' --mode "' + mode + '"' + \
+          ' --s3_usage "' + s3_usage + '"'
+    print("Running command: " + cmd)
+    try:
+        os.system(cmd)
+    except Exception as e:
+        msg = "Error during infer_on_pdf.py \nException:" + str(repr(e) + traceback.format_exc())
+        return Response(msg, status=500)
+    msg = "infer_on_pdf.py was executed without any error. Check the results please."
+    return Response(msg, status=200)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='coordinator server')
+    parser.add_argument('--port',
+                        type=int,
+                        default=2000,
+                        help='Port to use for the coordinator server')
+    args = parser.parse_args()
+    port = args.port
+    app.run(host="0.0.0.0", port=port)

data_extractor/code/esg_data_pipeline/Dockerfile

@@ -16,35 +16,25 @@ RUN apt-key del 7fa2af80
 RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
 
 # Added poppler-utils, default-jre installations
-RUN apt-get update && apt-get install -y git vim ninja-build poppler-utils default-jre \
+RUN apt-get update && apt-get install -y git wget vim ninja-build poppler-utils default-jre \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Install mmdetection
-RUN conda clean --all
-RUN git clone --branch v1.2.0 https://github.com/open-mmlab/mmdetection.git /mmdetection
-WORKDIR /mmdetection
-ENV FORCE_CUDA="1"
-RUN pip install cython --no-cache-dir
-RUN pip install "git+https://github.com/open-mmlab/cocoapi.git#subdirectory=pycocotools"
-RUN pip install --no-cache-dir -e .
-
-# Reinstall openmm-mmcv to use the right version, somehow the above installation
-# of mmdetection installs mmcv of an updated version which is not compatible for
-# mmdetection v1.2
-RUN pip install mmcv==0.5.7
-
 COPY ./esg_data_pipeline /app/code/esg_data_pipeline
+COPY ./s3_communication.py /app/code/esg_data_pipeline/esg_data_pipeline/s3_communication.py
 
-RUN chgrp -R 0 /app/code/esg_data_pipeline && chmod g=u /app/code/esg_data_pipeline
-RUN chmod -R a+x /app/code/esg_data_pipeline
+RUN chgrp -R 0 /app/code && chmod g=u /app/code
+RUN chmod -R 777 /app/code
 
 WORKDIR /app/code/esg_data_pipeline
 
 RUN pip install -e .
+
 RUN mkdir -p /app/server_logs
+RUN chmod -R 777 /app/server_logs
 
-USER 1234
+RUN mkdir -p /app/data
+RUN chmod -R 777 /app/data
 
 CMD ./entry.sh
 

data_extractor/code/esg_data_pipeline/entry.sh

@@ -8,4 +8,4 @@ python3 extraction_server.py > $log_file_path 2>&1 &
 #cd /esg_data_pipeline/notebooks
 #jupyter notebook --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.token='' --NotebookApp.password=''
 
-sleep infinity
+sleep infinity

data_extractor/code/esg_data_pipeline/esg_data_pipeline/components/text_curator.py

@@ -72,7 +72,7 @@ def run(self, extraction_folder, annotation_excels, output_folder):
 
         df_result = pd.DataFrame(examples_list).reset_index(drop=True)
         # Drop the unnecessary column.
-        df_result.drop(["Index"], axis=1, inplace=True)
+        # df_result.drop(["Index"], axis=1, inplace=True)
 
         # Map the KPI to KPI questions
         importlib.reload(kpi_mapping)